github.com/voedger/voedger@v0.0.0-20240520144910-273e84102129/pkg/iextengine/wazero/impl_test.go (about)

     1  /*
     2    - Copyright (c) 2023-present unTill Software Development Group B V.
     3      @author Michael Saigachenko
     4  */
     5  
     6  package iextenginewazero
     7  
     8  import (
     9  	"context"
    10  	"embed"
    11  	"fmt"
    12  	"net/url"
    13  	"testing"
    14  	"time"
    15  
    16  	"github.com/stretchr/testify/require"
    17  	"github.com/tetratelabs/wazero/sys"
    18  
    19  	"github.com/voedger/voedger/pkg/appdef"
    20  	"github.com/voedger/voedger/pkg/iextengine"
    21  	"github.com/voedger/voedger/pkg/iratesce"
    22  	"github.com/voedger/voedger/pkg/istorage/mem"
    23  	istorageimpl "github.com/voedger/voedger/pkg/istorage/provider"
    24  	"github.com/voedger/voedger/pkg/parser"
    25  	"github.com/voedger/voedger/pkg/state/safestate"
    26  	"github.com/voedger/voedger/pkg/sys/authnz"
    27  
    28  	"github.com/voedger/voedger/pkg/istructs"
    29  	"github.com/voedger/voedger/pkg/istructsmem"
    30  	payloads "github.com/voedger/voedger/pkg/itokens-payloads"
    31  	"github.com/voedger/voedger/pkg/itokensjwt"
    32  	"github.com/voedger/voedger/pkg/state"
    33  )
    34  
    35  var extIO = &mockIo{}
    36  var plogOffset istructs.Offset
    37  var wlogOffset istructs.Offset
    38  var newWorkspaceCmd = appdef.NewQName("sys", "NewWorkspace")
    39  var testView = appdef.NewQName(testPkg, "TestView")
    40  var dummyCommand = appdef.NewQName(testPkg, "Dummy")
    41  var dummyProj = appdef.NewQName(testPkg, "DummyProj")
    42  var testWorkspaceDescriptor = appdef.NewQName(testPkg, "RestaurantDescriptor")
    43  
    44  var testApp = istructs.AppQName_test1_app1
    45  
    46  const testPkg = "mypkg"
    47  const ws = istructs.WSID(1)
    48  const partition = istructs.PartitionID(1)
    49  
    50  func Test_BasicUsage(t *testing.T) {
    51  	// Test Consts
    52  	const intentsLimit = 5
    53  	const bundlesLimit = 5
    54  	require := require.New(t)
    55  	newOrderCmd := appdef.NewQName(testPkg, "NewOrder")
    56  	calcOrderedItemsProjector := appdef.NewQName(testPkg, "CalcOrderedItems")
    57  	orderedItemsView := appdef.NewQName(testPkg, "OrderedItems")
    58  
    59  	// Prepare app
    60  	app := appStructsFromSQL("github.com/untillpro/airs-bp3/packages/"+testPkg, `APPLICATION test();
    61  		WORKSPACE Restaurant (
    62  			DESCRIPTOR RestaurantDescriptor ();
    63  			TABLE Order INHERITS ODoc (
    64  				Year int32,
    65  				Month int32,
    66  				Day int32,
    67  				Waiter ref,
    68  				Items TABLE OrderItems (
    69  					Quantity int32,
    70  					SinglePrice currency,
    71  					Article ref
    72  				)
    73  			);
    74  			VIEW OrderedItems (
    75  				Year int32,
    76  				Month int32,
    77  				Day int32,
    78  				Amount currency,
    79  				PRIMARY KEY ((Year), Month, Day)
    80  			) AS RESULT OF CalcOrderedItems;
    81  			EXTENSION ENGINE WASM(
    82  				COMMAND NewOrder(Order);
    83  				PROJECTOR CalcOrderedItems AFTER EXECUTE ON NewOrder INTENTS(View(OrderedItems));
    84  			);
    85  		)
    86  		`,
    87  		func(cfg *istructsmem.AppConfigType) {
    88  			cfg.Resources.Add(istructsmem.NewCommandFunction(newOrderCmd, istructsmem.NullCommandExec))
    89  			cfg.AddAsyncProjectors(istructs.Projector{Name: calcOrderedItemsProjector})
    90  		})
    91  
    92  	// Build NewOrder event
    93  	reb := app.Events().GetNewRawEventBuilder(istructs.NewRawEventBuilderParams{
    94  		GenericRawEventBuilderParams: istructs.GenericRawEventBuilderParams{
    95  			Workspace:         ws,
    96  			HandlingPartition: partition,
    97  			PLogOffset:        plogOffset + 1,
    98  			QName:             newOrderCmd,
    99  			WLogOffset:        wlogOffset + 1,
   100  		},
   101  	})
   102  	orderBuilder := reb.ArgumentObjectBuilder()
   103  	orderBuilder.PutRecordID(appdef.SystemField_ID, 1)
   104  	orderBuilder.PutInt32("Year", 2023)
   105  	orderBuilder.PutInt32("Month", 1)
   106  	orderBuilder.PutInt32("Day", 1)
   107  	items := orderBuilder.ChildBuilder("Items")
   108  	items.PutRecordID(appdef.SystemField_ID, 2)
   109  	items.PutInt32("Quantity", 1)
   110  	items.PutInt64("SinglePrice", 100)
   111  	items = orderBuilder.ChildBuilder("Items")
   112  	items.PutRecordID(appdef.SystemField_ID, 3)
   113  	items.PutInt32("Quantity", 2)
   114  	items.PutInt64("SinglePrice", 50)
   115  	rawEvent, err := reb.BuildRawEvent()
   116  	if err != nil {
   117  		panic(err)
   118  	}
   119  	event, err := app.Events().PutPlog(rawEvent, nil, istructsmem.NewIDGenerator())
   120  	if err != nil {
   121  		panic(err)
   122  	}
   123  
   124  	eventFunc := func() istructs.IPLogEvent { return event }
   125  	cudFunc := func() istructs.ICUD { return reb.CUDBuilder() }
   126  	argFunc := func() istructs.IObject { return event.ArgumentObject() }
   127  	unloggedArgFunc := func() istructs.IObject { return nil }
   128  	appFunc := func() istructs.IAppStructs { return app }
   129  	wlogOffsetFunc := func() istructs.Offset { return event.WLogOffset() }
   130  
   131  	// Create states for Command processor and Actualizer
   132  	actualizerState := state.ProvideAsyncActualizerStateFactory()(context.Background(), appFunc, nil, state.SimpleWSIDFunc(ws), nil, nil, eventFunc, nil, nil, intentsLimit, bundlesLimit)
   133  	cmdProcState := state.ProvideCommandProcessorStateFactory()(context.Background(), appFunc, nil, state.SimpleWSIDFunc(ws), nil, cudFunc, nil, nil, intentsLimit, nil, argFunc, unloggedArgFunc, wlogOffsetFunc)
   134  
   135  	// Create extension package from WASM
   136  	ctx := context.Background()
   137  	moduleUrl := testModuleURL("./_testdata/basicusage/pkg.wasm")
   138  	packages := []iextengine.ExtensionPackage{
   139  		{
   140  			QualifiedName:  testPkg,
   141  			ModuleUrl:      moduleUrl,
   142  			ExtensionNames: []string{calcOrderedItemsProjector.Entity(), newOrderCmd.Entity()},
   143  		},
   144  	}
   145  
   146  	// Create extension engine
   147  	factory := ProvideExtensionEngineFactory(true)
   148  	engines, err := factory.New(ctx, app.AppQName(), packages, &iextengine.ExtEngineConfig{}, 1)
   149  	if err != nil {
   150  		panic(err)
   151  	}
   152  	extEngine := engines[0]
   153  	defer extEngine.Close(ctx)
   154  	//
   155  	// Invoke command
   156  	//
   157  	err = extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, newOrderCmd.Entity()), cmdProcState)
   158  	require.NoError(err)
   159  	//
   160  	// Invoke projector
   161  	//
   162  	err = extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, calcOrderedItemsProjector.Entity()), actualizerState)
   163  	require.NoError(err)
   164  	ready, err := actualizerState.ApplyIntents()
   165  	require.NoError(err)
   166  	require.False(ready)
   167  
   168  	// Invoke projector again
   169  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, calcOrderedItemsProjector.Entity()), actualizerState))
   170  	ready, err = actualizerState.ApplyIntents()
   171  	require.NoError(err)
   172  	require.False(ready)
   173  
   174  	// Flush bundles with intents
   175  	require.NoError(actualizerState.FlushBundles())
   176  
   177  	// Test view, must be calculated from 2 events
   178  	kb := app.ViewRecords().KeyBuilder(orderedItemsView)
   179  	kb.PartitionKey().PutInt32("Year", 2023)
   180  	kb.ClusteringColumns().PutInt32("Month", 1)
   181  	kb.ClusteringColumns().PutInt32("Day", 1)
   182  	value, err := app.ViewRecords().Get(ws, kb)
   183  
   184  	require.NoError(err)
   185  	require.NotNil(value)
   186  	require.Equal(int64(400), value.AsInt64("Amount"))
   187  }
   188  
   189  func appStructs(appDef appdef.IAppDefBuilder, prepareAppCfg appCfgCallback) istructs.IAppStructs {
   190  	cfgs := make(istructsmem.AppConfigsType, 1)
   191  	cfg := cfgs.AddConfig(istructs.AppQName_test1_app1, appDef)
   192  	cfg.SetNumAppWorkspaces(istructs.DefaultNumAppWorkspaces)
   193  	if prepareAppCfg != nil {
   194  		prepareAppCfg(cfg)
   195  		cfg.Resources.Add(istructsmem.NewCommandFunction(newWorkspaceCmd, istructsmem.NullCommandExec))
   196  	}
   197  
   198  	asf := mem.Provide()
   199  	storageProvider := istorageimpl.Provide(asf)
   200  	prov := istructsmem.Provide(
   201  		cfgs,
   202  		iratesce.TestBucketsFactory,
   203  		payloads.ProvideIAppTokensFactory(itokensjwt.TestTokensJWT()),
   204  		storageProvider)
   205  	structs, err := prov.AppStructs(istructs.AppQName_test1_app1)
   206  	if err != nil {
   207  		panic(err)
   208  	}
   209  	return structs
   210  }
   211  
   212  func requireMemStat(t *testing.T, wasmEngine *wazeroExtEngine, mallocs, frees, heapInUse uint32) {
   213  	m, err := wasmEngine.getMallocs(testPkg, context.Background())
   214  	require.NoError(t, err)
   215  	f, err := wasmEngine.getFrees(testPkg, context.Background())
   216  	require.NoError(t, err)
   217  	h, err := wasmEngine.getHeapinuse(testPkg, context.Background())
   218  	require.NoError(t, err)
   219  
   220  	require.Equal(t, mallocs, uint32(m))
   221  	require.Equal(t, frees, uint32(f))
   222  	require.Equal(t, heapInUse, uint32(h))
   223  }
   224  
   225  func requireMemStatEx(t *testing.T, wasmEngine *wazeroExtEngine, mallocs, frees, heapSys, heapInUse uint32) {
   226  	requireMemStat(t, wasmEngine, mallocs, frees, heapInUse)
   227  	h, err := wasmEngine.getHeapSys(testPkg, context.Background())
   228  	require.NoError(t, err)
   229  	require.Equal(t, heapSys, uint32(h))
   230  }
   231  
   232  func testFactoryHelper(ctx context.Context, moduleUrl *url.URL, funcs []string, cfg iextengine.ExtEngineConfig, compile bool) (iextengine.IExtensionEngine, error) {
   233  	packages := []iextengine.ExtensionPackage{
   234  		{
   235  			QualifiedName:  testPkg,
   236  			ModuleUrl:      moduleUrl,
   237  			ExtensionNames: funcs,
   238  		},
   239  	}
   240  	engines, err := ProvideExtensionEngineFactory(compile).New(ctx, testApp, packages, &cfg, 1)
   241  	if err != nil {
   242  		return nil, err
   243  	}
   244  	return engines[0], nil
   245  }
   246  
   247  func Test_Allocs_ManualGC(t *testing.T) {
   248  
   249  	const arrAppend = "arrAppend"
   250  	const arrReset = "arrReset"
   251  
   252  	require := require.New(t)
   253  	ctx := context.Background()
   254  	WasmPreallocatedBufferSize = 1000000
   255  	moduleUrl := testModuleURL("./_testdata/allocs/pkggc.wasm")
   256  	extEngine, err := testFactoryHelper(ctx, moduleUrl, []string{arrAppend, arrReset}, iextengine.ExtEngineConfig{}, false)
   257  	require.NoError(err)
   258  	defer extEngine.Close(ctx)
   259  
   260  	wasmEngine := extEngine.(*wazeroExtEngine)
   261  
   262  	const expectedHeapSize = uint32(1999568)
   263  
   264  	requireMemStatEx(t, wasmEngine, 1, 0, expectedHeapSize, WasmPreallocatedBufferSize)
   265  
   266  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend), extIO))
   267  	requireMemStatEx(t, wasmEngine, 3, 0, expectedHeapSize, WasmPreallocatedBufferSize+32)
   268  
   269  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend), extIO))
   270  	requireMemStatEx(t, wasmEngine, 5, 0, expectedHeapSize, WasmPreallocatedBufferSize+64)
   271  
   272  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend), extIO))
   273  	requireMemStatEx(t, wasmEngine, 7, 0, expectedHeapSize, WasmPreallocatedBufferSize+6*16)
   274  
   275  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrReset), extIO))
   276  	requireMemStatEx(t, wasmEngine, 7, 0, expectedHeapSize, WasmPreallocatedBufferSize+6*16)
   277  
   278  	require.NoError(wasmEngine.gc(testPkg, ctx))
   279  	requireMemStatEx(t, wasmEngine, 7, 6, expectedHeapSize, WasmPreallocatedBufferSize)
   280  }
   281  
   282  func Test_Allocs_AutoGC(t *testing.T) {
   283  
   284  	const arrAppend = "arrAppend"
   285  	const arrReset = "arrReset"
   286  	WasmPreallocatedBufferSize = 1000000
   287  	require := require.New(t)
   288  	ctx := context.Background()
   289  	moduleUrl := testModuleURL("./_testdata/allocs/pkggc.wasm")
   290  	extEngine, err := testFactoryHelper(ctx, moduleUrl, []string{arrAppend, arrReset}, iextengine.ExtEngineConfig{MemoryLimitPages: 0xffff}, false)
   291  	require.NoError(err)
   292  	defer extEngine.Close(ctx)
   293  
   294  	const expectedHeapSize = uint32(1999568)
   295  	var expectedAllocs = uint32(1)
   296  	var expectedFrees = uint32(0)
   297  	wasmEngine := extEngine.(*wazeroExtEngine)
   298  
   299  	requireMemStatEx(t, wasmEngine, expectedAllocs, expectedFrees, expectedHeapSize, WasmPreallocatedBufferSize)
   300  
   301  	calculatedHeapInUse := WasmPreallocatedBufferSize
   302  	for calculatedHeapInUse < expectedHeapSize-16 {
   303  		require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend), extIO))
   304  		require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrReset), extIO))
   305  		calculatedHeapInUse += 32
   306  		expectedAllocs += 2
   307  	}
   308  
   309  	// no GC has been called, and the heap is now full
   310  	requireMemStatEx(t, wasmEngine, expectedAllocs, expectedFrees, expectedHeapSize, expectedHeapSize-16)
   311  
   312  	// next call will trigger auto-GC
   313  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend), extIO))
   314  	expectedAllocs += 2
   315  	expectedFrees += expectedAllocs - 7
   316  	requireMemStat(t, wasmEngine, expectedAllocs, expectedFrees, WasmPreallocatedBufferSize+96)
   317  
   318  	// next call will not trigger auto-GC
   319  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrReset), extIO))
   320  	requireMemStat(t, wasmEngine, expectedAllocs, expectedFrees, WasmPreallocatedBufferSize+96) // stays the same
   321  
   322  }
   323  
   324  func Test_NoGc_MemoryOverflow(t *testing.T) {
   325  
   326  	const arrAppend = "arrAppend"
   327  	const arrReset = "arrReset"
   328  	WasmPreallocatedBufferSize = 1000000
   329  
   330  	require := require.New(t)
   331  	ctx := context.Background()
   332  
   333  	moduleUrl := testModuleURL("./_testdata/allocs/pkg.wasm")
   334  	extEngine, err := testFactoryHelper(ctx, moduleUrl, []string{}, iextengine.ExtEngineConfig{MemoryLimitPages: 0x10}, false)
   335  	require.ErrorContains(err, "the minimum limit of memory is: 1700000.0 bytes, requested limit is: 1048576.0")
   336  	require.Nil(extEngine)
   337  
   338  	extEngine, err = testFactoryHelper(ctx, moduleUrl, []string{arrAppend, arrReset}, iextengine.ExtEngineConfig{MemoryLimitPages: 0x20}, false)
   339  	require.NoError(err)
   340  	defer extEngine.Close(ctx)
   341  
   342  	wasmEngine := extEngine.(*wazeroExtEngine)
   343  	wasmEngine.autoRecover = false
   344  
   345  	var expectedAllocs = uint32(1)
   346  	var expectedFrees = uint32(0)
   347  
   348  	requireMemStatEx(t, wasmEngine, expectedAllocs, expectedFrees, WasmPreallocatedBufferSize, WasmPreallocatedBufferSize)
   349  
   350  	calculatedHeapInUse := WasmPreallocatedBufferSize
   351  	err = nil
   352  	for calculatedHeapInUse < 0x20*iextengine.MemoryPageSize {
   353  		err = wasmEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend), extIO)
   354  		if err != nil {
   355  			break
   356  		}
   357  		err = wasmEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrReset), extIO)
   358  		if err != nil {
   359  			break
   360  		}
   361  		calculatedHeapInUse += 32
   362  	}
   363  	require.EqualError(err, "panic: runtime error: out of memory")
   364  }
   365  
   366  func Test_SetLimitsExecutionInterval(t *testing.T) {
   367  
   368  	require := require.New(t)
   369  	ctx := context.Background()
   370  	moduleUrl := testModuleURL("./_testdata/allocs/pkg.wasm")
   371  	extEngine, err := testFactoryHelper(ctx, moduleUrl, []string{"longFunc"}, iextengine.ExtEngineConfig{}, false)
   372  	require.NoError(err)
   373  	defer extEngine.Close(ctx)
   374  
   375  	maxDuration := time.Millisecond * 50
   376  	ctxTimeout, cancel := context.WithTimeout(context.Background(), maxDuration)
   377  	defer cancel()
   378  
   379  	t0 := time.Now()
   380  	err = extEngine.Invoke(ctxTimeout, appdef.NewFullQName(testPkg, "longFunc"), extIO)
   381  
   382  	require.ErrorIs(err, sys.NewExitError(sys.ExitCodeDeadlineExceeded))
   383  	require.Less(time.Since(t0), maxDuration*4)
   384  }
   385  
   386  type panicsUnit struct {
   387  	name   string
   388  	expect string
   389  }
   390  
   391  func Test_HandlePanics(t *testing.T) {
   392  
   393  	WasmPreallocatedBufferSize = 1000000
   394  	tests := []panicsUnit{
   395  		{"incorrectStorageQname", "convert error: string «foo» to QName"},
   396  		{"incorrectEntityQname", "convert error: string «abc» to QName"},
   397  		{"unsupportedStorage", "unsupported storage"},
   398  		{"incorrectKeyBuilder", safestate.PanicIncorrectKeyBuilder},
   399  		{"canExistIncorrectKey", safestate.PanicIncorrectKeyBuilder},
   400  		{"mustExistIncorrectKey", safestate.PanicIncorrectKeyBuilder},
   401  		{"readIncorrectKeyBuilder", safestate.PanicIncorrectKeyBuilder},
   402  		{"incorrectKey", safestate.PanicIncorrectKey},
   403  		{"incorrectValue", safestate.PanicIncorrectValue},
   404  		{"incorrectValue2", safestate.PanicIncorrectValue},
   405  		{"incorrectValue3", safestate.PanicIncorrectValue},
   406  		{"mustExist", state.ErrNotExists.Error()},
   407  		{"incorrectKeyBuilderOnNewValue", safestate.PanicIncorrectKeyBuilder},
   408  		{"incorrectKeyBuilderOnUpdateValue", safestate.PanicIncorrectKeyBuilder},
   409  		{"incorrectValueOnUpdateValue", safestate.PanicIncorrectValue},
   410  		{"incorrectIntentId", safestate.PanicIncorrectIntent},
   411  		{"readPanic", safestate.PanicIncorrectValue},
   412  		{"readError", errTestIOError.Error()},
   413  		{"queryError", errTestIOError.Error()},
   414  		{"newValueError", errTestIOError.Error()},
   415  		{"updateValueError", errTestIOError.Error()},
   416  		{"asStringMemoryOverflow", "runtime error: out of memory"},
   417  	}
   418  
   419  	extNames := make([]string, 0, len(tests))
   420  	for _, test := range tests {
   421  		extNames = append(extNames, test.name)
   422  	}
   423  
   424  	require := require.New(t)
   425  	ctx := context.Background()
   426  	moduleUrl := testModuleURL("./_testdata/panics/pkg.wasm")
   427  	extEngine, err := testFactoryHelper(ctx, moduleUrl, extNames, iextengine.ExtEngineConfig{MemoryLimitPages: 0x20}, false)
   428  	require.NoError(err)
   429  	defer extEngine.Close(ctx)
   430  
   431  	for _, test := range tests {
   432  		e := extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, test.name), extIO)
   433  		require.ErrorContains(e, test.expect)
   434  	}
   435  }
   436  
   437  func Test_QueryValue(t *testing.T) {
   438  	const testQueryValue = "testQueryValue"
   439  
   440  	require := require.New(t)
   441  	ctx := context.Background()
   442  	moduleUrl := testModuleURL("./_testdata/tests/pkg.wasm")
   443  	extEngine, err := testFactoryHelper(ctx, moduleUrl, []string{testQueryValue}, iextengine.ExtEngineConfig{MemoryLimitPages: 0x20}, false)
   444  	require.NoError(err)
   445  	defer extEngine.Close(ctx)
   446  
   447  	err = extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, testQueryValue), extIO)
   448  	require.NoError(err)
   449  }
   450  
   451  func Test_RecoverEngine(t *testing.T) {
   452  
   453  	testWithPreallocatedBuffer := func(t *testing.T, preallocatedBufferSize uint32) {
   454  		t.Run(fmt.Sprintf("PreallocatedBufferSize=%d", preallocatedBufferSize), func(t *testing.T) {
   455  			WasmPreallocatedBufferSize = preallocatedBufferSize
   456  			const arrAppend2 = "arrAppend2"
   457  			require := require.New(t)
   458  			ctx := context.Background()
   459  			moduleUrl := testModuleURL("./_testdata/allocs/pkg.wasm")
   460  			extEngine, err := testFactoryHelper(ctx, moduleUrl, []string{arrAppend2}, iextengine.ExtEngineConfig{MemoryLimitPages: 0x20}, true)
   461  			require.NoError(err)
   462  			defer extEngine.Close(ctx)
   463  			we := extEngine.(*wazeroExtEngine)
   464  
   465  			require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend2), extIO))
   466  			require.Equal(0, we.numAutoRecovers) // no auto-recover during first invoke
   467  			heapInUseAfterFirstInvoke, err := we.getHeapinuse(testPkg, context.Background())
   468  			require.NoError(err)
   469  
   470  			for recoverNo := 0; recoverNo < 10; recoverNo++ { // 10 recover cycles
   471  				for run := 1; we.numAutoRecovers == recoverNo; { // run until auto-recover is triggered{
   472  					require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend2), extIO))
   473  					run++
   474  				}
   475  
   476  				require.Equal(recoverNo+1, we.numAutoRecovers)
   477  				heapInUseAfterRecover, err := we.getHeapinuse(testPkg, context.Background())
   478  				require.NoError(err)
   479  				require.Equal(heapInUseAfterRecover, heapInUseAfterFirstInvoke)
   480  			}
   481  		})
   482  	}
   483  
   484  	testWithPreallocatedBuffer(t, 1000000)
   485  	testWithPreallocatedBuffer(t, 900000)
   486  	testWithPreallocatedBuffer(t, 800000)
   487  	testWithPreallocatedBuffer(t, 700000)
   488  
   489  	testWithPreallocatedBuffer(t, 600000)
   490  	testWithPreallocatedBuffer(t, 500000)
   491  
   492  	testWithPreallocatedBuffer(t, 200000)
   493  	testWithPreallocatedBuffer(t, 100000)
   494  	testWithPreallocatedBuffer(t, WasmDefaultPreallocatedBufferSize)
   495  }
   496  
   497  func Test_RecoverEngine2(t *testing.T) {
   498  
   499  	WasmPreallocatedBufferSize = 1000000
   500  	const arrAppend2 = "arrAppend2"
   501  	require := require.New(t)
   502  	ctx := context.Background()
   503  	moduleUrl := testModuleURL("./_testdata/allocs/pkg.wasm")
   504  	extEngine, err := testFactoryHelper(ctx, moduleUrl, []string{arrAppend2}, iextengine.ExtEngineConfig{MemoryLimitPages: 0x20}, true)
   505  	require.NoError(err)
   506  	defer extEngine.Close(ctx)
   507  	we := extEngine.(*wazeroExtEngine)
   508  
   509  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend2), extIO))
   510  	require.Equal(0, we.numAutoRecovers)
   511  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend2), extIO))
   512  	require.Equal(0, we.numAutoRecovers)
   513  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend2), extIO))
   514  	require.Equal(0, we.numAutoRecovers)
   515  
   516  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend2), extIO))
   517  	require.Equal(1, we.numAutoRecovers)
   518  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend2), extIO))
   519  	require.Equal(1, we.numAutoRecovers)
   520  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend2), extIO))
   521  	require.Equal(1, we.numAutoRecovers)
   522  
   523  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend2), extIO))
   524  	require.Equal(2, we.numAutoRecovers)
   525  
   526  	// for i := 1; i <= 3; i++ {
   527  	// 	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend2), extIO))
   528  	// 	require.Equal(0, we.numAutoRecovers)
   529  	// 	printHeapInfo(t, we)
   530  	// }
   531  	// for i := 1; i <= 3; i++ {
   532  	// 	err = extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend2), extIO)
   533  	// 	require.Equal(1, we.numAutoRecovers)
   534  	// 	printHeapInfo(t, we)
   535  	// 	require.NoError(err)
   536  	// }
   537  	// require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend2), extIO))
   538  	// require.Equal(2, we.numAutoRecovers)
   539  }
   540  
   541  func Test_Read(t *testing.T) {
   542  	const testRead = "testRead"
   543  	require := require.New(t)
   544  	ctx := context.Background()
   545  	moduleUrl := testModuleURL("./_testdata/tests/pkg.wasm")
   546  	extEngine, err := testFactoryHelper(ctx, moduleUrl, []string{testRead}, iextengine.ExtEngineConfig{}, false)
   547  	require.NoError(err)
   548  	defer extEngine.Close(ctx)
   549  	err = extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, testRead), extIO)
   550  	require.NoError(err)
   551  }
   552  
   553  func Test_AsBytes(t *testing.T) {
   554  	const asBytes = "asBytes"
   555  	require := require.New(t)
   556  	ctx := context.Background()
   557  	moduleUrl := testModuleURL("./_testdata/tests/pkg.wasm")
   558  	extEngine, err := testFactoryHelper(ctx, moduleUrl, []string{asBytes}, iextengine.ExtEngineConfig{}, false)
   559  	require.NoError(err)
   560  	defer extEngine.Close(ctx)
   561  	wasmEngine := extEngine.(*wazeroExtEngine)
   562  	requireMemStatEx(t, wasmEngine, 1, 0, WasmPreallocatedBufferSize, WasmPreallocatedBufferSize)
   563  	err = extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, asBytes), extIO)
   564  	require.NoError(err)
   565  	requireMemStatEx(t, wasmEngine, 2, 0, WasmPreallocatedBufferSize+2000000, WasmPreallocatedBufferSize+2000000)
   566  }
   567  
   568  func Test_AsBytesOverflow(t *testing.T) {
   569  	WasmPreallocatedBufferSize = 1000000
   570  	const asBytes = "asBytes"
   571  	require := require.New(t)
   572  	ctx := context.Background()
   573  	moduleUrl := testModuleURL("./_testdata/tests/pkg.wasm")
   574  	extEngine, err := testFactoryHelper(ctx, moduleUrl, []string{asBytes}, iextengine.ExtEngineConfig{MemoryLimitPages: 0x20}, false)
   575  	require.NoError(err)
   576  	defer extEngine.Close(ctx)
   577  	wasmEngine := extEngine.(*wazeroExtEngine)
   578  	requireMemStatEx(t, wasmEngine, 1, 0, WasmPreallocatedBufferSize, WasmPreallocatedBufferSize)
   579  	err = extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, asBytes), extIO)
   580  	require.EqualError(err, "panic: runtime error: out of memory")
   581  }
   582  
   583  func Test_KeyPutQName(t *testing.T) {
   584  	const putQName = "keyPutQName"
   585  	require := require.New(t)
   586  	ctx := context.Background()
   587  	moduleUrl := testModuleURL("./_testdata/tests/pkg.wasm")
   588  	extEngine, err := testFactoryHelper(ctx, moduleUrl, []string{putQName}, iextengine.ExtEngineConfig{}, false)
   589  	require.NoError(err)
   590  	defer extEngine.Close(ctx)
   591  	wasmEngine := extEngine.(*wazeroExtEngine)
   592  	requireMemStatEx(t, wasmEngine, 1, 0, WasmPreallocatedBufferSize, WasmPreallocatedBufferSize)
   593  	err = extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, putQName), extIO)
   594  	require.NoError(err)
   595  }
   596  
   597  func Test_NoAllocs(t *testing.T) {
   598  	const testNoAllocs = "testNoAllocs"
   599  	extIO = &mockIo{}
   600  	projectorMode = false
   601  	require := require.New(t)
   602  	ctx := context.Background()
   603  	moduleUrl := testModuleURL("./_testdata/tests/pkg.wasm")
   604  	extEngine, err := testFactoryHelper(ctx, moduleUrl, []string{testNoAllocs}, iextengine.ExtEngineConfig{MemoryLimitPages: 0x20}, false)
   605  	require.NoError(err)
   606  	defer extEngine.Close(ctx)
   607  
   608  	wasmEngine := extEngine.(*wazeroExtEngine)
   609  	requireMemStatEx(t, wasmEngine, 1, 0, WasmPreallocatedBufferSize, WasmPreallocatedBufferSize)
   610  
   611  	err = extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, testNoAllocs), extIO)
   612  	require.NoError(err)
   613  
   614  	requireMemStatEx(t, wasmEngine, 1, 0, WasmPreallocatedBufferSize, WasmPreallocatedBufferSize)
   615  	require.Len(extIO.intents, 2)
   616  	v0 := extIO.intents[0].value.(*mockValueBuilder)
   617  
   618  	// new value
   619  	require.Equal("test@gmail.com", v0.items["from"])
   620  	require.Equal(int32(668), v0.items["port"])
   621  	require.Equal(appdef.NewQName(testPackageLocalPath, "test"), v0.items["qname"])
   622  	bytes := (v0.items["key"]).([]byte)
   623  	require.Len(bytes, 5)
   624  
   625  	v1 := extIO.intents[1].value.(*mockValueBuilder)
   626  	require.Equal(int32(12346), v1.items["offs"])
   627  	require.Equal("sys.InvitationAccepted", v1.items["qname"])
   628  }
   629  
   630  func Test_WithState(t *testing.T) {
   631  
   632  	require := require.New(t)
   633  	ctx := context.Background()
   634  
   635  	const extension = "incrementProjector"
   636  	const cc = "cc"
   637  	const pk = "pk"
   638  	const vv = "vv"
   639  	const intentsLimit = 5
   640  	const bundlesLimit = 5
   641  	const ws = istructs.WSID(1)
   642  
   643  	app := appStructsFromSQL(testPkg, `APPLICATION test();
   644  		WORKSPACE Restaurant (
   645  			DESCRIPTOR RestaurantDescriptor ();
   646  			VIEW TestView (
   647  				pk int32,
   648  				cc int32,
   649  				vv int32,
   650  				PRIMARY KEY ((pk), cc)
   651  			) AS RESULT OF DummyProj;
   652  			EXTENSION ENGINE WASM(
   653  				COMMAND Dummy();
   654  				PROJECTOR DummyProj AFTER EXECUTE ON Dummy INTENTS(View(TestView));
   655  			);
   656  		)`,
   657  		func(cfg *istructsmem.AppConfigType) {
   658  			cfg.Resources.Add(istructsmem.NewCommandFunction(dummyCommand, istructsmem.NullCommandExec))
   659  			cfg.AddAsyncProjectors(istructs.Projector{Name: dummyProj})
   660  		})
   661  
   662  	// build app
   663  	appFunc := func() istructs.IAppStructs { return app }
   664  	state := state.ProvideAsyncActualizerStateFactory()(context.Background(), appFunc, nil, state.SimpleWSIDFunc(ws), nil, nil, nil, nil, nil, intentsLimit, bundlesLimit)
   665  
   666  	// build packages
   667  	moduleUrl := testModuleURL("./_testdata/basicusage/pkg.wasm")
   668  	packages := []iextengine.ExtensionPackage{
   669  		{
   670  			QualifiedName:  testPkg,
   671  			ModuleUrl:      moduleUrl,
   672  			ExtensionNames: []string{extension},
   673  		},
   674  	}
   675  
   676  	// build extension engine
   677  	factory := ProvideExtensionEngineFactory(true)
   678  	engines, err := factory.New(ctx, app.AppQName(), packages, &iextengine.ExtEngineConfig{}, 1)
   679  	if err != nil {
   680  		panic(err)
   681  	}
   682  	extEngine := engines[0]
   683  	defer extEngine.Close(ctx)
   684  
   685  	// Invoke extension
   686  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, extension), state))
   687  	ready, err := state.ApplyIntents()
   688  	require.NoError(err)
   689  	require.False(ready)
   690  
   691  	// Invoke extension again
   692  	require.NoError(extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, extension), state))
   693  	ready, err = state.ApplyIntents()
   694  	require.NoError(err)
   695  	require.False(ready)
   696  
   697  	// Flush bundles with intents
   698  	require.NoError(state.FlushBundles())
   699  
   700  	// Test view
   701  	kb := app.ViewRecords().KeyBuilder(testView)
   702  	kb.PartitionKey().PutInt32(pk, 1)
   703  	kb.ClusteringColumns().PutInt32(cc, 1)
   704  	value, err := app.ViewRecords().Get(ws, kb)
   705  
   706  	require.NoError(err)
   707  	require.NotNil(value)
   708  	require.Equal(int32(2), value.AsInt32(vv))
   709  }
   710  
   711  func Test_StatePanic(t *testing.T) {
   712  
   713  	const intentsLimit = 5
   714  	const bundlesLimit = 5
   715  	const ws = istructs.WSID(1)
   716  
   717  	app := appStructsFromSQL(testPkg, `APPLICATION test();
   718  		WORKSPACE Restaurant (
   719  			DESCRIPTOR RestaurantDescriptor ();
   720  			VIEW TestView (
   721  				pk int32,
   722  				cc int32,
   723  				vv int32,
   724  				PRIMARY KEY ((pk), cc)
   725  			) AS RESULT OF DummyProj;
   726  			EXTENSION ENGINE WASM(
   727  				COMMAND Dummy();
   728  				PROJECTOR DummyProj AFTER EXECUTE ON Dummy INTENTS(View(TestView));
   729  			);
   730  		)`,
   731  		func(cfg *istructsmem.AppConfigType) {
   732  			cfg.Resources.Add(istructsmem.NewCommandFunction(dummyCommand, istructsmem.NullCommandExec))
   733  			cfg.AddAsyncProjectors(istructs.Projector{Name: dummyProj})
   734  		})
   735  	appFunc := func() istructs.IAppStructs { return app }
   736  	state := state.ProvideAsyncActualizerStateFactory()(context.Background(), appFunc, nil, state.SimpleWSIDFunc(ws), nil, nil, nil, nil, nil, intentsLimit, bundlesLimit)
   737  
   738  	const extname = "wrongFieldName"
   739  	const undefinedPackage = "undefinedPackage"
   740  
   741  	require := require.New(t)
   742  	ctx := context.Background()
   743  
   744  	moduleUrl := testModuleURL("./_testdata/panics/pkg.wasm")
   745  	packages := []iextengine.ExtensionPackage{
   746  		{
   747  			QualifiedName:  testPkg,
   748  			ModuleUrl:      moduleUrl,
   749  			ExtensionNames: []string{extname, undefinedPackage},
   750  		},
   751  	}
   752  	factory := ProvideExtensionEngineFactory(true)
   753  	engines, err := factory.New(ctx, app.AppQName(), packages, &iextengine.ExtEngineConfig{}, 1)
   754  	if err != nil {
   755  		panic(err)
   756  	}
   757  	extEngine := engines[0]
   758  	defer extEngine.Close(ctx)
   759  	//
   760  	// Invoke extension
   761  	//
   762  	err = extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, extname), state)
   763  	require.ErrorContains(err, "field «wrong» is not found")
   764  
   765  	//
   766  	// Invoke extension
   767  	//
   768  	err = extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, undefinedPackage), state)
   769  	require.ErrorContains(err, errUndefinedPackage("github.com/company/pkg").Error())
   770  }
   771  
   772  type (
   773  	appCfgCallback func(cfg *istructsmem.AppConfigType)
   774  )
   775  
   776  //go:embed sql_example_syspkg/*.vsql
   777  var sfs embed.FS
   778  
   779  func appStructsFromSQL(packagePath string, appdefSql string, prepareAppCfg appCfgCallback) istructs.IAppStructs {
   780  	plogOffset = istructs.Offset(123)
   781  	wlogOffset = istructs.Offset(42)
   782  	appDef := appdef.New()
   783  
   784  	fs, err := parser.ParseFile("file1.vsql", appdefSql)
   785  	if err != nil {
   786  		panic(err)
   787  	}
   788  
   789  	pkg, err := parser.BuildPackageSchema(packagePath, []*parser.FileSchemaAST{fs})
   790  	if err != nil {
   791  		panic(err)
   792  	}
   793  
   794  	pkgSys, err := parser.ParsePackageDir(appdef.SysPackage, sfs, "sql_example_syspkg")
   795  	if err != nil {
   796  		panic(err)
   797  	}
   798  
   799  	packages, err := parser.BuildAppSchema([]*parser.PackageSchemaAST{
   800  		pkgSys,
   801  		pkg,
   802  	})
   803  	if err != nil {
   804  		panic(err)
   805  	}
   806  
   807  	err = parser.BuildAppDefs(packages, appDef)
   808  	if err != nil {
   809  		panic(err)
   810  	}
   811  
   812  	app := appStructs(appDef, prepareAppCfg)
   813  
   814  	// Create workspace
   815  	rebWs := app.Events().GetNewRawEventBuilder(istructs.NewRawEventBuilderParams{
   816  		GenericRawEventBuilderParams: istructs.GenericRawEventBuilderParams{
   817  			Workspace:         ws,
   818  			HandlingPartition: partition,
   819  			PLogOffset:        plogOffset,
   820  			QName:             newWorkspaceCmd,
   821  			WLogOffset:        wlogOffset,
   822  		},
   823  	})
   824  	cud := rebWs.CUDBuilder().Create(authnz.QNameCDocWorkspaceDescriptor)
   825  	cud.PutRecordID(appdef.SystemField_ID, 1)
   826  	cud.PutQName("WSKind", testWorkspaceDescriptor)
   827  	rawWsEvent, err := rebWs.BuildRawEvent()
   828  	if err != nil {
   829  		panic(err)
   830  	}
   831  	wsEvent, err := app.Events().PutPlog(rawWsEvent, nil, istructsmem.NewIDGenerator())
   832  	if err != nil {
   833  		panic(err)
   834  	}
   835  	app.Records().Apply(wsEvent)
   836  
   837  	return app
   838  
   839  }
   840  
   841  func Test_Panic(t *testing.T) {
   842  	const arrAppend = "TestPanic"
   843  
   844  	require := require.New(t)
   845  	ctx := context.Background()
   846  	WasmPreallocatedBufferSize = 1000000
   847  	moduleUrl := testModuleURL("./_testdata/panics/pkg.wasm")
   848  	extEngine, err := testFactoryHelper(ctx, moduleUrl, []string{arrAppend}, iextengine.ExtEngineConfig{}, false)
   849  	require.NoError(err)
   850  	defer extEngine.Close(ctx)
   851  
   852  	err = extEngine.Invoke(context.Background(), appdef.NewFullQName(testPkg, arrAppend), extIO)
   853  	require.EqualError(err, "panic: goodbye, world")
   854  }