github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/pkg/loadtest/local_load_test.go (about)

     1  package loadtest
     2  
     3  import (
     4  	"context"
     5  	"math"
     6  	"net/http/httptest"
     7  	"os"
     8  	"testing"
     9  	"time"
    10  
    11  	"github.com/treeverse/lakefs/pkg/authentication"
    12  
    13  	"github.com/spf13/viper"
    14  	"github.com/treeverse/lakefs/pkg/actions"
    15  	"github.com/treeverse/lakefs/pkg/api"
    16  	"github.com/treeverse/lakefs/pkg/auth"
    17  	"github.com/treeverse/lakefs/pkg/auth/crypt"
    18  	authmodel "github.com/treeverse/lakefs/pkg/auth/model"
    19  	authparams "github.com/treeverse/lakefs/pkg/auth/params"
    20  	"github.com/treeverse/lakefs/pkg/auth/setup"
    21  	"github.com/treeverse/lakefs/pkg/block"
    22  	"github.com/treeverse/lakefs/pkg/catalog"
    23  	"github.com/treeverse/lakefs/pkg/config"
    24  	"github.com/treeverse/lakefs/pkg/kv"
    25  	"github.com/treeverse/lakefs/pkg/kv/kvparams"
    26  	"github.com/treeverse/lakefs/pkg/kv/kvtest"
    27  	"github.com/treeverse/lakefs/pkg/logging"
    28  	"github.com/treeverse/lakefs/pkg/stats"
    29  	"github.com/treeverse/lakefs/pkg/testutil"
    30  	"github.com/treeverse/lakefs/pkg/upload"
    31  	"github.com/treeverse/lakefs/pkg/version"
    32  )
    33  
    34  func TestLocalLoad(t *testing.T) {
    35  	if testing.Short() {
    36  		t.Skip("Skipping loadtest tests in short mode")
    37  	}
    38  
    39  	// Only once
    40  	ctx := context.Background()
    41  	viper.Set(config.BlockstoreTypeKey, block.BlockstoreTypeLocal)
    42  
    43  	conf, err := config.NewConfig("")
    44  	testutil.MustDo(t, "config", err)
    45  
    46  	superuser := &authmodel.SuperuserConfiguration{
    47  		User: authmodel.User{
    48  			CreatedAt: time.Now(),
    49  			Username:  "admin",
    50  		},
    51  	}
    52  
    53  	kvStore := kvtest.GetStore(ctx, t)
    54  	authService := auth.NewAuthService(kvStore, crypt.NewSecretStore([]byte("some secret")), authparams.ServiceCache{}, logging.ContextUnavailable().WithField("service", "auth"))
    55  	meta := auth.NewKVMetadataManager("local_load_test", conf.Installation.FixedID, conf.Database.Type, kvStore)
    56  
    57  	blockstoreType := os.Getenv(testutil.EnvKeyUseBlockAdapter)
    58  	if blockstoreType == "" {
    59  		blockstoreType = "mem"
    60  	}
    61  
    62  	blockAdapter := testutil.NewBlockAdapterByType(t, blockstoreType)
    63  	c, err := catalog.New(ctx, catalog.Config{
    64  		Config:       conf,
    65  		KVStore:      kvStore,
    66  		PathProvider: upload.DefaultPathProvider,
    67  	})
    68  	testutil.MustDo(t, "build catalog", err)
    69  
    70  	source := catalog.NewActionsSource(c)
    71  	outputWriter := catalog.NewActionsOutputWriter(c.BlockAdapter)
    72  
    73  	// wire actions
    74  	actionsService := actions.NewService(ctx, actions.NewActionsKVStore(kvStore), source, outputWriter, &actions.DecreasingIDGenerator{}, &stats.NullCollector{}, actions.Config{Enabled: true}, "")
    75  	c.SetHooksHandler(actionsService)
    76  
    77  	credentials, err := setup.CreateAdminUser(ctx, authService, conf, superuser)
    78  	testutil.Must(t, err)
    79  
    80  	authenticator := auth.NewBuiltinAuthenticator(authService)
    81  	kvParams, err := kvparams.NewConfig(conf)
    82  	testutil.Must(t, err)
    83  	migrator := kv.NewDatabaseMigrator(kvParams)
    84  	t.Cleanup(func() {
    85  		_ = c.Close()
    86  	})
    87  	auditChecker := version.NewDefaultAuditChecker(conf.Security.AuditCheckURL, "", nil)
    88  	authenticationService := authentication.NewDummyService()
    89  	handler := api.Serve(conf, c, authenticator, authService, authenticationService, blockAdapter, meta, migrator, &stats.NullCollector{}, nil, actionsService, auditChecker, logging.ContextUnavailable(), nil, nil, upload.DefaultPathProvider, stats.DefaultUsageReporter)
    90  
    91  	ts := httptest.NewServer(handler)
    92  	defer ts.Close()
    93  
    94  	testConfig := Config{
    95  		FreqPerSecond:    6,
    96  		Duration:         5 * time.Second,
    97  		MaxWorkers:       math.MaxInt64,
    98  		KeepRepo:         false,
    99  		Credentials:      *credentials,
   100  		ServerAddress:    ts.URL,
   101  		StorageNamespace: "mem://local/test/",
   102  		ShowProgress:     testing.Verbose(),
   103  	}
   104  	loader := NewLoader(testConfig)
   105  	err = loader.Run()
   106  	if err != nil {
   107  		t.Fatalf("Got error on test: %s", err)
   108  	}
   109  }