github.com/projecteru2/core@v0.0.0-20240321043226-06bcc1c23f58/cluster/calcium/create_test.go (about)

     1  package calcium
     2  
     3  import (
     4  	"context"
     5  	"testing"
     6  
     7  	"github.com/cockroachdb/errors"
     8  	"github.com/stretchr/testify/assert"
     9  	"github.com/stretchr/testify/mock"
    10  
    11  	enginemocks "github.com/projecteru2/core/engine/mocks"
    12  	enginetypes "github.com/projecteru2/core/engine/types"
    13  	lockmocks "github.com/projecteru2/core/lock/mocks"
    14  	resourcemocks "github.com/projecteru2/core/resource/mocks"
    15  	plugintypes "github.com/projecteru2/core/resource/plugins/types"
    16  	resourcetypes "github.com/projecteru2/core/resource/types"
    17  	storemocks "github.com/projecteru2/core/store/mocks"
    18  	"github.com/projecteru2/core/strategy"
    19  	"github.com/projecteru2/core/types"
    20  	"github.com/projecteru2/core/wal"
    21  	walmocks "github.com/projecteru2/core/wal/mocks"
    22  )
    23  
    24  func TestCreateWorkloadValidating(t *testing.T) {
    25  	c := NewTestCluster()
    26  	ctx := context.Background()
    27  	opts := &types.DeployOptions{
    28  		Name:    "deployname",
    29  		Podname: "somepod",
    30  		Image:   "image:todeploy",
    31  		Count:   1,
    32  		Entrypoint: &types.Entrypoint{
    33  			Name: "some-nice-entrypoint",
    34  		},
    35  		NodeFilter: &types.NodeFilter{},
    36  	}
    37  	// failed by validating
    38  	opts.Name = ""
    39  	_, err := c.CreateWorkload(ctx, opts)
    40  	assert.Error(t, err)
    41  	opts.Name = "deployname"
    42  
    43  	opts.Podname = ""
    44  	_, err = c.CreateWorkload(ctx, opts)
    45  	assert.Error(t, err)
    46  	opts.Podname = "somepod"
    47  
    48  	opts.Image = ""
    49  	_, err = c.CreateWorkload(ctx, opts)
    50  	assert.Error(t, err)
    51  	opts.Image = "image:todeploy"
    52  
    53  	opts.Count = 0
    54  	_, err = c.CreateWorkload(ctx, opts)
    55  	assert.Error(t, err)
    56  	opts.Count = 1
    57  
    58  	opts.Entrypoint.Name = "bad_entry_name"
    59  	_, err = c.CreateWorkload(ctx, opts)
    60  	assert.Error(t, err)
    61  	opts.Entrypoint.Name = "some-nice-entrypoint"
    62  }
    63  
    64  func TestCreateWorkloadTxn(t *testing.T) {
    65  	c, nodes := newCreateWorkloadCluster(t)
    66  	ctx := context.Background()
    67  	opts := &types.DeployOptions{
    68  		Name:           "zc:name",
    69  		Count:          2,
    70  		DeployStrategy: strategy.Auto,
    71  		Podname:        "p1",
    72  		Resources:      resourcetypes.Resources{},
    73  		Image:          "zc:test",
    74  		Entrypoint: &types.Entrypoint{
    75  			Name: "good-entrypoint",
    76  		},
    77  		NodeFilter: &types.NodeFilter{},
    78  	}
    79  
    80  	store := c.store.(*storemocks.Store)
    81  	rmgr := c.rmgr.(*resourcemocks.Manager)
    82  	mwal := &walmocks.WAL{}
    83  	c.wal = mwal
    84  	var walCommitted bool
    85  	commit := wal.Commit(func() error {
    86  		walCommitted = true
    87  		return nil
    88  	})
    89  	mwal.On("Log", mock.Anything, mock.Anything).Return(commit, nil)
    90  	node1, node2 := nodes[0], nodes[1]
    91  
    92  	// doAllocResource fails: GetNodesDeployCapacity
    93  	rmgr.On("GetNodesDeployCapacity", mock.Anything, mock.Anything, mock.Anything).Return(
    94  		nil, 0, types.ErrMockError,
    95  	).Once()
    96  	ch, err := c.CreateWorkload(ctx, opts)
    97  	assert.Nil(t, err)
    98  	cnt := 0
    99  	for m := range ch {
   100  		cnt++
   101  		assert.Error(t, m.Error, "key is empty")
   102  	}
   103  	assert.EqualValues(t, 1, cnt)
   104  	assert.True(t, walCommitted)
   105  	walCommitted = false
   106  	rmgr.On("GetNodesDeployCapacity", mock.Anything, mock.Anything, mock.Anything).Return(
   107  		map[string]*plugintypes.NodeDeployCapacity{
   108  			node1.Name: {
   109  				Capacity: 10,
   110  				Usage:    0.5,
   111  				Rate:     0.05,
   112  				Weight:   100,
   113  			},
   114  			node2.Name: {
   115  				Capacity: 10,
   116  				Usage:    0.5,
   117  				Rate:     0.05,
   118  				Weight:   100,
   119  			},
   120  		}, 20, nil,
   121  	)
   122  
   123  	// doAllocResource fails: GetDeployStatus
   124  	store.On("GetDeployStatus", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.Wrap(context.DeadlineExceeded, "GetDeployStatus")).Once()
   125  	ch, err = c.CreateWorkload(ctx, opts)
   126  	assert.Nil(t, err)
   127  	cnt = 0
   128  	for m := range ch {
   129  		cnt++
   130  		assert.ErrorIs(t, m.Error, context.DeadlineExceeded)
   131  		assert.Error(t, m.Error, "GetDeployStatus")
   132  	}
   133  	assert.EqualValues(t, 1, cnt)
   134  	assert.True(t, walCommitted)
   135  	walCommitted = false
   136  	store.On("GetDeployStatus", mock.Anything, mock.Anything, mock.Anything).Return(map[string]int{}, nil)
   137  
   138  	// doAllocResource fails: Alloc
   139  	rmgr.On("Alloc", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(
   140  		nil, nil, types.ErrMockError,
   141  	).Once()
   142  	ch, err = c.CreateWorkload(ctx, opts)
   143  	assert.Nil(t, err)
   144  	cnt = 0
   145  	for m := range ch {
   146  		cnt++
   147  		assert.Error(t, m.Error, "DeadlineExceeded")
   148  	}
   149  	assert.EqualValues(t, 1, cnt)
   150  	assert.True(t, walCommitted)
   151  	walCommitted = false
   152  	rmgr.On("Alloc", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(
   153  		[]resourcetypes.Resources{{}, {}},
   154  		[]resourcetypes.Resources{
   155  			{node1.Name: {}},
   156  			{node2.Name: {}},
   157  		},
   158  		nil,
   159  	)
   160  	rmgr.On("RollbackAlloc", mock.Anything, mock.Anything, mock.Anything).Return(nil)
   161  	store.On("GetNode",
   162  		mock.AnythingOfType("*context.timerCtx"),
   163  		mock.AnythingOfType("string"),
   164  	).Return(
   165  		func(_ context.Context, name string) (node *types.Node) {
   166  			node = node1
   167  			if name == "n2" {
   168  				node = node2
   169  			}
   170  			return
   171  		}, nil)
   172  	engine := node1.Engine.(*enginemocks.API)
   173  	engine.On("ImageLocalDigests", mock.Anything, mock.Anything).Return(nil, errors.Wrap(context.DeadlineExceeded, "ImageLocalDigest")).Twice()
   174  	engine.On("ImagePull", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.Wrap(context.DeadlineExceeded, "ImagePull")).Twice()
   175  	store.On("DeleteProcessing", mock.Anything, mock.Anything, mock.Anything).Return(nil)
   176  	ch, err = c.CreateWorkload(ctx, opts)
   177  	assert.Nil(t, err)
   178  	cnt = 0
   179  	for m := range ch {
   180  		cnt++
   181  		assert.Error(t, m.Error, "ImagePull")
   182  	}
   183  	assert.EqualValues(t, 2, cnt)
   184  	assert.True(t, walCommitted)
   185  
   186  	// doDeployOneWorkload fails: VirtualizationCreate
   187  	engine.On("ImageLocalDigests", mock.Anything, mock.Anything).Return([]string{""}, nil)
   188  	engine.On("ImageRemoteDigest", mock.Anything, mock.Anything).Return("", nil)
   189  	engine.On("VirtualizationCreate", mock.Anything, mock.Anything).Return(nil, errors.Wrap(context.DeadlineExceeded, "VirtualizationCreate")).Twice()
   190  	engine.On("VirtualizationRemove", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
   191  	store.On("ListNodeWorkloads", mock.Anything, mock.Anything, mock.Anything).Return(nil, types.ErrMockError)
   192  	walCommitted = false
   193  	ch, err = c.CreateWorkload(ctx, opts)
   194  	assert.Nil(t, err)
   195  	cnt = 0
   196  	for m := range ch {
   197  		cnt++
   198  		assert.Error(t, m.Error)
   199  		assert.True(t, errors.Is(m.Error, context.DeadlineExceeded))
   200  		assert.Error(t, m.Error, "VirtualizationCreate")
   201  	}
   202  	assert.EqualValues(t, 2, cnt)
   203  	assert.True(t, walCommitted)
   204  
   205  	// doCreateAndStartWorkload fails: AddWorkload
   206  	engine.On("VirtualizationCreate", mock.Anything, mock.Anything).Return(&enginetypes.VirtualizationCreated{ID: "c1"}, nil)
   207  	engine.On("VirtualizationStart", mock.Anything, mock.Anything).Return(nil)
   208  	engine.On("VirtualizationInspect", mock.Anything, mock.Anything).Return(&enginetypes.VirtualizationInfo{}, nil)
   209  	store.On("AddWorkload", mock.Anything, mock.Anything, mock.Anything).Return(errors.Wrap(context.DeadlineExceeded, "AddWorkload")).Twice()
   210  	walCommitted = false
   211  	ch, err = c.CreateWorkload(ctx, opts)
   212  	assert.Nil(t, err)
   213  	cnt = 0
   214  	for m := range ch {
   215  		cnt++
   216  		assert.Error(t, m.Error)
   217  		assert.True(t, errors.Is(m.Error, context.DeadlineExceeded))
   218  		assert.Error(t, m.Error, "AddWorkload")
   219  	}
   220  	assert.EqualValues(t, 2, cnt)
   221  	assert.True(t, walCommitted)
   222  
   223  	// doCreateAndStartWorkload fails: first time AddWorkload failed
   224  	engine.On("VirtualizationCreate", mock.Anything, mock.Anything).Return(&enginetypes.VirtualizationCreated{ID: "c1"}, nil)
   225  	engine.On("VirtualizationStart", mock.Anything, mock.Anything).Return(nil)
   226  	engine.On("VirtualizationInspect", mock.Anything, mock.Anything).Return(&enginetypes.VirtualizationInfo{}, nil)
   227  	store.On("AddWorkload", mock.Anything, mock.Anything, mock.Anything).Return(errors.Wrap(context.DeadlineExceeded, "AddWorkload2")).Once()
   228  	store.On("AddWorkload", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
   229  	walCommitted = false
   230  	ch, err = c.CreateWorkload(ctx, opts)
   231  	assert.Nil(t, err)
   232  	cnt = 0
   233  	errCnt := 0
   234  	for m := range ch {
   235  		cnt++
   236  		if m.Error != nil {
   237  			assert.Error(t, m.Error)
   238  			assert.True(t, errors.Is(m.Error, context.DeadlineExceeded))
   239  			assert.Error(t, m.Error, "AddWorkload2")
   240  			errCnt++
   241  		}
   242  	}
   243  	assert.EqualValues(t, 2, cnt)
   244  	assert.EqualValues(t, 1, errCnt)
   245  	assert.True(t, walCommitted)
   246  	store.AssertExpectations(t)
   247  	engine.AssertExpectations(t)
   248  }
   249  
   250  func TestCreateWorkloadIngorePullTxn(t *testing.T) {
   251  	c, nodes := newCreateWorkloadCluster(t)
   252  	ctx := context.Background()
   253  	opts := &types.DeployOptions{
   254  		Name:           "zc:name",
   255  		Count:          2,
   256  		DeployStrategy: strategy.Auto,
   257  		Podname:        "p1",
   258  		Resources:      resourcetypes.Resources{},
   259  		Image:          "zc:test",
   260  		Entrypoint: &types.Entrypoint{
   261  			Name: "good-entrypoint",
   262  		},
   263  		NodeFilter: &types.NodeFilter{},
   264  		IgnorePull: true,
   265  	}
   266  
   267  	store := c.store.(*storemocks.Store)
   268  	rmgr := c.rmgr.(*resourcemocks.Manager)
   269  	mwal := &walmocks.WAL{}
   270  	c.wal = mwal
   271  	var walCommitted bool
   272  	commit := wal.Commit(func() error {
   273  		walCommitted = true
   274  		return nil
   275  	})
   276  	mwal.On("Log", mock.Anything, mock.Anything).Return(commit, nil)
   277  	node1, node2 := nodes[0], nodes[1]
   278  
   279  	// doAllocResource fails: GetNodesDeployCapacity
   280  	rmgr.On("GetNodesDeployCapacity", mock.Anything, mock.Anything, mock.Anything).Return(
   281  		nil, 0, types.ErrMockError,
   282  	).Once()
   283  	ch, err := c.CreateWorkload(ctx, opts)
   284  	assert.Nil(t, err)
   285  	cnt := 0
   286  	for m := range ch {
   287  		cnt++
   288  		assert.Error(t, m.Error, "key is empty")
   289  	}
   290  	assert.EqualValues(t, 1, cnt)
   291  	assert.True(t, walCommitted)
   292  	walCommitted = false
   293  	rmgr.On("GetNodesDeployCapacity", mock.Anything, mock.Anything, mock.Anything).Return(
   294  		map[string]*plugintypes.NodeDeployCapacity{
   295  			node1.Name: {
   296  				Capacity: 10,
   297  				Usage:    0.5,
   298  				Rate:     0.05,
   299  				Weight:   100,
   300  			},
   301  			node2.Name: {
   302  				Capacity: 10,
   303  				Usage:    0.5,
   304  				Rate:     0.05,
   305  				Weight:   100,
   306  			},
   307  		}, 20, nil,
   308  	)
   309  
   310  	// doAllocResource fails: GetDeployStatus
   311  	store.On("GetDeployStatus", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.Wrap(context.DeadlineExceeded, "GetDeployStatus")).Once()
   312  	ch, err = c.CreateWorkload(ctx, opts)
   313  	assert.Nil(t, err)
   314  	cnt = 0
   315  	for m := range ch {
   316  		cnt++
   317  		assert.ErrorIs(t, m.Error, context.DeadlineExceeded)
   318  		assert.Error(t, m.Error, "GetDeployStatus")
   319  	}
   320  	assert.EqualValues(t, 1, cnt)
   321  	assert.True(t, walCommitted)
   322  	walCommitted = false
   323  	store.On("GetDeployStatus", mock.Anything, mock.Anything, mock.Anything).Return(map[string]int{}, nil)
   324  
   325  	// doAllocResource fails: Alloc
   326  	rmgr.On("Alloc", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(
   327  		nil, nil, types.ErrMockError,
   328  	).Once()
   329  	ch, err = c.CreateWorkload(ctx, opts)
   330  	assert.Nil(t, err)
   331  	cnt = 0
   332  	for m := range ch {
   333  		cnt++
   334  		assert.Error(t, m.Error, "DeadlineExceeded")
   335  	}
   336  	assert.EqualValues(t, 1, cnt)
   337  	assert.True(t, walCommitted)
   338  	walCommitted = false
   339  	rmgr.On("Alloc", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(
   340  		[]resourcetypes.Resources{{}, {}},
   341  		[]resourcetypes.Resources{
   342  			{node1.Name: {}},
   343  			{node2.Name: {}},
   344  		},
   345  		nil,
   346  	)
   347  	rmgr.On("RollbackAlloc", mock.Anything, mock.Anything, mock.Anything).Return(nil)
   348  	store.On("GetNode",
   349  		mock.AnythingOfType("*context.timerCtx"),
   350  		mock.AnythingOfType("string"),
   351  	).Return(
   352  		func(_ context.Context, name string) (node *types.Node) {
   353  			node = node1
   354  			if name == "n2" {
   355  				node = node2
   356  			}
   357  			return
   358  		}, nil)
   359  	engine := node1.Engine.(*enginemocks.API)
   360  	// engine.On("ImageLocalDigests", mock.Anything, mock.Anything).Return(nil, errors.Wrap(context.DeadlineExceeded, "ImageLocalDigest")).Twice()
   361  	// engine.On("ImagePull", mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.Wrap(context.DeadlineExceeded, "ImagePull")).Twice()
   362  	store.On("DeleteProcessing", mock.Anything, mock.Anything, mock.Anything).Return(nil)
   363  	// ch, err = c.CreateWorkload(ctx, opts)
   364  	// assert.Nil(t, err)
   365  	// cnt = 0
   366  	// for m := range ch {
   367  	// 	cnt++
   368  	// 	assert.Error(t, m.Error, "ImagePull")
   369  	// }
   370  	// assert.EqualValues(t, 2, cnt)
   371  	// assert.True(t, walCommitted)
   372  
   373  	// doDeployOneWorkload fails: VirtualizationCreate
   374  	// engine.On("ImageLocalDigests", mock.Anything, mock.Anything).Return([]string{""}, nil)
   375  	// engine.On("ImageRemoteDigest", mock.Anything, mock.Anything).Return("", nil)
   376  	engine.On("VirtualizationCreate", mock.Anything, mock.Anything).Return(nil, errors.Wrap(context.DeadlineExceeded, "VirtualizationCreate")).Twice()
   377  	engine.On("VirtualizationRemove", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
   378  	store.On("ListNodeWorkloads", mock.Anything, mock.Anything, mock.Anything).Return(nil, types.ErrMockError)
   379  	walCommitted = false
   380  	ch, err = c.CreateWorkload(ctx, opts)
   381  	assert.Nil(t, err)
   382  	cnt = 0
   383  	for m := range ch {
   384  		cnt++
   385  		assert.Error(t, m.Error)
   386  		assert.True(t, errors.Is(m.Error, context.DeadlineExceeded))
   387  		assert.Error(t, m.Error, "VirtualizationCreate")
   388  	}
   389  	assert.EqualValues(t, 2, cnt)
   390  	assert.True(t, walCommitted)
   391  
   392  	// doCreateAndStartWorkload fails: AddWorkload
   393  	engine.On("VirtualizationCreate", mock.Anything, mock.Anything).Return(&enginetypes.VirtualizationCreated{ID: "c1"}, nil)
   394  	engine.On("VirtualizationStart", mock.Anything, mock.Anything).Return(nil)
   395  	engine.On("VirtualizationInspect", mock.Anything, mock.Anything).Return(&enginetypes.VirtualizationInfo{}, nil)
   396  	store.On("AddWorkload", mock.Anything, mock.Anything, mock.Anything).Return(errors.Wrap(context.DeadlineExceeded, "AddWorkload")).Twice()
   397  	walCommitted = false
   398  	ch, err = c.CreateWorkload(ctx, opts)
   399  	assert.Nil(t, err)
   400  	cnt = 0
   401  	for m := range ch {
   402  		cnt++
   403  		assert.Error(t, m.Error)
   404  		assert.True(t, errors.Is(m.Error, context.DeadlineExceeded))
   405  		assert.Error(t, m.Error, "AddWorkload")
   406  	}
   407  	assert.EqualValues(t, 2, cnt)
   408  	assert.True(t, walCommitted)
   409  
   410  	// doCreateAndStartWorkload fails: first time AddWorkload failed
   411  	engine.On("VirtualizationCreate", mock.Anything, mock.Anything).Return(&enginetypes.VirtualizationCreated{ID: "c1"}, nil)
   412  	engine.On("VirtualizationStart", mock.Anything, mock.Anything).Return(nil)
   413  	engine.On("VirtualizationInspect", mock.Anything, mock.Anything).Return(&enginetypes.VirtualizationInfo{}, nil)
   414  	store.On("AddWorkload", mock.Anything, mock.Anything, mock.Anything).Return(errors.Wrap(context.DeadlineExceeded, "AddWorkload2")).Once()
   415  	store.On("AddWorkload", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
   416  	walCommitted = false
   417  	ch, err = c.CreateWorkload(ctx, opts)
   418  	assert.Nil(t, err)
   419  	cnt = 0
   420  	errCnt := 0
   421  	for m := range ch {
   422  		cnt++
   423  		if m.Error != nil {
   424  			assert.Error(t, m.Error)
   425  			assert.True(t, errors.Is(m.Error, context.DeadlineExceeded))
   426  			assert.Error(t, m.Error, "AddWorkload2")
   427  			errCnt++
   428  		}
   429  	}
   430  	assert.EqualValues(t, 2, cnt)
   431  	assert.EqualValues(t, 1, errCnt)
   432  	assert.True(t, walCommitted)
   433  	store.AssertExpectations(t)
   434  	engine.AssertExpectations(t)
   435  }
   436  
   437  func newCreateWorkloadCluster(_ *testing.T) (*Calcium, []*types.Node) {
   438  	c := NewTestCluster()
   439  
   440  	engine := &enginemocks.API{}
   441  	node1 := &types.Node{
   442  		NodeMeta: types.NodeMeta{
   443  			Name: "n1",
   444  		},
   445  		Engine: engine,
   446  	}
   447  	node2 := &types.Node{
   448  		NodeMeta: types.NodeMeta{
   449  			Name: "n2",
   450  		},
   451  		Engine: engine,
   452  	}
   453  	nodes := []*types.Node{node1, node2}
   454  
   455  	// for processing
   456  	store := c.store.(*storemocks.Store)
   457  	store.On("CreateProcessing", mock.Anything, mock.Anything, mock.Anything).Return(nil)
   458  	store.On("DeleteProcessing", mock.Anything, mock.Anything, mock.Anything).Return(nil)
   459  
   460  	// for lock
   461  	lock := &lockmocks.DistributedLock{}
   462  	lock.On("Lock", mock.Anything).Return(context.Background(), nil)
   463  	lock.On("Unlock", mock.Anything).Return(nil)
   464  	store.On("CreateLock", mock.Anything, mock.Anything).Return(lock, nil)
   465  
   466  	// for get node
   467  	store.On("GetNodesByPod", mock.Anything, mock.Anything).Return(nodes, nil)
   468  	store.On("GetNode", mock.Anything, mock.Anything).Return(
   469  		func(_ context.Context, name string) (node *types.Node) {
   470  			node = node1
   471  			if name == "n2" {
   472  				node = node2
   473  			}
   474  			return
   475  		}, nil)
   476  
   477  	store.On("RemoveWorkload", mock.Anything, mock.Anything).Return(nil)
   478  
   479  	return c, nodes
   480  }