github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/client/allocrunner/tasklifecycle/coordinator_test.go (about)

     1  package tasklifecycle
     2  
     3  import (
     4  	"testing"
     5  	"time"
     6  
     7  	"github.com/hashicorp/nomad/ci"
     8  	"github.com/hashicorp/nomad/helper/testlog"
     9  	"github.com/hashicorp/nomad/nomad/mock"
    10  	"github.com/hashicorp/nomad/nomad/structs"
    11  )
    12  
    13  func TestCoordinator_OnlyMainApp(t *testing.T) {
    14  	ci.Parallel(t)
    15  
    16  	alloc := mock.Alloc()
    17  	tasks := alloc.Job.TaskGroups[0].Tasks
    18  	task := tasks[0]
    19  	logger := testlog.HCLogger(t)
    20  
    21  	shutdownCh := make(chan struct{})
    22  	defer close(shutdownCh)
    23  	coord := NewCoordinator(logger, tasks, shutdownCh)
    24  
    25  	// Tasks starts blocked.
    26  	RequireTaskBlocked(t, coord, task)
    27  
    28  	// When main is pending it's allowed to run.
    29  	states := map[string]*structs.TaskState{
    30  		task.Name: {
    31  			State:  structs.TaskStatePending,
    32  			Failed: false,
    33  		},
    34  	}
    35  	coord.TaskStateUpdated(states)
    36  	RequireTaskAllowed(t, coord, task)
    37  
    38  	// After main is running, main tasks are still allowed to run.
    39  	states = map[string]*structs.TaskState{
    40  		task.Name: {
    41  			State:  structs.TaskStateRunning,
    42  			Failed: false,
    43  		},
    44  	}
    45  	coord.TaskStateUpdated(states)
    46  	RequireTaskAllowed(t, coord, task)
    47  }
    48  
    49  func TestCoordinator_PrestartRunsBeforeMain(t *testing.T) {
    50  	ci.Parallel(t)
    51  
    52  	logger := testlog.HCLogger(t)
    53  
    54  	alloc := mock.LifecycleAlloc()
    55  	tasks := alloc.Job.TaskGroups[0].Tasks
    56  
    57  	mainTask := tasks[0]
    58  	sideTask := tasks[1]
    59  	initTask := tasks[2]
    60  
    61  	// Only use the tasks that we care about.
    62  	tasks = []*structs.Task{mainTask, sideTask, initTask}
    63  
    64  	shutdownCh := make(chan struct{})
    65  	defer close(shutdownCh)
    66  	coord := NewCoordinator(logger, tasks, shutdownCh)
    67  
    68  	// All tasks start blocked.
    69  	RequireTaskBlocked(t, coord, initTask)
    70  	RequireTaskBlocked(t, coord, sideTask)
    71  	RequireTaskBlocked(t, coord, mainTask)
    72  
    73  	// Set initial state, prestart tasks are allowed to run.
    74  	states := map[string]*structs.TaskState{
    75  		initTask.Name: {
    76  			State:  structs.TaskStatePending,
    77  			Failed: false,
    78  		},
    79  		sideTask.Name: {
    80  			State:  structs.TaskStatePending,
    81  			Failed: false,
    82  		},
    83  		mainTask.Name: {
    84  			State:  structs.TaskStatePending,
    85  			Failed: false,
    86  		},
    87  	}
    88  	coord.TaskStateUpdated(states)
    89  	RequireTaskAllowed(t, coord, initTask)
    90  	RequireTaskAllowed(t, coord, sideTask)
    91  	RequireTaskBlocked(t, coord, mainTask)
    92  
    93  	// Sidecar task is running, main is blocked.
    94  	states = map[string]*structs.TaskState{
    95  		initTask.Name: {
    96  			State:  structs.TaskStatePending,
    97  			Failed: false,
    98  		},
    99  		sideTask.Name: {
   100  			State:  structs.TaskStateRunning,
   101  			Failed: false,
   102  		},
   103  		mainTask.Name: {
   104  			State:  structs.TaskStatePending,
   105  			Failed: false,
   106  		},
   107  	}
   108  	coord.TaskStateUpdated(states)
   109  	RequireTaskAllowed(t, coord, initTask)
   110  	RequireTaskAllowed(t, coord, sideTask)
   111  	RequireTaskBlocked(t, coord, mainTask)
   112  
   113  	// Init task is running, main is blocked.
   114  	states = map[string]*structs.TaskState{
   115  		initTask.Name: {
   116  			State:  structs.TaskStateRunning,
   117  			Failed: false,
   118  		},
   119  		sideTask.Name: {
   120  			State:  structs.TaskStateRunning,
   121  			Failed: false,
   122  		},
   123  		mainTask.Name: {
   124  			State:  structs.TaskStatePending,
   125  			Failed: false,
   126  		},
   127  	}
   128  	coord.TaskStateUpdated(states)
   129  	RequireTaskAllowed(t, coord, initTask)
   130  	RequireTaskAllowed(t, coord, sideTask)
   131  	RequireTaskBlocked(t, coord, mainTask)
   132  
   133  	// Init task is done, main is now allowed to run.
   134  	states = map[string]*structs.TaskState{
   135  		initTask.Name: {
   136  			State:  structs.TaskStateDead,
   137  			Failed: false,
   138  		},
   139  		sideTask.Name: {
   140  			State:  structs.TaskStateRunning,
   141  			Failed: false,
   142  		},
   143  		mainTask.Name: {
   144  			State:  structs.TaskStatePending,
   145  			Failed: false,
   146  		},
   147  	}
   148  	coord.TaskStateUpdated(states)
   149  	RequireTaskBlocked(t, coord, initTask)
   150  	RequireTaskAllowed(t, coord, sideTask)
   151  	RequireTaskAllowed(t, coord, mainTask)
   152  }
   153  
   154  func TestCoordinator_MainRunsAfterManyInitTasks(t *testing.T) {
   155  	ci.Parallel(t)
   156  
   157  	logger := testlog.HCLogger(t)
   158  
   159  	alloc := mock.LifecycleAlloc()
   160  	alloc.Job = mock.VariableLifecycleJob(structs.Resources{CPU: 100, MemoryMB: 256}, 1, 2, 0)
   161  	tasks := alloc.Job.TaskGroups[0].Tasks
   162  
   163  	mainTask := tasks[0]
   164  	init1Task := tasks[1]
   165  	init2Task := tasks[2]
   166  
   167  	// Only use the tasks that we care about.
   168  	tasks = []*structs.Task{mainTask, init1Task, init2Task}
   169  
   170  	shutdownCh := make(chan struct{})
   171  	defer close(shutdownCh)
   172  	coord := NewCoordinator(logger, tasks, shutdownCh)
   173  
   174  	// All tasks start blocked.
   175  	RequireTaskBlocked(t, coord, init1Task)
   176  	RequireTaskBlocked(t, coord, init2Task)
   177  	RequireTaskBlocked(t, coord, mainTask)
   178  
   179  	// Set initial state, prestart tasks are allowed to run, main is blocked.
   180  	states := map[string]*structs.TaskState{
   181  		init1Task.Name: {
   182  			State:  structs.TaskStatePending,
   183  			Failed: false,
   184  		},
   185  		init2Task.Name: {
   186  			State:  structs.TaskStatePending,
   187  			Failed: false,
   188  		},
   189  		mainTask.Name: {
   190  			State:  structs.TaskStatePending,
   191  			Failed: false,
   192  		},
   193  	}
   194  	coord.TaskStateUpdated(states)
   195  	RequireTaskAllowed(t, coord, init1Task)
   196  	RequireTaskAllowed(t, coord, init2Task)
   197  	RequireTaskBlocked(t, coord, mainTask)
   198  
   199  	// Init tasks complete, main is allowed to run.
   200  	states = map[string]*structs.TaskState{
   201  		init1Task.Name: {
   202  			State:      structs.TaskStateDead,
   203  			Failed:     false,
   204  			StartedAt:  time.Now(),
   205  			FinishedAt: time.Now(),
   206  		},
   207  		init2Task.Name: {
   208  			State:     structs.TaskStateDead,
   209  			Failed:    false,
   210  			StartedAt: time.Now(),
   211  		},
   212  		mainTask.Name: {
   213  			State:  structs.TaskStatePending,
   214  			Failed: false,
   215  		},
   216  	}
   217  	coord.TaskStateUpdated(states)
   218  	RequireTaskBlocked(t, coord, init1Task)
   219  	RequireTaskBlocked(t, coord, init2Task)
   220  	RequireTaskAllowed(t, coord, mainTask)
   221  }
   222  
   223  func TestCoordinator_FailedInitTask(t *testing.T) {
   224  	ci.Parallel(t)
   225  
   226  	logger := testlog.HCLogger(t)
   227  
   228  	alloc := mock.LifecycleAlloc()
   229  	alloc.Job = mock.VariableLifecycleJob(structs.Resources{CPU: 100, MemoryMB: 256}, 1, 2, 0)
   230  	tasks := alloc.Job.TaskGroups[0].Tasks
   231  
   232  	mainTask := tasks[0]
   233  	init1Task := tasks[1]
   234  	init2Task := tasks[2]
   235  
   236  	// Only use the tasks that we care about.
   237  	tasks = []*structs.Task{mainTask, init1Task, init2Task}
   238  
   239  	shutdownCh := make(chan struct{})
   240  	defer close(shutdownCh)
   241  	coord := NewCoordinator(logger, tasks, shutdownCh)
   242  
   243  	// All tasks start blocked.
   244  	RequireTaskBlocked(t, coord, init1Task)
   245  	RequireTaskBlocked(t, coord, init2Task)
   246  	RequireTaskBlocked(t, coord, mainTask)
   247  
   248  	// Set initial state, prestart tasks are allowed to run, main is blocked.
   249  	states := map[string]*structs.TaskState{
   250  		init1Task.Name: {
   251  			State:  structs.TaskStatePending,
   252  			Failed: false,
   253  		},
   254  		init2Task.Name: {
   255  			State:  structs.TaskStatePending,
   256  			Failed: false,
   257  		},
   258  		mainTask.Name: {
   259  			State:  structs.TaskStatePending,
   260  			Failed: false,
   261  		},
   262  	}
   263  	coord.TaskStateUpdated(states)
   264  	RequireTaskAllowed(t, coord, init1Task)
   265  	RequireTaskAllowed(t, coord, init2Task)
   266  	RequireTaskBlocked(t, coord, mainTask)
   267  
   268  	// Init task dies, main is still blocked.
   269  	states = map[string]*structs.TaskState{
   270  		init1Task.Name: {
   271  			State:      structs.TaskStateDead,
   272  			Failed:     false,
   273  			StartedAt:  time.Now(),
   274  			FinishedAt: time.Now(),
   275  		},
   276  		init2Task.Name: {
   277  			State:     structs.TaskStateDead,
   278  			Failed:    true,
   279  			StartedAt: time.Now(),
   280  		},
   281  		mainTask.Name: {
   282  			State:  structs.TaskStatePending,
   283  			Failed: false,
   284  		},
   285  	}
   286  	coord.TaskStateUpdated(states)
   287  	RequireTaskAllowed(t, coord, init1Task)
   288  	RequireTaskAllowed(t, coord, init2Task)
   289  	RequireTaskBlocked(t, coord, mainTask)
   290  }
   291  
   292  func TestCoordinator_SidecarNeverStarts(t *testing.T) {
   293  	ci.Parallel(t)
   294  
   295  	logger := testlog.HCLogger(t)
   296  
   297  	alloc := mock.LifecycleAlloc()
   298  	tasks := alloc.Job.TaskGroups[0].Tasks
   299  
   300  	mainTask := tasks[0]
   301  	sideTask := tasks[1]
   302  	initTask := tasks[2]
   303  
   304  	// Only use the tasks that we care about.
   305  	tasks = []*structs.Task{mainTask, sideTask, initTask}
   306  
   307  	shutdownCh := make(chan struct{})
   308  	defer close(shutdownCh)
   309  	coord := NewCoordinator(logger, tasks, shutdownCh)
   310  
   311  	// All tasks start blocked.
   312  	RequireTaskBlocked(t, coord, initTask)
   313  	RequireTaskBlocked(t, coord, sideTask)
   314  	RequireTaskBlocked(t, coord, mainTask)
   315  
   316  	// Set initial state, prestart tasks are allowed to run, main is blocked.
   317  	states := map[string]*structs.TaskState{
   318  		initTask.Name: {
   319  			State:  structs.TaskStatePending,
   320  			Failed: false,
   321  		},
   322  		sideTask.Name: {
   323  			State:  structs.TaskStatePending,
   324  			Failed: false,
   325  		},
   326  		mainTask.Name: {
   327  			State:  structs.TaskStatePending,
   328  			Failed: false,
   329  		},
   330  	}
   331  	coord.TaskStateUpdated(states)
   332  	RequireTaskAllowed(t, coord, initTask)
   333  	RequireTaskAllowed(t, coord, sideTask)
   334  	RequireTaskBlocked(t, coord, mainTask)
   335  
   336  	// Init completes, but sidecar not yet.
   337  	states = map[string]*structs.TaskState{
   338  		initTask.Name: {
   339  			State:      structs.TaskStateDead,
   340  			Failed:     false,
   341  			StartedAt:  time.Now(),
   342  			FinishedAt: time.Now(),
   343  		},
   344  		sideTask.Name: {
   345  			State:  structs.TaskStatePending,
   346  			Failed: false,
   347  		},
   348  		mainTask.Name: {
   349  			State:  structs.TaskStatePending,
   350  			Failed: false,
   351  		},
   352  	}
   353  	coord.TaskStateUpdated(states)
   354  	RequireTaskAllowed(t, coord, initTask)
   355  	RequireTaskAllowed(t, coord, sideTask)
   356  	RequireTaskBlocked(t, coord, mainTask)
   357  }
   358  
   359  func TestCoordinator_PoststartStartsAfterMain(t *testing.T) {
   360  	ci.Parallel(t)
   361  
   362  	logger := testlog.HCLogger(t)
   363  
   364  	alloc := mock.LifecycleAlloc()
   365  	tasks := alloc.Job.TaskGroups[0].Tasks
   366  
   367  	mainTask := tasks[0]
   368  	sideTask := tasks[1]
   369  	postTask := tasks[2]
   370  
   371  	// Only use the tasks that we care about.
   372  	tasks = []*structs.Task{mainTask, sideTask, postTask}
   373  
   374  	// Make the the third task is a poststart hook
   375  	postTask.Lifecycle.Hook = structs.TaskLifecycleHookPoststart
   376  
   377  	shutdownCh := make(chan struct{})
   378  	defer close(shutdownCh)
   379  	coord := NewCoordinator(logger, tasks, shutdownCh)
   380  
   381  	// All tasks start blocked.
   382  	RequireTaskBlocked(t, coord, sideTask)
   383  	RequireTaskBlocked(t, coord, mainTask)
   384  	RequireTaskBlocked(t, coord, postTask)
   385  
   386  	// Set initial state, prestart tasks are allowed to run, main and poststart
   387  	// are blocked.
   388  	states := map[string]*structs.TaskState{
   389  		sideTask.Name: {
   390  			State:  structs.TaskStatePending,
   391  			Failed: false,
   392  		},
   393  		mainTask.Name: {
   394  			State:  structs.TaskStatePending,
   395  			Failed: false,
   396  		},
   397  		postTask.Name: {
   398  			State:  structs.TaskStatePending,
   399  			Failed: false,
   400  		},
   401  	}
   402  	coord.TaskStateUpdated(states)
   403  	RequireTaskAllowed(t, coord, sideTask)
   404  	RequireTaskBlocked(t, coord, mainTask)
   405  	RequireTaskBlocked(t, coord, postTask)
   406  
   407  	// Sidecar and main running, poststart allowed to run.
   408  	states = map[string]*structs.TaskState{
   409  		sideTask.Name: {
   410  			State:     structs.TaskStateRunning,
   411  			Failed:    false,
   412  			StartedAt: time.Now(),
   413  		},
   414  		mainTask.Name: {
   415  			State:     structs.TaskStateRunning,
   416  			Failed:    false,
   417  			StartedAt: time.Now(),
   418  		},
   419  		postTask.Name: {
   420  			State:  structs.TaskStatePending,
   421  			Failed: false,
   422  		},
   423  	}
   424  	coord.TaskStateUpdated(states)
   425  	RequireTaskAllowed(t, coord, sideTask)
   426  	RequireTaskAllowed(t, coord, mainTask)
   427  	RequireTaskAllowed(t, coord, postTask)
   428  }
   429  
   430  func TestCoordinator_Restore(t *testing.T) {
   431  	ci.Parallel(t)
   432  
   433  	task := mock.Job().TaskGroups[0].Tasks[0]
   434  
   435  	preEphemeral := task.Copy()
   436  	preEphemeral.Name = "pre_ephemeral"
   437  	preEphemeral.Lifecycle = &structs.TaskLifecycleConfig{
   438  		Hook:    structs.TaskLifecycleHookPrestart,
   439  		Sidecar: false,
   440  	}
   441  
   442  	preSide := task.Copy()
   443  	preSide.Name = "pre_side"
   444  	preSide.Lifecycle = &structs.TaskLifecycleConfig{
   445  		Hook:    structs.TaskLifecycleHookPrestart,
   446  		Sidecar: true,
   447  	}
   448  
   449  	main := task.Copy()
   450  	main.Name = "main"
   451  	main.Lifecycle = nil
   452  
   453  	postEphemeral := task.Copy()
   454  	postEphemeral.Name = "post_ephemeral"
   455  	postEphemeral.Lifecycle = &structs.TaskLifecycleConfig{
   456  		Hook:    structs.TaskLifecycleHookPoststart,
   457  		Sidecar: false,
   458  	}
   459  
   460  	postSide := task.Copy()
   461  	postSide.Name = "post_side"
   462  	postSide.Lifecycle = &structs.TaskLifecycleConfig{
   463  		Hook:    structs.TaskLifecycleHookPoststart,
   464  		Sidecar: true,
   465  	}
   466  
   467  	poststop := task.Copy()
   468  	poststop.Name = "poststop"
   469  	poststop.Lifecycle = &structs.TaskLifecycleConfig{
   470  		Hook:    structs.TaskLifecycleHookPoststop,
   471  		Sidecar: false,
   472  	}
   473  
   474  	testCases := []struct {
   475  		name       string
   476  		tasks      []*structs.Task
   477  		tasksState map[string]*structs.TaskState
   478  		testFn     func(*testing.T, *Coordinator)
   479  	}{
   480  		{
   481  			name:  "prestart ephemeral running",
   482  			tasks: []*structs.Task{preEphemeral, preSide, main},
   483  			tasksState: map[string]*structs.TaskState{
   484  				preEphemeral.Name: {State: structs.TaskStateRunning},
   485  				preSide.Name:      {State: structs.TaskStateRunning},
   486  				main.Name:         {State: structs.TaskStatePending},
   487  			},
   488  			testFn: func(t *testing.T, c *Coordinator) {
   489  				RequireTaskBlocked(t, c, main)
   490  
   491  				RequireTaskAllowed(t, c, preEphemeral)
   492  				RequireTaskAllowed(t, c, preSide)
   493  			},
   494  		},
   495  		{
   496  			name:  "prestart ephemeral complete",
   497  			tasks: []*structs.Task{preEphemeral, preSide, main},
   498  			tasksState: map[string]*structs.TaskState{
   499  				preEphemeral.Name: {State: structs.TaskStateDead},
   500  				preSide.Name:      {State: structs.TaskStateRunning},
   501  				main.Name:         {State: structs.TaskStatePending},
   502  			},
   503  			testFn: func(t *testing.T, c *Coordinator) {
   504  				RequireTaskBlocked(t, c, preEphemeral)
   505  
   506  				RequireTaskAllowed(t, c, preSide)
   507  				RequireTaskAllowed(t, c, main)
   508  			},
   509  		},
   510  		{
   511  			name:  "main running",
   512  			tasks: []*structs.Task{main},
   513  			tasksState: map[string]*structs.TaskState{
   514  				main.Name: {State: structs.TaskStateRunning},
   515  			},
   516  			testFn: func(t *testing.T, c *Coordinator) {
   517  				RequireTaskAllowed(t, c, main)
   518  			},
   519  		},
   520  		{
   521  			name:  "poststart with sidecar",
   522  			tasks: []*structs.Task{main, postEphemeral, postSide},
   523  			tasksState: map[string]*structs.TaskState{
   524  				main.Name:          {State: structs.TaskStateRunning},
   525  				postEphemeral.Name: {State: structs.TaskStateDead},
   526  				postSide.Name:      {State: structs.TaskStateRunning},
   527  			},
   528  			testFn: func(t *testing.T, c *Coordinator) {
   529  				RequireTaskBlocked(t, c, postEphemeral)
   530  
   531  				RequireTaskAllowed(t, c, main)
   532  				RequireTaskAllowed(t, c, postSide)
   533  			},
   534  		},
   535  		{
   536  			name:  "poststop running",
   537  			tasks: []*structs.Task{main, poststop},
   538  			tasksState: map[string]*structs.TaskState{
   539  				main.Name:     {State: structs.TaskStateDead},
   540  				poststop.Name: {State: structs.TaskStateRunning},
   541  			},
   542  			testFn: func(t *testing.T, c *Coordinator) {
   543  				RequireTaskBlocked(t, c, main)
   544  
   545  				RequireTaskAllowed(t, c, poststop)
   546  			},
   547  		},
   548  	}
   549  
   550  	for _, tc := range testCases {
   551  		t.Run(tc.name, func(t *testing.T) {
   552  			shutdownCh := make(chan struct{})
   553  			defer close(shutdownCh)
   554  
   555  			c := NewCoordinator(testlog.HCLogger(t), tc.tasks, shutdownCh)
   556  			c.Restore(tc.tasksState)
   557  			tc.testFn(t, c)
   558  		})
   559  	}
   560  }