github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/nomad/state/events_test.go (about)

     1  package state
     2  
     3  import (
     4  	"testing"
     5  	"time"
     6  
     7  	memdb "github.com/hashicorp/go-memdb"
     8  	"github.com/hashicorp/nomad/helper"
     9  	"github.com/hashicorp/nomad/helper/uuid"
    10  	"github.com/hashicorp/nomad/nomad/mock"
    11  	"github.com/hashicorp/nomad/nomad/structs"
    12  	"github.com/stretchr/testify/require"
    13  )
    14  
    15  // TestEventFromChange_SingleEventPerTable ensures that only a single event is
    16  // created per table per memdb.Change
    17  func TestEventFromChange_SingleEventPerTable(t *testing.T) {
    18  	t.Parallel()
    19  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
    20  	defer s.StopEventBroker()
    21  
    22  	changes := Changes{
    23  		Index:   100,
    24  		MsgType: structs.JobRegisterRequestType,
    25  		Changes: memdb.Changes{
    26  			{
    27  				Table:  "job_version",
    28  				Before: mock.Job(),
    29  				After:  mock.Job(),
    30  			},
    31  			{
    32  				Table:  "jobs",
    33  				Before: mock.Job(),
    34  				After:  mock.Job(),
    35  			},
    36  		},
    37  	}
    38  
    39  	out := eventsFromChanges(s.db.ReadTxn(), changes)
    40  	require.Len(t, out.Events, 1)
    41  	require.Equal(t, out.Events[0].Type, structs.TypeJobRegistered)
    42  }
    43  
    44  func TestEventFromChange_ACLTokenSecretID(t *testing.T) {
    45  	t.Parallel()
    46  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
    47  	defer s.StopEventBroker()
    48  
    49  	token := mock.ACLToken()
    50  	require.NotEmpty(t, token.SecretID)
    51  
    52  	// Create
    53  	changes := Changes{
    54  		Index:   100,
    55  		MsgType: structs.NodeRegisterRequestType,
    56  		Changes: memdb.Changes{
    57  			{
    58  				Table:  "acl_token",
    59  				Before: nil,
    60  				After:  token,
    61  			},
    62  		},
    63  	}
    64  
    65  	out := eventsFromChanges(s.db.ReadTxn(), changes)
    66  	require.Len(t, out.Events, 1)
    67  	// Ensure original value not altered
    68  	require.NotEmpty(t, token.SecretID)
    69  
    70  	aclTokenEvent, ok := out.Events[0].Payload.(*structs.ACLTokenEvent)
    71  	require.True(t, ok)
    72  	require.Empty(t, aclTokenEvent.ACLToken.SecretID)
    73  
    74  	require.Equal(t, token.SecretID, aclTokenEvent.SecretID())
    75  
    76  	// Delete
    77  	changes = Changes{
    78  		Index:   100,
    79  		MsgType: structs.NodeDeregisterRequestType,
    80  		Changes: memdb.Changes{
    81  			{
    82  				Table:  "acl_token",
    83  				Before: token,
    84  				After:  nil,
    85  			},
    86  		},
    87  	}
    88  
    89  	out2 := eventsFromChanges(s.db.ReadTxn(), changes)
    90  	require.Len(t, out2.Events, 1)
    91  
    92  	tokenEvent2, ok := out2.Events[0].Payload.(*structs.ACLTokenEvent)
    93  	require.True(t, ok)
    94  	require.Empty(t, tokenEvent2.ACLToken.SecretID)
    95  }
    96  
    97  // TestEventFromChange_NodeSecretID ensures that a node's secret ID is not
    98  // included in a node event
    99  func TestEventFromChange_NodeSecretID(t *testing.T) {
   100  	t.Parallel()
   101  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
   102  	defer s.StopEventBroker()
   103  
   104  	node := mock.Node()
   105  	require.NotEmpty(t, node.SecretID)
   106  
   107  	// Create
   108  	changes := Changes{
   109  		Index:   100,
   110  		MsgType: structs.NodeRegisterRequestType,
   111  		Changes: memdb.Changes{
   112  			{
   113  				Table:  "nodes",
   114  				Before: nil,
   115  				After:  node,
   116  			},
   117  		},
   118  	}
   119  
   120  	out := eventsFromChanges(s.db.ReadTxn(), changes)
   121  	require.Len(t, out.Events, 1)
   122  
   123  	nodeEvent, ok := out.Events[0].Payload.(*structs.NodeStreamEvent)
   124  	require.True(t, ok)
   125  	require.Empty(t, nodeEvent.Node.SecretID)
   126  
   127  	// Delete
   128  	changes = Changes{
   129  		Index:   100,
   130  		MsgType: structs.NodeDeregisterRequestType,
   131  		Changes: memdb.Changes{
   132  			{
   133  				Table:  "nodes",
   134  				Before: node,
   135  				After:  nil,
   136  			},
   137  		},
   138  	}
   139  
   140  	out2 := eventsFromChanges(s.db.ReadTxn(), changes)
   141  	require.Len(t, out2.Events, 1)
   142  
   143  	nodeEvent2, ok := out2.Events[0].Payload.(*structs.NodeStreamEvent)
   144  	require.True(t, ok)
   145  	require.Empty(t, nodeEvent2.Node.SecretID)
   146  }
   147  
   148  func TestEventsFromChanges_DeploymentUpdate(t *testing.T) {
   149  	t.Parallel()
   150  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
   151  	defer s.StopEventBroker()
   152  
   153  	// setup
   154  	setupTx := s.db.WriteTxn(10)
   155  
   156  	j := mock.Job()
   157  	e := mock.Eval()
   158  	e.JobID = j.ID
   159  
   160  	d := mock.Deployment()
   161  	d.JobID = j.ID
   162  
   163  	require.NoError(t, s.upsertJobImpl(10, j, false, setupTx))
   164  	require.NoError(t, s.upsertDeploymentImpl(10, d, setupTx))
   165  
   166  	setupTx.Txn.Commit()
   167  
   168  	msgType := structs.DeploymentStatusUpdateRequestType
   169  
   170  	req := &structs.DeploymentStatusUpdateRequest{
   171  		DeploymentUpdate: &structs.DeploymentStatusUpdate{
   172  			DeploymentID:      d.ID,
   173  			Status:            structs.DeploymentStatusPaused,
   174  			StatusDescription: structs.DeploymentStatusDescriptionPaused,
   175  		},
   176  		Eval: e,
   177  		// Exlude Job and assert its added
   178  	}
   179  
   180  	require.NoError(t, s.UpdateDeploymentStatus(msgType, 100, req))
   181  
   182  	events := WaitForEvents(t, s, 100, 1, 1*time.Second)
   183  	require.Len(t, events, 2)
   184  
   185  	got := events[0]
   186  	require.Equal(t, uint64(100), got.Index)
   187  	require.Equal(t, d.ID, got.Key)
   188  
   189  	de := got.Payload.(*structs.DeploymentEvent)
   190  	require.Equal(t, structs.DeploymentStatusPaused, de.Deployment.Status)
   191  	require.Contains(t, got.FilterKeys, j.ID)
   192  }
   193  
   194  func TestEventsFromChanges_DeploymentPromotion(t *testing.T) {
   195  	t.Parallel()
   196  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
   197  	defer s.StopEventBroker()
   198  
   199  	// setup
   200  	setupTx := s.db.WriteTxn(10)
   201  
   202  	j := mock.Job()
   203  	tg1 := j.TaskGroups[0]
   204  	tg2 := tg1.Copy()
   205  	tg2.Name = "foo"
   206  	j.TaskGroups = append(j.TaskGroups, tg2)
   207  	require.NoError(t, s.upsertJobImpl(10, j, false, setupTx))
   208  
   209  	d := mock.Deployment()
   210  	d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion
   211  	d.JobID = j.ID
   212  	d.TaskGroups = map[string]*structs.DeploymentState{
   213  		"web": {
   214  			DesiredTotal:    10,
   215  			DesiredCanaries: 1,
   216  		},
   217  		"foo": {
   218  			DesiredTotal:    10,
   219  			DesiredCanaries: 1,
   220  		},
   221  	}
   222  	require.NoError(t, s.upsertDeploymentImpl(10, d, setupTx))
   223  
   224  	// create set of allocs
   225  	c1 := mock.Alloc()
   226  	c1.JobID = j.ID
   227  	c1.DeploymentID = d.ID
   228  	d.TaskGroups[c1.TaskGroup].PlacedCanaries = append(d.TaskGroups[c1.TaskGroup].PlacedCanaries, c1.ID)
   229  	c1.DeploymentStatus = &structs.AllocDeploymentStatus{
   230  		Healthy: helper.BoolToPtr(true),
   231  	}
   232  	c2 := mock.Alloc()
   233  	c2.JobID = j.ID
   234  	c2.DeploymentID = d.ID
   235  	d.TaskGroups[c2.TaskGroup].PlacedCanaries = append(d.TaskGroups[c2.TaskGroup].PlacedCanaries, c2.ID)
   236  	c2.TaskGroup = tg2.Name
   237  	c2.DeploymentStatus = &structs.AllocDeploymentStatus{
   238  		Healthy: helper.BoolToPtr(true),
   239  	}
   240  
   241  	require.NoError(t, s.upsertAllocsImpl(10, []*structs.Allocation{c1, c2}, setupTx))
   242  
   243  	// commit setup transaction
   244  	setupTx.Txn.Commit()
   245  
   246  	e := mock.Eval()
   247  	// Request to promote canaries
   248  	msgType := structs.DeploymentPromoteRequestType
   249  	req := &structs.ApplyDeploymentPromoteRequest{
   250  		DeploymentPromoteRequest: structs.DeploymentPromoteRequest{
   251  			DeploymentID: d.ID,
   252  			All:          true,
   253  		},
   254  		Eval: e,
   255  	}
   256  
   257  	require.NoError(t, s.UpdateDeploymentPromotion(msgType, 100, req))
   258  
   259  	events := WaitForEvents(t, s, 100, 1, 1*time.Second)
   260  	require.Len(t, events, 4)
   261  
   262  	got := events[0]
   263  	require.Equal(t, uint64(100), got.Index)
   264  	require.Equal(t, d.ID, got.Key)
   265  
   266  	de := got.Payload.(*structs.DeploymentEvent)
   267  	require.Equal(t, structs.DeploymentStatusRunning, de.Deployment.Status)
   268  	require.Equal(t, structs.TypeDeploymentPromotion, got.Type)
   269  }
   270  
   271  func TestEventsFromChanges_DeploymentAllocHealthRequestType(t *testing.T) {
   272  	t.Parallel()
   273  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
   274  	defer s.StopEventBroker()
   275  
   276  	// setup
   277  	setupTx := s.db.WriteTxn(10)
   278  
   279  	j := mock.Job()
   280  	tg1 := j.TaskGroups[0]
   281  	tg2 := tg1.Copy()
   282  	tg2.Name = "foo"
   283  	j.TaskGroups = append(j.TaskGroups, tg2)
   284  	require.NoError(t, s.upsertJobImpl(10, j, false, setupTx))
   285  
   286  	d := mock.Deployment()
   287  	d.StatusDescription = structs.DeploymentStatusDescriptionRunningNeedsPromotion
   288  	d.JobID = j.ID
   289  	d.TaskGroups = map[string]*structs.DeploymentState{
   290  		"web": {
   291  			DesiredTotal:    10,
   292  			DesiredCanaries: 1,
   293  		},
   294  		"foo": {
   295  			DesiredTotal:    10,
   296  			DesiredCanaries: 1,
   297  		},
   298  	}
   299  	require.NoError(t, s.upsertDeploymentImpl(10, d, setupTx))
   300  
   301  	// create set of allocs
   302  	c1 := mock.Alloc()
   303  	c1.JobID = j.ID
   304  	c1.DeploymentID = d.ID
   305  	d.TaskGroups[c1.TaskGroup].PlacedCanaries = append(d.TaskGroups[c1.TaskGroup].PlacedCanaries, c1.ID)
   306  	c1.DeploymentStatus = &structs.AllocDeploymentStatus{
   307  		Healthy: helper.BoolToPtr(true),
   308  	}
   309  	c2 := mock.Alloc()
   310  	c2.JobID = j.ID
   311  	c2.DeploymentID = d.ID
   312  	d.TaskGroups[c2.TaskGroup].PlacedCanaries = append(d.TaskGroups[c2.TaskGroup].PlacedCanaries, c2.ID)
   313  	c2.TaskGroup = tg2.Name
   314  	c2.DeploymentStatus = &structs.AllocDeploymentStatus{
   315  		Healthy: helper.BoolToPtr(true),
   316  	}
   317  
   318  	require.NoError(t, s.upsertAllocsImpl(10, []*structs.Allocation{c1, c2}, setupTx))
   319  
   320  	// Commit setup
   321  	setupTx.Commit()
   322  
   323  	msgType := structs.DeploymentAllocHealthRequestType
   324  
   325  	req := &structs.ApplyDeploymentAllocHealthRequest{
   326  		DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{
   327  			DeploymentID:           d.ID,
   328  			HealthyAllocationIDs:   []string{c1.ID},
   329  			UnhealthyAllocationIDs: []string{c2.ID},
   330  		},
   331  		DeploymentUpdate: &structs.DeploymentStatusUpdate{
   332  			DeploymentID: d.ID,
   333  		},
   334  	}
   335  
   336  	require.NoError(t, s.UpdateDeploymentAllocHealth(msgType, 100, req))
   337  
   338  	events := WaitForEvents(t, s, 100, 1, 1*time.Second)
   339  	require.Len(t, events, 3)
   340  
   341  	var allocEvents []structs.Event
   342  	var deploymentEvent []structs.Event
   343  	for _, e := range events {
   344  		if e.Topic == structs.TopicAllocation {
   345  			allocEvents = append(allocEvents, e)
   346  		} else if e.Topic == structs.TopicDeployment {
   347  			deploymentEvent = append(deploymentEvent, e)
   348  		}
   349  	}
   350  
   351  	require.Len(t, allocEvents, 2)
   352  	for _, e := range allocEvents {
   353  		require.Equal(t, 100, int(e.Index))
   354  		require.Equal(t, structs.TypeDeploymentAllocHealth, e.Type)
   355  		require.Equal(t, structs.TopicAllocation, e.Topic)
   356  	}
   357  
   358  	require.Len(t, deploymentEvent, 1)
   359  	for _, e := range deploymentEvent {
   360  		require.Equal(t, 100, int(e.Index))
   361  		require.Equal(t, structs.TypeDeploymentAllocHealth, e.Type)
   362  		require.Equal(t, structs.TopicDeployment, e.Topic)
   363  		require.Equal(t, d.ID, e.Key)
   364  	}
   365  }
   366  
   367  func TestEventsFromChanges_UpsertNodeEventsType(t *testing.T) {
   368  	t.Parallel()
   369  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
   370  	defer s.StopEventBroker()
   371  
   372  	// setup
   373  	n1 := mock.Node()
   374  	n2 := mock.Node()
   375  
   376  	require.NoError(t, s.UpsertNode(structs.MsgTypeTestSetup, 10, n1))
   377  	require.NoError(t, s.UpsertNode(structs.MsgTypeTestSetup, 12, n2))
   378  
   379  	msgType := structs.UpsertNodeEventsType
   380  	req := &structs.EmitNodeEventsRequest{
   381  		NodeEvents: map[string][]*structs.NodeEvent{
   382  			n1.ID: {
   383  				{
   384  					Message: "update",
   385  				},
   386  			},
   387  			n2.ID: {
   388  				{
   389  					Message: "update",
   390  				},
   391  			},
   392  		},
   393  	}
   394  
   395  	require.NoError(t, s.UpsertNodeEvents(msgType, 100, req.NodeEvents))
   396  	events := WaitForEvents(t, s, 100, 1, 1*time.Second)
   397  	require.Len(t, events, 2)
   398  
   399  	for _, e := range events {
   400  		require.Equal(t, structs.TopicNode, e.Topic)
   401  		require.Equal(t, structs.TypeNodeEvent, e.Type)
   402  		event := e.Payload.(*structs.NodeStreamEvent)
   403  		require.Equal(t, "update", event.Node.Events[len(event.Node.Events)-1].Message)
   404  	}
   405  
   406  }
   407  
   408  func TestEventsFromChanges_NodeUpdateStatusRequest(t *testing.T) {
   409  	t.Parallel()
   410  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
   411  	defer s.StopEventBroker()
   412  
   413  	// setup
   414  	n1 := mock.Node()
   415  
   416  	require.NoError(t, s.UpsertNode(structs.MsgTypeTestSetup, 10, n1))
   417  
   418  	updated := time.Now()
   419  	msgType := structs.NodeUpdateStatusRequestType
   420  	req := &structs.NodeUpdateStatusRequest{
   421  		NodeID:    n1.ID,
   422  		Status:    structs.NodeStatusDown,
   423  		UpdatedAt: updated.UnixNano(),
   424  		NodeEvent: &structs.NodeEvent{Message: "down"},
   425  	}
   426  
   427  	require.NoError(t, s.UpdateNodeStatus(msgType, 100, req.NodeID, req.Status, req.UpdatedAt, req.NodeEvent))
   428  	events := WaitForEvents(t, s, 100, 1, 1*time.Second)
   429  	require.Len(t, events, 1)
   430  
   431  	e := events[0]
   432  	require.Equal(t, structs.TopicNode, e.Topic)
   433  	require.Equal(t, structs.TypeNodeEvent, e.Type)
   434  	event := e.Payload.(*structs.NodeStreamEvent)
   435  	require.Equal(t, "down", event.Node.Events[len(event.Node.Events)-1].Message)
   436  	require.Equal(t, structs.NodeStatusDown, event.Node.Status)
   437  }
   438  
   439  func TestEventsFromChanges_EvalUpdateRequestType(t *testing.T) {
   440  	t.Parallel()
   441  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
   442  	defer s.StopEventBroker()
   443  
   444  	// setup
   445  	e1 := mock.Eval()
   446  
   447  	require.NoError(t, s.UpsertEvals(structs.MsgTypeTestSetup, 10, []*structs.Evaluation{e1}))
   448  
   449  	e2 := mock.Eval()
   450  	e2.ID = e1.ID
   451  	e2.JobID = e1.JobID
   452  	e2.Status = structs.EvalStatusBlocked
   453  
   454  	msgType := structs.EvalUpdateRequestType
   455  	req := &structs.EvalUpdateRequest{
   456  		Evals: []*structs.Evaluation{e2},
   457  	}
   458  
   459  	require.NoError(t, s.UpsertEvals(msgType, 100, req.Evals))
   460  
   461  	events := WaitForEvents(t, s, 100, 1, 1*time.Second)
   462  	require.Len(t, events, 1)
   463  
   464  	e := events[0]
   465  	require.Equal(t, structs.TopicEvaluation, e.Topic)
   466  	require.Equal(t, structs.TypeEvalUpdated, e.Type)
   467  	require.Contains(t, e.FilterKeys, e2.JobID)
   468  	require.Contains(t, e.FilterKeys, e2.DeploymentID)
   469  	event := e.Payload.(*structs.EvaluationEvent)
   470  	require.Equal(t, "blocked", event.Evaluation.Status)
   471  }
   472  
   473  func TestEventsFromChanges_ApplyPlanResultsRequestType(t *testing.T) {
   474  	t.Parallel()
   475  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
   476  	defer s.StopEventBroker()
   477  
   478  	// setup
   479  	alloc := mock.Alloc()
   480  	alloc2 := mock.Alloc()
   481  	job := alloc.Job
   482  	alloc.Job = nil
   483  	alloc2.Job = nil
   484  
   485  	d := mock.Deployment()
   486  	alloc.DeploymentID = d.ID
   487  	alloc2.DeploymentID = d.ID
   488  
   489  	require.NoError(t, s.UpsertJob(structs.MsgTypeTestSetup, 9, job))
   490  
   491  	eval := mock.Eval()
   492  	eval.JobID = job.ID
   493  
   494  	// Create an eval
   495  	require.NoError(t, s.UpsertEvals(structs.MsgTypeTestSetup, 10, []*structs.Evaluation{eval}))
   496  
   497  	msgType := structs.ApplyPlanResultsRequestType
   498  	req := &structs.ApplyPlanResultsRequest{
   499  		AllocUpdateRequest: structs.AllocUpdateRequest{
   500  			Alloc: []*structs.Allocation{alloc, alloc2},
   501  			Job:   job,
   502  		},
   503  		Deployment: d,
   504  		EvalID:     eval.ID,
   505  	}
   506  
   507  	require.NoError(t, s.UpsertPlanResults(msgType, 100, req))
   508  
   509  	events := WaitForEvents(t, s, 100, 1, 1*time.Second)
   510  	require.Len(t, events, 5)
   511  
   512  	var allocs []structs.Event
   513  	var evals []structs.Event
   514  	var jobs []structs.Event
   515  	var deploys []structs.Event
   516  	for _, e := range events {
   517  		if e.Topic == structs.TopicAllocation {
   518  			allocs = append(allocs, e)
   519  		} else if e.Topic == structs.TopicEvaluation {
   520  			evals = append(evals, e)
   521  		} else if e.Topic == structs.TopicJob {
   522  			jobs = append(jobs, e)
   523  		} else if e.Topic == structs.TopicDeployment {
   524  			deploys = append(deploys, e)
   525  		}
   526  		require.Equal(t, structs.TypePlanResult, e.Type)
   527  	}
   528  	require.Len(t, allocs, 2)
   529  	require.Len(t, evals, 1)
   530  	require.Len(t, jobs, 1)
   531  	require.Len(t, deploys, 1)
   532  }
   533  
   534  func TestEventsFromChanges_BatchNodeUpdateDrainRequestType(t *testing.T) {
   535  	t.Parallel()
   536  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
   537  	defer s.StopEventBroker()
   538  
   539  	// setup
   540  	n1 := mock.Node()
   541  	n2 := mock.Node()
   542  
   543  	require.NoError(t, s.UpsertNode(structs.MsgTypeTestSetup, 10, n1))
   544  	require.NoError(t, s.UpsertNode(structs.MsgTypeTestSetup, 11, n2))
   545  
   546  	updated := time.Now()
   547  	msgType := structs.BatchNodeUpdateDrainRequestType
   548  
   549  	expectedDrain := &structs.DrainStrategy{
   550  		DrainSpec: structs.DrainSpec{
   551  			Deadline: -1 * time.Second,
   552  		},
   553  	}
   554  	event := &structs.NodeEvent{
   555  		Message:   "Drain strategy enabled",
   556  		Subsystem: structs.NodeEventSubsystemDrain,
   557  		Timestamp: time.Now(),
   558  	}
   559  	req := structs.BatchNodeUpdateDrainRequest{
   560  		Updates: map[string]*structs.DrainUpdate{
   561  			n1.ID: {
   562  				DrainStrategy: expectedDrain,
   563  			},
   564  			n2.ID: {
   565  				DrainStrategy: expectedDrain,
   566  			},
   567  		},
   568  		NodeEvents: map[string]*structs.NodeEvent{
   569  			n1.ID: event,
   570  			n2.ID: event,
   571  		},
   572  		UpdatedAt: updated.UnixNano(),
   573  	}
   574  
   575  	require.NoError(t, s.BatchUpdateNodeDrain(msgType, 100, req.UpdatedAt, req.Updates, req.NodeEvents))
   576  
   577  	events := WaitForEvents(t, s, 100, 1, 1*time.Second)
   578  	require.Len(t, events, 2)
   579  
   580  	for _, e := range events {
   581  		require.Equal(t, 100, int(e.Index))
   582  		require.Equal(t, structs.TypeNodeDrain, e.Type)
   583  		require.Equal(t, structs.TopicNode, e.Topic)
   584  		ne := e.Payload.(*structs.NodeStreamEvent)
   585  		require.Equal(t, event.Message, ne.Node.Events[len(ne.Node.Events)-1].Message)
   586  	}
   587  }
   588  
   589  func TestEventsFromChanges_NodeUpdateEligibilityRequestType(t *testing.T) {
   590  	t.Parallel()
   591  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
   592  	defer s.StopEventBroker()
   593  
   594  	// setup
   595  	n1 := mock.Node()
   596  
   597  	require.NoError(t, s.UpsertNode(structs.MsgTypeTestSetup, 10, n1))
   598  
   599  	msgType := structs.NodeUpdateEligibilityRequestType
   600  
   601  	event := &structs.NodeEvent{
   602  		Message:   "Node marked as ineligible",
   603  		Subsystem: structs.NodeEventSubsystemCluster,
   604  		Timestamp: time.Now(),
   605  	}
   606  
   607  	req := structs.NodeUpdateEligibilityRequest{
   608  		NodeID:      n1.ID,
   609  		NodeEvent:   event,
   610  		Eligibility: structs.NodeSchedulingIneligible,
   611  		UpdatedAt:   time.Now().UnixNano(),
   612  	}
   613  
   614  	require.NoError(t, s.UpdateNodeEligibility(msgType, 100, req.NodeID, req.Eligibility, req.UpdatedAt, req.NodeEvent))
   615  
   616  	events := WaitForEvents(t, s, 100, 1, 1*time.Second)
   617  	require.Len(t, events, 1)
   618  
   619  	for _, e := range events {
   620  		require.Equal(t, 100, int(e.Index))
   621  		require.Equal(t, structs.TypeNodeDrain, e.Type)
   622  		require.Equal(t, structs.TopicNode, e.Topic)
   623  		ne := e.Payload.(*structs.NodeStreamEvent)
   624  		require.Equal(t, event.Message, ne.Node.Events[len(ne.Node.Events)-1].Message)
   625  		require.Equal(t, structs.NodeSchedulingIneligible, ne.Node.SchedulingEligibility)
   626  	}
   627  }
   628  
   629  func TestEventsFromChanges_AllocUpdateDesiredTransitionRequestType(t *testing.T) {
   630  	t.Parallel()
   631  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
   632  	defer s.StopEventBroker()
   633  
   634  	alloc := mock.Alloc()
   635  
   636  	require.Nil(t, s.UpsertJob(structs.MsgTypeTestSetup, 10, alloc.Job))
   637  	require.Nil(t, s.UpsertAllocs(structs.MsgTypeTestSetup, 11, []*structs.Allocation{alloc}))
   638  
   639  	msgType := structs.AllocUpdateDesiredTransitionRequestType
   640  
   641  	eval := &structs.Evaluation{
   642  		ID:             uuid.Generate(),
   643  		Namespace:      alloc.Namespace,
   644  		Priority:       alloc.Job.Priority,
   645  		Type:           alloc.Job.Type,
   646  		TriggeredBy:    structs.EvalTriggerNodeDrain,
   647  		JobID:          alloc.Job.ID,
   648  		JobModifyIndex: alloc.Job.ModifyIndex,
   649  		Status:         structs.EvalStatusPending,
   650  	}
   651  	evals := []*structs.Evaluation{eval}
   652  
   653  	req := &structs.AllocUpdateDesiredTransitionRequest{
   654  		Allocs: map[string]*structs.DesiredTransition{
   655  			alloc.ID: {Migrate: helper.BoolToPtr(true)},
   656  		},
   657  		Evals: evals,
   658  	}
   659  
   660  	require.NoError(t, s.UpdateAllocsDesiredTransitions(msgType, 100, req.Allocs, req.Evals))
   661  
   662  	events := WaitForEvents(t, s, 100, 1, 1*time.Second)
   663  	require.Len(t, events, 2)
   664  
   665  	var allocs []structs.Event
   666  	var evalEvents []structs.Event
   667  	for _, e := range events {
   668  		if e.Topic == structs.TopicEvaluation {
   669  			evalEvents = append(evalEvents, e)
   670  		} else if e.Topic == structs.TopicAllocation {
   671  			allocs = append(allocs, e)
   672  		} else {
   673  			require.Fail(t, "unexpected event type")
   674  		}
   675  
   676  		require.Equal(t, structs.TypeAllocationUpdateDesiredStatus, e.Type)
   677  	}
   678  
   679  	require.Len(t, allocs, 1)
   680  	require.Len(t, evalEvents, 1)
   681  }
   682  
   683  func TestEventsFromChanges_JobBatchDeregisterRequestType(t *testing.T) {
   684  	// TODO Job batch deregister logic mostly occurs in the FSM
   685  	t.SkipNow()
   686  
   687  }
   688  func TestEventsFromChanges_AllocClientUpdateRequestType(t *testing.T) {
   689  	t.SkipNow()
   690  }
   691  
   692  func TestEventsFromChanges_AllocUpdateRequestType(t *testing.T) {
   693  	t.SkipNow()
   694  }
   695  
   696  func TestEventsFromChanges_JobDeregisterRequestType(t *testing.T) {
   697  	t.SkipNow()
   698  }
   699  
   700  func TestEventsFromChanges_WithDeletion(t *testing.T) {
   701  	t.Parallel()
   702  
   703  	changes := Changes{
   704  		Index: uint64(1),
   705  		Changes: memdb.Changes{
   706  			{
   707  				Table:  "jobs",
   708  				Before: &structs.Job{},
   709  				After:  &structs.Job{},
   710  			},
   711  			{
   712  				Table:  "jobs",
   713  				Before: &structs.Job{},
   714  				After:  nil, // deleted
   715  			},
   716  		},
   717  		MsgType: structs.JobDeregisterRequestType,
   718  	}
   719  
   720  	event := eventsFromChanges(nil, changes)
   721  	require.NotNil(t, event)
   722  
   723  	require.Len(t, event.Events, 1)
   724  }
   725  
   726  func TestEventsFromChanges_WithNodeDeregistration(t *testing.T) {
   727  	t.Parallel()
   728  
   729  	before := &structs.Node{
   730  		ID:         "some-id",
   731  		Datacenter: "some-datacenter",
   732  	}
   733  
   734  	changes := Changes{
   735  		Index: uint64(1),
   736  		Changes: memdb.Changes{
   737  			{
   738  				Table:  "nodes",
   739  				Before: before,
   740  				After:  nil, // deleted
   741  			},
   742  		},
   743  		MsgType: structs.NodeDeregisterRequestType,
   744  	}
   745  
   746  	actual := eventsFromChanges(nil, changes)
   747  	require.NotNil(t, actual)
   748  
   749  	require.Len(t, actual.Events, 1)
   750  
   751  	event := actual.Events[0]
   752  
   753  	require.Equal(t, structs.TypeNodeDeregistration, event.Type)
   754  	require.Equal(t, uint64(1), event.Index)
   755  	require.Equal(t, structs.TopicNode, event.Topic)
   756  	require.Equal(t, "some-id", event.Key)
   757  
   758  	require.Len(t, event.FilterKeys, 0)
   759  
   760  	nodeEvent, ok := event.Payload.(*structs.NodeStreamEvent)
   761  	require.True(t, ok)
   762  	require.Equal(t, *before, *nodeEvent.Node)
   763  }
   764  
   765  func TestNodeEventsFromChanges(t *testing.T) {
   766  	cases := []struct {
   767  		Name       string
   768  		MsgType    structs.MessageType
   769  		Setup      func(s *StateStore, tx *txn) error
   770  		Mutate     func(s *StateStore, tx *txn) error
   771  		WantEvents []structs.Event
   772  		WantTopic  structs.Topic
   773  	}{
   774  		{
   775  			MsgType:   structs.NodeRegisterRequestType,
   776  			WantTopic: structs.TopicNode,
   777  			Name:      "node registered",
   778  			Mutate: func(s *StateStore, tx *txn) error {
   779  				return upsertNodeTxn(tx, tx.Index, testNode())
   780  			},
   781  			WantEvents: []structs.Event{{
   782  				Topic: structs.TopicNode,
   783  				Type:  structs.TypeNodeRegistration,
   784  				Key:   testNodeID(),
   785  				Index: 100,
   786  				Payload: &structs.NodeStreamEvent{
   787  					Node: testNode(),
   788  				},
   789  			}},
   790  		},
   791  		{
   792  			MsgType:   structs.NodeRegisterRequestType,
   793  			WantTopic: structs.TopicNode,
   794  			Name:      "node registered initializing",
   795  			Mutate: func(s *StateStore, tx *txn) error {
   796  				return upsertNodeTxn(tx, tx.Index, testNode(nodeNotReady))
   797  			},
   798  			WantEvents: []structs.Event{{
   799  				Topic: structs.TopicNode,
   800  				Type:  structs.TypeNodeRegistration,
   801  				Key:   testNodeID(),
   802  				Index: 100,
   803  				Payload: &structs.NodeStreamEvent{
   804  					Node: testNode(nodeNotReady),
   805  				},
   806  			}},
   807  		},
   808  		{
   809  			MsgType:   structs.NodeDeregisterRequestType,
   810  			WantTopic: structs.TopicNode,
   811  			Name:      "node deregistered",
   812  			Setup: func(s *StateStore, tx *txn) error {
   813  				return upsertNodeTxn(tx, tx.Index, testNode())
   814  			},
   815  			Mutate: func(s *StateStore, tx *txn) error {
   816  				return deleteNodeTxn(tx, tx.Index, []string{testNodeID()})
   817  			},
   818  			WantEvents: []structs.Event{{
   819  				Topic: structs.TopicNode,
   820  				Type:  structs.TypeNodeDeregistration,
   821  				Key:   testNodeID(),
   822  				Index: 100,
   823  				Payload: &structs.NodeStreamEvent{
   824  					Node: testNode(),
   825  				},
   826  			}},
   827  		},
   828  		{
   829  			MsgType:   structs.NodeDeregisterRequestType,
   830  			WantTopic: structs.TopicNode,
   831  			Name:      "batch node deregistered",
   832  			Setup: func(s *StateStore, tx *txn) error {
   833  				require.NoError(t, upsertNodeTxn(tx, tx.Index, testNode()))
   834  				return upsertNodeTxn(tx, tx.Index, testNode(nodeIDTwo))
   835  			},
   836  			Mutate: func(s *StateStore, tx *txn) error {
   837  				return deleteNodeTxn(tx, tx.Index, []string{testNodeID(), testNodeIDTwo()})
   838  			},
   839  			WantEvents: []structs.Event{
   840  				{
   841  					Topic: structs.TopicNode,
   842  					Type:  structs.TypeNodeDeregistration,
   843  					Key:   testNodeID(),
   844  					Index: 100,
   845  					Payload: &structs.NodeStreamEvent{
   846  						Node: testNode(),
   847  					},
   848  				},
   849  				{
   850  					Topic: structs.TopicNode,
   851  					Type:  structs.TypeNodeDeregistration,
   852  					Key:   testNodeIDTwo(),
   853  					Index: 100,
   854  					Payload: &structs.NodeStreamEvent{
   855  						Node: testNode(nodeIDTwo),
   856  					},
   857  				},
   858  			},
   859  		},
   860  		{
   861  			MsgType:   structs.UpsertNodeEventsType,
   862  			WantTopic: structs.TopicNode,
   863  			Name:      "batch node events upserted",
   864  			Setup: func(s *StateStore, tx *txn) error {
   865  				require.NoError(t, upsertNodeTxn(tx, tx.Index, testNode()))
   866  				return upsertNodeTxn(tx, tx.Index, testNode(nodeIDTwo))
   867  			},
   868  			Mutate: func(s *StateStore, tx *txn) error {
   869  				eventFn := func(id string) []*structs.NodeEvent {
   870  					return []*structs.NodeEvent{
   871  						{
   872  							Message:   "test event one",
   873  							Subsystem: "Cluster",
   874  							Details: map[string]string{
   875  								"NodeID": id,
   876  							},
   877  						},
   878  						{
   879  							Message:   "test event two",
   880  							Subsystem: "Cluster",
   881  							Details: map[string]string{
   882  								"NodeID": id,
   883  							},
   884  						},
   885  					}
   886  				}
   887  				require.NoError(t, s.upsertNodeEvents(tx.Index, testNodeID(), eventFn(testNodeID()), tx))
   888  				return s.upsertNodeEvents(tx.Index, testNodeIDTwo(), eventFn(testNodeIDTwo()), tx)
   889  			},
   890  			WantEvents: []structs.Event{
   891  				{
   892  					Topic: structs.TopicNode,
   893  					Type:  structs.TypeNodeEvent,
   894  					Key:   testNodeID(),
   895  					Index: 100,
   896  					Payload: &structs.NodeStreamEvent{
   897  						Node: testNode(),
   898  					},
   899  				},
   900  				{
   901  					Topic: structs.TopicNode,
   902  					Type:  structs.TypeNodeEvent,
   903  					Key:   testNodeIDTwo(),
   904  					Index: 100,
   905  					Payload: &structs.NodeStreamEvent{
   906  						Node: testNode(nodeIDTwo),
   907  					},
   908  				},
   909  			},
   910  		},
   911  	}
   912  
   913  	for _, tc := range cases {
   914  		t.Run(tc.Name, func(t *testing.T) {
   915  			s := TestStateStoreCfg(t, TestStateStorePublisher(t))
   916  			defer s.StopEventBroker()
   917  
   918  			if tc.Setup != nil {
   919  				// Bypass publish mechanism for setup
   920  				setupTx := s.db.WriteTxn(10)
   921  				require.NoError(t, tc.Setup(s, setupTx))
   922  				setupTx.Txn.Commit()
   923  			}
   924  
   925  			tx := s.db.WriteTxnMsgT(tc.MsgType, 100)
   926  			require.NoError(t, tc.Mutate(s, tx))
   927  
   928  			changes := Changes{Changes: tx.Changes(), Index: 100, MsgType: tc.MsgType}
   929  			got := eventsFromChanges(tx, changes)
   930  
   931  			require.NotNil(t, got)
   932  
   933  			require.Equal(t, len(tc.WantEvents), len(got.Events))
   934  			for idx, g := range got.Events {
   935  				// assert equality of shared fields
   936  
   937  				want := tc.WantEvents[idx]
   938  				require.Equal(t, want.Index, g.Index)
   939  				require.Equal(t, want.Key, g.Key)
   940  				require.Equal(t, want.Topic, g.Topic)
   941  
   942  				switch tc.MsgType {
   943  				case structs.NodeRegisterRequestType:
   944  					requireNodeRegistrationEventEqual(t, tc.WantEvents[idx], g)
   945  				case structs.NodeDeregisterRequestType:
   946  					requireNodeDeregistrationEventEqual(t, tc.WantEvents[idx], g)
   947  				case structs.UpsertNodeEventsType:
   948  					requireNodeEventEqual(t, tc.WantEvents[idx], g)
   949  				default:
   950  					require.Fail(t, "unhandled message type")
   951  				}
   952  			}
   953  		})
   954  	}
   955  }
   956  
   957  func TestNodeDrainEventFromChanges(t *testing.T) {
   958  	t.Parallel()
   959  	s := TestStateStoreCfg(t, TestStateStorePublisher(t))
   960  	defer s.StopEventBroker()
   961  
   962  	// setup
   963  	setupTx := s.db.WriteTxn(10)
   964  
   965  	node := mock.Node()
   966  	alloc1 := mock.Alloc()
   967  	alloc2 := mock.Alloc()
   968  	alloc1.NodeID = node.ID
   969  	alloc2.NodeID = node.ID
   970  
   971  	require.NoError(t, upsertNodeTxn(setupTx, 10, node))
   972  	require.NoError(t, s.upsertAllocsImpl(100, []*structs.Allocation{alloc1, alloc2}, setupTx))
   973  	setupTx.Txn.Commit()
   974  
   975  	// changes
   976  	tx := s.db.WriteTxn(100)
   977  
   978  	strat := &structs.DrainStrategy{
   979  		DrainSpec: structs.DrainSpec{
   980  			Deadline:         10 * time.Minute,
   981  			IgnoreSystemJobs: false,
   982  		},
   983  		StartedAt: time.Now(),
   984  	}
   985  	markEligible := false
   986  	updatedAt := time.Now()
   987  	event := &structs.NodeEvent{}
   988  
   989  	require.NoError(t, s.updateNodeDrainImpl(tx, 100, node.ID, strat, markEligible, updatedAt.UnixNano(), event))
   990  	changes := Changes{Changes: tx.Changes(), Index: 100, MsgType: structs.NodeUpdateDrainRequestType}
   991  	got := eventsFromChanges(tx, changes)
   992  
   993  	require.Len(t, got.Events, 1)
   994  
   995  	require.Equal(t, structs.TopicNode, got.Events[0].Topic)
   996  	require.Equal(t, structs.TypeNodeDrain, got.Events[0].Type)
   997  	require.Equal(t, uint64(100), got.Events[0].Index)
   998  
   999  	nodeEvent, ok := got.Events[0].Payload.(*structs.NodeStreamEvent)
  1000  	require.True(t, ok)
  1001  
  1002  	require.Equal(t, structs.NodeSchedulingIneligible, nodeEvent.Node.SchedulingEligibility)
  1003  	require.Equal(t, strat, nodeEvent.Node.DrainStrategy)
  1004  }
  1005  
  1006  func requireNodeRegistrationEventEqual(t *testing.T, want, got structs.Event) {
  1007  	t.Helper()
  1008  
  1009  	wantPayload := want.Payload.(*structs.NodeStreamEvent)
  1010  	gotPayload := got.Payload.(*structs.NodeStreamEvent)
  1011  
  1012  	// Check payload equality for the fields that we can easily control
  1013  	require.Equal(t, wantPayload.Node.Status, gotPayload.Node.Status)
  1014  	require.Equal(t, wantPayload.Node.ID, gotPayload.Node.ID)
  1015  	require.NotEqual(t, wantPayload.Node.Events, gotPayload.Node.Events)
  1016  }
  1017  
  1018  func requireNodeDeregistrationEventEqual(t *testing.T, want, got structs.Event) {
  1019  	t.Helper()
  1020  
  1021  	wantPayload := want.Payload.(*structs.NodeStreamEvent)
  1022  	gotPayload := got.Payload.(*structs.NodeStreamEvent)
  1023  
  1024  	require.Equal(t, wantPayload.Node.ID, gotPayload.Node.ID)
  1025  	require.NotEqual(t, wantPayload.Node.Events, gotPayload.Node.Events)
  1026  }
  1027  
  1028  func requireNodeEventEqual(t *testing.T, want, got structs.Event) {
  1029  	gotPayload := got.Payload.(*structs.NodeStreamEvent)
  1030  
  1031  	require.Len(t, gotPayload.Node.Events, 3)
  1032  }
  1033  
  1034  type nodeOpts func(n *structs.Node)
  1035  
  1036  func nodeNotReady(n *structs.Node) {
  1037  	n.Status = structs.NodeStatusInit
  1038  }
  1039  
  1040  func nodeIDTwo(n *structs.Node) {
  1041  	n.ID = testNodeIDTwo()
  1042  }
  1043  
  1044  func testNode(opts ...nodeOpts) *structs.Node {
  1045  	n := mock.Node()
  1046  	n.ID = testNodeID()
  1047  
  1048  	n.SecretID = "ab9812d3-6a21-40d3-973d-d9d2174a23ee"
  1049  
  1050  	for _, opt := range opts {
  1051  		opt(n)
  1052  	}
  1053  	return n
  1054  }
  1055  
  1056  func testNodeID() string {
  1057  	return "9d5741c1-3899-498a-98dd-eb3c05665863"
  1058  }
  1059  
  1060  func testNodeIDTwo() string {
  1061  	return "694ff31d-8c59-4030-ac83-e15692560c8d"
  1062  }