github.com/blixtra/nomad@v0.7.2-0.20171221000451-da9a1d7bb050/nomad/eval_broker_test.go (about)

     1  package nomad
     2  
     3  import (
     4  	"fmt"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/hashicorp/nomad/nomad/mock"
     9  	"github.com/hashicorp/nomad/nomad/structs"
    10  	"github.com/hashicorp/nomad/testutil"
    11  )
    12  
    13  var (
    14  	defaultSched = []string{
    15  		structs.JobTypeService,
    16  		structs.JobTypeBatch,
    17  	}
    18  )
    19  
    20  func testBrokerConfig() *Config {
    21  	config := DefaultConfig()
    22  
    23  	// Tune the Nack timeout
    24  	config.EvalNackTimeout = 5 * time.Second
    25  
    26  	// Tune the Nack delay
    27  	config.EvalNackInitialReenqueueDelay = 5 * time.Millisecond
    28  	config.EvalNackSubsequentReenqueueDelay = 50 * time.Millisecond
    29  	return config
    30  }
    31  
    32  func testBroker(t *testing.T, timeout time.Duration) *EvalBroker {
    33  	config := testBrokerConfig()
    34  
    35  	if timeout != 0 {
    36  		config.EvalNackTimeout = timeout
    37  	}
    38  
    39  	return testBrokerFromConfig(t, config)
    40  }
    41  
    42  func testBrokerFromConfig(t *testing.T, c *Config) *EvalBroker {
    43  	b, err := NewEvalBroker(c.EvalNackTimeout, c.EvalNackInitialReenqueueDelay, c.EvalNackSubsequentReenqueueDelay, 3)
    44  	if err != nil {
    45  		t.Fatalf("err: %v", err)
    46  	}
    47  
    48  	return b
    49  }
    50  
    51  func TestEvalBroker_Enqueue_Dequeue_Nack_Ack(t *testing.T) {
    52  	t.Parallel()
    53  	b := testBroker(t, 0)
    54  
    55  	// Enqueue, but broker is disabled!
    56  	eval := mock.Eval()
    57  	b.Enqueue(eval)
    58  
    59  	// Verify nothing was done
    60  	stats := b.Stats()
    61  	if stats.TotalReady != 0 {
    62  		t.Fatalf("bad: %#v", stats)
    63  	}
    64  
    65  	if b.Enabled() {
    66  		t.Fatalf("should not be enabled")
    67  	}
    68  
    69  	// Enable the broker, and enqueue
    70  	b.SetEnabled(true)
    71  	b.Enqueue(eval)
    72  
    73  	// Double enqueue is a no-op
    74  	b.Enqueue(eval)
    75  
    76  	if !b.Enabled() {
    77  		t.Fatalf("should be enabled")
    78  	}
    79  
    80  	// Verify enqueue is done
    81  	stats = b.Stats()
    82  	if stats.TotalReady != 1 {
    83  		t.Fatalf("bad: %#v", stats)
    84  	}
    85  	if stats.ByScheduler[eval.Type].Ready != 1 {
    86  		t.Fatalf("bad: %#v", stats)
    87  	}
    88  
    89  	// Dequeue should work
    90  	out, token, err := b.Dequeue(defaultSched, time.Second)
    91  	if err != nil {
    92  		t.Fatalf("err: %v", err)
    93  	}
    94  	if out != eval {
    95  		t.Fatalf("bad : %#v", out)
    96  	}
    97  
    98  	tokenOut, ok := b.Outstanding(out.ID)
    99  	if !ok {
   100  		t.Fatalf("should be outstanding")
   101  	}
   102  	if tokenOut != token {
   103  		t.Fatalf("Bad: %#v %#v", token, tokenOut)
   104  	}
   105  
   106  	// OutstandingReset should verify the token
   107  	err = b.OutstandingReset("nope", "foo")
   108  	if err != ErrNotOutstanding {
   109  		t.Fatalf("err: %v", err)
   110  	}
   111  	err = b.OutstandingReset(out.ID, "foo")
   112  	if err != ErrTokenMismatch {
   113  		t.Fatalf("err: %v", err)
   114  	}
   115  	err = b.OutstandingReset(out.ID, tokenOut)
   116  	if err != nil {
   117  		t.Fatalf("err: %v", err)
   118  	}
   119  
   120  	// Check the stats
   121  	stats = b.Stats()
   122  	if stats.TotalReady != 0 {
   123  		t.Fatalf("bad: %#v", stats)
   124  	}
   125  	if stats.TotalUnacked != 1 {
   126  		t.Fatalf("bad: %#v", stats)
   127  	}
   128  	if stats.ByScheduler[eval.Type].Ready != 0 {
   129  		t.Fatalf("bad: %#v", stats)
   130  	}
   131  	if stats.ByScheduler[eval.Type].Unacked != 1 {
   132  		t.Fatalf("bad: %#v", stats)
   133  	}
   134  
   135  	// Nack with wrong token should fail
   136  	err = b.Nack(eval.ID, "foobarbaz")
   137  	if err == nil {
   138  		t.Fatalf("should fail to nack")
   139  	}
   140  
   141  	// Nack back into the queue
   142  	err = b.Nack(eval.ID, token)
   143  	if err != nil {
   144  		t.Fatalf("err: %v", err)
   145  	}
   146  
   147  	if _, ok := b.Outstanding(out.ID); ok {
   148  		t.Fatalf("should not be outstanding")
   149  	}
   150  
   151  	// Check the stats
   152  	testutil.WaitForResult(func() (bool, error) {
   153  		stats = b.Stats()
   154  		if stats.TotalReady != 1 {
   155  			return false, fmt.Errorf("bad: %#v", stats)
   156  		}
   157  		if stats.TotalUnacked != 0 {
   158  			return false, fmt.Errorf("bad: %#v", stats)
   159  		}
   160  		if stats.TotalWaiting != 0 {
   161  			return false, fmt.Errorf("bad: %#v", stats)
   162  		}
   163  		if stats.ByScheduler[eval.Type].Ready != 1 {
   164  			return false, fmt.Errorf("bad: %#v", stats)
   165  		}
   166  		if stats.ByScheduler[eval.Type].Unacked != 0 {
   167  			return false, fmt.Errorf("bad: %#v", stats)
   168  		}
   169  
   170  		return true, nil
   171  	}, func(e error) {
   172  		t.Fatal(e)
   173  	})
   174  
   175  	// Dequeue should work again
   176  	out2, token2, err := b.Dequeue(defaultSched, time.Second)
   177  	if err != nil {
   178  		t.Fatalf("err: %v", err)
   179  	}
   180  	if out2 != eval {
   181  		t.Fatalf("bad : %#v", out2)
   182  	}
   183  	if token2 == token {
   184  		t.Fatalf("should get a new token")
   185  	}
   186  
   187  	tokenOut2, ok := b.Outstanding(out.ID)
   188  	if !ok {
   189  		t.Fatalf("should be outstanding")
   190  	}
   191  	if tokenOut2 != token2 {
   192  		t.Fatalf("Bad: %#v %#v", token2, tokenOut2)
   193  	}
   194  
   195  	// Ack with wrong token
   196  	err = b.Ack(eval.ID, "zip")
   197  	if err == nil {
   198  		t.Fatalf("should fail to ack")
   199  	}
   200  
   201  	// Ack finally
   202  	err = b.Ack(eval.ID, token2)
   203  	if err != nil {
   204  		t.Fatalf("err: %v", err)
   205  	}
   206  
   207  	if _, ok := b.Outstanding(out.ID); ok {
   208  		t.Fatalf("should not be outstanding")
   209  	}
   210  
   211  	// Check the stats
   212  	stats = b.Stats()
   213  	if stats.TotalReady != 0 {
   214  		t.Fatalf("bad: %#v", stats)
   215  	}
   216  	if stats.TotalUnacked != 0 {
   217  		t.Fatalf("bad: %#v", stats)
   218  	}
   219  	if stats.ByScheduler[eval.Type].Ready != 0 {
   220  		t.Fatalf("bad: %#v", stats)
   221  	}
   222  	if stats.ByScheduler[eval.Type].Unacked != 0 {
   223  		t.Fatalf("bad: %#v", stats)
   224  	}
   225  }
   226  
   227  func TestEvalBroker_Nack_Delay(t *testing.T) {
   228  	t.Parallel()
   229  	b := testBroker(t, 0)
   230  
   231  	// Enqueue, but broker is disabled!
   232  	b.SetEnabled(true)
   233  	eval := mock.Eval()
   234  	b.Enqueue(eval)
   235  
   236  	// Dequeue should work
   237  	out, token, err := b.Dequeue(defaultSched, time.Second)
   238  	if err != nil {
   239  		t.Fatalf("err: %v", err)
   240  	}
   241  	if out != eval {
   242  		t.Fatalf("bad : %#v", out)
   243  	}
   244  
   245  	// Nack back into the queue
   246  	err = b.Nack(eval.ID, token)
   247  	if err != nil {
   248  		t.Fatalf("err: %v", err)
   249  	}
   250  
   251  	if _, ok := b.Outstanding(out.ID); ok {
   252  		t.Fatalf("should not be outstanding")
   253  	}
   254  
   255  	// Check the stats to ensure that it is waiting
   256  	stats := b.Stats()
   257  	if stats.TotalReady != 0 {
   258  		t.Fatalf("bad: %#v", stats)
   259  	}
   260  	if stats.TotalUnacked != 0 {
   261  		t.Fatalf("bad: %#v", stats)
   262  	}
   263  	if stats.TotalWaiting != 1 {
   264  		t.Fatalf("bad: %#v", stats)
   265  	}
   266  	if stats.ByScheduler[eval.Type].Ready != 0 {
   267  		t.Fatalf("bad: %#v", stats)
   268  	}
   269  	if stats.ByScheduler[eval.Type].Unacked != 0 {
   270  		t.Fatalf("bad: %#v", stats)
   271  	}
   272  
   273  	// Now wait for it to be re-enqueued
   274  	testutil.WaitForResult(func() (bool, error) {
   275  		stats = b.Stats()
   276  		if stats.TotalReady != 1 {
   277  			return false, fmt.Errorf("bad: %#v", stats)
   278  		}
   279  		if stats.TotalUnacked != 0 {
   280  			return false, fmt.Errorf("bad: %#v", stats)
   281  		}
   282  		if stats.TotalWaiting != 0 {
   283  			return false, fmt.Errorf("bad: %#v", stats)
   284  		}
   285  		if stats.ByScheduler[eval.Type].Ready != 1 {
   286  			return false, fmt.Errorf("bad: %#v", stats)
   287  		}
   288  		if stats.ByScheduler[eval.Type].Unacked != 0 {
   289  			return false, fmt.Errorf("bad: %#v", stats)
   290  		}
   291  
   292  		return true, nil
   293  	}, func(e error) {
   294  		t.Fatal(e)
   295  	})
   296  
   297  	// Dequeue should work again
   298  	out2, token2, err := b.Dequeue(defaultSched, time.Second)
   299  	if err != nil {
   300  		t.Fatalf("err: %v", err)
   301  	}
   302  	if out2 != eval {
   303  		t.Fatalf("bad : %#v", out2)
   304  	}
   305  	if token2 == token {
   306  		t.Fatalf("should get a new token")
   307  	}
   308  
   309  	// Capture the time
   310  	start := time.Now()
   311  
   312  	// Nack back into the queue
   313  	err = b.Nack(eval.ID, token2)
   314  	if err != nil {
   315  		t.Fatalf("err: %v", err)
   316  	}
   317  
   318  	// Now wait for it to be re-enqueued
   319  	testutil.WaitForResult(func() (bool, error) {
   320  		stats = b.Stats()
   321  		if stats.TotalReady != 1 {
   322  			return false, fmt.Errorf("bad: %#v", stats)
   323  		}
   324  		if stats.TotalUnacked != 0 {
   325  			return false, fmt.Errorf("bad: %#v", stats)
   326  		}
   327  		if stats.TotalWaiting != 0 {
   328  			return false, fmt.Errorf("bad: %#v", stats)
   329  		}
   330  		if stats.ByScheduler[eval.Type].Ready != 1 {
   331  			return false, fmt.Errorf("bad: %#v", stats)
   332  		}
   333  		if stats.ByScheduler[eval.Type].Unacked != 0 {
   334  			return false, fmt.Errorf("bad: %#v", stats)
   335  		}
   336  
   337  		return true, nil
   338  	}, func(e error) {
   339  		t.Fatal(e)
   340  	})
   341  
   342  	delay := time.Now().Sub(start)
   343  	if delay < b.subsequentNackDelay {
   344  		t.Fatalf("bad: delay was %v; want at least %v", delay, b.subsequentNackDelay)
   345  	}
   346  
   347  	// Dequeue should work again
   348  	out3, token3, err := b.Dequeue(defaultSched, time.Second)
   349  	if err != nil {
   350  		t.Fatalf("err: %v", err)
   351  	}
   352  	if out3 != eval {
   353  		t.Fatalf("bad : %#v", out3)
   354  	}
   355  	if token3 == token || token3 == token2 {
   356  		t.Fatalf("should get a new token")
   357  	}
   358  
   359  	// Ack finally
   360  	err = b.Ack(eval.ID, token3)
   361  	if err != nil {
   362  		t.Fatalf("err: %v", err)
   363  	}
   364  
   365  	if _, ok := b.Outstanding(out.ID); ok {
   366  		t.Fatalf("should not be outstanding")
   367  	}
   368  
   369  	// Check the stats
   370  	stats = b.Stats()
   371  	if stats.TotalReady != 0 {
   372  		t.Fatalf("bad: %#v", stats)
   373  	}
   374  	if stats.TotalUnacked != 0 {
   375  		t.Fatalf("bad: %#v", stats)
   376  	}
   377  	if stats.ByScheduler[eval.Type].Ready != 0 {
   378  		t.Fatalf("bad: %#v", stats)
   379  	}
   380  	if stats.ByScheduler[eval.Type].Unacked != 0 {
   381  		t.Fatalf("bad: %#v", stats)
   382  	}
   383  }
   384  
   385  func TestEvalBroker_Serialize_DuplicateJobID(t *testing.T) {
   386  	t.Parallel()
   387  	b := testBroker(t, 0)
   388  	b.SetEnabled(true)
   389  
   390  	ns1 := "namespace-one"
   391  	ns2 := "namespace-two"
   392  	eval := mock.Eval()
   393  	eval.Namespace = ns1
   394  	b.Enqueue(eval)
   395  
   396  	eval2 := mock.Eval()
   397  	eval2.JobID = eval.JobID
   398  	eval2.Namespace = ns1
   399  	eval2.CreateIndex = eval.CreateIndex + 1
   400  	b.Enqueue(eval2)
   401  
   402  	eval3 := mock.Eval()
   403  	eval3.JobID = eval.JobID
   404  	eval3.Namespace = ns1
   405  	eval3.CreateIndex = eval.CreateIndex + 2
   406  	b.Enqueue(eval3)
   407  
   408  	eval4 := mock.Eval()
   409  	eval4.JobID = eval.JobID
   410  	eval4.Namespace = ns2
   411  	eval4.CreateIndex = eval.CreateIndex + 3
   412  	b.Enqueue(eval4)
   413  
   414  	eval5 := mock.Eval()
   415  	eval5.JobID = eval.JobID
   416  	eval5.Namespace = ns2
   417  	eval5.CreateIndex = eval.CreateIndex + 4
   418  	b.Enqueue(eval5)
   419  
   420  	stats := b.Stats()
   421  	if stats.TotalReady != 2 {
   422  		t.Fatalf("bad: %#v", stats)
   423  	}
   424  	if stats.TotalBlocked != 3 {
   425  		t.Fatalf("bad: %#v", stats)
   426  	}
   427  
   428  	// Dequeue should work
   429  	out, token, err := b.Dequeue(defaultSched, time.Second)
   430  	if err != nil {
   431  		t.Fatalf("err: %v", err)
   432  	}
   433  	if out != eval {
   434  		t.Fatalf("bad : %#v", out)
   435  	}
   436  
   437  	// Check the stats
   438  	stats = b.Stats()
   439  	if stats.TotalReady != 1 {
   440  		t.Fatalf("bad: %#v", stats)
   441  	}
   442  	if stats.TotalUnacked != 1 {
   443  		t.Fatalf("bad: %#v", stats)
   444  	}
   445  	if stats.TotalBlocked != 3 {
   446  		t.Fatalf("bad: %#v", stats)
   447  	}
   448  
   449  	// Ack out
   450  	err = b.Ack(eval.ID, token)
   451  	if err != nil {
   452  		t.Fatalf("err: %v", err)
   453  	}
   454  
   455  	// Check the stats
   456  	stats = b.Stats()
   457  	if stats.TotalReady != 2 {
   458  		t.Fatalf("bad: %#v", stats)
   459  	}
   460  	if stats.TotalUnacked != 0 {
   461  		t.Fatalf("bad: %#v", stats)
   462  	}
   463  	if stats.TotalBlocked != 2 {
   464  		t.Fatalf("bad: %#v", stats)
   465  	}
   466  
   467  	// Dequeue should work
   468  	out, token, err = b.Dequeue(defaultSched, time.Second)
   469  	if err != nil {
   470  		t.Fatalf("err: %v", err)
   471  	}
   472  	if out != eval2 {
   473  		t.Fatalf("bad : %#v", out)
   474  	}
   475  
   476  	// Check the stats
   477  	stats = b.Stats()
   478  	if stats.TotalReady != 1 {
   479  		t.Fatalf("bad: %#v", stats)
   480  	}
   481  	if stats.TotalUnacked != 1 {
   482  		t.Fatalf("bad: %#v", stats)
   483  	}
   484  	if stats.TotalBlocked != 2 {
   485  		t.Fatalf("bad: %#v", stats)
   486  	}
   487  
   488  	// Ack out
   489  	err = b.Ack(eval2.ID, token)
   490  	if err != nil {
   491  		t.Fatalf("err: %v", err)
   492  	}
   493  
   494  	// Check the stats
   495  	stats = b.Stats()
   496  	if stats.TotalReady != 2 {
   497  		t.Fatalf("bad: %#v", stats)
   498  	}
   499  	if stats.TotalUnacked != 0 {
   500  		t.Fatalf("bad: %#v", stats)
   501  	}
   502  	if stats.TotalBlocked != 1 {
   503  		t.Fatalf("bad: %#v", stats)
   504  	}
   505  
   506  	// Dequeue should work
   507  	out, token, err = b.Dequeue(defaultSched, time.Second)
   508  	if err != nil {
   509  		t.Fatalf("err: %v", err)
   510  	}
   511  	if out != eval3 {
   512  		t.Fatalf("bad : %#v", out)
   513  	}
   514  
   515  	// Check the stats
   516  	stats = b.Stats()
   517  	if stats.TotalReady != 1 {
   518  		t.Fatalf("bad: %#v", stats)
   519  	}
   520  	if stats.TotalUnacked != 1 {
   521  		t.Fatalf("bad: %#v", stats)
   522  	}
   523  	if stats.TotalBlocked != 1 {
   524  		t.Fatalf("bad: %#v", stats)
   525  	}
   526  
   527  	// Ack out
   528  	err = b.Ack(eval3.ID, token)
   529  	if err != nil {
   530  		t.Fatalf("err: %v", err)
   531  	}
   532  
   533  	// Check the stats
   534  	stats = b.Stats()
   535  	if stats.TotalReady != 1 {
   536  		t.Fatalf("bad: %#v", stats)
   537  	}
   538  	if stats.TotalUnacked != 0 {
   539  		t.Fatalf("bad: %#v", stats)
   540  	}
   541  	if stats.TotalBlocked != 1 {
   542  		t.Fatalf("bad: %#v", stats)
   543  	}
   544  
   545  	// Dequeue should work
   546  	out, token, err = b.Dequeue(defaultSched, time.Second)
   547  	if err != nil {
   548  		t.Fatalf("err: %v", err)
   549  	}
   550  	if out != eval4 {
   551  		t.Fatalf("bad : %#v", out)
   552  	}
   553  
   554  	// Check the stats
   555  	stats = b.Stats()
   556  	if stats.TotalReady != 0 {
   557  		t.Fatalf("bad: %#v", stats)
   558  	}
   559  	if stats.TotalUnacked != 1 {
   560  		t.Fatalf("bad: %#v", stats)
   561  	}
   562  	if stats.TotalBlocked != 1 {
   563  		t.Fatalf("bad: %#v", stats)
   564  	}
   565  
   566  	// Ack out
   567  	err = b.Ack(eval4.ID, token)
   568  	if err != nil {
   569  		t.Fatalf("err: %v", err)
   570  	}
   571  
   572  	// Check the stats
   573  	stats = b.Stats()
   574  	if stats.TotalReady != 1 {
   575  		t.Fatalf("bad: %#v", stats)
   576  	}
   577  	if stats.TotalUnacked != 0 {
   578  		t.Fatalf("bad: %#v", stats)
   579  	}
   580  	if stats.TotalBlocked != 0 {
   581  		t.Fatalf("bad: %#v", stats)
   582  	}
   583  
   584  	// Dequeue should work
   585  	out, token, err = b.Dequeue(defaultSched, time.Second)
   586  	if err != nil {
   587  		t.Fatalf("err: %v", err)
   588  	}
   589  	if out != eval5 {
   590  		t.Fatalf("bad : %#v", out)
   591  	}
   592  
   593  	// Check the stats
   594  	stats = b.Stats()
   595  	if stats.TotalReady != 0 {
   596  		t.Fatalf("bad: %#v", stats)
   597  	}
   598  	if stats.TotalUnacked != 1 {
   599  		t.Fatalf("bad: %#v", stats)
   600  	}
   601  	if stats.TotalBlocked != 0 {
   602  		t.Fatalf("bad: %#v", stats)
   603  	}
   604  
   605  	// Ack out
   606  	err = b.Ack(eval5.ID, token)
   607  	if err != nil {
   608  		t.Fatalf("err: %v", err)
   609  	}
   610  
   611  	// Check the stats
   612  	stats = b.Stats()
   613  	if stats.TotalReady != 0 {
   614  		t.Fatalf("bad: %#v", stats)
   615  	}
   616  	if stats.TotalUnacked != 0 {
   617  		t.Fatalf("bad: %#v", stats)
   618  	}
   619  	if stats.TotalBlocked != 0 {
   620  		t.Fatalf("bad: %#v", stats)
   621  	}
   622  }
   623  
   624  func TestEvalBroker_Enqueue_Disable(t *testing.T) {
   625  	t.Parallel()
   626  	b := testBroker(t, 0)
   627  
   628  	// Enqueue
   629  	eval := mock.Eval()
   630  	b.SetEnabled(true)
   631  	b.Enqueue(eval)
   632  
   633  	// Flush via SetEnabled
   634  	b.SetEnabled(false)
   635  
   636  	// Check the stats
   637  	stats := b.Stats()
   638  	if stats.TotalReady != 0 {
   639  		t.Fatalf("bad: %#v", stats)
   640  	}
   641  	if stats.TotalUnacked != 0 {
   642  		t.Fatalf("bad: %#v", stats)
   643  	}
   644  	if _, ok := stats.ByScheduler[eval.Type]; ok {
   645  		t.Fatalf("bad: %#v", stats)
   646  	}
   647  }
   648  
   649  func TestEvalBroker_Dequeue_Timeout(t *testing.T) {
   650  	t.Parallel()
   651  	b := testBroker(t, 0)
   652  	b.SetEnabled(true)
   653  
   654  	start := time.Now()
   655  	out, _, err := b.Dequeue(defaultSched, 5*time.Millisecond)
   656  	end := time.Now()
   657  
   658  	if err != nil {
   659  		t.Fatalf("err: %v", err)
   660  	}
   661  	if out != nil {
   662  		t.Fatalf("unexpected: %#v", out)
   663  	}
   664  
   665  	if diff := end.Sub(start); diff < 5*time.Millisecond {
   666  		t.Fatalf("bad: %#v", diff)
   667  	}
   668  }
   669  
   670  func TestEvalBroker_Dequeue_Empty_Timeout(t *testing.T) {
   671  	t.Parallel()
   672  	b := testBroker(t, 0)
   673  	b.SetEnabled(true)
   674  	doneCh := make(chan struct{}, 1)
   675  
   676  	go func() {
   677  		out, _, err := b.Dequeue(defaultSched, 0)
   678  		if err != nil {
   679  			t.Fatalf("err: %v", err)
   680  		}
   681  		if out == nil {
   682  			t.Fatal("Expect an eval")
   683  		}
   684  		doneCh <- struct{}{}
   685  	}()
   686  
   687  	// Sleep for a little bit
   688  	select {
   689  	case <-time.After(5 * time.Millisecond):
   690  	case <-doneCh:
   691  		t.Fatalf("Dequeue(0) should block")
   692  	}
   693  
   694  	// Enqueue to unblock the dequeue.
   695  	eval := mock.Eval()
   696  	b.Enqueue(eval)
   697  
   698  	select {
   699  	case <-doneCh:
   700  		return
   701  	case <-time.After(5 * time.Millisecond):
   702  		t.Fatal("timeout: Dequeue(0) should return after enqueue")
   703  	}
   704  }
   705  
   706  // Ensure higher priority dequeued first
   707  func TestEvalBroker_Dequeue_Priority(t *testing.T) {
   708  	t.Parallel()
   709  	b := testBroker(t, 0)
   710  	b.SetEnabled(true)
   711  
   712  	eval1 := mock.Eval()
   713  	eval1.Priority = 10
   714  	b.Enqueue(eval1)
   715  
   716  	eval2 := mock.Eval()
   717  	eval2.Priority = 30
   718  	b.Enqueue(eval2)
   719  
   720  	eval3 := mock.Eval()
   721  	eval3.Priority = 20
   722  	b.Enqueue(eval3)
   723  
   724  	out1, _, _ := b.Dequeue(defaultSched, time.Second)
   725  	if out1 != eval2 {
   726  		t.Fatalf("bad: %#v", out1)
   727  	}
   728  
   729  	out2, _, _ := b.Dequeue(defaultSched, time.Second)
   730  	if out2 != eval3 {
   731  		t.Fatalf("bad: %#v", out2)
   732  	}
   733  
   734  	out3, _, _ := b.Dequeue(defaultSched, time.Second)
   735  	if out3 != eval1 {
   736  		t.Fatalf("bad: %#v", out3)
   737  	}
   738  }
   739  
   740  // Ensure FIFO at fixed priority
   741  func TestEvalBroker_Dequeue_FIFO(t *testing.T) {
   742  	t.Parallel()
   743  	b := testBroker(t, 0)
   744  	b.SetEnabled(true)
   745  	NUM := 100
   746  
   747  	for i := 0; i < NUM; i++ {
   748  		eval1 := mock.Eval()
   749  		eval1.CreateIndex = uint64(i)
   750  		eval1.ModifyIndex = uint64(i)
   751  		b.Enqueue(eval1)
   752  	}
   753  
   754  	for i := 0; i < NUM; i++ {
   755  		out1, _, _ := b.Dequeue(defaultSched, time.Second)
   756  		if out1.CreateIndex != uint64(i) {
   757  			t.Fatalf("bad: %d %#v", i, out1)
   758  		}
   759  	}
   760  }
   761  
   762  // Ensure fairness between schedulers
   763  func TestEvalBroker_Dequeue_Fairness(t *testing.T) {
   764  	t.Parallel()
   765  	b := testBroker(t, 0)
   766  	b.SetEnabled(true)
   767  	NUM := 1000
   768  
   769  	for i := 0; i < NUM; i++ {
   770  		eval1 := mock.Eval()
   771  		if i < (NUM / 2) {
   772  			eval1.Type = structs.JobTypeService
   773  		} else {
   774  			eval1.Type = structs.JobTypeBatch
   775  		}
   776  		b.Enqueue(eval1)
   777  	}
   778  
   779  	counter := 0
   780  	for i := 0; i < NUM; i++ {
   781  		out1, _, _ := b.Dequeue(defaultSched, time.Second)
   782  
   783  		switch out1.Type {
   784  		case structs.JobTypeService:
   785  			if counter < 0 {
   786  				counter = 0
   787  			}
   788  			counter += 1
   789  		case structs.JobTypeBatch:
   790  			if counter > 0 {
   791  				counter = 0
   792  			}
   793  			counter -= 1
   794  		}
   795  
   796  		// This will fail randomly at times. It is very hard to
   797  		// test deterministically that its acting randomly.
   798  		if counter >= 250 || counter <= -250 {
   799  			t.Fatalf("unlikely sequence: %d", counter)
   800  		}
   801  	}
   802  }
   803  
   804  // Ensure we get unblocked
   805  func TestEvalBroker_Dequeue_Blocked(t *testing.T) {
   806  	t.Parallel()
   807  	b := testBroker(t, 0)
   808  	b.SetEnabled(true)
   809  
   810  	// Start with a blocked dequeue
   811  	outCh := make(chan *structs.Evaluation, 1)
   812  	go func() {
   813  		start := time.Now()
   814  		out, _, err := b.Dequeue(defaultSched, time.Second)
   815  		end := time.Now()
   816  		outCh <- out
   817  		if err != nil {
   818  			t.Fatalf("err: %v", err)
   819  		}
   820  		if d := end.Sub(start); d < 5*time.Millisecond {
   821  			t.Fatalf("bad: %v", d)
   822  		}
   823  	}()
   824  
   825  	// Wait for a bit
   826  	time.Sleep(5 * time.Millisecond)
   827  
   828  	// Enqueue
   829  	eval := mock.Eval()
   830  	b.Enqueue(eval)
   831  
   832  	// Ensure dequeue
   833  	select {
   834  	case out := <-outCh:
   835  		if out != eval {
   836  			t.Fatalf("bad: %v", out)
   837  		}
   838  	case <-time.After(time.Second):
   839  		t.Fatalf("timeout")
   840  	}
   841  }
   842  
   843  // Ensure we nack in a timely manner
   844  func TestEvalBroker_Nack_Timeout(t *testing.T) {
   845  	t.Parallel()
   846  	b := testBroker(t, 5*time.Millisecond)
   847  	b.SetEnabled(true)
   848  
   849  	// Enqueue
   850  	eval := mock.Eval()
   851  	b.Enqueue(eval)
   852  
   853  	// Dequeue
   854  	out, _, err := b.Dequeue(defaultSched, time.Second)
   855  	start := time.Now()
   856  	if err != nil {
   857  		t.Fatalf("err: %v", err)
   858  	}
   859  	if out != eval {
   860  		t.Fatalf("bad: %v", out)
   861  	}
   862  
   863  	// Dequeue, should block on Nack timer
   864  	out, _, err = b.Dequeue(defaultSched, time.Second)
   865  	end := time.Now()
   866  	if err != nil {
   867  		t.Fatalf("err: %v", err)
   868  	}
   869  	if out != eval {
   870  		t.Fatalf("bad: %v", out)
   871  	}
   872  
   873  	// Check the nack timer
   874  	if diff := end.Sub(start); diff < 5*time.Millisecond {
   875  		t.Fatalf("bad: %#v", diff)
   876  	}
   877  }
   878  
   879  // Ensure we nack in a timely manner
   880  func TestEvalBroker_Nack_TimeoutReset(t *testing.T) {
   881  	t.Parallel()
   882  	b := testBroker(t, 50*time.Millisecond)
   883  	b.SetEnabled(true)
   884  
   885  	// Enqueue
   886  	eval := mock.Eval()
   887  	b.Enqueue(eval)
   888  
   889  	// Dequeue
   890  	out, token, err := b.Dequeue(defaultSched, time.Second)
   891  	start := time.Now()
   892  	if err != nil {
   893  		t.Fatalf("err: %v", err)
   894  	}
   895  	if out != eval {
   896  		t.Fatalf("bad: %v", out)
   897  	}
   898  
   899  	// Reset in 20 milliseconds
   900  	time.Sleep(20 * time.Millisecond)
   901  	if err := b.OutstandingReset(out.ID, token); err != nil {
   902  		t.Fatalf("err: %v", err)
   903  	}
   904  
   905  	// Dequeue, should block on Nack timer
   906  	out, _, err = b.Dequeue(defaultSched, time.Second)
   907  	end := time.Now()
   908  	if err != nil {
   909  		t.Fatalf("err: %v", err)
   910  	}
   911  	if out != eval {
   912  		t.Fatalf("bad: %v", out)
   913  	}
   914  
   915  	// Check the nack timer
   916  	if diff := end.Sub(start); diff < 75*time.Millisecond {
   917  		t.Fatalf("bad: %#v", diff)
   918  	}
   919  }
   920  
   921  func TestEvalBroker_PauseResumeNackTimeout(t *testing.T) {
   922  	t.Parallel()
   923  	b := testBroker(t, 50*time.Millisecond)
   924  	b.SetEnabled(true)
   925  
   926  	// Enqueue
   927  	eval := mock.Eval()
   928  	b.Enqueue(eval)
   929  
   930  	// Dequeue
   931  	out, token, err := b.Dequeue(defaultSched, time.Second)
   932  	start := time.Now()
   933  	if err != nil {
   934  		t.Fatalf("err: %v", err)
   935  	}
   936  	if out != eval {
   937  		t.Fatalf("bad: %v", out)
   938  	}
   939  
   940  	// Pause in 20 milliseconds
   941  	time.Sleep(20 * time.Millisecond)
   942  	if err := b.PauseNackTimeout(out.ID, token); err != nil {
   943  		t.Fatalf("err: %v", err)
   944  	}
   945  
   946  	go func() {
   947  		time.Sleep(20 * time.Millisecond)
   948  		if err := b.ResumeNackTimeout(out.ID, token); err != nil {
   949  			t.Fatalf("err: %v", err)
   950  		}
   951  	}()
   952  
   953  	// Dequeue, should block until the timer is resumed
   954  	out, _, err = b.Dequeue(defaultSched, time.Second)
   955  	end := time.Now()
   956  	if err != nil {
   957  		t.Fatalf("err: %v", err)
   958  	}
   959  	if out != eval {
   960  		t.Fatalf("bad: %v", out)
   961  	}
   962  
   963  	// Check the nack timer
   964  	if diff := end.Sub(start); diff < 95*time.Millisecond {
   965  		t.Fatalf("bad: %#v", diff)
   966  	}
   967  }
   968  
   969  func TestEvalBroker_DeliveryLimit(t *testing.T) {
   970  	t.Parallel()
   971  	b := testBroker(t, 0)
   972  	b.SetEnabled(true)
   973  
   974  	eval := mock.Eval()
   975  	b.Enqueue(eval)
   976  
   977  	for i := 0; i < 3; i++ {
   978  		// Dequeue should work
   979  		out, token, err := b.Dequeue(defaultSched, time.Second)
   980  		if err != nil {
   981  			t.Fatalf("err: %v", err)
   982  		}
   983  		if out != eval {
   984  			t.Fatalf("bad : %#v", out)
   985  		}
   986  
   987  		// Nack with wrong token should fail
   988  		err = b.Nack(eval.ID, token)
   989  		if err != nil {
   990  			t.Fatalf("err: %v", err)
   991  		}
   992  	}
   993  
   994  	// Check the stats
   995  	stats := b.Stats()
   996  	if stats.TotalReady != 1 {
   997  		t.Fatalf("bad: %#v", stats)
   998  	}
   999  	if stats.TotalUnacked != 0 {
  1000  		t.Fatalf("bad: %#v", stats)
  1001  	}
  1002  	if stats.ByScheduler[failedQueue].Ready != 1 {
  1003  		t.Fatalf("bad: %#v", stats)
  1004  	}
  1005  	if stats.ByScheduler[failedQueue].Unacked != 0 {
  1006  		t.Fatalf("bad: %#v", stats)
  1007  	}
  1008  
  1009  	// Dequeue from failed queue
  1010  	out, token, err := b.Dequeue([]string{failedQueue}, time.Second)
  1011  	if err != nil {
  1012  		t.Fatalf("err: %v", err)
  1013  	}
  1014  	if out != eval {
  1015  		t.Fatalf("bad : %#v", out)
  1016  	}
  1017  
  1018  	// Check the stats
  1019  	stats = b.Stats()
  1020  	if stats.TotalReady != 0 {
  1021  		t.Fatalf("bad: %#v", stats)
  1022  	}
  1023  	if stats.TotalUnacked != 1 {
  1024  		t.Fatalf("bad: %#v", stats)
  1025  	}
  1026  	if stats.ByScheduler[failedQueue].Ready != 0 {
  1027  		t.Fatalf("bad: %#v", stats)
  1028  	}
  1029  	if stats.ByScheduler[failedQueue].Unacked != 1 {
  1030  		t.Fatalf("bad: %#v", stats)
  1031  	}
  1032  
  1033  	// Ack finally
  1034  	err = b.Ack(out.ID, token)
  1035  	if err != nil {
  1036  		t.Fatalf("err: %v", err)
  1037  	}
  1038  
  1039  	if _, ok := b.Outstanding(out.ID); ok {
  1040  		t.Fatalf("should not be outstanding")
  1041  	}
  1042  
  1043  	// Check the stats
  1044  	stats = b.Stats()
  1045  	if stats.TotalReady != 0 {
  1046  		t.Fatalf("bad: %#v", stats)
  1047  	}
  1048  	if stats.TotalUnacked != 0 {
  1049  		t.Fatalf("bad: %#v", stats)
  1050  	}
  1051  	if stats.ByScheduler[failedQueue].Ready != 0 {
  1052  		t.Fatalf("bad: %#v", stats.ByScheduler[failedQueue])
  1053  	}
  1054  	if stats.ByScheduler[failedQueue].Unacked != 0 {
  1055  		t.Fatalf("bad: %#v", stats.ByScheduler[failedQueue])
  1056  	}
  1057  }
  1058  
  1059  func TestEvalBroker_AckAtDeliveryLimit(t *testing.T) {
  1060  	t.Parallel()
  1061  	b := testBroker(t, 0)
  1062  	b.SetEnabled(true)
  1063  
  1064  	eval := mock.Eval()
  1065  	b.Enqueue(eval)
  1066  
  1067  	for i := 0; i < 3; i++ {
  1068  		// Dequeue should work
  1069  		out, token, err := b.Dequeue(defaultSched, time.Second)
  1070  		if err != nil {
  1071  			t.Fatalf("err: %v", err)
  1072  		}
  1073  		if out != eval {
  1074  			t.Fatalf("bad : %#v", out)
  1075  		}
  1076  
  1077  		if i == 2 {
  1078  			b.Ack(eval.ID, token)
  1079  		} else {
  1080  			// Nack with wrong token should fail
  1081  			err = b.Nack(eval.ID, token)
  1082  			if err != nil {
  1083  				t.Fatalf("err: %v", err)
  1084  			}
  1085  		}
  1086  	}
  1087  
  1088  	// Check the stats
  1089  	stats := b.Stats()
  1090  	if stats.TotalReady != 0 {
  1091  		t.Fatalf("bad: %#v", stats)
  1092  	}
  1093  	if stats.TotalUnacked != 0 {
  1094  		t.Fatalf("bad: %#v", stats)
  1095  	}
  1096  	if _, ok := stats.ByScheduler[failedQueue]; ok {
  1097  		t.Fatalf("bad: %#v", stats)
  1098  	}
  1099  }
  1100  
  1101  // Ensure fairness between schedulers
  1102  func TestEvalBroker_Wait(t *testing.T) {
  1103  	t.Parallel()
  1104  	b := testBroker(t, 0)
  1105  	b.SetEnabled(true)
  1106  
  1107  	// Create an eval that should wait
  1108  	eval := mock.Eval()
  1109  	eval.Wait = 10 * time.Millisecond
  1110  	b.Enqueue(eval)
  1111  
  1112  	// Verify waiting
  1113  	stats := b.Stats()
  1114  	if stats.TotalReady != 0 {
  1115  		t.Fatalf("bad: %#v", stats)
  1116  	}
  1117  	if stats.TotalWaiting != 1 {
  1118  		t.Fatalf("bad: %#v", stats)
  1119  	}
  1120  
  1121  	// Let the wait elapse
  1122  	time.Sleep(20 * time.Millisecond)
  1123  
  1124  	// Verify ready
  1125  	stats = b.Stats()
  1126  	if stats.TotalReady != 1 {
  1127  		t.Fatalf("bad: %#v", stats)
  1128  	}
  1129  	if stats.TotalWaiting != 0 {
  1130  		t.Fatalf("bad: %#v", stats)
  1131  	}
  1132  
  1133  	// Dequeue should work
  1134  	out, _, err := b.Dequeue(defaultSched, time.Second)
  1135  	if err != nil {
  1136  		t.Fatalf("err: %v", err)
  1137  	}
  1138  	if out != eval {
  1139  		t.Fatalf("bad : %#v", out)
  1140  	}
  1141  }
  1142  
  1143  // Ensure that priority is taken into account when enqueueing many evaluations.
  1144  func TestEvalBroker_EnqueueAll_Dequeue_Fair(t *testing.T) {
  1145  	t.Parallel()
  1146  	b := testBroker(t, 0)
  1147  	b.SetEnabled(true)
  1148  
  1149  	// Start with a blocked dequeue
  1150  	outCh := make(chan *structs.Evaluation, 1)
  1151  	go func() {
  1152  		start := time.Now()
  1153  		out, _, err := b.Dequeue(defaultSched, time.Second)
  1154  		end := time.Now()
  1155  		outCh <- out
  1156  		if err != nil {
  1157  			t.Fatalf("err: %v", err)
  1158  		}
  1159  		if d := end.Sub(start); d < 5*time.Millisecond {
  1160  			t.Fatalf("bad: %v", d)
  1161  		}
  1162  	}()
  1163  
  1164  	// Wait for a bit
  1165  	time.Sleep(5 * time.Millisecond)
  1166  
  1167  	// Enqueue
  1168  	evals := make(map[*structs.Evaluation]string, 8)
  1169  	expectedPriority := 90
  1170  	for i := 10; i <= expectedPriority; i += 10 {
  1171  		eval := mock.Eval()
  1172  		eval.Priority = i
  1173  		evals[eval] = ""
  1174  
  1175  	}
  1176  	b.EnqueueAll(evals)
  1177  
  1178  	// Ensure dequeue
  1179  	select {
  1180  	case out := <-outCh:
  1181  		if out.Priority != expectedPriority {
  1182  			t.Fatalf("bad: %v", out)
  1183  		}
  1184  	case <-time.After(time.Second):
  1185  		t.Fatalf("timeout")
  1186  	}
  1187  }
  1188  
  1189  func TestEvalBroker_EnqueueAll_Requeue_Ack(t *testing.T) {
  1190  	t.Parallel()
  1191  	b := testBroker(t, 0)
  1192  	b.SetEnabled(true)
  1193  
  1194  	// Create the evaluation, enqueue and dequeue
  1195  	eval := mock.Eval()
  1196  	b.Enqueue(eval)
  1197  
  1198  	out, token, err := b.Dequeue(defaultSched, time.Second)
  1199  	if err != nil {
  1200  		t.Fatalf("err: %v", err)
  1201  	}
  1202  	if out != eval {
  1203  		t.Fatalf("bad : %#v", out)
  1204  	}
  1205  
  1206  	// Requeue the same evaluation.
  1207  	b.EnqueueAll(map[*structs.Evaluation]string{eval: token})
  1208  
  1209  	// The stats should show one unacked
  1210  	stats := b.Stats()
  1211  	if stats.TotalReady != 0 {
  1212  		t.Fatalf("bad: %#v", stats)
  1213  	}
  1214  	if stats.TotalUnacked != 1 {
  1215  		t.Fatalf("bad: %#v", stats)
  1216  	}
  1217  
  1218  	// Ack the evaluation.
  1219  	if err := b.Ack(eval.ID, token); err != nil {
  1220  		t.Fatalf("err: %v", err)
  1221  	}
  1222  
  1223  	// Check stats again as this should cause the re-enqueued one to transition
  1224  	// into the ready state
  1225  	stats = b.Stats()
  1226  	if stats.TotalReady != 1 {
  1227  		t.Fatalf("bad: %#v", stats)
  1228  	}
  1229  	if stats.TotalUnacked != 0 {
  1230  		t.Fatalf("bad: %#v", stats)
  1231  	}
  1232  
  1233  	// Another dequeue should be successful
  1234  	out2, token2, err := b.Dequeue(defaultSched, time.Second)
  1235  	if err != nil {
  1236  		t.Fatalf("err: %v", err)
  1237  	}
  1238  	if out2 != eval {
  1239  		t.Fatalf("bad : %#v", out)
  1240  	}
  1241  	if token == token2 {
  1242  		t.Fatalf("bad : %s and %s", token, token2)
  1243  	}
  1244  }
  1245  
  1246  func TestEvalBroker_EnqueueAll_Requeue_Nack(t *testing.T) {
  1247  	t.Parallel()
  1248  	b := testBroker(t, 0)
  1249  	b.SetEnabled(true)
  1250  
  1251  	// Create the evaluation, enqueue and dequeue
  1252  	eval := mock.Eval()
  1253  	b.Enqueue(eval)
  1254  
  1255  	out, token, err := b.Dequeue(defaultSched, time.Second)
  1256  	if err != nil {
  1257  		t.Fatalf("err: %v", err)
  1258  	}
  1259  	if out != eval {
  1260  		t.Fatalf("bad : %#v", out)
  1261  	}
  1262  
  1263  	// Requeue the same evaluation.
  1264  	b.EnqueueAll(map[*structs.Evaluation]string{eval: token})
  1265  
  1266  	// The stats should show one unacked
  1267  	stats := b.Stats()
  1268  	if stats.TotalReady != 0 {
  1269  		t.Fatalf("bad: %#v", stats)
  1270  	}
  1271  	if stats.TotalUnacked != 1 {
  1272  		t.Fatalf("bad: %#v", stats)
  1273  	}
  1274  
  1275  	// Nack the evaluation.
  1276  	if err := b.Nack(eval.ID, token); err != nil {
  1277  		t.Fatalf("err: %v", err)
  1278  	}
  1279  
  1280  	// Check stats again as this should cause the re-enqueued one to be dropped
  1281  	testutil.WaitForResult(func() (bool, error) {
  1282  		stats = b.Stats()
  1283  		if stats.TotalReady != 1 {
  1284  			return false, fmt.Errorf("bad: %#v", stats)
  1285  		}
  1286  		if stats.TotalUnacked != 0 {
  1287  			return false, fmt.Errorf("bad: %#v", stats)
  1288  		}
  1289  		if len(b.requeue) != 0 {
  1290  			return false, fmt.Errorf("bad: %#v", b.requeue)
  1291  		}
  1292  
  1293  		return true, nil
  1294  	}, func(e error) {
  1295  		t.Fatal(e)
  1296  	})
  1297  }