github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/e2e/consul/on_update.go (about)

     1  package consul
     2  
     3  import (
     4  	"fmt"
     5  	"time"
     6  
     7  	"github.com/hashicorp/nomad/e2e/e2eutil"
     8  	e2e "github.com/hashicorp/nomad/e2e/e2eutil"
     9  	"github.com/hashicorp/nomad/e2e/framework"
    10  	"github.com/hashicorp/nomad/helper/uuid"
    11  	"github.com/hashicorp/nomad/testutil"
    12  )
    13  
    14  type OnUpdateChecksTest struct {
    15  	framework.TC
    16  	jobIDs []string
    17  }
    18  
    19  func (tc *OnUpdateChecksTest) BeforeAll(f *framework.F) {
    20  	// Ensure cluster has leader before running tests
    21  	e2eutil.WaitForLeader(f.T(), tc.Nomad())
    22  	// Ensure that we have at least 1 client node in ready state
    23  	e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1)
    24  }
    25  
    26  func (tc *OnUpdateChecksTest) AfterEach(f *framework.F) {
    27  	nomadClient := tc.Nomad()
    28  	j := nomadClient.Jobs()
    29  
    30  	for _, id := range tc.jobIDs {
    31  		j.Deregister(id, true, nil)
    32  	}
    33  	_, err := e2eutil.Command("nomad", "system", "gc")
    34  	f.NoError(err)
    35  }
    36  
    37  // TestOnUpdateCheck_IgnoreWarning_IgnoreErrors ensures that deployments
    38  // complete successfully with service checks that warn and error when on_update
    39  // is specified to ignore either.
    40  func (tc *OnUpdateChecksTest) TestOnUpdateCheck_IgnoreWarning_IgnoreErrors(f *framework.F) {
    41  	uuid := uuid.Generate()
    42  	jobID := fmt.Sprintf("on-update-%s", uuid[0:8])
    43  	tc.jobIDs = append(tc.jobIDs, jobID)
    44  
    45  	f.NoError(
    46  		e2eutil.Register(jobID, "consul/input/on_update.nomad"),
    47  		"should have registered successfully",
    48  	)
    49  
    50  	wc := &e2eutil.WaitConfig{
    51  		Interval: 1 * time.Second,
    52  		Retries:  60,
    53  	}
    54  	f.NoError(
    55  		e2eutil.WaitForLastDeploymentStatus(jobID, "", "successful", wc),
    56  		"deployment should have completed successfully",
    57  	)
    58  
    59  	// register update with on_update = ignore
    60  	// this check errors, deployment should still be successful
    61  	f.NoError(
    62  		e2eutil.Register(jobID, "consul/input/on_update_2.nomad"),
    63  		"should have registered successfully",
    64  	)
    65  
    66  	f.NoError(
    67  		e2eutil.WaitForLastDeploymentStatus(jobID, "", "successful", wc),
    68  		"deployment should have completed successfully",
    69  	)
    70  }
    71  
    72  // TestOnUpdate_CheckRestart ensures that a service check set to ignore
    73  // warnings still follows the check_restart stanza if the task becomes
    74  // unhealthy after a deployment is successful
    75  func (tc *OnUpdateChecksTest) TestOnUpdate_CheckRestart(f *framework.F) {
    76  	uuid := uuid.Generate()
    77  	jobID := fmt.Sprintf("on-update-restart-%s", uuid[0:8])
    78  	tc.jobIDs = append(tc.jobIDs, jobID)
    79  
    80  	f.NoError(
    81  		e2eutil.Register(jobID, "consul/input/on_update_check_restart.nomad"),
    82  		"should have registered successfully",
    83  	)
    84  
    85  	wc := &e2eutil.WaitConfig{
    86  		Interval: 1 * time.Second,
    87  		Retries:  60,
    88  	}
    89  	f.NoError(
    90  		e2eutil.WaitForLastDeploymentStatus(jobID, "", "successful", wc),
    91  		"deployment should have completed successfully",
    92  	)
    93  
    94  	// register update with on_update = ignore
    95  	// this check errors, deployment should still be successful
    96  	f.NoError(
    97  		e2eutil.Register(jobID, "consul/input/on_update_2.nomad"),
    98  		"should have registered successfully",
    99  	)
   100  
   101  	f.NoError(
   102  		e2eutil.WaitForLastDeploymentStatus(jobID, "", "successful", wc),
   103  		"deployment should have completed successfully",
   104  	)
   105  
   106  	interval, retries := wc.OrDefault()
   107  	// Wait for and ensure that allocation restarted
   108  	testutil.WaitForResultRetries(retries, func() (bool, error) {
   109  		time.Sleep(interval)
   110  		allocs, err := e2e.AllocTaskEventsForJob(jobID, "")
   111  		if err != nil {
   112  			return false, err
   113  		}
   114  
   115  		for allocID, allocEvents := range allocs {
   116  			var allocRestarted bool
   117  			for _, events := range allocEvents {
   118  				if events["Type"] == "Restart Signaled" {
   119  					allocRestarted = true
   120  				}
   121  			}
   122  			if allocRestarted {
   123  				return true, nil
   124  			}
   125  			return false, fmt.Errorf("alloc %s expected to restart", allocID)
   126  		}
   127  
   128  		return true, nil
   129  	}, func(err error) {
   130  		f.NoError(err)
   131  	})
   132  }