github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/consul/on_update.go (about)

     1  package consul
     2  
     3  import (
     4  	"fmt"
     5  	"time"
     6  
     7  	"github.com/hashicorp/nomad/e2e/e2eutil"
     8  	"github.com/hashicorp/nomad/e2e/framework"
     9  	"github.com/hashicorp/nomad/helper/uuid"
    10  	"github.com/hashicorp/nomad/testutil"
    11  )
    12  
    13  type OnUpdateChecksTest struct {
    14  	framework.TC
    15  	jobIDs []string
    16  }
    17  
    18  func (tc *OnUpdateChecksTest) BeforeAll(f *framework.F) {
    19  	// Ensure cluster has leader before running tests
    20  	e2eutil.WaitForLeader(f.T(), tc.Nomad())
    21  	// Ensure that we have at least 1 client node in ready state
    22  	e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1)
    23  }
    24  
    25  func (tc *OnUpdateChecksTest) AfterEach(f *framework.F) {
    26  	nomadClient := tc.Nomad()
    27  	j := nomadClient.Jobs()
    28  
    29  	for _, id := range tc.jobIDs {
    30  		j.Deregister(id, true, nil)
    31  	}
    32  	_, err := e2eutil.Command("nomad", "system", "gc")
    33  	f.NoError(err)
    34  }
    35  
    36  // TestOnUpdateCheck_IgnoreWarning_IgnoreErrors ensures that deployments
    37  // complete successfully with service checks that warn and error when on_update
    38  // is specified to ignore either.
    39  func (tc *OnUpdateChecksTest) TestOnUpdateCheck_IgnoreWarning_IgnoreErrors(f *framework.F) {
    40  	uuid := uuid.Generate()
    41  	jobID := fmt.Sprintf("on-update-%s", uuid[0:8])
    42  	tc.jobIDs = append(tc.jobIDs, jobID)
    43  
    44  	f.NoError(
    45  		e2eutil.Register(jobID, "consul/input/on_update.nomad"),
    46  		"should have registered successfully",
    47  	)
    48  
    49  	wc := &e2eutil.WaitConfig{
    50  		Interval: 1 * time.Second,
    51  		Retries:  60,
    52  	}
    53  	f.NoError(
    54  		e2eutil.WaitForLastDeploymentStatus(jobID, "", "successful", wc),
    55  		"deployment should have completed successfully",
    56  	)
    57  
    58  	// register update with on_update = ignore
    59  	// this check errors, deployment should still be successful
    60  	f.NoError(
    61  		e2eutil.Register(jobID, "consul/input/on_update_2.nomad"),
    62  		"should have registered successfully",
    63  	)
    64  
    65  	f.NoError(
    66  		e2eutil.WaitForLastDeploymentStatus(jobID, "", "successful", wc),
    67  		"deployment should have completed successfully",
    68  	)
    69  }
    70  
    71  // TestOnUpdate_CheckRestart ensures that a service check set to ignore
    72  // warnings still follows the check_restart stanza if the task becomes
    73  // unhealthy after a deployment is successful.  on_update_check_restart has a
    74  // script check that should report as a warning status for the deployment to
    75  // become healthy. The script check then reports unhealthy and the
    76  // check_restart policy should restart the task
    77  func (tc *OnUpdateChecksTest) TestOnUpdate_CheckRestart(f *framework.F) {
    78  	uuid := uuid.Generate()
    79  	jobID := fmt.Sprintf("on-update-restart-%s", uuid[0:8])
    80  	tc.jobIDs = append(tc.jobIDs, jobID)
    81  
    82  	f.NoError(
    83  		e2eutil.Register(jobID, "consul/input/on_update_check_restart.nomad"),
    84  		"should have registered successfully",
    85  	)
    86  
    87  	wc := &e2eutil.WaitConfig{
    88  		Interval: 1 * time.Second,
    89  		Retries:  60,
    90  	}
    91  	f.NoError(
    92  		e2eutil.WaitForLastDeploymentStatus(jobID, "", "successful", wc),
    93  		"deployment should have completed successfully",
    94  	)
    95  
    96  	// Wait for and ensure that allocation restarted
    97  	testutil.WaitForResultRetries(wc.Retries, func() (bool, error) {
    98  		time.Sleep(wc.Interval)
    99  		allocs, err := e2eutil.AllocTaskEventsForJob(jobID, "")
   100  		if err != nil {
   101  			return false, err
   102  		}
   103  
   104  		for allocID, allocEvents := range allocs {
   105  			var allocRestarted bool
   106  			var eventTypes []string
   107  			for _, events := range allocEvents {
   108  				eventTypes = append(eventTypes, events["Type"])
   109  				if events["Type"] == "Restart Signaled" {
   110  					allocRestarted = true
   111  				}
   112  			}
   113  			if allocRestarted {
   114  				return true, nil
   115  			}
   116  			return false, fmt.Errorf("alloc %s expected to restart got %v", allocID, eventTypes)
   117  		}
   118  
   119  		return true, nil
   120  	}, func(err error) {
   121  		f.NoError(err)
   122  	})
   123  }