github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/connect/multi_service.go (about)

     1  package connect
     2  
     3  import (
     4  	"fmt"
     5  	"strings"
     6  	"time"
     7  
     8  	consulapi "github.com/hashicorp/consul/api"
     9  	"github.com/hashicorp/nomad/api"
    10  	"github.com/hashicorp/nomad/e2e/framework"
    11  	"github.com/hashicorp/nomad/helper/uuid"
    12  	"github.com/hashicorp/nomad/jobspec"
    13  	"github.com/hashicorp/nomad/testutil"
    14  	"github.com/kr/pretty"
    15  	"github.com/stretchr/testify/require"
    16  )
    17  
    18  // TestMultiServiceConnect tests running multiple envoy sidecars in the same allocation.
    19  func (tc *ConnectE2ETest) TestMultiServiceConnect(f *framework.F) {
    20  	t := f.T()
    21  	uuid := uuid.Generate()
    22  	jobID := "connect" + uuid[0:8]
    23  	tc.jobIds = append(tc.jobIds, jobID)
    24  	jobapi := tc.Nomad().Jobs()
    25  
    26  	job, err := jobspec.ParseFile("connect/input/multi-service.nomad")
    27  	require.NoError(t, err)
    28  	job.ID = &jobID
    29  
    30  	resp, _, err := jobapi.Register(job, nil)
    31  	require.NoError(t, err)
    32  	require.NotNil(t, resp)
    33  	require.Zero(t, resp.Warnings)
    34  
    35  EVAL:
    36  	qopts := &api.QueryOptions{
    37  		WaitIndex: resp.EvalCreateIndex,
    38  	}
    39  	evalapi := tc.Nomad().Evaluations()
    40  	eval, qmeta, err := evalapi.Info(resp.EvalID, qopts)
    41  	require.NoError(t, err)
    42  	qopts.WaitIndex = qmeta.LastIndex
    43  
    44  	switch eval.Status {
    45  	case "pending":
    46  		goto EVAL
    47  	case "complete":
    48  		// Ok!
    49  	case "failed", "canceled", "blocked":
    50  		require.Failf(t, "expected complete status", "eval %s\n%s", eval.Status, pretty.Sprint(eval))
    51  	default:
    52  		require.Failf(t, "expected complete status", "unknown eval status: %s\n%s", eval.Status, pretty.Sprint(eval))
    53  	}
    54  
    55  	// Assert there were 0 placement failures
    56  	require.Zero(t, eval.FailedTGAllocs, pretty.Sprint(eval.FailedTGAllocs))
    57  	require.Len(t, eval.QueuedAllocations, 1, pretty.Sprint(eval.QueuedAllocations))
    58  
    59  	// Assert allocs are running
    60  	for i := 0; i < 20; i++ {
    61  		allocs, qmeta, err := evalapi.Allocations(eval.ID, qopts)
    62  		require.NoError(t, err)
    63  		require.Len(t, allocs, 1)
    64  		qopts.WaitIndex = qmeta.LastIndex
    65  
    66  		running := 0
    67  		for _, alloc := range allocs {
    68  			switch alloc.ClientStatus {
    69  			case "running":
    70  				running++
    71  			case "pending":
    72  				// keep trying
    73  			default:
    74  				require.Failf(t, "alloc failed", "alloc: %s", pretty.Sprint(alloc))
    75  			}
    76  		}
    77  
    78  		if running == len(allocs) {
    79  			break
    80  		}
    81  
    82  		time.Sleep(500 * time.Millisecond)
    83  	}
    84  
    85  	allocs, _, err := evalapi.Allocations(eval.ID, qopts)
    86  	require.NoError(t, err)
    87  	allocIDs := make(map[string]bool, 1)
    88  	for _, a := range allocs {
    89  		if a.ClientStatus != "running" || a.DesiredStatus != "run" {
    90  			require.Failf(t, "expected running status", "alloc %s (%s) terminal; client=%s desired=%s", a.TaskGroup, a.ID, a.ClientStatus, a.DesiredStatus)
    91  		}
    92  		allocIDs[a.ID] = true
    93  	}
    94  
    95  	// Check Consul service health
    96  	agentapi := tc.Consul().Agent()
    97  
    98  	failing := map[string]*consulapi.AgentCheck{}
    99  	testutil.WaitForResultRetries(60, func() (bool, error) {
   100  		defer time.Sleep(time.Second)
   101  
   102  		checks, err := agentapi.Checks()
   103  		require.NoError(t, err)
   104  
   105  		// Filter out checks for other services
   106  		for cid, check := range checks {
   107  			found := false
   108  			for allocID := range allocIDs {
   109  				if strings.Contains(check.ServiceID, allocID) {
   110  					found = true
   111  					break
   112  				}
   113  			}
   114  
   115  			if !found {
   116  				delete(checks, cid)
   117  			}
   118  		}
   119  
   120  		// Ensure checks are all passing
   121  		failing = map[string]*consulapi.AgentCheck{}
   122  		for _, check := range checks {
   123  			if check.Status != "passing" {
   124  				failing[check.CheckID] = check
   125  				break
   126  			}
   127  		}
   128  
   129  		if len(failing) == 0 {
   130  			return true, nil
   131  		}
   132  
   133  		t.Logf("still %d checks not passing", len(failing))
   134  		return false, fmt.Errorf("checks are not passing %v %v", len(failing), pretty.Sprint(failing))
   135  	}, func(e error) {
   136  		require.NoError(t, err)
   137  	})
   138  
   139  	require.Len(t, failing, 0, pretty.Sprint(failing))
   140  }