github.com/hernad/nomad@v1.6.112/e2e/connect/multi_service.go (about)

     1  // Copyright (c) HashiCorp, Inc.
     2  // SPDX-License-Identifier: MPL-2.0
     3  
     4  package connect
     5  
     6  import (
     7  	"fmt"
     8  	"strings"
     9  	"time"
    10  
    11  	consulapi "github.com/hashicorp/consul/api"
    12  	"github.com/hernad/nomad/api"
    13  	"github.com/hernad/nomad/e2e/framework"
    14  	"github.com/hernad/nomad/helper/uuid"
    15  	"github.com/hernad/nomad/jobspec"
    16  	"github.com/hernad/nomad/testutil"
    17  	"github.com/kr/pretty"
    18  	"github.com/stretchr/testify/require"
    19  )
    20  
    21  // TestMultiServiceConnect tests running multiple envoy sidecars in the same allocation.
    22  func (tc *ConnectE2ETest) TestMultiServiceConnect(f *framework.F) {
    23  	t := f.T()
    24  	uuid := uuid.Generate()
    25  	jobID := "connect" + uuid[0:8]
    26  	tc.jobIds = append(tc.jobIds, jobID)
    27  	jobapi := tc.Nomad().Jobs()
    28  
    29  	job, err := jobspec.ParseFile("connect/input/multi-service.nomad")
    30  	require.NoError(t, err)
    31  	job.ID = &jobID
    32  
    33  	resp, _, err := jobapi.Register(job, nil)
    34  	require.NoError(t, err)
    35  	require.NotNil(t, resp)
    36  	require.Zero(t, resp.Warnings)
    37  
    38  EVAL:
    39  	qopts := &api.QueryOptions{
    40  		WaitIndex: resp.EvalCreateIndex,
    41  	}
    42  	evalapi := tc.Nomad().Evaluations()
    43  	eval, qmeta, err := evalapi.Info(resp.EvalID, qopts)
    44  	require.NoError(t, err)
    45  	qopts.WaitIndex = qmeta.LastIndex
    46  
    47  	switch eval.Status {
    48  	case "pending":
    49  		goto EVAL
    50  	case "complete":
    51  		// Ok!
    52  	case "failed", "canceled", "blocked":
    53  		require.Failf(t, "expected complete status", "eval %s\n%s", eval.Status, pretty.Sprint(eval))
    54  	default:
    55  		require.Failf(t, "expected complete status", "unknown eval status: %s\n%s", eval.Status, pretty.Sprint(eval))
    56  	}
    57  
    58  	// Assert there were 0 placement failures
    59  	require.Zero(t, eval.FailedTGAllocs, pretty.Sprint(eval.FailedTGAllocs))
    60  	require.Len(t, eval.QueuedAllocations, 1, pretty.Sprint(eval.QueuedAllocations))
    61  
    62  	// Assert allocs are running
    63  	for i := 0; i < 20; i++ {
    64  		allocs, qmeta, err := evalapi.Allocations(eval.ID, qopts)
    65  		require.NoError(t, err)
    66  		require.Len(t, allocs, 1)
    67  		qopts.WaitIndex = qmeta.LastIndex
    68  
    69  		running := 0
    70  		for _, alloc := range allocs {
    71  			switch alloc.ClientStatus {
    72  			case "running":
    73  				running++
    74  			case "pending":
    75  				// keep trying
    76  			default:
    77  				require.Failf(t, "alloc failed", "alloc: %s", pretty.Sprint(alloc))
    78  			}
    79  		}
    80  
    81  		if running == len(allocs) {
    82  			break
    83  		}
    84  
    85  		time.Sleep(500 * time.Millisecond)
    86  	}
    87  
    88  	allocs, _, err := evalapi.Allocations(eval.ID, qopts)
    89  	require.NoError(t, err)
    90  	allocIDs := make(map[string]bool, 1)
    91  	for _, a := range allocs {
    92  		if a.ClientStatus != "running" || a.DesiredStatus != "run" {
    93  			require.Failf(t, "expected running status", "alloc %s (%s) terminal; client=%s desired=%s", a.TaskGroup, a.ID, a.ClientStatus, a.DesiredStatus)
    94  		}
    95  		allocIDs[a.ID] = true
    96  	}
    97  
    98  	// Check Consul service health
    99  	agentapi := tc.Consul().Agent()
   100  
   101  	failing := map[string]*consulapi.AgentCheck{}
   102  	testutil.WaitForResultRetries(60, func() (bool, error) {
   103  		defer time.Sleep(time.Second)
   104  
   105  		checks, err := agentapi.Checks()
   106  		require.NoError(t, err)
   107  
   108  		// Filter out checks for other services
   109  		for cid, check := range checks {
   110  			found := false
   111  			for allocID := range allocIDs {
   112  				if strings.Contains(check.ServiceID, allocID) {
   113  					found = true
   114  					break
   115  				}
   116  			}
   117  
   118  			if !found {
   119  				delete(checks, cid)
   120  			}
   121  		}
   122  
   123  		// Ensure checks are all passing
   124  		failing = map[string]*consulapi.AgentCheck{}
   125  		for _, check := range checks {
   126  			if check.Status != "passing" {
   127  				failing[check.CheckID] = check
   128  				break
   129  			}
   130  		}
   131  
   132  		if len(failing) == 0 {
   133  			return true, nil
   134  		}
   135  
   136  		t.Logf("still %d checks not passing", len(failing))
   137  		return false, fmt.Errorf("checks are not passing %v %v", len(failing), pretty.Sprint(failing))
   138  	}, func(e error) {
   139  		require.NoError(t, err)
   140  	})
   141  
   142  	require.Len(t, failing, 0, pretty.Sprint(failing))
   143  }