github.com/emate/nomad@v0.8.2-wo-binpacking/nomad/client_alloc_endpoint_test.go (about)

     1  package nomad
     2  
     3  import (
     4  	"fmt"
     5  	"testing"
     6  
     7  	msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
     8  	"github.com/hashicorp/nomad/acl"
     9  	"github.com/hashicorp/nomad/client"
    10  	"github.com/hashicorp/nomad/client/config"
    11  	cstructs "github.com/hashicorp/nomad/client/structs"
    12  	"github.com/hashicorp/nomad/helper/uuid"
    13  	"github.com/hashicorp/nomad/nomad/mock"
    14  	"github.com/hashicorp/nomad/nomad/structs"
    15  	"github.com/hashicorp/nomad/testutil"
    16  	"github.com/stretchr/testify/require"
    17  )
    18  
    19  func TestClientAllocations_GarbageCollectAll_Local(t *testing.T) {
    20  	t.Parallel()
    21  	require := require.New(t)
    22  
    23  	// Start a server and client
    24  	s := TestServer(t, nil)
    25  	defer s.Shutdown()
    26  	codec := rpcClient(t, s)
    27  	testutil.WaitForLeader(t, s.RPC)
    28  
    29  	c := client.TestClient(t, func(c *config.Config) {
    30  		c.Servers = []string{s.config.RPCAddr.String()}
    31  	})
    32  	defer c.Shutdown()
    33  
    34  	testutil.WaitForResult(func() (bool, error) {
    35  		nodes := s.connectedNodes()
    36  		return len(nodes) == 1, nil
    37  	}, func(err error) {
    38  		t.Fatalf("should have a clients")
    39  	})
    40  
    41  	// Make the request without having a node-id
    42  	req := &structs.NodeSpecificRequest{
    43  		QueryOptions: structs.QueryOptions{Region: "global"},
    44  	}
    45  
    46  	// Fetch the response
    47  	var resp structs.GenericResponse
    48  	err := msgpackrpc.CallWithCodec(codec, "ClientAllocations.GarbageCollectAll", req, &resp)
    49  	require.NotNil(err)
    50  	require.Contains(err.Error(), "missing")
    51  
    52  	// Fetch the response setting the node id
    53  	req.NodeID = c.NodeID()
    54  	var resp2 structs.GenericResponse
    55  	err = msgpackrpc.CallWithCodec(codec, "ClientAllocations.GarbageCollectAll", req, &resp2)
    56  	require.Nil(err)
    57  }
    58  
    59  func TestClientAllocations_GarbageCollectAll_Local_ACL(t *testing.T) {
    60  	t.Parallel()
    61  	require := require.New(t)
    62  
    63  	// Start a server
    64  	s, root := TestACLServer(t, nil)
    65  	defer s.Shutdown()
    66  	codec := rpcClient(t, s)
    67  	testutil.WaitForLeader(t, s.RPC)
    68  
    69  	// Create a bad token
    70  	policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityReadFS})
    71  	tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad)
    72  
    73  	policyGood := mock.NodePolicy(acl.PolicyWrite)
    74  	tokenGood := mock.CreatePolicyAndToken(t, s.State(), 1009, "valid2", policyGood)
    75  
    76  	cases := []struct {
    77  		Name          string
    78  		Token         string
    79  		ExpectedError string
    80  	}{
    81  		{
    82  			Name:          "bad token",
    83  			Token:         tokenBad.SecretID,
    84  			ExpectedError: structs.ErrPermissionDenied.Error(),
    85  		},
    86  		{
    87  			Name:          "good token",
    88  			Token:         tokenGood.SecretID,
    89  			ExpectedError: "Unknown node",
    90  		},
    91  		{
    92  			Name:          "root token",
    93  			Token:         root.SecretID,
    94  			ExpectedError: "Unknown node",
    95  		},
    96  	}
    97  
    98  	for _, c := range cases {
    99  		t.Run(c.Name, func(t *testing.T) {
   100  
   101  			// Make the request without having a node-id
   102  			req := &structs.NodeSpecificRequest{
   103  				NodeID: uuid.Generate(),
   104  				QueryOptions: structs.QueryOptions{
   105  					AuthToken: c.Token,
   106  					Region:    "global",
   107  				},
   108  			}
   109  
   110  			// Fetch the response
   111  			var resp structs.GenericResponse
   112  			err := msgpackrpc.CallWithCodec(codec, "ClientAllocations.GarbageCollectAll", req, &resp)
   113  			require.NotNil(err)
   114  			require.Contains(err.Error(), c.ExpectedError)
   115  		})
   116  	}
   117  }
   118  
   119  func TestClientAllocations_GarbageCollectAll_NoNode(t *testing.T) {
   120  	t.Parallel()
   121  	require := require.New(t)
   122  
   123  	// Start a server and client
   124  	s := TestServer(t, nil)
   125  	defer s.Shutdown()
   126  	codec := rpcClient(t, s)
   127  	testutil.WaitForLeader(t, s.RPC)
   128  
   129  	// Make the request without having a node-id
   130  	req := &structs.NodeSpecificRequest{
   131  		NodeID:       uuid.Generate(),
   132  		QueryOptions: structs.QueryOptions{Region: "global"},
   133  	}
   134  
   135  	// Fetch the response
   136  	var resp structs.GenericResponse
   137  	err := msgpackrpc.CallWithCodec(codec, "ClientAllocations.GarbageCollectAll", req, &resp)
   138  	require.NotNil(err)
   139  	require.Contains(err.Error(), "Unknown node")
   140  }
   141  
   142  func TestClientAllocations_GarbageCollectAll_OldNode(t *testing.T) {
   143  	t.Parallel()
   144  	require := require.New(t)
   145  
   146  	// Start a server and fake an old client
   147  	s := TestServer(t, nil)
   148  	defer s.Shutdown()
   149  	state := s.State()
   150  	codec := rpcClient(t, s)
   151  	testutil.WaitForLeader(t, s.RPC)
   152  
   153  	// Test for an old version error
   154  	node := mock.Node()
   155  	node.Attributes["nomad.version"] = "0.7.1"
   156  	require.Nil(state.UpsertNode(1005, node))
   157  
   158  	req := &structs.NodeSpecificRequest{
   159  		NodeID:       node.ID,
   160  		QueryOptions: structs.QueryOptions{Region: "global"},
   161  	}
   162  
   163  	var resp structs.GenericResponse
   164  	err := msgpackrpc.CallWithCodec(codec, "ClientAllocations.GarbageCollectAll", req, &resp)
   165  	require.True(structs.IsErrNodeLacksRpc(err))
   166  
   167  	// Test for a missing version error
   168  	delete(node.Attributes, "nomad.version")
   169  	require.Nil(state.UpsertNode(1006, node))
   170  
   171  	err = msgpackrpc.CallWithCodec(codec, "ClientAllocations.GarbageCollectAll", req, &resp)
   172  	require.True(structs.IsErrUnknownNomadVersion(err))
   173  }
   174  
   175  func TestClientAllocations_GarbageCollectAll_Remote(t *testing.T) {
   176  	t.Parallel()
   177  	require := require.New(t)
   178  
   179  	// Start a server and client
   180  	s1 := TestServer(t, nil)
   181  	defer s1.Shutdown()
   182  	s2 := TestServer(t, func(c *Config) {
   183  		c.DevDisableBootstrap = true
   184  	})
   185  	defer s2.Shutdown()
   186  	TestJoin(t, s1, s2)
   187  	testutil.WaitForLeader(t, s1.RPC)
   188  	testutil.WaitForLeader(t, s2.RPC)
   189  	codec := rpcClient(t, s2)
   190  
   191  	c := client.TestClient(t, func(c *config.Config) {
   192  		c.Servers = []string{s2.config.RPCAddr.String()}
   193  		c.GCDiskUsageThreshold = 100.0
   194  	})
   195  	defer c.Shutdown()
   196  
   197  	testutil.WaitForResult(func() (bool, error) {
   198  		nodes := s2.connectedNodes()
   199  		return len(nodes) == 1, nil
   200  	}, func(err error) {
   201  		t.Fatalf("should have a clients")
   202  	})
   203  
   204  	// Force remove the connection locally in case it exists
   205  	s1.nodeConnsLock.Lock()
   206  	delete(s1.nodeConns, c.NodeID())
   207  	s1.nodeConnsLock.Unlock()
   208  
   209  	// Make the request
   210  	req := &structs.NodeSpecificRequest{
   211  		NodeID:       c.NodeID(),
   212  		QueryOptions: structs.QueryOptions{Region: "global"},
   213  	}
   214  
   215  	// Fetch the response
   216  	var resp cstructs.ClientStatsResponse
   217  	err := msgpackrpc.CallWithCodec(codec, "ClientAllocations.GarbageCollectAll", req, &resp)
   218  	require.Nil(err)
   219  }
   220  
   221  func TestClientAllocations_GarbageCollect_OldNode(t *testing.T) {
   222  	t.Parallel()
   223  	require := require.New(t)
   224  
   225  	// Start a server and fake an old client
   226  	s := TestServer(t, nil)
   227  	defer s.Shutdown()
   228  	state := s.State()
   229  	codec := rpcClient(t, s)
   230  	testutil.WaitForLeader(t, s.RPC)
   231  
   232  	// Test for an old version error
   233  	node := mock.Node()
   234  	node.Attributes["nomad.version"] = "0.7.1"
   235  	require.Nil(state.UpsertNode(1005, node))
   236  
   237  	alloc := mock.Alloc()
   238  	alloc.NodeID = node.ID
   239  	require.Nil(state.UpsertAllocs(1006, []*structs.Allocation{alloc}))
   240  
   241  	req := &structs.AllocSpecificRequest{
   242  		AllocID: alloc.ID,
   243  		QueryOptions: structs.QueryOptions{
   244  			Region:    "global",
   245  			Namespace: structs.DefaultNamespace,
   246  		},
   247  	}
   248  
   249  	var resp structs.GenericResponse
   250  	err := msgpackrpc.CallWithCodec(codec, "ClientAllocations.GarbageCollect", req, &resp)
   251  	require.True(structs.IsErrNodeLacksRpc(err), err.Error())
   252  
   253  	// Test for a missing version error
   254  	delete(node.Attributes, "nomad.version")
   255  	require.Nil(state.UpsertNode(1007, node))
   256  
   257  	err = msgpackrpc.CallWithCodec(codec, "ClientAllocations.GarbageCollect", req, &resp)
   258  	require.True(structs.IsErrUnknownNomadVersion(err), err.Error())
   259  }
   260  
   261  func TestClientAllocations_GarbageCollect_Local(t *testing.T) {
   262  	t.Parallel()
   263  	require := require.New(t)
   264  
   265  	// Start a server and client
   266  	s := TestServer(t, nil)
   267  	defer s.Shutdown()
   268  	codec := rpcClient(t, s)
   269  	testutil.WaitForLeader(t, s.RPC)
   270  
   271  	c := client.TestClient(t, func(c *config.Config) {
   272  		c.Servers = []string{s.config.RPCAddr.String()}
   273  		c.GCDiskUsageThreshold = 100.0
   274  	})
   275  	defer c.Shutdown()
   276  
   277  	// Force an allocation onto the node
   278  	a := mock.Alloc()
   279  	a.Job.Type = structs.JobTypeBatch
   280  	a.NodeID = c.NodeID()
   281  	a.Job.TaskGroups[0].Count = 1
   282  	a.Job.TaskGroups[0].Tasks[0] = &structs.Task{
   283  		Name:   "web",
   284  		Driver: "mock_driver",
   285  		Config: map[string]interface{}{
   286  			"run_for": "2s",
   287  		},
   288  		LogConfig: structs.DefaultLogConfig(),
   289  		Resources: &structs.Resources{
   290  			CPU:      500,
   291  			MemoryMB: 256,
   292  		},
   293  	}
   294  
   295  	testutil.WaitForResult(func() (bool, error) {
   296  		nodes := s.connectedNodes()
   297  		return len(nodes) == 1, nil
   298  	}, func(err error) {
   299  		t.Fatalf("should have a clients")
   300  	})
   301  
   302  	// Upsert the allocation
   303  	state := s.State()
   304  	require.Nil(state.UpsertJob(999, a.Job))
   305  	require.Nil(state.UpsertAllocs(1003, []*structs.Allocation{a}))
   306  
   307  	// Wait for the client to run the allocation
   308  	testutil.WaitForResult(func() (bool, error) {
   309  		alloc, err := state.AllocByID(nil, a.ID)
   310  		if err != nil {
   311  			return false, err
   312  		}
   313  		if alloc == nil {
   314  			return false, fmt.Errorf("unknown alloc")
   315  		}
   316  		if alloc.ClientStatus != structs.AllocClientStatusComplete {
   317  			return false, fmt.Errorf("alloc client status: %v", alloc.ClientStatus)
   318  		}
   319  
   320  		return true, nil
   321  	}, func(err error) {
   322  		t.Fatalf("Alloc on node %q not finished: %v", c.NodeID(), err)
   323  	})
   324  
   325  	// Make the request without having an alloc id
   326  	req := &structs.AllocSpecificRequest{
   327  		QueryOptions: structs.QueryOptions{Region: "global"},
   328  	}
   329  
   330  	// Fetch the response
   331  	var resp structs.GenericResponse
   332  	err := msgpackrpc.CallWithCodec(codec, "ClientAllocations.GarbageCollect", req, &resp)
   333  	require.NotNil(err)
   334  	require.Contains(err.Error(), "missing")
   335  
   336  	// Fetch the response setting the node id
   337  	req.AllocID = a.ID
   338  	var resp2 structs.GenericResponse
   339  	err = msgpackrpc.CallWithCodec(codec, "ClientAllocations.GarbageCollect", req, &resp2)
   340  	require.Nil(err)
   341  }
   342  
   343  func TestClientAllocations_GarbageCollect_Local_ACL(t *testing.T) {
   344  	t.Parallel()
   345  	require := require.New(t)
   346  
   347  	// Start a server
   348  	s, root := TestACLServer(t, nil)
   349  	defer s.Shutdown()
   350  	codec := rpcClient(t, s)
   351  	testutil.WaitForLeader(t, s.RPC)
   352  
   353  	// Create a bad token
   354  	policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityReadFS})
   355  	tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad)
   356  
   357  	policyGood := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})
   358  	tokenGood := mock.CreatePolicyAndToken(t, s.State(), 1009, "valid2", policyGood)
   359  
   360  	cases := []struct {
   361  		Name          string
   362  		Token         string
   363  		ExpectedError string
   364  	}{
   365  		{
   366  			Name:          "bad token",
   367  			Token:         tokenBad.SecretID,
   368  			ExpectedError: structs.ErrPermissionDenied.Error(),
   369  		},
   370  		{
   371  			Name:          "good token",
   372  			Token:         tokenGood.SecretID,
   373  			ExpectedError: structs.ErrUnknownAllocationPrefix,
   374  		},
   375  		{
   376  			Name:          "root token",
   377  			Token:         root.SecretID,
   378  			ExpectedError: structs.ErrUnknownAllocationPrefix,
   379  		},
   380  	}
   381  
   382  	for _, c := range cases {
   383  		t.Run(c.Name, func(t *testing.T) {
   384  
   385  			// Make the request without having a node-id
   386  			req := &structs.AllocSpecificRequest{
   387  				AllocID: uuid.Generate(),
   388  				QueryOptions: structs.QueryOptions{
   389  					AuthToken: c.Token,
   390  					Region:    "global",
   391  					Namespace: structs.DefaultNamespace,
   392  				},
   393  			}
   394  
   395  			// Fetch the response
   396  			var resp structs.GenericResponse
   397  			err := msgpackrpc.CallWithCodec(codec, "ClientAllocations.GarbageCollect", req, &resp)
   398  			require.NotNil(err)
   399  			require.Contains(err.Error(), c.ExpectedError)
   400  		})
   401  	}
   402  }
   403  
   404  func TestClientAllocations_GarbageCollect_Remote(t *testing.T) {
   405  	t.Parallel()
   406  	require := require.New(t)
   407  
   408  	// Start a server and client
   409  	s1 := TestServer(t, nil)
   410  	defer s1.Shutdown()
   411  	s2 := TestServer(t, func(c *Config) {
   412  		c.DevDisableBootstrap = true
   413  	})
   414  	defer s2.Shutdown()
   415  	TestJoin(t, s1, s2)
   416  	testutil.WaitForLeader(t, s1.RPC)
   417  	testutil.WaitForLeader(t, s2.RPC)
   418  	codec := rpcClient(t, s2)
   419  
   420  	c := client.TestClient(t, func(c *config.Config) {
   421  		c.Servers = []string{s2.config.RPCAddr.String()}
   422  		c.GCDiskUsageThreshold = 100.0
   423  	})
   424  	defer c.Shutdown()
   425  
   426  	// Force an allocation onto the node
   427  	a := mock.Alloc()
   428  	a.Job.Type = structs.JobTypeBatch
   429  	a.NodeID = c.NodeID()
   430  	a.Job.TaskGroups[0].Count = 1
   431  	a.Job.TaskGroups[0].Tasks[0] = &structs.Task{
   432  		Name:   "web",
   433  		Driver: "mock_driver",
   434  		Config: map[string]interface{}{
   435  			"run_for": "2s",
   436  		},
   437  		LogConfig: structs.DefaultLogConfig(),
   438  		Resources: &structs.Resources{
   439  			CPU:      500,
   440  			MemoryMB: 256,
   441  		},
   442  	}
   443  	testutil.WaitForResult(func() (bool, error) {
   444  		nodes := s2.connectedNodes()
   445  		return len(nodes) == 1, nil
   446  	}, func(err error) {
   447  		t.Fatalf("should have a clients")
   448  	})
   449  
   450  	// Upsert the allocation
   451  	state1 := s1.State()
   452  	state2 := s2.State()
   453  	require.Nil(state1.UpsertJob(999, a.Job))
   454  	require.Nil(state1.UpsertAllocs(1003, []*structs.Allocation{a}))
   455  	require.Nil(state2.UpsertJob(999, a.Job))
   456  	require.Nil(state2.UpsertAllocs(1003, []*structs.Allocation{a}))
   457  
   458  	// Wait for the client to run the allocation
   459  	testutil.WaitForResult(func() (bool, error) {
   460  		alloc, err := state2.AllocByID(nil, a.ID)
   461  		if err != nil {
   462  			return false, err
   463  		}
   464  		if alloc == nil {
   465  			return false, fmt.Errorf("unknown alloc")
   466  		}
   467  		if alloc.ClientStatus != structs.AllocClientStatusComplete {
   468  			return false, fmt.Errorf("alloc client status: %v", alloc.ClientStatus)
   469  		}
   470  
   471  		return true, nil
   472  	}, func(err error) {
   473  		t.Fatalf("Alloc on node %q not finished: %v", c.NodeID(), err)
   474  	})
   475  
   476  	// Force remove the connection locally in case it exists
   477  	s1.nodeConnsLock.Lock()
   478  	delete(s1.nodeConns, c.NodeID())
   479  	s1.nodeConnsLock.Unlock()
   480  
   481  	// Make the request
   482  	req := &structs.AllocSpecificRequest{
   483  		AllocID:      a.ID,
   484  		QueryOptions: structs.QueryOptions{Region: "global"},
   485  	}
   486  
   487  	// Fetch the response
   488  	var resp cstructs.ClientStatsResponse
   489  	err := msgpackrpc.CallWithCodec(codec, "ClientAllocations.GarbageCollect", req, &resp)
   490  	require.Nil(err)
   491  }
   492  
   493  func TestClientAllocations_Stats_OldNode(t *testing.T) {
   494  	t.Parallel()
   495  	require := require.New(t)
   496  
   497  	// Start a server and fake an old client
   498  	s := TestServer(t, nil)
   499  	defer s.Shutdown()
   500  	state := s.State()
   501  	codec := rpcClient(t, s)
   502  	testutil.WaitForLeader(t, s.RPC)
   503  
   504  	// Test for an old version error
   505  	node := mock.Node()
   506  	node.Attributes["nomad.version"] = "0.7.1"
   507  	require.Nil(state.UpsertNode(1005, node))
   508  
   509  	alloc := mock.Alloc()
   510  	alloc.NodeID = node.ID
   511  	require.Nil(state.UpsertAllocs(1006, []*structs.Allocation{alloc}))
   512  
   513  	req := &structs.AllocSpecificRequest{
   514  		AllocID: alloc.ID,
   515  		QueryOptions: structs.QueryOptions{
   516  			Region: "global",
   517  		},
   518  	}
   519  
   520  	var resp structs.GenericResponse
   521  	err := msgpackrpc.CallWithCodec(codec, "ClientAllocations.Stats", req, &resp)
   522  	require.True(structs.IsErrNodeLacksRpc(err), err.Error())
   523  
   524  	// Test for a missing version error
   525  	delete(node.Attributes, "nomad.version")
   526  	require.Nil(state.UpsertNode(1007, node))
   527  
   528  	err = msgpackrpc.CallWithCodec(codec, "ClientAllocations.Stats", req, &resp)
   529  	require.True(structs.IsErrUnknownNomadVersion(err), err.Error())
   530  }
   531  
   532  func TestClientAllocations_Stats_Local(t *testing.T) {
   533  	t.Parallel()
   534  	require := require.New(t)
   535  
   536  	// Start a server and client
   537  	s := TestServer(t, nil)
   538  	defer s.Shutdown()
   539  	codec := rpcClient(t, s)
   540  	testutil.WaitForLeader(t, s.RPC)
   541  
   542  	c := client.TestClient(t, func(c *config.Config) {
   543  		c.Servers = []string{s.config.RPCAddr.String()}
   544  	})
   545  	defer c.Shutdown()
   546  
   547  	// Force an allocation onto the node
   548  	a := mock.Alloc()
   549  	a.Job.Type = structs.JobTypeBatch
   550  	a.NodeID = c.NodeID()
   551  	a.Job.TaskGroups[0].Count = 1
   552  	a.Job.TaskGroups[0].Tasks[0] = &structs.Task{
   553  		Name:   "web",
   554  		Driver: "mock_driver",
   555  		Config: map[string]interface{}{
   556  			"run_for": "2s",
   557  		},
   558  		LogConfig: structs.DefaultLogConfig(),
   559  		Resources: &structs.Resources{
   560  			CPU:      500,
   561  			MemoryMB: 256,
   562  		},
   563  	}
   564  
   565  	testutil.WaitForResult(func() (bool, error) {
   566  		nodes := s.connectedNodes()
   567  		return len(nodes) == 1, nil
   568  	}, func(err error) {
   569  		t.Fatalf("should have a clients")
   570  	})
   571  
   572  	// Upsert the allocation
   573  	state := s.State()
   574  	require.Nil(state.UpsertJob(999, a.Job))
   575  	require.Nil(state.UpsertAllocs(1003, []*structs.Allocation{a}))
   576  
   577  	// Wait for the client to run the allocation
   578  	testutil.WaitForResult(func() (bool, error) {
   579  		alloc, err := state.AllocByID(nil, a.ID)
   580  		if err != nil {
   581  			return false, err
   582  		}
   583  		if alloc == nil {
   584  			return false, fmt.Errorf("unknown alloc")
   585  		}
   586  		if alloc.ClientStatus != structs.AllocClientStatusComplete {
   587  			return false, fmt.Errorf("alloc client status: %v", alloc.ClientStatus)
   588  		}
   589  
   590  		return true, nil
   591  	}, func(err error) {
   592  		t.Fatalf("Alloc on node %q not finished: %v", c.NodeID(), err)
   593  	})
   594  
   595  	// Make the request without having an alloc id
   596  	req := &structs.AllocSpecificRequest{
   597  		QueryOptions: structs.QueryOptions{Region: "global"},
   598  	}
   599  
   600  	// Fetch the response
   601  	var resp cstructs.AllocStatsResponse
   602  	err := msgpackrpc.CallWithCodec(codec, "ClientAllocations.Stats", req, &resp)
   603  	require.NotNil(err)
   604  	require.Contains(err.Error(), "missing")
   605  
   606  	// Fetch the response setting the node id
   607  	req.AllocID = a.ID
   608  	var resp2 cstructs.AllocStatsResponse
   609  	err = msgpackrpc.CallWithCodec(codec, "ClientAllocations.Stats", req, &resp2)
   610  	require.Nil(err)
   611  	require.NotNil(resp2.Stats)
   612  }
   613  
   614  func TestClientAllocations_Stats_Local_ACL(t *testing.T) {
   615  	t.Parallel()
   616  	require := require.New(t)
   617  
   618  	// Start a server
   619  	s, root := TestACLServer(t, nil)
   620  	defer s.Shutdown()
   621  	codec := rpcClient(t, s)
   622  	testutil.WaitForLeader(t, s.RPC)
   623  
   624  	// Create a bad token
   625  	policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityReadFS})
   626  	tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad)
   627  
   628  	policyGood := mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})
   629  	tokenGood := mock.CreatePolicyAndToken(t, s.State(), 1009, "valid2", policyGood)
   630  
   631  	cases := []struct {
   632  		Name          string
   633  		Token         string
   634  		ExpectedError string
   635  	}{
   636  		{
   637  			Name:          "bad token",
   638  			Token:         tokenBad.SecretID,
   639  			ExpectedError: structs.ErrPermissionDenied.Error(),
   640  		},
   641  		{
   642  			Name:          "good token",
   643  			Token:         tokenGood.SecretID,
   644  			ExpectedError: structs.ErrUnknownAllocationPrefix,
   645  		},
   646  		{
   647  			Name:          "root token",
   648  			Token:         root.SecretID,
   649  			ExpectedError: structs.ErrUnknownAllocationPrefix,
   650  		},
   651  	}
   652  
   653  	for _, c := range cases {
   654  		t.Run(c.Name, func(t *testing.T) {
   655  
   656  			// Make the request without having a node-id
   657  			req := &structs.AllocSpecificRequest{
   658  				AllocID: uuid.Generate(),
   659  				QueryOptions: structs.QueryOptions{
   660  					AuthToken: c.Token,
   661  					Region:    "global",
   662  					Namespace: structs.DefaultNamespace,
   663  				},
   664  			}
   665  
   666  			// Fetch the response
   667  			var resp cstructs.AllocStatsResponse
   668  			err := msgpackrpc.CallWithCodec(codec, "ClientAllocations.Stats", req, &resp)
   669  			require.NotNil(err)
   670  			require.Contains(err.Error(), c.ExpectedError)
   671  		})
   672  	}
   673  }
   674  
   675  func TestClientAllocations_Stats_Remote(t *testing.T) {
   676  	t.Parallel()
   677  	require := require.New(t)
   678  
   679  	// Start a server and client
   680  	s1 := TestServer(t, nil)
   681  	defer s1.Shutdown()
   682  	s2 := TestServer(t, func(c *Config) {
   683  		c.DevDisableBootstrap = true
   684  	})
   685  	defer s2.Shutdown()
   686  	TestJoin(t, s1, s2)
   687  	testutil.WaitForLeader(t, s1.RPC)
   688  	testutil.WaitForLeader(t, s2.RPC)
   689  	codec := rpcClient(t, s2)
   690  
   691  	c := client.TestClient(t, func(c *config.Config) {
   692  		c.Servers = []string{s2.config.RPCAddr.String()}
   693  	})
   694  	defer c.Shutdown()
   695  
   696  	// Force an allocation onto the node
   697  	a := mock.Alloc()
   698  	a.Job.Type = structs.JobTypeBatch
   699  	a.NodeID = c.NodeID()
   700  	a.Job.TaskGroups[0].Count = 1
   701  	a.Job.TaskGroups[0].Tasks[0] = &structs.Task{
   702  		Name:   "web",
   703  		Driver: "mock_driver",
   704  		Config: map[string]interface{}{
   705  			"run_for": "2s",
   706  		},
   707  		LogConfig: structs.DefaultLogConfig(),
   708  		Resources: &structs.Resources{
   709  			CPU:      500,
   710  			MemoryMB: 256,
   711  		},
   712  	}
   713  	testutil.WaitForResult(func() (bool, error) {
   714  		nodes := s2.connectedNodes()
   715  		return len(nodes) == 1, nil
   716  	}, func(err error) {
   717  		t.Fatalf("should have a clients")
   718  	})
   719  
   720  	// Upsert the allocation
   721  	state1 := s1.State()
   722  	state2 := s2.State()
   723  	require.Nil(state1.UpsertJob(999, a.Job))
   724  	require.Nil(state1.UpsertAllocs(1003, []*structs.Allocation{a}))
   725  	require.Nil(state2.UpsertJob(999, a.Job))
   726  	require.Nil(state2.UpsertAllocs(1003, []*structs.Allocation{a}))
   727  
   728  	// Wait for the client to run the allocation
   729  	testutil.WaitForResult(func() (bool, error) {
   730  		alloc, err := state2.AllocByID(nil, a.ID)
   731  		if err != nil {
   732  			return false, err
   733  		}
   734  		if alloc == nil {
   735  			return false, fmt.Errorf("unknown alloc")
   736  		}
   737  		if alloc.ClientStatus != structs.AllocClientStatusComplete {
   738  			return false, fmt.Errorf("alloc client status: %v", alloc.ClientStatus)
   739  		}
   740  
   741  		return true, nil
   742  	}, func(err error) {
   743  		t.Fatalf("Alloc on node %q not finished: %v", c.NodeID(), err)
   744  	})
   745  
   746  	// Force remove the connection locally in case it exists
   747  	s1.nodeConnsLock.Lock()
   748  	delete(s1.nodeConns, c.NodeID())
   749  	s1.nodeConnsLock.Unlock()
   750  
   751  	// Make the request
   752  	req := &structs.AllocSpecificRequest{
   753  		AllocID:      a.ID,
   754  		QueryOptions: structs.QueryOptions{Region: "global"},
   755  	}
   756  
   757  	// Fetch the response
   758  	var resp cstructs.AllocStatsResponse
   759  	err := msgpackrpc.CallWithCodec(codec, "ClientAllocations.Stats", req, &resp)
   760  	require.Nil(err)
   761  	require.NotNil(resp.Stats)
   762  }