github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/integration/service/create_test.go (about)

     1  package service // import "github.com/demonoid81/moby/integration/service"
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"strings"
     8  	"testing"
     9  	"time"
    10  
    11  	"github.com/demonoid81/moby/api/types"
    12  	"github.com/demonoid81/moby/api/types/filters"
    13  	swarmtypes "github.com/demonoid81/moby/api/types/swarm"
    14  	"github.com/demonoid81/moby/api/types/versions"
    15  	"github.com/demonoid81/moby/client"
    16  	"github.com/demonoid81/moby/errdefs"
    17  	"github.com/demonoid81/moby/integration/internal/network"
    18  	"github.com/demonoid81/moby/integration/internal/swarm"
    19  	"github.com/demonoid81/moby/testutil/daemon"
    20  	"gotest.tools/v3/assert"
    21  	is "gotest.tools/v3/assert/cmp"
    22  	"gotest.tools/v3/poll"
    23  	"gotest.tools/v3/skip"
    24  )
    25  
    26  func TestServiceCreateInit(t *testing.T) {
    27  	defer setupTest(t)()
    28  	t.Run("daemonInitDisabled", testServiceCreateInit(false))
    29  	t.Run("daemonInitEnabled", testServiceCreateInit(true))
    30  }
    31  
    32  func testServiceCreateInit(daemonEnabled bool) func(t *testing.T) {
    33  	return func(t *testing.T) {
    34  		var ops = []daemon.Option{}
    35  
    36  		if daemonEnabled {
    37  			ops = append(ops, daemon.WithInit())
    38  		}
    39  		d := swarm.NewSwarm(t, testEnv, ops...)
    40  		defer d.Stop(t)
    41  		client := d.NewClientT(t)
    42  		defer client.Close()
    43  
    44  		booleanTrue := true
    45  		booleanFalse := false
    46  
    47  		serviceID := swarm.CreateService(t, d)
    48  		poll.WaitOn(t, swarm.RunningTasksCount(client, serviceID, 1), swarm.ServicePoll)
    49  		i := inspectServiceContainer(t, client, serviceID)
    50  		// HostConfig.Init == nil means that it delegates to daemon configuration
    51  		assert.Check(t, i.HostConfig.Init == nil)
    52  
    53  		serviceID = swarm.CreateService(t, d, swarm.ServiceWithInit(&booleanTrue))
    54  		poll.WaitOn(t, swarm.RunningTasksCount(client, serviceID, 1), swarm.ServicePoll)
    55  		i = inspectServiceContainer(t, client, serviceID)
    56  		assert.Check(t, is.Equal(true, *i.HostConfig.Init))
    57  
    58  		serviceID = swarm.CreateService(t, d, swarm.ServiceWithInit(&booleanFalse))
    59  		poll.WaitOn(t, swarm.RunningTasksCount(client, serviceID, 1), swarm.ServicePoll)
    60  		i = inspectServiceContainer(t, client, serviceID)
    61  		assert.Check(t, is.Equal(false, *i.HostConfig.Init))
    62  	}
    63  }
    64  
    65  func inspectServiceContainer(t *testing.T, client client.APIClient, serviceID string) types.ContainerJSON {
    66  	t.Helper()
    67  	filter := filters.NewArgs()
    68  	filter.Add("label", fmt.Sprintf("com.docker.swarm.service.id=%s", serviceID))
    69  	containers, err := client.ContainerList(context.Background(), types.ContainerListOptions{Filters: filter})
    70  	assert.NilError(t, err)
    71  	assert.Check(t, is.Len(containers, 1))
    72  
    73  	i, err := client.ContainerInspect(context.Background(), containers[0].ID)
    74  	assert.NilError(t, err)
    75  	return i
    76  }
    77  
    78  func TestCreateServiceMultipleTimes(t *testing.T) {
    79  	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
    80  	defer setupTest(t)()
    81  	d := swarm.NewSwarm(t, testEnv)
    82  	defer d.Stop(t)
    83  	client := d.NewClientT(t)
    84  	defer client.Close()
    85  	ctx := context.Background()
    86  
    87  	overlayName := "overlay1_" + t.Name()
    88  	overlayID := network.CreateNoError(ctx, t, client, overlayName,
    89  		network.WithCheckDuplicate(),
    90  		network.WithDriver("overlay"),
    91  	)
    92  
    93  	var instances uint64 = 4
    94  
    95  	serviceName := "TestService_" + t.Name()
    96  	serviceSpec := []swarm.ServiceSpecOpt{
    97  		swarm.ServiceWithReplicas(instances),
    98  		swarm.ServiceWithName(serviceName),
    99  		swarm.ServiceWithNetwork(overlayName),
   100  	}
   101  
   102  	serviceID := swarm.CreateService(t, d, serviceSpec...)
   103  	poll.WaitOn(t, swarm.RunningTasksCount(client, serviceID, instances), swarm.ServicePoll)
   104  
   105  	_, _, err := client.ServiceInspectWithRaw(context.Background(), serviceID, types.ServiceInspectOptions{})
   106  	assert.NilError(t, err)
   107  
   108  	err = client.ServiceRemove(context.Background(), serviceID)
   109  	assert.NilError(t, err)
   110  
   111  	poll.WaitOn(t, swarm.NoTasksForService(ctx, client, serviceID), swarm.ServicePoll)
   112  
   113  	serviceID2 := swarm.CreateService(t, d, serviceSpec...)
   114  	poll.WaitOn(t, swarm.RunningTasksCount(client, serviceID2, instances), swarm.ServicePoll)
   115  
   116  	err = client.ServiceRemove(context.Background(), serviceID2)
   117  	assert.NilError(t, err)
   118  
   119  	// we can't just wait on no tasks for the service, counter-intuitively.
   120  	// Tasks may briefly exist but not show up, if they are are in the process
   121  	// of being deallocated. To avoid this case, we should retry network remove
   122  	// a few times, to give tasks time to be deallcoated
   123  	poll.WaitOn(t, swarm.NoTasksForService(ctx, client, serviceID2), swarm.ServicePoll)
   124  
   125  	for retry := 0; retry < 5; retry++ {
   126  		err = client.NetworkRemove(context.Background(), overlayID)
   127  		// TODO(dperny): using strings.Contains for error checking is awful,
   128  		// but so is the fact that swarm functions don't return errdefs errors.
   129  		// I don't have time at this moment to fix the latter, so I guess I'll
   130  		// go with the former.
   131  		//
   132  		// The full error we're looking for is something like this:
   133  		//
   134  		// Error response from daemon: rpc error: code = FailedPrecondition desc = network %v is in use by task %v
   135  		//
   136  		// The safest way to catch this, I think, will be to match on "is in
   137  		// use by", as this is an uninterrupted string that best identifies
   138  		// this error.
   139  		if err == nil || !strings.Contains(err.Error(), "is in use by") {
   140  			// if there is no error, or the error isn't this kind of error,
   141  			// then we'll break the loop body, and either fail the test or
   142  			// continue.
   143  			break
   144  		}
   145  	}
   146  	assert.NilError(t, err)
   147  
   148  	poll.WaitOn(t, network.IsRemoved(context.Background(), client, overlayID), poll.WithTimeout(1*time.Minute), poll.WithDelay(10*time.Second))
   149  }
   150  
   151  func TestCreateServiceConflict(t *testing.T) {
   152  	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
   153  	defer setupTest(t)()
   154  	d := swarm.NewSwarm(t, testEnv)
   155  	defer d.Stop(t)
   156  	c := d.NewClientT(t)
   157  	defer c.Close()
   158  	ctx := context.Background()
   159  
   160  	serviceName := "TestService_" + t.Name()
   161  	serviceSpec := []swarm.ServiceSpecOpt{
   162  		swarm.ServiceWithName(serviceName),
   163  	}
   164  
   165  	swarm.CreateService(t, d, serviceSpec...)
   166  
   167  	spec := swarm.CreateServiceSpec(t, serviceSpec...)
   168  	_, err := c.ServiceCreate(ctx, spec, types.ServiceCreateOptions{})
   169  	assert.Check(t, errdefs.IsConflict(err))
   170  	assert.ErrorContains(t, err, "service "+serviceName+" already exists")
   171  }
   172  
   173  func TestCreateServiceMaxReplicas(t *testing.T) {
   174  	defer setupTest(t)()
   175  	d := swarm.NewSwarm(t, testEnv)
   176  	defer d.Stop(t)
   177  	client := d.NewClientT(t)
   178  	defer client.Close()
   179  
   180  	var maxReplicas uint64 = 2
   181  	serviceSpec := []swarm.ServiceSpecOpt{
   182  		swarm.ServiceWithReplicas(maxReplicas),
   183  		swarm.ServiceWithMaxReplicas(maxReplicas),
   184  	}
   185  
   186  	serviceID := swarm.CreateService(t, d, serviceSpec...)
   187  	poll.WaitOn(t, swarm.RunningTasksCount(client, serviceID, maxReplicas), swarm.ServicePoll)
   188  
   189  	_, _, err := client.ServiceInspectWithRaw(context.Background(), serviceID, types.ServiceInspectOptions{})
   190  	assert.NilError(t, err)
   191  }
   192  
   193  func TestCreateWithDuplicateNetworkNames(t *testing.T) {
   194  	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
   195  	defer setupTest(t)()
   196  	d := swarm.NewSwarm(t, testEnv)
   197  	defer d.Stop(t)
   198  	client := d.NewClientT(t)
   199  	defer client.Close()
   200  	ctx := context.Background()
   201  
   202  	name := "foo_" + t.Name()
   203  	n1 := network.CreateNoError(ctx, t, client, name, network.WithDriver("bridge"))
   204  	n2 := network.CreateNoError(ctx, t, client, name, network.WithDriver("bridge"))
   205  
   206  	// Duplicates with name but with different driver
   207  	n3 := network.CreateNoError(ctx, t, client, name, network.WithDriver("overlay"))
   208  
   209  	// Create Service with the same name
   210  	var instances uint64 = 1
   211  
   212  	serviceName := "top_" + t.Name()
   213  	serviceID := swarm.CreateService(t, d,
   214  		swarm.ServiceWithReplicas(instances),
   215  		swarm.ServiceWithName(serviceName),
   216  		swarm.ServiceWithNetwork(name),
   217  	)
   218  
   219  	poll.WaitOn(t, swarm.RunningTasksCount(client, serviceID, instances), swarm.ServicePoll)
   220  
   221  	resp, _, err := client.ServiceInspectWithRaw(ctx, serviceID, types.ServiceInspectOptions{})
   222  	assert.NilError(t, err)
   223  	assert.Check(t, is.Equal(n3, resp.Spec.TaskTemplate.Networks[0].Target))
   224  
   225  	// Remove Service, and wait for its tasks to be removed
   226  	err = client.ServiceRemove(ctx, serviceID)
   227  	assert.NilError(t, err)
   228  	poll.WaitOn(t, swarm.NoTasksForService(ctx, client, serviceID), swarm.ServicePoll)
   229  
   230  	// Remove networks
   231  	err = client.NetworkRemove(context.Background(), n3)
   232  	assert.NilError(t, err)
   233  
   234  	err = client.NetworkRemove(context.Background(), n2)
   235  	assert.NilError(t, err)
   236  
   237  	err = client.NetworkRemove(context.Background(), n1)
   238  	assert.NilError(t, err)
   239  
   240  	// Make sure networks have been destroyed.
   241  	poll.WaitOn(t, network.IsRemoved(context.Background(), client, n3), poll.WithTimeout(1*time.Minute), poll.WithDelay(10*time.Second))
   242  	poll.WaitOn(t, network.IsRemoved(context.Background(), client, n2), poll.WithTimeout(1*time.Minute), poll.WithDelay(10*time.Second))
   243  	poll.WaitOn(t, network.IsRemoved(context.Background(), client, n1), poll.WithTimeout(1*time.Minute), poll.WithDelay(10*time.Second))
   244  }
   245  
   246  func TestCreateServiceSecretFileMode(t *testing.T) {
   247  	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
   248  	defer setupTest(t)()
   249  	d := swarm.NewSwarm(t, testEnv)
   250  	defer d.Stop(t)
   251  	client := d.NewClientT(t)
   252  	defer client.Close()
   253  
   254  	ctx := context.Background()
   255  	secretName := "TestSecret_" + t.Name()
   256  	secretResp, err := client.SecretCreate(ctx, swarmtypes.SecretSpec{
   257  		Annotations: swarmtypes.Annotations{
   258  			Name: secretName,
   259  		},
   260  		Data: []byte("TESTSECRET"),
   261  	})
   262  	assert.NilError(t, err)
   263  
   264  	var instances uint64 = 1
   265  	serviceName := "TestService_" + t.Name()
   266  	serviceID := swarm.CreateService(t, d,
   267  		swarm.ServiceWithReplicas(instances),
   268  		swarm.ServiceWithName(serviceName),
   269  		swarm.ServiceWithCommand([]string{"/bin/sh", "-c", "ls -l /etc/secret || /bin/top"}),
   270  		swarm.ServiceWithSecret(&swarmtypes.SecretReference{
   271  			File: &swarmtypes.SecretReferenceFileTarget{
   272  				Name: "/etc/secret",
   273  				UID:  "0",
   274  				GID:  "0",
   275  				Mode: 0777,
   276  			},
   277  			SecretID:   secretResp.ID,
   278  			SecretName: secretName,
   279  		}),
   280  	)
   281  
   282  	poll.WaitOn(t, swarm.RunningTasksCount(client, serviceID, instances), swarm.ServicePoll)
   283  
   284  	filter := filters.NewArgs()
   285  	filter.Add("service", serviceID)
   286  	tasks, err := client.TaskList(ctx, types.TaskListOptions{
   287  		Filters: filter,
   288  	})
   289  	assert.NilError(t, err)
   290  	assert.Check(t, is.Equal(len(tasks), 1))
   291  
   292  	body, err := client.ContainerLogs(ctx, tasks[0].Status.ContainerStatus.ContainerID, types.ContainerLogsOptions{
   293  		ShowStdout: true,
   294  	})
   295  	assert.NilError(t, err)
   296  	defer body.Close()
   297  
   298  	content, err := ioutil.ReadAll(body)
   299  	assert.NilError(t, err)
   300  	assert.Check(t, is.Contains(string(content), "-rwxrwxrwx"))
   301  
   302  	err = client.ServiceRemove(ctx, serviceID)
   303  	assert.NilError(t, err)
   304  	poll.WaitOn(t, swarm.NoTasksForService(ctx, client, serviceID), swarm.ServicePoll)
   305  
   306  	err = client.SecretRemove(ctx, secretName)
   307  	assert.NilError(t, err)
   308  }
   309  
   310  func TestCreateServiceConfigFileMode(t *testing.T) {
   311  	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
   312  	defer setupTest(t)()
   313  	d := swarm.NewSwarm(t, testEnv)
   314  	defer d.Stop(t)
   315  	client := d.NewClientT(t)
   316  	defer client.Close()
   317  
   318  	ctx := context.Background()
   319  	configName := "TestConfig_" + t.Name()
   320  	configResp, err := client.ConfigCreate(ctx, swarmtypes.ConfigSpec{
   321  		Annotations: swarmtypes.Annotations{
   322  			Name: configName,
   323  		},
   324  		Data: []byte("TESTCONFIG"),
   325  	})
   326  	assert.NilError(t, err)
   327  
   328  	var instances uint64 = 1
   329  	serviceName := "TestService_" + t.Name()
   330  	serviceID := swarm.CreateService(t, d,
   331  		swarm.ServiceWithName(serviceName),
   332  		swarm.ServiceWithCommand([]string{"/bin/sh", "-c", "ls -l /etc/config || /bin/top"}),
   333  		swarm.ServiceWithReplicas(instances),
   334  		swarm.ServiceWithConfig(&swarmtypes.ConfigReference{
   335  			File: &swarmtypes.ConfigReferenceFileTarget{
   336  				Name: "/etc/config",
   337  				UID:  "0",
   338  				GID:  "0",
   339  				Mode: 0777,
   340  			},
   341  			ConfigID:   configResp.ID,
   342  			ConfigName: configName,
   343  		}),
   344  	)
   345  
   346  	poll.WaitOn(t, swarm.RunningTasksCount(client, serviceID, instances))
   347  
   348  	filter := filters.NewArgs()
   349  	filter.Add("service", serviceID)
   350  	tasks, err := client.TaskList(ctx, types.TaskListOptions{
   351  		Filters: filter,
   352  	})
   353  	assert.NilError(t, err)
   354  	assert.Check(t, is.Equal(len(tasks), 1))
   355  
   356  	body, err := client.ContainerLogs(ctx, tasks[0].Status.ContainerStatus.ContainerID, types.ContainerLogsOptions{
   357  		ShowStdout: true,
   358  	})
   359  	assert.NilError(t, err)
   360  	defer body.Close()
   361  
   362  	content, err := ioutil.ReadAll(body)
   363  	assert.NilError(t, err)
   364  	assert.Check(t, is.Contains(string(content), "-rwxrwxrwx"))
   365  
   366  	err = client.ServiceRemove(ctx, serviceID)
   367  	assert.NilError(t, err)
   368  	poll.WaitOn(t, swarm.NoTasksForService(ctx, client, serviceID))
   369  
   370  	err = client.ConfigRemove(ctx, configName)
   371  	assert.NilError(t, err)
   372  }
   373  
   374  // TestServiceCreateSysctls tests that a service created with sysctl options in
   375  // the ContainerSpec correctly applies those options.
   376  //
   377  // To test this, we're going to create a service with the sysctl option
   378  //
   379  //   {"net.ipv4.ip_nonlocal_bind": "0"}
   380  //
   381  // We'll get the service's tasks to get the container ID, and then we'll
   382  // inspect the container. If the output of the container inspect contains the
   383  // sysctl option with the correct value, we can assume that the sysctl has been
   384  // plumbed correctly.
   385  //
   386  // Next, we'll remove that service and create a new service with that option
   387  // set to 1. This means that no matter what the default is, we can be confident
   388  // that the sysctl option is applying as intended.
   389  //
   390  // Additionally, we'll do service and task inspects to verify that the inspect
   391  // output includes the desired sysctl option.
   392  //
   393  // We're using net.ipv4.ip_nonlocal_bind because it's something that I'm fairly
   394  // confident won't be modified by the container runtime, and won't blow
   395  // anything up in the test environment
   396  func TestCreateServiceSysctls(t *testing.T) {
   397  	skip.If(
   398  		t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"),
   399  		"setting service sysctls is unsupported before api v1.40",
   400  	)
   401  
   402  	defer setupTest(t)()
   403  	d := swarm.NewSwarm(t, testEnv)
   404  	defer d.Stop(t)
   405  	client := d.NewClientT(t)
   406  	defer client.Close()
   407  
   408  	ctx := context.Background()
   409  
   410  	// run thie block twice, so that no matter what the default value of
   411  	// net.ipv4.ip_nonlocal_bind is, we can verify that setting the sysctl
   412  	// options works
   413  	for _, expected := range []string{"0", "1"} {
   414  
   415  		// store the map we're going to be using everywhere.
   416  		expectedSysctls := map[string]string{"net.ipv4.ip_nonlocal_bind": expected}
   417  
   418  		// Create the service with the sysctl options
   419  		var instances uint64 = 1
   420  		serviceID := swarm.CreateService(t, d,
   421  			swarm.ServiceWithSysctls(expectedSysctls),
   422  		)
   423  
   424  		// wait for the service to converge to 1 running task as expected
   425  		poll.WaitOn(t, swarm.RunningTasksCount(client, serviceID, instances))
   426  
   427  		// we're going to check 3 things:
   428  		//
   429  		//   1. Does the container, when inspected, have the sysctl option set?
   430  		//   2. Does the task have the sysctl in the spec?
   431  		//   3. Does the service have the sysctl in the spec?
   432  		//
   433  		// if all 3 of these things are true, we know that the sysctl has been
   434  		// plumbed correctly through the engine.
   435  		//
   436  		// We don't actually have to get inside the container and check its
   437  		// logs or anything. If we see the sysctl set on the container inspect,
   438  		// we know that the sysctl is plumbed correctly. everything below that
   439  		// level has been tested elsewhere. (thanks @thaJeztah, because an
   440  		// earlier version of this test had to get container logs and was much
   441  		// more complex)
   442  
   443  		// get all of the tasks of the service, so we can get the container
   444  		filter := filters.NewArgs()
   445  		filter.Add("service", serviceID)
   446  		tasks, err := client.TaskList(ctx, types.TaskListOptions{
   447  			Filters: filter,
   448  		})
   449  		assert.NilError(t, err)
   450  		assert.Check(t, is.Equal(len(tasks), 1))
   451  
   452  		// verify that the container has the sysctl option set
   453  		ctnr, err := client.ContainerInspect(ctx, tasks[0].Status.ContainerStatus.ContainerID)
   454  		assert.NilError(t, err)
   455  		assert.DeepEqual(t, ctnr.HostConfig.Sysctls, expectedSysctls)
   456  
   457  		// verify that the task has the sysctl option set in the task object
   458  		assert.DeepEqual(t, tasks[0].Spec.ContainerSpec.Sysctls, expectedSysctls)
   459  
   460  		// verify that the service also has the sysctl set in the spec.
   461  		service, _, err := client.ServiceInspectWithRaw(ctx, serviceID, types.ServiceInspectOptions{})
   462  		assert.NilError(t, err)
   463  		assert.DeepEqual(t,
   464  			service.Spec.TaskTemplate.ContainerSpec.Sysctls, expectedSysctls,
   465  		)
   466  	}
   467  }
   468  
   469  // TestServiceCreateCapabilities tests that a service created with capabilities options in
   470  // the ContainerSpec correctly applies those options.
   471  //
   472  // To test this, we're going to create a service with the capabilities option
   473  //
   474  //   []string{"CAP_NET_RAW", "CAP_SYS_CHROOT"}
   475  //
   476  // We'll get the service's tasks to get the container ID, and then we'll
   477  // inspect the container. If the output of the container inspect contains the
   478  // capabilities option with the correct value, we can assume that the capabilities has been
   479  // plumbed correctly.
   480  func TestCreateServiceCapabilities(t *testing.T) {
   481  	skip.If(
   482  		t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.41"),
   483  		"setting service capabilities is unsupported before api v1.41",
   484  	)
   485  
   486  	defer setupTest(t)()
   487  	d := swarm.NewSwarm(t, testEnv)
   488  	defer d.Stop(t)
   489  	client := d.NewClientT(t)
   490  	defer client.Close()
   491  
   492  	ctx := context.Background()
   493  
   494  	// store the map we're going to be using everywhere.
   495  	expectedCapabilities := []string{"CAP_NET_RAW", "CAP_SYS_CHROOT"}
   496  
   497  	// Create the service with the capabilities options
   498  	var instances uint64 = 1
   499  	serviceID := swarm.CreateService(t, d,
   500  		swarm.ServiceWithCapabilities(expectedCapabilities),
   501  	)
   502  
   503  	// wait for the service to converge to 1 running task as expected
   504  	poll.WaitOn(t, swarm.RunningTasksCount(client, serviceID, instances))
   505  
   506  	// we're going to check 3 things:
   507  	//
   508  	//   1. Does the container, when inspected, have the capabilities option set?
   509  	//   2. Does the task have the capabilities in the spec?
   510  	//   3. Does the service have the capabilities in the spec?
   511  	//
   512  	// if all 3 of these things are true, we know that the capabilities has been
   513  	// plumbed correctly through the engine.
   514  	//
   515  	// We don't actually have to get inside the container and check its
   516  	// logs or anything. If we see the capabilities set on the container inspect,
   517  	// we know that the capabilities is plumbed correctly. everything below that
   518  	// level has been tested elsewhere.
   519  
   520  	// get all of the tasks of the service, so we can get the container
   521  	filter := filters.NewArgs()
   522  	filter.Add("service", serviceID)
   523  	tasks, err := client.TaskList(ctx, types.TaskListOptions{
   524  		Filters: filter,
   525  	})
   526  	assert.NilError(t, err)
   527  	assert.Check(t, is.Equal(len(tasks), 1))
   528  
   529  	// verify that the container has the capabilities option set
   530  	ctnr, err := client.ContainerInspect(ctx, tasks[0].Status.ContainerStatus.ContainerID)
   531  	assert.NilError(t, err)
   532  	assert.DeepEqual(t, ctnr.HostConfig.Capabilities, expectedCapabilities)
   533  
   534  	// verify that the task has the capabilities option set in the task object
   535  	assert.DeepEqual(t, tasks[0].Spec.ContainerSpec.Capabilities, expectedCapabilities)
   536  
   537  	// verify that the service also has the capabilities set in the spec.
   538  	service, _, err := client.ServiceInspectWithRaw(ctx, serviceID, types.ServiceInspectOptions{})
   539  	assert.NilError(t, err)
   540  	assert.DeepEqual(t,
   541  		service.Spec.TaskTemplate.ContainerSpec.Capabilities, expectedCapabilities,
   542  	)
   543  }