github.com/moby/docker@v26.1.3+incompatible/integration-cli/docker_cli_swarm_test.go (about) 1 //go:build !windows 2 3 package main 4 5 import ( 6 "bytes" 7 "context" 8 "encoding/json" 9 "encoding/pem" 10 "fmt" 11 "net/http" 12 "net/http/httptest" 13 "os" 14 "path/filepath" 15 "runtime" 16 "strings" 17 "testing" 18 "time" 19 20 "github.com/cloudflare/cfssl/helpers" 21 "github.com/docker/docker/api/types/swarm" 22 "github.com/docker/docker/integration-cli/checker" 23 "github.com/docker/docker/integration-cli/cli" 24 "github.com/docker/docker/integration-cli/daemon" 25 "github.com/docker/docker/libnetwork/driverapi" 26 "github.com/docker/docker/libnetwork/ipamapi" 27 remoteipam "github.com/docker/docker/libnetwork/ipams/remote/api" 28 "github.com/docker/docker/pkg/plugins" 29 "github.com/docker/docker/testutil" 30 testdaemon "github.com/docker/docker/testutil/daemon" 31 "github.com/moby/swarmkit/v2/ca/keyutils" 32 "github.com/vishvananda/netlink" 33 "gotest.tools/v3/assert" 34 "gotest.tools/v3/fs" 35 "gotest.tools/v3/icmd" 36 "gotest.tools/v3/poll" 37 ) 38 39 func (s *DockerSwarmSuite) TestSwarmUpdate(c *testing.T) { 40 ctx := testutil.GetContext(c) 41 d := s.AddDaemon(ctx, c, true, true) 42 43 getSpec := func() swarm.Spec { 44 sw := d.GetSwarm(c) 45 return sw.Spec 46 } 47 48 out, err := d.Cmd("swarm", "update", "--cert-expiry", "30h", "--dispatcher-heartbeat", "11s") 49 assert.NilError(c, err, out) 50 51 spec := getSpec() 52 assert.Equal(c, spec.CAConfig.NodeCertExpiry, 30*time.Hour) 53 assert.Equal(c, spec.Dispatcher.HeartbeatPeriod, 11*time.Second) 54 55 // setting anything under 30m for cert-expiry is not allowed 56 out, err = d.Cmd("swarm", "update", "--cert-expiry", "15m") 57 assert.ErrorContains(c, err, "") 58 assert.Assert(c, strings.Contains(out, "minimum certificate expiry time")) 59 spec = getSpec() 60 assert.Equal(c, spec.CAConfig.NodeCertExpiry, 30*time.Hour) 61 62 // passing an external CA (this is without starting a root rotation) does not fail 63 cli.Docker(cli.Args("swarm", "update", "--external-ca", "protocol=cfssl,url=https://something.org", 64 "--external-ca", "protocol=cfssl,url=https://somethingelse.org,cacert=fixtures/https/ca.pem"), 65 cli.Daemon(d)).Assert(c, icmd.Success) 66 67 expected, err := os.ReadFile("fixtures/https/ca.pem") 68 assert.NilError(c, err) 69 70 spec = getSpec() 71 assert.Equal(c, len(spec.CAConfig.ExternalCAs), 2) 72 assert.Equal(c, spec.CAConfig.ExternalCAs[0].CACert, "") 73 assert.Equal(c, spec.CAConfig.ExternalCAs[1].CACert, string(expected)) 74 75 // passing an invalid external CA fails 76 tempFile := fs.NewFile(c, "testfile", fs.WithContent("fakecert")) 77 defer tempFile.Remove() 78 79 result := cli.Docker(cli.Args("swarm", "update", 80 "--external-ca", fmt.Sprintf("protocol=cfssl,url=https://something.org,cacert=%s", tempFile.Path())), 81 cli.Daemon(d)) 82 result.Assert(c, icmd.Expected{ 83 ExitCode: 125, 84 Err: "must be in PEM format", 85 }) 86 } 87 88 func (s *DockerSwarmSuite) TestSwarmInit(c *testing.T) { 89 ctx := testutil.GetContext(c) 90 d := s.AddDaemon(ctx, c, false, false) 91 92 getSpec := func() swarm.Spec { 93 sw := d.GetSwarm(c) 94 return sw.Spec 95 } 96 97 // passing an invalid external CA fails 98 tempFile := fs.NewFile(c, "testfile", fs.WithContent("fakecert")) 99 defer tempFile.Remove() 100 101 result := cli.Docker(cli.Args("swarm", "init", "--cert-expiry", "30h", "--dispatcher-heartbeat", "11s", 102 "--external-ca", fmt.Sprintf("protocol=cfssl,url=https://somethingelse.org,cacert=%s", tempFile.Path())), 103 cli.Daemon(d)) 104 result.Assert(c, icmd.Expected{ 105 ExitCode: 125, 106 Err: "must be in PEM format", 107 }) 108 109 cli.Docker(cli.Args("swarm", "init", "--cert-expiry", "30h", "--dispatcher-heartbeat", "11s", 110 "--external-ca", "protocol=cfssl,url=https://something.org", 111 "--external-ca", "protocol=cfssl,url=https://somethingelse.org,cacert=fixtures/https/ca.pem"), 112 cli.Daemon(d)).Assert(c, icmd.Success) 113 114 expected, err := os.ReadFile("fixtures/https/ca.pem") 115 assert.NilError(c, err) 116 117 spec := getSpec() 118 assert.Equal(c, spec.CAConfig.NodeCertExpiry, 30*time.Hour) 119 assert.Equal(c, spec.Dispatcher.HeartbeatPeriod, 11*time.Second) 120 assert.Equal(c, len(spec.CAConfig.ExternalCAs), 2) 121 assert.Equal(c, spec.CAConfig.ExternalCAs[0].CACert, "") 122 assert.Equal(c, spec.CAConfig.ExternalCAs[1].CACert, string(expected)) 123 124 assert.Assert(c, d.SwarmLeave(ctx, c, true) == nil) 125 cli.Docker(cli.Args("swarm", "init"), cli.Daemon(d)).Assert(c, icmd.Success) 126 127 spec = getSpec() 128 assert.Equal(c, spec.CAConfig.NodeCertExpiry, 90*24*time.Hour) 129 assert.Equal(c, spec.Dispatcher.HeartbeatPeriod, 5*time.Second) 130 } 131 132 func (s *DockerSwarmSuite) TestSwarmInitIPv6(c *testing.T) { 133 testRequires(c, IPv6) 134 ctx := testutil.GetContext(c) 135 d1 := s.AddDaemon(ctx, c, false, false) 136 cli.Docker(cli.Args("swarm", "init", "--listen-add", "::1"), cli.Daemon(d1)).Assert(c, icmd.Success) 137 138 d2 := s.AddDaemon(ctx, c, false, false) 139 cli.Docker(cli.Args("swarm", "join", "::1"), cli.Daemon(d2)).Assert(c, icmd.Success) 140 141 out := cli.Docker(cli.Args("info"), cli.Daemon(d2)).Assert(c, icmd.Success).Combined() 142 assert.Assert(c, strings.Contains(out, "Swarm: active")) 143 } 144 145 func (s *DockerSwarmSuite) TestSwarmInitUnspecifiedAdvertiseAddr(c *testing.T) { 146 ctx := testutil.GetContext(c) 147 d := s.AddDaemon(ctx, c, false, false) 148 out, err := d.Cmd("swarm", "init", "--advertise-addr", "0.0.0.0") 149 assert.ErrorContains(c, err, "") 150 assert.Assert(c, strings.Contains(out, "advertise address must be a non-zero IP address")) 151 } 152 153 func (s *DockerSwarmSuite) TestSwarmIncompatibleDaemon(c *testing.T) { 154 ctx := testutil.GetContext(c) 155 // init swarm mode and stop a daemon 156 d := s.AddDaemon(ctx, c, true, true) 157 info := d.SwarmInfo(ctx, c) 158 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 159 d.Stop(c) 160 161 // start a daemon with --live-restore 162 err := d.StartWithError("--live-restore") 163 assert.ErrorContains(c, err, "") 164 content, err := d.ReadLogFile() 165 assert.NilError(c, err) 166 assert.Assert(c, strings.Contains(string(content), "--live-restore daemon configuration is incompatible with swarm mode")) 167 // restart for teardown 168 d.StartNode(c) 169 } 170 171 func (s *DockerSwarmSuite) TestSwarmServiceTemplatingHostname(c *testing.T) { 172 ctx := testutil.GetContext(c) 173 d := s.AddDaemon(ctx, c, true, true) 174 hostname, err := d.Cmd("node", "inspect", "--format", "{{.Description.Hostname}}", "self") 175 assert.Assert(c, err == nil, hostname) 176 177 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "test", "--hostname", "{{.Service.Name}}-{{.Task.Slot}}-{{.Node.Hostname}}", "busybox", "top") 178 assert.NilError(c, err, out) 179 180 // make sure task has been deployed. 181 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 182 183 containers := d.ActiveContainers(testutil.GetContext(c), c) 184 out, err = d.Cmd("inspect", "--type", "container", "--format", "{{.Config.Hostname}}", containers[0]) 185 assert.NilError(c, err, out) 186 assert.Equal(c, strings.Split(out, "\n")[0], "test-1-"+strings.Split(hostname, "\n")[0], "hostname with templating invalid") 187 } 188 189 // Test case for #24270 190 func (s *DockerSwarmSuite) TestSwarmServiceListFilter(c *testing.T) { 191 ctx := testutil.GetContext(c) 192 d := s.AddDaemon(ctx, c, true, true) 193 194 name1 := "redis-cluster-md5" 195 name2 := "redis-cluster" 196 name3 := "other-cluster" 197 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name1, "busybox", "top") 198 assert.NilError(c, err, out) 199 assert.Assert(c, strings.TrimSpace(out) != "") 200 201 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name2, "busybox", "top") 202 assert.NilError(c, err, out) 203 assert.Assert(c, strings.TrimSpace(out) != "") 204 205 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name3, "busybox", "top") 206 assert.NilError(c, err, out) 207 assert.Assert(c, strings.TrimSpace(out) != "") 208 209 filter1 := "name=redis-cluster-md5" 210 filter2 := "name=redis-cluster" 211 212 // We search checker.Contains with `name+" "` to prevent prefix only. 213 out, err = d.Cmd("service", "ls", "--filter", filter1) 214 assert.NilError(c, err, out) 215 assert.Assert(c, strings.Contains(out, name1+" "), out) 216 assert.Assert(c, !strings.Contains(out, name2+" "), out) 217 assert.Assert(c, !strings.Contains(out, name3+" "), out) 218 out, err = d.Cmd("service", "ls", "--filter", filter2) 219 assert.NilError(c, err, out) 220 assert.Assert(c, strings.Contains(out, name1+" "), out) 221 assert.Assert(c, strings.Contains(out, name2+" "), out) 222 assert.Assert(c, !strings.Contains(out, name3+" "), out) 223 out, err = d.Cmd("service", "ls") 224 assert.NilError(c, err, out) 225 assert.Assert(c, strings.Contains(out, name1+" "), out) 226 assert.Assert(c, strings.Contains(out, name2+" "), out) 227 assert.Assert(c, strings.Contains(out, name3+" "), out) 228 } 229 230 func (s *DockerSwarmSuite) TestSwarmNodeListFilter(c *testing.T) { 231 ctx := testutil.GetContext(c) 232 d := s.AddDaemon(ctx, c, true, true) 233 234 out, err := d.Cmd("node", "inspect", "--format", "{{ .Description.Hostname }}", "self") 235 assert.NilError(c, err, out) 236 assert.Assert(c, strings.TrimSpace(out) != "") 237 name := strings.TrimSpace(out) 238 239 filter := "name=" + name[:4] 240 241 out, err = d.Cmd("node", "ls", "--filter", filter) 242 assert.NilError(c, err, out) 243 assert.Assert(c, strings.Contains(out, name), out) 244 out, err = d.Cmd("node", "ls", "--filter", "name=none") 245 assert.NilError(c, err, out) 246 assert.Assert(c, !strings.Contains(out, name), out) 247 } 248 249 func (s *DockerSwarmSuite) TestSwarmNodeTaskListFilter(c *testing.T) { 250 ctx := testutil.GetContext(c) 251 d := s.AddDaemon(ctx, c, true, true) 252 253 name := "redis-cluster-md5" 254 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--replicas=3", "busybox", "top") 255 assert.NilError(c, err, out) 256 assert.Assert(c, strings.TrimSpace(out) != "") 257 258 // make sure task has been deployed. 259 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(3)), poll.WithTimeout(defaultReconciliationTimeout)) 260 261 filter := "name=redis-cluster" 262 263 out, err = d.Cmd("node", "ps", "--filter", filter, "self") 264 assert.NilError(c, err, out) 265 assert.Assert(c, strings.Contains(out, name+".1"), out) 266 assert.Assert(c, strings.Contains(out, name+".2"), out) 267 assert.Assert(c, strings.Contains(out, name+".3"), out) 268 out, err = d.Cmd("node", "ps", "--filter", "name=none", "self") 269 assert.NilError(c, err, out) 270 assert.Assert(c, !strings.Contains(out, name+".1"), out) 271 assert.Assert(c, !strings.Contains(out, name+".2"), out) 272 assert.Assert(c, !strings.Contains(out, name+".3"), out) 273 } 274 275 // Test case for #25375 276 func (s *DockerSwarmSuite) TestSwarmPublishAdd(c *testing.T) { 277 ctx := testutil.GetContext(c) 278 d := s.AddDaemon(ctx, c, true, true) 279 280 name := "top" 281 // this first command does not have to be retried because service creates 282 // don't return out of sequence errors. 283 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--label", "x=y", "busybox", "top") 284 assert.NilError(c, err, out) 285 assert.Assert(c, strings.TrimSpace(out) != "") 286 287 out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", name) 288 assert.NilError(c, err, out) 289 290 out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", name) 291 assert.NilError(c, err, out) 292 293 _, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", "--publish-add", "80:20", name) 294 assert.ErrorContains(c, err, "") 295 296 // this last command does not have to be retried because service inspect 297 // does not return out of sequence errors. 298 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.EndpointSpec.Ports }}", name) 299 assert.NilError(c, err, out) 300 assert.Equal(c, strings.TrimSpace(out), "[{ tcp 80 80 ingress}]") 301 } 302 303 func (s *DockerSwarmSuite) TestSwarmServiceWithGroup(c *testing.T) { 304 ctx := testutil.GetContext(c) 305 d := s.AddDaemon(ctx, c, true, true) 306 307 name := "top" 308 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--user", "root:root", "--group", "wheel", "--group", "audio", "--group", "staff", "--group", "777", "busybox", "top") 309 assert.NilError(c, err, out) 310 assert.Assert(c, strings.TrimSpace(out) != "") 311 312 // make sure task has been deployed. 313 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 314 315 out, err = d.Cmd("ps", "-q") 316 assert.NilError(c, err, out) 317 assert.Assert(c, strings.TrimSpace(out) != "") 318 319 container := strings.TrimSpace(out) 320 321 out, err = d.Cmd("exec", container, "id") 322 assert.NilError(c, err, out) 323 assert.Equal(c, strings.TrimSpace(out), "uid=0(root) gid=0(root) groups=0(root),10(wheel),29(audio),50(staff),777") 324 } 325 326 func (s *DockerSwarmSuite) TestSwarmContainerAutoStart(c *testing.T) { 327 ctx := testutil.GetContext(c) 328 d := s.AddDaemon(ctx, c, true, true) 329 330 out, err := d.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 331 assert.NilError(c, err, out) 332 assert.Assert(c, strings.TrimSpace(out) != "") 333 334 out, err = d.Cmd("run", "-id", "--restart=always", "--net=foo", "--name=test", "busybox", "top") 335 assert.NilError(c, err, out) 336 assert.Assert(c, strings.TrimSpace(out) != "") 337 338 out, err = d.Cmd("ps", "-q") 339 assert.NilError(c, err, out) 340 assert.Assert(c, strings.TrimSpace(out) != "") 341 342 d.RestartNode(c) 343 344 out, err = d.Cmd("ps", "-q") 345 assert.NilError(c, err, out) 346 assert.Assert(c, strings.TrimSpace(out) != "") 347 } 348 349 func (s *DockerSwarmSuite) TestSwarmContainerEndpointOptions(c *testing.T) { 350 ctx := testutil.GetContext(c) 351 d := s.AddDaemon(ctx, c, true, true) 352 353 out, err := d.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 354 assert.NilError(c, err, out) 355 assert.Assert(c, strings.TrimSpace(out) != "") 356 357 out, err = d.Cmd("run", "-d", "--net=foo", "--name=first", "--net-alias=first-alias", "busybox:glibc", "top") 358 assert.NilError(c, err, out) 359 360 out, err = d.Cmd("run", "-d", "--net=foo", "--name=second", "busybox:glibc", "top") 361 assert.NilError(c, err, out) 362 363 out, err = d.Cmd("run", "-d", "--net=foo", "--net-alias=third-alias", "busybox:glibc", "top") 364 assert.NilError(c, err, out) 365 366 // ping first container and its alias, also ping third and anonymous container by its alias 367 out, err = d.Cmd("exec", "second", "ping", "-c", "1", "first") 368 assert.NilError(c, err, out) 369 out, err = d.Cmd("exec", "second", "ping", "-c", "1", "first-alias") 370 assert.NilError(c, err, out) 371 out, err = d.Cmd("exec", "second", "ping", "-c", "1", "third-alias") 372 assert.NilError(c, err, out) 373 } 374 375 func (s *DockerSwarmSuite) TestSwarmContainerAttachByNetworkId(c *testing.T) { 376 ctx := testutil.GetContext(c) 377 d := s.AddDaemon(ctx, c, true, true) 378 379 out, err := d.Cmd("network", "create", "--attachable", "-d", "overlay", "testnet") 380 assert.NilError(c, err, out) 381 assert.Assert(c, strings.TrimSpace(out) != "") 382 networkID := strings.TrimSpace(out) 383 384 out, err = d.Cmd("run", "-d", "--net", networkID, "busybox", "top") 385 assert.NilError(c, err, out) 386 cID := strings.TrimSpace(out) 387 d.WaitRun(cID) 388 389 out, err = d.Cmd("rm", "-f", cID) 390 assert.NilError(c, err, out) 391 392 out, err = d.Cmd("network", "rm", "testnet") 393 assert.NilError(c, err, out) 394 395 checkNetwork := func(*testing.T) (interface{}, string) { 396 out, err := d.Cmd("network", "ls") 397 assert.NilError(c, err) 398 return out, "" 399 } 400 401 poll.WaitOn(c, pollCheck(c, checkNetwork, checker.Not(checker.Contains("testnet"))), poll.WithTimeout(3*time.Second)) 402 } 403 404 func (s *DockerSwarmSuite) TestOverlayAttachable(c *testing.T) { 405 ctx := testutil.GetContext(c) 406 d := s.AddDaemon(ctx, c, true, true) 407 408 out, err := d.Cmd("network", "create", "-d", "overlay", "--attachable", "ovnet") 409 assert.NilError(c, err, out) 410 411 // validate attachable 412 out, err = d.Cmd("network", "inspect", "--format", "{{json .Attachable}}", "ovnet") 413 assert.NilError(c, err, out) 414 assert.Equal(c, strings.TrimSpace(out), "true") 415 416 // validate containers can attach to this overlay network 417 out, err = d.Cmd("run", "-d", "--network", "ovnet", "--name", "c1", "busybox", "top") 418 assert.NilError(c, err, out) 419 420 // redo validation, there was a bug that the value of attachable changes after 421 // containers attach to the network 422 out, err = d.Cmd("network", "inspect", "--format", "{{json .Attachable}}", "ovnet") 423 assert.NilError(c, err, out) 424 assert.Equal(c, strings.TrimSpace(out), "true") 425 } 426 427 func (s *DockerSwarmSuite) TestOverlayAttachableOnSwarmLeave(c *testing.T) { 428 ctx := testutil.GetContext(c) 429 d := s.AddDaemon(ctx, c, true, true) 430 431 // Create an attachable swarm network 432 nwName := "attovl" 433 out, err := d.Cmd("network", "create", "-d", "overlay", "--attachable", nwName) 434 assert.NilError(c, err, out) 435 436 // Connect a container to the network 437 out, err = d.Cmd("run", "-d", "--network", nwName, "--name", "c1", "busybox", "top") 438 assert.NilError(c, err, out) 439 440 // Leave the swarm 441 assert.Assert(c, d.SwarmLeave(ctx, c, true) == nil) 442 443 // Check the container is disconnected 444 out, err = d.Cmd("inspect", "c1", "--format", "{{.NetworkSettings.Networks."+nwName+"}}") 445 assert.NilError(c, err, out) 446 assert.Equal(c, strings.TrimSpace(out), "<no value>") 447 448 // Check the network is gone 449 out, err = d.Cmd("network", "ls", "--format", "{{.Name}}") 450 assert.NilError(c, err, out) 451 assert.Assert(c, !strings.Contains(out, nwName), out) 452 } 453 454 func (s *DockerSwarmSuite) TestOverlayAttachableReleaseResourcesOnFailure(c *testing.T) { 455 ctx := testutil.GetContext(c) 456 d := s.AddDaemon(ctx, c, true, true) 457 458 // Create attachable network 459 out, err := d.Cmd("network", "create", "-d", "overlay", "--attachable", "--subnet", "10.10.9.0/24", "ovnet") 460 assert.NilError(c, err, out) 461 462 // Attach a container with specific IP 463 out, err = d.Cmd("run", "-d", "--network", "ovnet", "--name", "c1", "--ip", "10.10.9.33", "busybox", "top") 464 assert.NilError(c, err, out) 465 466 // Attempt to attach another container with same IP, must fail 467 out, err = d.Cmd("run", "-d", "--network", "ovnet", "--name", "c2", "--ip", "10.10.9.33", "busybox", "top") 468 assert.ErrorContains(c, err, "", out) 469 470 // Remove first container 471 out, err = d.Cmd("rm", "-f", "c1") 472 assert.NilError(c, err, out) 473 474 // Verify the network can be removed, no phantom network attachment task left over 475 out, err = d.Cmd("network", "rm", "ovnet") 476 assert.NilError(c, err, out) 477 } 478 479 func (s *DockerSwarmSuite) TestSwarmIngressNetwork(c *testing.T) { 480 ctx := testutil.GetContext(c) 481 d := s.AddDaemon(ctx, c, true, true) 482 483 // Ingress network can be removed 484 removeNetwork := func(name string) *icmd.Result { 485 return cli.Docker( 486 cli.Args("-H", d.Sock(), "network", "rm", name), 487 cli.WithStdin(strings.NewReader("Y"))) 488 } 489 490 result := removeNetwork("ingress") 491 result.Assert(c, icmd.Success) 492 493 // And recreated 494 out, err := d.Cmd("network", "create", "-d", "overlay", "--ingress", "new-ingress") 495 assert.NilError(c, err, out) 496 497 // But only one is allowed 498 out, err = d.Cmd("network", "create", "-d", "overlay", "--ingress", "another-ingress") 499 assert.ErrorContains(c, err, "") 500 assert.Assert(c, strings.Contains(strings.TrimSpace(out), "is already present"), out) 501 // It cannot be removed if it is being used 502 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "srv1", "-p", "9000:8000", "busybox", "top") 503 assert.NilError(c, err, out) 504 505 result = removeNetwork("new-ingress") 506 result.Assert(c, icmd.Expected{ 507 ExitCode: 1, 508 Err: "ingress network cannot be removed because service", 509 }) 510 511 // But it can be removed once no more services depend on it 512 out, err = d.Cmd("service", "update", "--detach", "--publish-rm", "9000:8000", "srv1") 513 assert.NilError(c, err, out) 514 515 result = removeNetwork("new-ingress") 516 result.Assert(c, icmd.Success) 517 518 // A service which needs the ingress network cannot be created if no ingress is present 519 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "srv2", "-p", "500:500", "busybox", "top") 520 assert.ErrorContains(c, err, "") 521 assert.Assert(c, strings.Contains(strings.TrimSpace(out), "no ingress network is present"), out) 522 // An existing service cannot be updated to use the ingress nw if the nw is not present 523 out, err = d.Cmd("service", "update", "--detach", "--publish-add", "9000:8000", "srv1") 524 assert.ErrorContains(c, err, "") 525 assert.Assert(c, strings.Contains(strings.TrimSpace(out), "no ingress network is present"), out) 526 // But services which do not need routing mesh can be created regardless 527 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "srv3", "--endpoint-mode", "dnsrr", "busybox", "top") 528 assert.NilError(c, err, out) 529 } 530 531 func (s *DockerSwarmSuite) TestSwarmCreateServiceWithNoIngressNetwork(c *testing.T) { 532 ctx := testutil.GetContext(c) 533 d := s.AddDaemon(ctx, c, true, true) 534 535 // Remove ingress network 536 result := cli.Docker( 537 cli.Args("-H", d.Sock(), "network", "rm", "ingress"), 538 cli.WithStdin(strings.NewReader("Y"))) 539 result.Assert(c, icmd.Success) 540 541 // Create a overlay network and launch a service on it 542 // Make sure nothing panics because ingress network is missing 543 out, err := d.Cmd("network", "create", "-d", "overlay", "another-network") 544 assert.NilError(c, err, out) 545 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "srv4", "--network", "another-network", "busybox", "top") 546 assert.NilError(c, err, out) 547 } 548 549 // Test case for #24108, also the case from: 550 // https://github.com/docker/docker/pull/24620#issuecomment-233715656 551 func (s *DockerSwarmSuite) TestSwarmTaskListFilter(c *testing.T) { 552 ctx := testutil.GetContext(c) 553 d := s.AddDaemon(ctx, c, true, true) 554 555 name := "redis-cluster-md5" 556 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--replicas=3", "busybox", "top") 557 assert.NilError(c, err, out) 558 assert.Assert(c, strings.TrimSpace(out) != "") 559 560 filter := "name=redis-cluster" 561 562 checkNumTasks := func(*testing.T) (interface{}, string) { 563 out, err := d.Cmd("service", "ps", "--filter", filter, name) 564 assert.NilError(c, err, out) 565 return len(strings.Split(out, "\n")) - 2, "" // includes header and nl in last line 566 } 567 568 // wait until all tasks have been created 569 poll.WaitOn(c, pollCheck(c, checkNumTasks, checker.Equals(3)), poll.WithTimeout(defaultReconciliationTimeout)) 570 571 out, err = d.Cmd("service", "ps", "--filter", filter, name) 572 assert.NilError(c, err, out) 573 assert.Assert(c, strings.Contains(out, name+".1"), out) 574 assert.Assert(c, strings.Contains(out, name+".2"), out) 575 assert.Assert(c, strings.Contains(out, name+".3"), out) 576 out, err = d.Cmd("service", "ps", "--filter", "name="+name+".1", name) 577 assert.NilError(c, err, out) 578 assert.Assert(c, strings.Contains(out, name+".1"), out) 579 assert.Assert(c, !strings.Contains(out, name+".2"), out) 580 assert.Assert(c, !strings.Contains(out, name+".3"), out) 581 out, err = d.Cmd("service", "ps", "--filter", "name=none", name) 582 assert.NilError(c, err, out) 583 assert.Assert(c, !strings.Contains(out, name+".1"), out) 584 assert.Assert(c, !strings.Contains(out, name+".2"), out) 585 assert.Assert(c, !strings.Contains(out, name+".3"), out) 586 name = "redis-cluster-sha1" 587 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--mode=global", "busybox", "top") 588 assert.NilError(c, err, out) 589 assert.Assert(c, strings.TrimSpace(out) != "") 590 591 poll.WaitOn(c, pollCheck(c, checkNumTasks, checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 592 593 filter = "name=redis-cluster" 594 out, err = d.Cmd("service", "ps", "--filter", filter, name) 595 assert.NilError(c, err, out) 596 assert.Assert(c, strings.Contains(out, name), out) 597 out, err = d.Cmd("service", "ps", "--filter", "name="+name, name) 598 assert.NilError(c, err, out) 599 assert.Assert(c, strings.Contains(out, name), out) 600 out, err = d.Cmd("service", "ps", "--filter", "name=none", name) 601 assert.NilError(c, err, out) 602 assert.Assert(c, !strings.Contains(out, name), out) 603 } 604 605 func (s *DockerSwarmSuite) TestPsListContainersFilterIsTask(c *testing.T) { 606 ctx := testutil.GetContext(c) 607 d := s.AddDaemon(ctx, c, true, true) 608 609 // Create a bare container 610 out, err := d.Cmd("run", "-d", "--name=bare-container", "busybox", "top") 611 assert.NilError(c, err, out) 612 bareID := strings.TrimSpace(out)[:12] 613 // Create a service 614 name := "busybox-top" 615 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "busybox", "top") 616 assert.NilError(c, err, out) 617 assert.Assert(c, strings.TrimSpace(out) != "") 618 619 // make sure task has been deployed. 620 poll.WaitOn(c, pollCheck(c, d.CheckServiceRunningTasks(ctx, name), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 621 622 // Filter non-tasks 623 out, err = d.Cmd("ps", "-a", "-q", "--filter=is-task=false") 624 assert.NilError(c, err, out) 625 psOut := strings.TrimSpace(out) 626 assert.Equal(c, psOut, bareID, fmt.Sprintf("Expected id %s, got %s for is-task label, output %q", bareID, psOut, out)) 627 628 // Filter tasks 629 out, err = d.Cmd("ps", "-a", "-q", "--filter=is-task=true") 630 assert.NilError(c, err, out) 631 lines := strings.Split(strings.Trim(out, "\n "), "\n") 632 assert.Equal(c, len(lines), 1) 633 assert.Assert(c, lines[0] != bareID, "Expected not %s, but got it for is-task label, output %q", bareID, out) 634 } 635 636 const ( 637 globalNetworkPlugin = "global-network-plugin" 638 globalIPAMPlugin = "global-ipam-plugin" 639 ) 640 641 func setupRemoteGlobalNetworkPlugin(c *testing.T, mux *http.ServeMux, url, netDrv, ipamDrv string) { 642 mux.HandleFunc("/Plugin.Activate", func(w http.ResponseWriter, r *http.Request) { 643 w.Header().Set("Content-Type", plugins.VersionMimetype) 644 fmt.Fprintf(w, `{"Implements": ["%s", "%s"]}`, driverapi.NetworkPluginEndpointType, ipamapi.PluginEndpointType) 645 }) 646 647 // Network driver implementation 648 mux.HandleFunc(fmt.Sprintf("/%s.GetCapabilities", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 649 w.Header().Set("Content-Type", plugins.VersionMimetype) 650 fmt.Fprintf(w, `{"Scope":"global"}`) 651 }) 652 653 mux.HandleFunc(fmt.Sprintf("/%s.AllocateNetwork", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 654 err := json.NewDecoder(r.Body).Decode(&remoteDriverNetworkRequest) 655 if err != nil { 656 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 657 return 658 } 659 w.Header().Set("Content-Type", plugins.VersionMimetype) 660 fmt.Fprintf(w, "null") 661 }) 662 663 mux.HandleFunc(fmt.Sprintf("/%s.FreeNetwork", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 664 w.Header().Set("Content-Type", plugins.VersionMimetype) 665 fmt.Fprintf(w, "null") 666 }) 667 668 mux.HandleFunc(fmt.Sprintf("/%s.CreateNetwork", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 669 err := json.NewDecoder(r.Body).Decode(&remoteDriverNetworkRequest) 670 if err != nil { 671 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 672 return 673 } 674 w.Header().Set("Content-Type", plugins.VersionMimetype) 675 fmt.Fprintf(w, "null") 676 }) 677 678 mux.HandleFunc(fmt.Sprintf("/%s.DeleteNetwork", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 679 w.Header().Set("Content-Type", plugins.VersionMimetype) 680 fmt.Fprintf(w, "null") 681 }) 682 683 mux.HandleFunc(fmt.Sprintf("/%s.CreateEndpoint", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 684 w.Header().Set("Content-Type", plugins.VersionMimetype) 685 fmt.Fprintf(w, `{"Interface":{"MacAddress":"a0:b1:c2:d3:e4:f5"}}`) 686 }) 687 688 mux.HandleFunc(fmt.Sprintf("/%s.Join", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 689 w.Header().Set("Content-Type", plugins.VersionMimetype) 690 691 veth := &netlink.Veth{ 692 LinkAttrs: netlink.LinkAttrs{Name: "randomIfName", TxQLen: 0}, PeerName: "cnt0", 693 } 694 if err := netlink.LinkAdd(veth); err != nil { 695 fmt.Fprintf(w, `{"Error":"failed to add veth pair: `+err.Error()+`"}`) 696 } else { 697 fmt.Fprintf(w, `{"InterfaceName":{ "SrcName":"cnt0", "DstPrefix":"veth"}}`) 698 } 699 }) 700 701 mux.HandleFunc(fmt.Sprintf("/%s.Leave", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 702 w.Header().Set("Content-Type", plugins.VersionMimetype) 703 fmt.Fprintf(w, "null") 704 }) 705 706 mux.HandleFunc(fmt.Sprintf("/%s.DeleteEndpoint", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 707 w.Header().Set("Content-Type", plugins.VersionMimetype) 708 if link, err := netlink.LinkByName("cnt0"); err == nil { 709 netlink.LinkDel(link) 710 } 711 fmt.Fprintf(w, "null") 712 }) 713 714 // IPAM Driver implementation 715 var ( 716 poolRequest remoteipam.RequestPoolRequest 717 poolReleaseReq remoteipam.ReleasePoolRequest 718 addressRequest remoteipam.RequestAddressRequest 719 addressReleaseReq remoteipam.ReleaseAddressRequest 720 lAS = "localAS" 721 gAS = "globalAS" 722 pool = "172.28.0.0/16" 723 poolID = lAS + "/" + pool 724 gw = "172.28.255.254/16" 725 ) 726 727 mux.HandleFunc(fmt.Sprintf("/%s.GetDefaultAddressSpaces", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 728 w.Header().Set("Content-Type", plugins.VersionMimetype) 729 fmt.Fprintf(w, `{"LocalDefaultAddressSpace":"`+lAS+`", "GlobalDefaultAddressSpace": "`+gAS+`"}`) 730 }) 731 732 mux.HandleFunc(fmt.Sprintf("/%s.RequestPool", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 733 err := json.NewDecoder(r.Body).Decode(&poolRequest) 734 if err != nil { 735 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 736 return 737 } 738 w.Header().Set("Content-Type", plugins.VersionMimetype) 739 if poolRequest.AddressSpace != lAS && poolRequest.AddressSpace != gAS { 740 fmt.Fprintf(w, `{"Error":"Unknown address space in pool request: `+poolRequest.AddressSpace+`"}`) 741 } else if poolRequest.Pool != "" && poolRequest.Pool != pool { 742 fmt.Fprintf(w, `{"Error":"Cannot handle explicit pool requests yet"}`) 743 } else { 744 fmt.Fprintf(w, `{"PoolID":"`+poolID+`", "Pool":"`+pool+`"}`) 745 } 746 }) 747 748 mux.HandleFunc(fmt.Sprintf("/%s.RequestAddress", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 749 err := json.NewDecoder(r.Body).Decode(&addressRequest) 750 if err != nil { 751 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 752 return 753 } 754 w.Header().Set("Content-Type", plugins.VersionMimetype) 755 // make sure libnetwork is now querying on the expected pool id 756 if addressRequest.PoolID != poolID { 757 fmt.Fprintf(w, `{"Error":"unknown pool id"}`) 758 } else if addressRequest.Address != "" { 759 fmt.Fprintf(w, `{"Error":"Cannot handle explicit address requests yet"}`) 760 } else { 761 fmt.Fprintf(w, `{"Address":"`+gw+`"}`) 762 } 763 }) 764 765 mux.HandleFunc(fmt.Sprintf("/%s.ReleaseAddress", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 766 err := json.NewDecoder(r.Body).Decode(&addressReleaseReq) 767 if err != nil { 768 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 769 return 770 } 771 w.Header().Set("Content-Type", plugins.VersionMimetype) 772 // make sure libnetwork is now asking to release the expected address from the expected poolid 773 if addressRequest.PoolID != poolID { 774 fmt.Fprintf(w, `{"Error":"unknown pool id"}`) 775 } else if addressReleaseReq.Address != gw { 776 fmt.Fprintf(w, `{"Error":"unknown address"}`) 777 } else { 778 fmt.Fprintf(w, "null") 779 } 780 }) 781 782 mux.HandleFunc(fmt.Sprintf("/%s.ReleasePool", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 783 err := json.NewDecoder(r.Body).Decode(&poolReleaseReq) 784 if err != nil { 785 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 786 return 787 } 788 w.Header().Set("Content-Type", plugins.VersionMimetype) 789 // make sure libnetwork is now asking to release the expected poolid 790 if addressRequest.PoolID != poolID { 791 fmt.Fprintf(w, `{"Error":"unknown pool id"}`) 792 } else { 793 fmt.Fprintf(w, "null") 794 } 795 }) 796 797 err := os.MkdirAll("/etc/docker/plugins", 0o755) 798 assert.NilError(c, err) 799 800 fileName := fmt.Sprintf("/etc/docker/plugins/%s.spec", netDrv) 801 err = os.WriteFile(fileName, []byte(url), 0o644) 802 assert.NilError(c, err) 803 804 ipamFileName := fmt.Sprintf("/etc/docker/plugins/%s.spec", ipamDrv) 805 err = os.WriteFile(ipamFileName, []byte(url), 0o644) 806 assert.NilError(c, err) 807 } 808 809 func (s *DockerSwarmSuite) TestSwarmNetworkPlugin(c *testing.T) { 810 ctx := testutil.GetContext(c) 811 mux := http.NewServeMux() 812 s.server = httptest.NewServer(mux) 813 assert.Assert(c, s.server != nil) // check that HTTP server has started 814 setupRemoteGlobalNetworkPlugin(c, mux, s.server.URL, globalNetworkPlugin, globalIPAMPlugin) 815 defer func() { 816 s.server.Close() 817 err := os.RemoveAll("/etc/docker/plugins") 818 assert.NilError(c, err) 819 }() 820 821 d := s.AddDaemon(ctx, c, true, true) 822 823 out, err := d.Cmd("network", "create", "-d", globalNetworkPlugin, "foo") 824 assert.ErrorContains(c, err, "", out) 825 assert.Assert(c, strings.Contains(out, "not supported in swarm mode"), out) 826 } 827 828 // Test case for #24712 829 func (s *DockerSwarmSuite) TestSwarmServiceEnvFile(c *testing.T) { 830 ctx := testutil.GetContext(c) 831 d := s.AddDaemon(ctx, c, true, true) 832 833 path := filepath.Join(d.Folder, "env.txt") 834 err := os.WriteFile(path, []byte("VAR1=A\nVAR2=A\n"), 0o644) 835 assert.NilError(c, err) 836 837 name := "worker" 838 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--env-file", path, "--env", "VAR1=B", "--env", "VAR1=C", "--env", "VAR2=", "--env", "VAR2", "--name", name, "busybox", "top") 839 assert.NilError(c, err, out) 840 assert.Assert(c, strings.TrimSpace(out) != "") 841 842 // The complete env is [VAR1=A VAR2=A VAR1=B VAR1=C VAR2= VAR2] and duplicates will be removed => [VAR1=C VAR2] 843 out, err = d.Cmd("inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.Env }}", name) 844 assert.NilError(c, err, out) 845 assert.Assert(c, strings.Contains(out, "[VAR1=C VAR2]"), out) 846 } 847 848 func (s *DockerSwarmSuite) TestSwarmServiceTTY(c *testing.T) { 849 ctx := testutil.GetContext(c) 850 d := s.AddDaemon(ctx, c, true, true) 851 852 name := "top" 853 854 ttyCheck := "if [ -t 0 ]; then echo TTY > /status; else echo none > /status; fi; exec top" 855 856 // Without --tty 857 expectedOutput := "none" 858 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "busybox", "sh", "-c", ttyCheck) 859 assert.NilError(c, err, out) 860 861 // Make sure task has been deployed. 862 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 863 864 // We need to get the container id. 865 out, err = d.Cmd("ps", "-q", "--no-trunc") 866 assert.NilError(c, err, out) 867 id := strings.TrimSpace(out) 868 869 out, err = d.Cmd("exec", id, "cat", "/status") 870 assert.NilError(c, err, out) 871 assert.Assert(c, strings.Contains(out, expectedOutput), "Expected '%s', but got %q", expectedOutput, out) 872 // Remove service 873 out, err = d.Cmd("service", "rm", name) 874 assert.NilError(c, err, out) 875 // Make sure container has been destroyed. 876 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(0)), poll.WithTimeout(defaultReconciliationTimeout)) 877 878 // With --tty 879 expectedOutput = "TTY" 880 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--tty", "busybox", "sh", "-c", ttyCheck) 881 assert.NilError(c, err, out) 882 883 // Make sure task has been deployed. 884 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 885 886 // We need to get the container id. 887 out, err = d.Cmd("ps", "-q", "--no-trunc") 888 assert.NilError(c, err, out) 889 id = strings.TrimSpace(out) 890 891 out, err = d.Cmd("exec", id, "cat", "/status") 892 assert.NilError(c, err, out) 893 assert.Assert(c, strings.Contains(out, expectedOutput), "Expected '%s', but got %q", expectedOutput, out) 894 } 895 896 func (s *DockerSwarmSuite) TestSwarmServiceTTYUpdate(c *testing.T) { 897 ctx := testutil.GetContext(c) 898 d := s.AddDaemon(ctx, c, true, true) 899 900 // Create a service 901 name := "top" 902 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "busybox", "top") 903 assert.NilError(c, err, out) 904 905 // Make sure task has been deployed. 906 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 907 908 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.TTY }}", name) 909 assert.NilError(c, err, out) 910 assert.Equal(c, strings.TrimSpace(out), "false") 911 912 out, err = d.Cmd("service", "update", "--detach", "--tty", name) 913 assert.NilError(c, err, out) 914 915 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.TTY }}", name) 916 assert.NilError(c, err, out) 917 assert.Equal(c, strings.TrimSpace(out), "true") 918 } 919 920 func (s *DockerSwarmSuite) TestSwarmServiceNetworkUpdate(c *testing.T) { 921 ctx := testutil.GetContext(c) 922 d := s.AddDaemon(ctx, c, true, true) 923 924 result := icmd.RunCmd(d.Command("network", "create", "-d", "overlay", "foo")) 925 result.Assert(c, icmd.Success) 926 fooNetwork := strings.TrimSpace(result.Combined()) 927 928 result = icmd.RunCmd(d.Command("network", "create", "-d", "overlay", "bar")) 929 result.Assert(c, icmd.Success) 930 barNetwork := strings.TrimSpace(result.Combined()) 931 932 result = icmd.RunCmd(d.Command("network", "create", "-d", "overlay", "baz")) 933 result.Assert(c, icmd.Success) 934 bazNetwork := strings.TrimSpace(result.Combined()) 935 936 // Create a service 937 name := "top" 938 result = icmd.RunCmd(d.Command("service", "create", "--detach", "--no-resolve-image", "--network", "foo", "--network", "bar", "--name", name, "busybox", "top")) 939 result.Assert(c, icmd.Success) 940 941 // Make sure task has been deployed. 942 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskNetworks(ctx), checker.DeepEquals(map[string]int{fooNetwork: 1, barNetwork: 1})), poll.WithTimeout(defaultReconciliationTimeout)) 943 944 // Remove a network 945 result = icmd.RunCmd(d.Command("service", "update", "--detach", "--network-rm", "foo", name)) 946 result.Assert(c, icmd.Success) 947 948 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskNetworks(ctx), checker.DeepEquals(map[string]int{barNetwork: 1})), poll.WithTimeout(defaultReconciliationTimeout)) 949 950 // Add a network 951 result = icmd.RunCmd(d.Command("service", "update", "--detach", "--network-add", "baz", name)) 952 result.Assert(c, icmd.Success) 953 954 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskNetworks(ctx), checker.DeepEquals(map[string]int{barNetwork: 1, bazNetwork: 1})), poll.WithTimeout(defaultReconciliationTimeout)) 955 } 956 957 func (s *DockerSwarmSuite) TestDNSConfig(c *testing.T) { 958 ctx := testutil.GetContext(c) 959 d := s.AddDaemon(ctx, c, true, true) 960 961 // Create a service 962 name := "top" 963 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--dns=1.2.3.4", "--dns-search=example.com", "--dns-option=timeout:3", "busybox", "top") 964 assert.NilError(c, err, out) 965 966 // Make sure task has been deployed. 967 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 968 969 // We need to get the container id. 970 out, err = d.Cmd("ps", "-a", "-q", "--no-trunc") 971 assert.NilError(c, err, out) 972 id := strings.TrimSpace(out) 973 974 // Compare against expected output. 975 expectedOutput1 := "nameserver 1.2.3.4" 976 expectedOutput2 := "search example.com" 977 expectedOutput3 := "options timeout:3" 978 out, err = d.Cmd("exec", id, "cat", "/etc/resolv.conf") 979 assert.NilError(c, err, out) 980 assert.Assert(c, strings.Contains(out, expectedOutput1), "Expected '%s', but got %q", expectedOutput1, out) 981 assert.Assert(c, strings.Contains(out, expectedOutput2), "Expected '%s', but got %q", expectedOutput2, out) 982 assert.Assert(c, strings.Contains(out, expectedOutput3), "Expected '%s', but got %q", expectedOutput3, out) 983 } 984 985 func (s *DockerSwarmSuite) TestDNSConfigUpdate(c *testing.T) { 986 ctx := testutil.GetContext(c) 987 d := s.AddDaemon(ctx, c, true, true) 988 989 // Create a service 990 name := "top" 991 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "busybox", "top") 992 assert.NilError(c, err, out) 993 994 // Make sure task has been deployed. 995 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 996 997 out, err = d.Cmd("service", "update", "--detach", "--dns-add=1.2.3.4", "--dns-search-add=example.com", "--dns-option-add=timeout:3", name) 998 assert.NilError(c, err, out) 999 1000 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.DNSConfig }}", name) 1001 assert.NilError(c, err, out) 1002 assert.Equal(c, strings.TrimSpace(out), "{[1.2.3.4] [example.com] [timeout:3]}") 1003 } 1004 1005 func getNodeStatus(c *testing.T, d *daemon.Daemon) swarm.LocalNodeState { 1006 ctx := testutil.GetContext(c) 1007 info := d.SwarmInfo(ctx, c) 1008 return info.LocalNodeState 1009 } 1010 1011 func checkKeyIsEncrypted(d *daemon.Daemon) func(*testing.T) (interface{}, string) { 1012 return func(c *testing.T) (interface{}, string) { 1013 keyBytes, err := os.ReadFile(filepath.Join(d.Folder, "root", "swarm", "certificates", "swarm-node.key")) 1014 if err != nil { 1015 return fmt.Errorf("error reading key: %v", err), "" 1016 } 1017 1018 keyBlock, _ := pem.Decode(keyBytes) 1019 if keyBlock == nil { 1020 return fmt.Errorf("invalid PEM-encoded private key"), "" 1021 } 1022 1023 return keyutils.IsEncryptedPEMBlock(keyBlock), "" 1024 } 1025 } 1026 1027 func checkSwarmLockedToUnlocked(ctx context.Context, c *testing.T, d *daemon.Daemon) { 1028 // Wait for the PEM file to become unencrypted 1029 poll.WaitOn(c, pollCheck(c, checkKeyIsEncrypted(d), checker.Equals(false)), poll.WithTimeout(defaultReconciliationTimeout)) 1030 1031 d.RestartNode(c) 1032 poll.WaitOn(c, pollCheck(c, d.CheckLocalNodeState(ctx), checker.Equals(swarm.LocalNodeStateActive)), poll.WithTimeout(time.Second)) 1033 } 1034 1035 func checkSwarmUnlockedToLocked(ctx context.Context, c *testing.T, d *daemon.Daemon) { 1036 // Wait for the PEM file to become encrypted 1037 poll.WaitOn(c, pollCheck(c, checkKeyIsEncrypted(d), checker.Equals(true)), poll.WithTimeout(defaultReconciliationTimeout)) 1038 1039 d.RestartNode(c) 1040 poll.WaitOn(c, pollCheck(c, d.CheckLocalNodeState(ctx), checker.Equals(swarm.LocalNodeStateLocked)), poll.WithTimeout(time.Second)) 1041 } 1042 1043 func (s *DockerSwarmSuite) TestUnlockEngineAndUnlockedSwarm(c *testing.T) { 1044 ctx := testutil.GetContext(c) 1045 d := s.AddDaemon(ctx, c, false, false) 1046 1047 // unlocking a normal engine should return an error - it does not even ask for the key 1048 cmd := d.Command("swarm", "unlock") 1049 result := icmd.RunCmd(cmd) 1050 result.Assert(c, icmd.Expected{ 1051 ExitCode: 1, 1052 }) 1053 out := result.Combined() 1054 assert.Assert(c, strings.Contains(result.Combined(), "Error: This node is not part of a swarm"), out) 1055 assert.Assert(c, !strings.Contains(result.Combined(), "Please enter unlock key"), out) 1056 out, err := d.Cmd("swarm", "init") 1057 assert.NilError(c, err, out) 1058 1059 // unlocking an unlocked swarm should return an error - it does not even ask for the key 1060 cmd = d.Command("swarm", "unlock") 1061 result = icmd.RunCmd(cmd) 1062 result.Assert(c, icmd.Expected{ 1063 ExitCode: 1, 1064 }) 1065 out = result.Combined() 1066 assert.Assert(c, strings.Contains(result.Combined(), "Error: swarm is not locked"), out) 1067 assert.Assert(c, !strings.Contains(result.Combined(), "Please enter unlock key"), out) 1068 } 1069 1070 func (s *DockerSwarmSuite) TestSwarmInitLocked(c *testing.T) { 1071 ctx := testutil.GetContext(c) 1072 d := s.AddDaemon(ctx, c, false, false) 1073 1074 outs, err := d.Cmd("swarm", "init", "--autolock") 1075 assert.Assert(c, err == nil, outs) 1076 unlockKey := getUnlockKey(d, c, outs) 1077 1078 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1079 1080 // It starts off locked 1081 d.RestartNode(c) 1082 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateLocked) 1083 1084 cmd := d.Command("swarm", "unlock") 1085 cmd.Stdin = bytes.NewBufferString("wrong-secret-key") 1086 icmd.RunCmd(cmd).Assert(c, icmd.Expected{ 1087 ExitCode: 1, 1088 Err: "invalid key", 1089 }) 1090 1091 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateLocked) 1092 1093 cmd = d.Command("swarm", "unlock") 1094 cmd.Stdin = bytes.NewBufferString(unlockKey) 1095 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1096 1097 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1098 1099 outs, err = d.Cmd("node", "ls") 1100 assert.Assert(c, err == nil, outs) 1101 assert.Assert(c, !strings.Contains(outs, "Swarm is encrypted and needs to be unlocked"), outs) 1102 outs, err = d.Cmd("swarm", "update", "--autolock=false") 1103 assert.Assert(c, err == nil, outs) 1104 1105 checkSwarmLockedToUnlocked(ctx, c, d) 1106 1107 outs, err = d.Cmd("node", "ls") 1108 assert.Assert(c, err == nil, outs) 1109 assert.Assert(c, !strings.Contains(outs, "Swarm is encrypted and needs to be unlocked"), outs) 1110 } 1111 1112 func (s *DockerSwarmSuite) TestSwarmLeaveLocked(c *testing.T) { 1113 ctx := testutil.GetContext(c) 1114 d := s.AddDaemon(ctx, c, false, false) 1115 1116 outs, err := d.Cmd("swarm", "init", "--autolock") 1117 assert.Assert(c, err == nil, outs) 1118 1119 // It starts off locked 1120 d.RestartNode(c) 1121 1122 info := d.SwarmInfo(ctx, c) 1123 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateLocked) 1124 1125 outs, _ = d.Cmd("node", "ls") 1126 assert.Assert(c, strings.Contains(outs, "Swarm is encrypted and needs to be unlocked"), outs) 1127 // `docker swarm leave` a locked swarm without --force will return an error 1128 outs, _ = d.Cmd("swarm", "leave") 1129 assert.Assert(c, strings.Contains(outs, "Swarm is encrypted and locked."), outs) 1130 // It is OK for user to leave a locked swarm with --force 1131 outs, err = d.Cmd("swarm", "leave", "--force") 1132 assert.Assert(c, err == nil, outs) 1133 1134 info = d.SwarmInfo(ctx, c) 1135 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive) 1136 1137 outs, err = d.Cmd("swarm", "init") 1138 assert.Assert(c, err == nil, outs) 1139 1140 info = d.SwarmInfo(ctx, c) 1141 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 1142 } 1143 1144 func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *testing.T) { 1145 ctx := testutil.GetContext(c) 1146 d1 := s.AddDaemon(ctx, c, true, true) 1147 d2 := s.AddDaemon(ctx, c, true, true) 1148 d3 := s.AddDaemon(ctx, c, true, true) 1149 1150 // they start off unlocked 1151 d2.RestartNode(c) 1152 assert.Equal(c, getNodeStatus(c, d2), swarm.LocalNodeStateActive) 1153 1154 // stop this one so it does not get autolock info 1155 d2.Stop(c) 1156 1157 // enable autolock 1158 outs, err := d1.Cmd("swarm", "update", "--autolock") 1159 assert.Assert(c, err == nil, outs) 1160 unlockKey := getUnlockKey(d1, c, outs) 1161 1162 // The ones that got the cluster update should be set to locked 1163 for _, d := range []*daemon.Daemon{d1, d3} { 1164 checkSwarmUnlockedToLocked(ctx, c, d) 1165 1166 cmd := d.Command("swarm", "unlock") 1167 cmd.Stdin = bytes.NewBufferString(unlockKey) 1168 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1169 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1170 } 1171 1172 // d2 never got the cluster update, so it is still set to unlocked 1173 d2.StartNode(c) 1174 assert.Equal(c, getNodeStatus(c, d2), swarm.LocalNodeStateActive) 1175 1176 // d2 is now set to lock 1177 checkSwarmUnlockedToLocked(ctx, c, d2) 1178 1179 // leave it locked, and set the cluster to no longer autolock 1180 outs, err = d1.Cmd("swarm", "update", "--autolock=false") 1181 assert.Assert(c, err == nil, "out: %v", outs) 1182 1183 // the ones that got the update are now set to unlocked 1184 for _, d := range []*daemon.Daemon{d1, d3} { 1185 checkSwarmLockedToUnlocked(ctx, c, d) 1186 } 1187 1188 // d2 still locked 1189 assert.Equal(c, getNodeStatus(c, d2), swarm.LocalNodeStateLocked) 1190 1191 // unlock it 1192 cmd := d2.Command("swarm", "unlock") 1193 cmd.Stdin = bytes.NewBufferString(unlockKey) 1194 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1195 assert.Equal(c, getNodeStatus(c, d2), swarm.LocalNodeStateActive) 1196 1197 // once it's caught up, d2 is set to not be locked 1198 checkSwarmLockedToUnlocked(ctx, c, d2) 1199 1200 // managers who join now are never set to locked in the first place 1201 d4 := s.AddDaemon(ctx, c, true, true) 1202 d4.RestartNode(c) 1203 assert.Equal(c, getNodeStatus(c, d4), swarm.LocalNodeStateActive) 1204 } 1205 1206 func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *testing.T) { 1207 ctx := testutil.GetContext(c) 1208 d1 := s.AddDaemon(ctx, c, true, true) 1209 1210 // enable autolock 1211 outs, err := d1.Cmd("swarm", "update", "--autolock") 1212 assert.Assert(c, err == nil, "out: %v", outs) 1213 unlockKey := getUnlockKey(d1, c, outs) 1214 1215 // joined workers start off unlocked 1216 d2 := s.AddDaemon(ctx, c, true, false) 1217 d2.RestartNode(c) 1218 poll.WaitOn(c, pollCheck(c, d2.CheckLocalNodeState(ctx), checker.Equals(swarm.LocalNodeStateActive)), poll.WithTimeout(time.Second)) 1219 1220 // promote worker 1221 outs, err = d1.Cmd("node", "promote", d2.NodeID()) 1222 assert.NilError(c, err) 1223 assert.Assert(c, strings.Contains(outs, "promoted to a manager in the swarm"), outs) 1224 // join new manager node 1225 d3 := s.AddDaemon(ctx, c, true, true) 1226 1227 // both new nodes are locked 1228 for _, d := range []*daemon.Daemon{d2, d3} { 1229 checkSwarmUnlockedToLocked(ctx, c, d) 1230 1231 cmd := d.Command("swarm", "unlock") 1232 cmd.Stdin = bytes.NewBufferString(unlockKey) 1233 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1234 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1235 } 1236 1237 // demote manager back to worker - workers are not locked 1238 outs, err = d1.Cmd("node", "demote", d3.NodeID()) 1239 assert.NilError(c, err) 1240 assert.Assert(c, strings.Contains(outs, "demoted in the swarm"), outs) 1241 // Wait for it to actually be demoted, for the key and cert to be replaced. 1242 // Then restart and assert that the node is not locked. If we don't wait for the cert 1243 // to be replaced, then the node still has the manager TLS key which is still locked 1244 // (because we never want a manager TLS key to be on disk unencrypted if the cluster 1245 // is set to autolock) 1246 poll.WaitOn(c, pollCheck(c, d3.CheckControlAvailable(ctx), checker.False()), poll.WithTimeout(defaultReconciliationTimeout)) 1247 poll.WaitOn(c, pollCheck(c, func(c *testing.T) (interface{}, string) { 1248 certBytes, err := os.ReadFile(filepath.Join(d3.Folder, "root", "swarm", "certificates", "swarm-node.crt")) 1249 if err != nil { 1250 return "", fmt.Sprintf("error: %v", err) 1251 } 1252 certs, err := helpers.ParseCertificatesPEM(certBytes) 1253 if err == nil && len(certs) > 0 && len(certs[0].Subject.OrganizationalUnit) > 0 { 1254 return certs[0].Subject.OrganizationalUnit[0], "" 1255 } 1256 return "", "could not get organizational unit from certificate" 1257 }, checker.Equals("swarm-worker")), poll.WithTimeout(defaultReconciliationTimeout)) 1258 1259 // by now, it should *never* be locked on restart 1260 d3.RestartNode(c) 1261 poll.WaitOn(c, pollCheck(c, d3.CheckLocalNodeState(ctx), checker.Equals(swarm.LocalNodeStateActive)), poll.WithTimeout(time.Second)) 1262 } 1263 1264 const swarmIsEncryptedMsg = "Swarm is encrypted and needs to be unlocked" 1265 1266 func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *testing.T) { 1267 ctx := testutil.GetContext(c) 1268 d := s.AddDaemon(ctx, c, true, true) 1269 1270 outs, err := d.Cmd("swarm", "update", "--autolock") 1271 assert.Assert(c, err == nil, "out: %v", outs) 1272 unlockKey := getUnlockKey(d, c, outs) 1273 1274 // Rotate multiple times 1275 for i := 0; i != 3; i++ { 1276 outs, err = d.Cmd("swarm", "unlock-key", "-q", "--rotate") 1277 assert.Assert(c, err == nil, "out: %v", outs) 1278 // Strip \n 1279 newUnlockKey := outs[:len(outs)-1] 1280 assert.Assert(c, newUnlockKey != "") 1281 assert.Assert(c, newUnlockKey != unlockKey) 1282 1283 d.RestartNode(c) 1284 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateLocked) 1285 1286 unlock := func(d *daemon.Daemon, key string) *icmd.Result { 1287 cmd := d.Command("swarm", "unlock") 1288 cmd.Stdin = strings.NewReader(key) 1289 return icmd.RunCmd(cmd) 1290 } 1291 1292 outs, _ = d.Cmd("node", "ls") 1293 assert.Assert(c, strings.Contains(outs, swarmIsEncryptedMsg), outs) 1294 1295 result := unlock(d, unlockKey) 1296 if result.Error == nil { 1297 // On occasion, the daemon may not have finished 1298 // rotating the KEK before restarting. The test is 1299 // intentionally written to explore this behavior. 1300 // When this happens, unlocking with the old key will 1301 // succeed. If we wait for the rotation to happen and 1302 // restart again, the new key should be required this 1303 // time. 1304 1305 // Wait for the rotation to happen 1306 // Since there are multiple rotations, we need to wait until for the number of rotations we are currently on to be reflected in the logs 1307 // This is a little janky... its depending on specific log messages AND these are debug logs... but it is the best we can do for now. 1308 matcher := testdaemon.ScanLogsMatchCount(testdaemon.ScanLogsMatchString("successfully rotated KEK"), i+1) 1309 poll.WaitOn(c, d.PollCheckLogs(ctx, matcher), poll.WithDelay(3*time.Second), poll.WithTimeout(time.Minute)) 1310 d.Restart(c) 1311 1312 d.RestartNode(c) 1313 1314 result = unlock(d, unlockKey) 1315 } 1316 result.Assert(c, icmd.Expected{ 1317 ExitCode: 1, 1318 Err: "invalid key", 1319 }) 1320 1321 outs, _ = d.Cmd("node", "ls") 1322 assert.Assert(c, strings.Contains(outs, swarmIsEncryptedMsg), outs) 1323 unlock(d, newUnlockKey).Assert(c, icmd.Success) 1324 1325 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1326 1327 checkNodeLs := func(t poll.LogT) poll.Result { 1328 // an issue sometimes prevents leader to be available right away 1329 out, err := d.Cmd("node", "ls") 1330 if err != nil { 1331 return poll.Continue("error running node ls: %v: %s", err, out) 1332 } 1333 return poll.Success() 1334 } 1335 poll.WaitOn(c, checkNodeLs, poll.WithDelay(3*time.Second), poll.WithTimeout(time.Minute)) 1336 1337 unlockKey = newUnlockKey 1338 } 1339 } 1340 1341 // This differs from `TestSwarmRotateUnlockKey` because that one rotates a single node, which is the leader. 1342 // This one keeps the leader up, and asserts that other manager nodes in the cluster also have their unlock 1343 // key rotated. 1344 func (s *DockerSwarmSuite) TestSwarmClusterRotateUnlockKey(c *testing.T) { 1345 if runtime.GOARCH == "s390x" { 1346 c.Skip("Disabled on s390x") 1347 } 1348 if runtime.GOARCH == "ppc64le" { 1349 c.Skip("Disabled on ppc64le") 1350 } 1351 ctx := testutil.GetContext(c) 1352 1353 d1 := s.AddDaemon(ctx, c, true, true) // leader - don't restart this one, we don't want leader election delays 1354 d2 := s.AddDaemon(ctx, c, true, true) 1355 d3 := s.AddDaemon(ctx, c, true, true) 1356 1357 outs, err := d1.Cmd("swarm", "update", "--autolock") 1358 assert.Assert(c, err == nil, outs) 1359 unlockKey := getUnlockKey(d1, c, outs) 1360 1361 // Rotate multiple times 1362 for i := 0; i != 3; i++ { 1363 outs, err = d1.Cmd("swarm", "unlock-key", "-q", "--rotate") 1364 assert.Assert(c, err == nil, outs) 1365 // Strip \n 1366 newUnlockKey := outs[:len(outs)-1] 1367 assert.Assert(c, newUnlockKey != "") 1368 assert.Assert(c, newUnlockKey != unlockKey) 1369 1370 d2.RestartNode(c) 1371 d3.RestartNode(c) 1372 1373 unlock := func(d *daemon.Daemon, key string) *icmd.Result { 1374 cmd := d.Command("swarm", "unlock") 1375 cmd.Stdin = strings.NewReader(key) 1376 return icmd.RunCmd(cmd) 1377 } 1378 1379 const swarmIsEncryptedMsg = "Swarm is encrypted and needs to be unlocked" 1380 1381 for _, d := range []*daemon.Daemon{d2, d3} { 1382 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateLocked) 1383 1384 outs, _ := d.Cmd("node", "ls") 1385 assert.Assert(c, strings.Contains(outs, swarmIsEncryptedMsg), outs) 1386 1387 // unlock with the original key should fail 1388 // Use poll here because the daemon may not have finished 1389 result := unlock(d, unlockKey) 1390 if result.Error == nil { 1391 // On occasion, the daemon may not have finished 1392 // rotating the KEK before restarting. The test is 1393 // intentionally written to explore this behavior. 1394 // When this happens, unlocking with the old key will 1395 // succeed. If we wait for the rotation to happen and 1396 // restart again, the new key should be required this 1397 // time. 1398 1399 // Wait for the rotation to happen 1400 // Since there are multiple rotations, we need to wait until for the number of rotations we are currently on to be reflected in the logs 1401 // This is a little janky... its depending on specific log messages AND these are debug logs... but it is the best we can do for now. 1402 matcher := testdaemon.ScanLogsMatchCount(testdaemon.ScanLogsMatchString("successfully rotated KEK"), i+1) 1403 poll.WaitOn(c, d.PollCheckLogs(ctx, matcher), poll.WithDelay(3*time.Second), poll.WithTimeout(time.Minute)) 1404 d.Restart(c) 1405 1406 result = unlock(d, unlockKey) 1407 } 1408 result.Assert(c, icmd.Expected{ 1409 ExitCode: 1, 1410 Err: "invalid key", 1411 }) 1412 1413 outs, _ = d.Cmd("node", "ls") 1414 assert.Assert(c, strings.Contains(outs, swarmIsEncryptedMsg), outs) 1415 1416 // now unlock with the rotated key, this should succeed 1417 unlock(d, newUnlockKey).Assert(c, icmd.Success) 1418 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1419 1420 checkNodeLs := func(t poll.LogT) poll.Result { 1421 // an issue sometimes prevents leader to be available right away 1422 out, err := d.Cmd("node", "ls") 1423 if err != nil { 1424 return poll.Continue("error running node ls: %v: %s", err, out) 1425 } 1426 return poll.Success() 1427 } 1428 poll.WaitOn(c, checkNodeLs, poll.WithDelay(3*time.Second), poll.WithTimeout(time.Minute)) 1429 } 1430 1431 unlockKey = newUnlockKey 1432 } 1433 } 1434 1435 func (s *DockerSwarmSuite) TestSwarmAlternateLockUnlock(c *testing.T) { 1436 ctx := testutil.GetContext(c) 1437 d := s.AddDaemon(ctx, c, true, true) 1438 1439 for i := 0; i < 2; i++ { 1440 // set to lock 1441 outs, err := d.Cmd("swarm", "update", "--autolock") 1442 assert.Assert(c, err == nil, "out: %v", outs) 1443 assert.Assert(c, strings.Contains(outs, "docker swarm unlock"), outs) 1444 unlockKey := getUnlockKey(d, c, outs) 1445 1446 checkSwarmUnlockedToLocked(ctx, c, d) 1447 1448 cmd := d.Command("swarm", "unlock") 1449 cmd.Stdin = bytes.NewBufferString(unlockKey) 1450 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1451 1452 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1453 1454 outs, err = d.Cmd("swarm", "update", "--autolock=false") 1455 assert.Assert(c, err == nil, "out: %v", outs) 1456 1457 checkSwarmLockedToUnlocked(ctx, c, d) 1458 } 1459 } 1460 1461 func (s *DockerSwarmSuite) TestExtraHosts(c *testing.T) { 1462 ctx := testutil.GetContext(c) 1463 d := s.AddDaemon(ctx, c, true, true) 1464 1465 // Create a service 1466 name := "top" 1467 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--host=example.com:1.2.3.4", "busybox", "top") 1468 assert.NilError(c, err, out) 1469 1470 // Make sure task has been deployed. 1471 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 1472 1473 // We need to get the container id. 1474 out, err = d.Cmd("ps", "-a", "-q", "--no-trunc") 1475 assert.NilError(c, err, out) 1476 id := strings.TrimSpace(out) 1477 1478 // Compare against expected output. 1479 expectedOutput := "1.2.3.4\texample.com" 1480 out, err = d.Cmd("exec", id, "cat", "/etc/hosts") 1481 assert.NilError(c, err, out) 1482 assert.Assert(c, strings.Contains(out, expectedOutput), "Expected '%s', but got %q", expectedOutput, out) 1483 } 1484 1485 func (s *DockerSwarmSuite) TestSwarmManagerAddress(c *testing.T) { 1486 ctx := testutil.GetContext(c) 1487 d1 := s.AddDaemon(ctx, c, true, true) 1488 d2 := s.AddDaemon(ctx, c, true, false) 1489 d3 := s.AddDaemon(ctx, c, true, false) 1490 1491 // Manager Addresses will always show Node 1's address 1492 expectedOutput := fmt.Sprintf("127.0.0.1:%d", d1.SwarmPort) 1493 1494 out, err := d1.Cmd("info", "--format", "{{ (index .Swarm.RemoteManagers 0).Addr }}") 1495 assert.NilError(c, err, out) 1496 assert.Assert(c, strings.Contains(out, expectedOutput), out) 1497 1498 out, err = d2.Cmd("info", "--format", "{{ (index .Swarm.RemoteManagers 0).Addr }}") 1499 assert.NilError(c, err, out) 1500 assert.Assert(c, strings.Contains(out, expectedOutput), out) 1501 1502 out, err = d3.Cmd("info", "--format", "{{ (index .Swarm.RemoteManagers 0).Addr }}") 1503 assert.NilError(c, err, out) 1504 assert.Assert(c, strings.Contains(out, expectedOutput), out) 1505 } 1506 1507 func (s *DockerSwarmSuite) TestSwarmNetworkIPAMOptions(c *testing.T) { 1508 ctx := testutil.GetContext(c) 1509 d := s.AddDaemon(ctx, c, true, true) 1510 1511 out, err := d.Cmd("network", "create", "-d", "overlay", "--ipam-opt", "foo=bar", "foo") 1512 assert.NilError(c, err, out) 1513 assert.Assert(c, strings.TrimSpace(out) != "") 1514 1515 out, err = d.Cmd("network", "inspect", "--format", "{{.IPAM.Options}}", "foo") 1516 out = strings.TrimSpace(out) 1517 assert.NilError(c, err, out) 1518 assert.Assert(c, strings.Contains(out, "foo:bar"), out) 1519 assert.Assert(c, strings.Contains(out, "com.docker.network.ipam.serial:true"), out) 1520 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--network=foo", "--name", "top", "busybox", "top") 1521 assert.NilError(c, err, out) 1522 1523 // make sure task has been deployed. 1524 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 1525 1526 out, err = d.Cmd("network", "inspect", "--format", "{{.IPAM.Options}}", "foo") 1527 assert.NilError(c, err, out) 1528 assert.Assert(c, strings.Contains(out, "foo:bar"), out) 1529 assert.Assert(c, strings.Contains(out, "com.docker.network.ipam.serial:true"), out) 1530 } 1531 1532 // Test case for issue #27866, which did not allow NW name that is the prefix of a swarm NW ID. 1533 // e.g. if the ingress ID starts with "n1", it was impossible to create a NW named "n1". 1534 func (s *DockerSwarmSuite) TestSwarmNetworkCreateIssue27866(c *testing.T) { 1535 ctx := testutil.GetContext(c) 1536 d := s.AddDaemon(ctx, c, true, true) 1537 out, err := d.Cmd("network", "inspect", "-f", "{{.Id}}", "ingress") 1538 assert.NilError(c, err, "out: %v", out) 1539 ingressID := strings.TrimSpace(out) 1540 assert.Assert(c, ingressID != "") 1541 1542 // create a network of which name is the prefix of the ID of an overlay network 1543 // (ingressID in this case) 1544 newNetName := ingressID[0:2] 1545 out, err = d.Cmd("network", "create", "--driver", "overlay", newNetName) 1546 // In #27866, it was failing because of "network with name %s already exists" 1547 assert.NilError(c, err, "out: %v", out) 1548 out, err = d.Cmd("network", "rm", newNetName) 1549 assert.NilError(c, err, "out: %v", out) 1550 } 1551 1552 // Test case for https://github.com/docker/docker/pull/27938#issuecomment-265768303 1553 // This test creates two networks with the same name sequentially, with various drivers. 1554 // Since the operations in this test are done sequentially, the 2nd call should fail with 1555 // "network with name FOO already exists". 1556 // Note that it is to ok have multiple networks with the same name if the operations are done 1557 // in parallel. (#18864) 1558 func (s *DockerSwarmSuite) TestSwarmNetworkCreateDup(c *testing.T) { 1559 ctx := testutil.GetContext(c) 1560 d := s.AddDaemon(ctx, c, true, true) 1561 drivers := []string{"bridge", "overlay"} 1562 for i, driver1 := range drivers { 1563 for _, driver2 := range drivers { 1564 c.Run(fmt.Sprintf("driver %s then %s", driver1, driver2), func(c *testing.T) { 1565 nwName := fmt.Sprintf("network-test-%d", i) 1566 out, err := d.Cmd("network", "create", "--driver", driver1, nwName) 1567 assert.NilError(c, err, "out: %v", out) 1568 out, err = d.Cmd("network", "create", "--driver", driver2, nwName) 1569 assert.Assert(c, strings.Contains(out, fmt.Sprintf("network with name %s already exists", nwName)), out) 1570 assert.ErrorContains(c, err, "") 1571 out, err = d.Cmd("network", "rm", nwName) 1572 assert.NilError(c, err, "out: %v", out) 1573 }) 1574 } 1575 } 1576 } 1577 1578 func (s *DockerSwarmSuite) TestSwarmPublishDuplicatePorts(c *testing.T) { 1579 ctx := testutil.GetContext(c) 1580 d := s.AddDaemon(ctx, c, true, true) 1581 1582 out, err := d.Cmd("service", "create", "--no-resolve-image", "--detach=true", "--publish", "5005:80", "--publish", "5006:80", "--publish", "80", "--publish", "80", "busybox", "top") 1583 assert.NilError(c, err, out) 1584 id := strings.TrimSpace(out) 1585 1586 // make sure task has been deployed. 1587 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 1588 1589 // Total len = 4, with 2 dynamic ports and 2 non-dynamic ports 1590 // Dynamic ports are likely to be 30000 and 30001 but doesn't matter 1591 out, err = d.Cmd("service", "inspect", "--format", "{{.Endpoint.Ports}} len={{len .Endpoint.Ports}}", id) 1592 assert.NilError(c, err, out) 1593 assert.Assert(c, strings.Contains(out, "len=4"), out) 1594 assert.Assert(c, strings.Contains(out, "{ tcp 80 5005 ingress}"), out) 1595 assert.Assert(c, strings.Contains(out, "{ tcp 80 5006 ingress}"), out) 1596 } 1597 1598 func (s *DockerSwarmSuite) TestSwarmJoinWithDrain(c *testing.T) { 1599 ctx := testutil.GetContext(c) 1600 d := s.AddDaemon(ctx, c, true, true) 1601 1602 out, err := d.Cmd("node", "ls") 1603 assert.NilError(c, err) 1604 assert.Assert(c, !strings.Contains(out, "Drain"), out) 1605 out, err = d.Cmd("swarm", "join-token", "-q", "manager") 1606 assert.NilError(c, err) 1607 assert.Assert(c, strings.TrimSpace(out) != "") 1608 1609 token := strings.TrimSpace(out) 1610 1611 d1 := s.AddDaemon(ctx, c, false, false) 1612 1613 out, err = d1.Cmd("swarm", "join", "--availability=drain", "--token", token, d.SwarmListenAddr()) 1614 assert.NilError(c, err) 1615 assert.Assert(c, strings.TrimSpace(out) != "") 1616 1617 out, err = d.Cmd("node", "ls") 1618 assert.NilError(c, err) 1619 assert.Assert(c, strings.Contains(out, "Drain"), out) 1620 out, err = d1.Cmd("node", "ls") 1621 assert.NilError(c, err) 1622 assert.Assert(c, strings.Contains(out, "Drain"), out) 1623 } 1624 1625 func (s *DockerSwarmSuite) TestSwarmInitWithDrain(c *testing.T) { 1626 ctx := testutil.GetContext(c) 1627 d := s.AddDaemon(ctx, c, false, false) 1628 1629 out, err := d.Cmd("swarm", "init", "--availability", "drain") 1630 assert.NilError(c, err, "out: %v", out) 1631 1632 out, err = d.Cmd("node", "ls") 1633 assert.NilError(c, err) 1634 assert.Assert(c, strings.Contains(out, "Drain")) 1635 } 1636 1637 func (s *DockerSwarmSuite) TestSwarmReadonlyRootfs(c *testing.T) { 1638 testRequires(c, DaemonIsLinux, UserNamespaceROMount) 1639 ctx := testutil.GetContext(c) 1640 1641 d := s.AddDaemon(ctx, c, true, true) 1642 1643 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "top", "--read-only", "busybox", "top") 1644 assert.NilError(c, err, out) 1645 1646 // make sure task has been deployed. 1647 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 1648 1649 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.ReadOnly }}", "top") 1650 assert.NilError(c, err, out) 1651 assert.Equal(c, strings.TrimSpace(out), "true") 1652 1653 containers := d.ActiveContainers(testutil.GetContext(c), c) 1654 out, err = d.Cmd("inspect", "--type", "container", "--format", "{{.HostConfig.ReadonlyRootfs}}", containers[0]) 1655 assert.NilError(c, err, out) 1656 assert.Equal(c, strings.TrimSpace(out), "true") 1657 } 1658 1659 func (s *DockerSwarmSuite) TestSwarmStopSignal(c *testing.T) { 1660 ctx := testutil.GetContext(c) 1661 testRequires(c, DaemonIsLinux, UserNamespaceROMount) 1662 1663 d := s.AddDaemon(ctx, c, true, true) 1664 1665 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "top", "--stop-signal=SIGHUP", "busybox", "top") 1666 assert.NilError(c, err, out) 1667 1668 // make sure task has been deployed. 1669 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 1670 1671 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.StopSignal }}", "top") 1672 assert.NilError(c, err, out) 1673 assert.Equal(c, strings.TrimSpace(out), "SIGHUP") 1674 1675 containers := d.ActiveContainers(testutil.GetContext(c), c) 1676 out, err = d.Cmd("inspect", "--type", "container", "--format", "{{.Config.StopSignal}}", containers[0]) 1677 assert.NilError(c, err, out) 1678 assert.Equal(c, strings.TrimSpace(out), "SIGHUP") 1679 1680 out, err = d.Cmd("service", "update", "--detach", "--stop-signal=SIGUSR1", "top") 1681 assert.NilError(c, err, out) 1682 1683 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.StopSignal }}", "top") 1684 assert.NilError(c, err, out) 1685 assert.Equal(c, strings.TrimSpace(out), "SIGUSR1") 1686 } 1687 1688 func (s *DockerSwarmSuite) TestSwarmServiceLsFilterMode(c *testing.T) { 1689 ctx := testutil.GetContext(c) 1690 d := s.AddDaemon(ctx, c, true, true) 1691 1692 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "top1", "busybox", "top") 1693 assert.NilError(c, err, out) 1694 assert.Assert(c, strings.TrimSpace(out) != "") 1695 1696 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "top2", "--mode=global", "busybox", "top") 1697 assert.NilError(c, err, out) 1698 assert.Assert(c, strings.TrimSpace(out) != "") 1699 1700 // make sure task has been deployed. 1701 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(2)), poll.WithTimeout(defaultReconciliationTimeout)) 1702 1703 out, err = d.Cmd("service", "ls") 1704 assert.NilError(c, err, out) 1705 assert.Assert(c, strings.Contains(out, "top1"), out) 1706 assert.Assert(c, strings.Contains(out, "top2"), out) 1707 assert.Assert(c, !strings.Contains(out, "localnet"), out) 1708 out, err = d.Cmd("service", "ls", "--filter", "mode=global") 1709 assert.Assert(c, !strings.Contains(out, "top1"), out) 1710 assert.Assert(c, strings.Contains(out, "top2"), out) 1711 assert.NilError(c, err, out) 1712 1713 out, err = d.Cmd("service", "ls", "--filter", "mode=replicated") 1714 assert.NilError(c, err, out) 1715 assert.Assert(c, strings.Contains(out, "top1"), out) 1716 assert.Assert(c, !strings.Contains(out, "top2"), out) 1717 } 1718 1719 func (s *DockerSwarmSuite) TestSwarmInitUnspecifiedDataPathAddr(c *testing.T) { 1720 ctx := testutil.GetContext(c) 1721 d := s.AddDaemon(ctx, c, false, false) 1722 1723 out, err := d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0") 1724 assert.ErrorContains(c, err, "") 1725 assert.Assert(c, strings.Contains(out, "data path address must be a non-zero IP"), out) 1726 out, err = d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0:2000") 1727 assert.ErrorContains(c, err, "") 1728 assert.Assert(c, strings.Contains(out, "data path address must be a non-zero IP"), out) 1729 } 1730 1731 func (s *DockerSwarmSuite) TestSwarmJoinLeave(c *testing.T) { 1732 ctx := testutil.GetContext(c) 1733 d := s.AddDaemon(ctx, c, true, true) 1734 1735 out, err := d.Cmd("swarm", "join-token", "-q", "worker") 1736 assert.NilError(c, err) 1737 assert.Assert(c, strings.TrimSpace(out) != "") 1738 1739 token := strings.TrimSpace(out) 1740 1741 // Verify that back to back join/leave does not cause panics 1742 d1 := s.AddDaemon(ctx, c, false, false) 1743 for i := 0; i < 10; i++ { 1744 out, err = d1.Cmd("swarm", "join", "--token", token, d.SwarmListenAddr()) 1745 assert.NilError(c, err) 1746 assert.Assert(c, strings.TrimSpace(out) != "") 1747 1748 _, err = d1.Cmd("swarm", "leave") 1749 assert.NilError(c, err) 1750 } 1751 } 1752 1753 const defaultRetryCount = 10 1754 1755 func waitForEvent(c *testing.T, d *daemon.Daemon, since string, filter string, event string, retry int) string { 1756 if retry < 1 { 1757 c.Fatalf("retry count %d is invalid. It should be no less than 1", retry) 1758 return "" 1759 } 1760 var out string 1761 for i := 0; i < retry; i++ { 1762 until := daemonUnixTime(c) 1763 var err error 1764 if len(filter) > 0 { 1765 out, err = d.Cmd("events", "--since", since, "--until", until, filter) 1766 } else { 1767 out, err = d.Cmd("events", "--since", since, "--until", until) 1768 } 1769 assert.NilError(c, err, out) 1770 if strings.Contains(out, event) { 1771 return strings.TrimSpace(out) 1772 } 1773 // no need to sleep after last retry 1774 if i < retry-1 { 1775 time.Sleep(200 * time.Millisecond) 1776 } 1777 } 1778 c.Fatalf("docker events output '%s' doesn't contain event '%s'", out, event) 1779 return "" 1780 } 1781 1782 func (s *DockerSwarmSuite) TestSwarmClusterEventsSource(c *testing.T) { 1783 ctx := testutil.GetContext(c) 1784 d1 := s.AddDaemon(ctx, c, true, true) 1785 d2 := s.AddDaemon(ctx, c, true, true) 1786 d3 := s.AddDaemon(ctx, c, true, false) 1787 1788 // create a network 1789 out, err := d1.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 1790 assert.NilError(c, err, out) 1791 networkID := strings.TrimSpace(out) 1792 assert.Assert(c, networkID != "") 1793 1794 // d1, d2 are managers that can get swarm events 1795 waitForEvent(c, d1, "0", "-f scope=swarm", "network create "+networkID, defaultRetryCount) 1796 waitForEvent(c, d2, "0", "-f scope=swarm", "network create "+networkID, defaultRetryCount) 1797 1798 // d3 is a worker, not able to get cluster events 1799 out = waitForEvent(c, d3, "0", "-f scope=swarm", "", 1) 1800 assert.Assert(c, !strings.Contains(out, "network create "), out) 1801 } 1802 1803 func (s *DockerSwarmSuite) TestSwarmClusterEventsScope(c *testing.T) { 1804 ctx := testutil.GetContext(c) 1805 d := s.AddDaemon(ctx, c, true, true) 1806 1807 // create a service 1808 out, err := d.Cmd("service", "create", "--no-resolve-image", "--name", "test", "--detach=false", "busybox", "top") 1809 assert.NilError(c, err, out) 1810 serviceID := strings.Split(out, "\n")[0] 1811 1812 // scope swarm filters cluster events 1813 out = waitForEvent(c, d, "0", "-f scope=swarm", "service create "+serviceID, defaultRetryCount) 1814 assert.Assert(c, !strings.Contains(out, "container create "), out) 1815 // all events are returned if scope is not specified 1816 waitForEvent(c, d, "0", "", "service create "+serviceID, 1) 1817 waitForEvent(c, d, "0", "", "container create ", defaultRetryCount) 1818 1819 // scope local only shows non-cluster events 1820 out = waitForEvent(c, d, "0", "-f scope=local", "container create ", 1) 1821 assert.Assert(c, !strings.Contains(out, "service create "), out) 1822 } 1823 1824 func (s *DockerSwarmSuite) TestSwarmClusterEventsType(c *testing.T) { 1825 ctx := testutil.GetContext(c) 1826 d := s.AddDaemon(ctx, c, true, true) 1827 1828 // create a service 1829 out, err := d.Cmd("service", "create", "--no-resolve-image", "--name", "test", "--detach=false", "busybox", "top") 1830 assert.NilError(c, err, out) 1831 serviceID := strings.Split(out, "\n")[0] 1832 1833 // create a network 1834 out, err = d.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 1835 assert.NilError(c, err, out) 1836 networkID := strings.TrimSpace(out) 1837 assert.Assert(c, networkID != "") 1838 1839 // filter by service 1840 out = waitForEvent(c, d, "0", "-f type=service", "service create "+serviceID, defaultRetryCount) 1841 assert.Assert(c, !strings.Contains(out, "network create"), out) 1842 // filter by network 1843 out = waitForEvent(c, d, "0", "-f type=network", "network create "+networkID, defaultRetryCount) 1844 assert.Assert(c, !strings.Contains(out, "service create"), out) 1845 } 1846 1847 func (s *DockerSwarmSuite) TestSwarmClusterEventsService(c *testing.T) { 1848 ctx := testutil.GetContext(c) 1849 d := s.AddDaemon(ctx, c, true, true) 1850 1851 // create a service 1852 out, err := d.Cmd("service", "create", "--no-resolve-image", "--name", "test", "--detach=false", "busybox", "top") 1853 assert.NilError(c, err, out) 1854 serviceID := strings.Split(out, "\n")[0] 1855 1856 // validate service create event 1857 waitForEvent(c, d, "0", "-f scope=swarm", "service create "+serviceID, defaultRetryCount) 1858 1859 t1 := daemonUnixTime(c) 1860 out, err = d.Cmd("service", "update", "--force", "--detach=false", "test") 1861 assert.NilError(c, err, out) 1862 1863 // wait for service update start 1864 out = waitForEvent(c, d, t1, "-f scope=swarm", "service update "+serviceID, defaultRetryCount) 1865 assert.Assert(c, strings.Contains(out, "updatestate.new=updating"), out) 1866 // allow service update complete. This is a service with 1 instance 1867 time.Sleep(400 * time.Millisecond) 1868 out = waitForEvent(c, d, t1, "-f scope=swarm", "service update "+serviceID, defaultRetryCount) 1869 assert.Assert(c, strings.Contains(out, "updatestate.new=completed, updatestate.old=updating"), out) 1870 // scale service 1871 t2 := daemonUnixTime(c) 1872 out, err = d.Cmd("service", "scale", "test=3") 1873 assert.NilError(c, err, out) 1874 1875 out = waitForEvent(c, d, t2, "-f scope=swarm", "service update "+serviceID, defaultRetryCount) 1876 assert.Assert(c, strings.Contains(out, "replicas.new=3, replicas.old=1"), out) 1877 // remove service 1878 t3 := daemonUnixTime(c) 1879 out, err = d.Cmd("service", "rm", "test") 1880 assert.NilError(c, err, out) 1881 1882 waitForEvent(c, d, t3, "-f scope=swarm", "service remove "+serviceID, defaultRetryCount) 1883 } 1884 1885 func (s *DockerSwarmSuite) TestSwarmClusterEventsNode(c *testing.T) { 1886 ctx := testutil.GetContext(c) 1887 d1 := s.AddDaemon(ctx, c, true, true) 1888 s.AddDaemon(ctx, c, true, true) 1889 d3 := s.AddDaemon(ctx, c, true, true) 1890 1891 d3ID := d3.NodeID() 1892 waitForEvent(c, d1, "0", "-f scope=swarm", "node create "+d3ID, defaultRetryCount) 1893 1894 t1 := daemonUnixTime(c) 1895 out, err := d1.Cmd("node", "update", "--availability=pause", d3ID) 1896 assert.NilError(c, err, out) 1897 1898 // filter by type 1899 out = waitForEvent(c, d1, t1, "-f type=node", "node update "+d3ID, defaultRetryCount) 1900 assert.Assert(c, strings.Contains(out, "availability.new=pause, availability.old=active"), out) 1901 t2 := daemonUnixTime(c) 1902 out, err = d1.Cmd("node", "demote", d3ID) 1903 assert.NilError(c, err, out) 1904 1905 waitForEvent(c, d1, t2, "-f type=node", "node update "+d3ID, defaultRetryCount) 1906 1907 t3 := daemonUnixTime(c) 1908 out, err = d1.Cmd("node", "rm", "-f", d3ID) 1909 assert.NilError(c, err, out) 1910 1911 // filter by scope 1912 waitForEvent(c, d1, t3, "-f scope=swarm", "node remove "+d3ID, defaultRetryCount) 1913 } 1914 1915 func (s *DockerSwarmSuite) TestSwarmClusterEventsNetwork(c *testing.T) { 1916 ctx := testutil.GetContext(c) 1917 d := s.AddDaemon(ctx, c, true, true) 1918 1919 // create a network 1920 out, err := d.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 1921 assert.NilError(c, err, out) 1922 networkID := strings.TrimSpace(out) 1923 1924 waitForEvent(c, d, "0", "-f scope=swarm", "network create "+networkID, defaultRetryCount) 1925 1926 // remove network 1927 t1 := daemonUnixTime(c) 1928 out, err = d.Cmd("network", "rm", "foo") 1929 assert.NilError(c, err, out) 1930 1931 // filtered by network 1932 waitForEvent(c, d, t1, "-f type=network", "network remove "+networkID, defaultRetryCount) 1933 } 1934 1935 func (s *DockerSwarmSuite) TestSwarmClusterEventsSecret(c *testing.T) { 1936 ctx := testutil.GetContext(c) 1937 d := s.AddDaemon(ctx, c, true, true) 1938 1939 testName := "test_secret" 1940 id := d.CreateSecret(c, swarm.SecretSpec{ 1941 Annotations: swarm.Annotations{ 1942 Name: testName, 1943 }, 1944 Data: []byte("TESTINGDATA"), 1945 }) 1946 assert.Assert(c, id != "", "secrets: %s", id) 1947 1948 waitForEvent(c, d, "0", "-f scope=swarm", "secret create "+id, defaultRetryCount) 1949 1950 t1 := daemonUnixTime(c) 1951 d.DeleteSecret(c, id) 1952 // filtered by secret 1953 waitForEvent(c, d, t1, "-f type=secret", "secret remove "+id, defaultRetryCount) 1954 } 1955 1956 func (s *DockerSwarmSuite) TestSwarmClusterEventsConfig(c *testing.T) { 1957 ctx := testutil.GetContext(c) 1958 d := s.AddDaemon(ctx, c, true, true) 1959 1960 testName := "test_config" 1961 id := d.CreateConfig(c, swarm.ConfigSpec{ 1962 Annotations: swarm.Annotations{ 1963 Name: testName, 1964 }, 1965 Data: []byte("TESTINGDATA"), 1966 }) 1967 assert.Assert(c, id != "", "configs: %s", id) 1968 1969 waitForEvent(c, d, "0", "-f scope=swarm", "config create "+id, defaultRetryCount) 1970 1971 t1 := daemonUnixTime(c) 1972 d.DeleteConfig(c, id) 1973 // filtered by config 1974 waitForEvent(c, d, t1, "-f type=config", "config remove "+id, defaultRetryCount) 1975 } 1976 1977 func getUnlockKey(d *daemon.Daemon, c *testing.T, autolockOutput string) string { 1978 unlockKey, err := d.Cmd("swarm", "unlock-key", "-q") 1979 assert.Assert(c, err == nil, unlockKey) 1980 unlockKey = strings.TrimSuffix(unlockKey, "\n") 1981 1982 // Check that "docker swarm init --autolock" or "docker swarm update --autolock" 1983 // contains all the expected strings, including the unlock key 1984 assert.Assert(c, strings.Contains(autolockOutput, "docker swarm unlock"), autolockOutput) 1985 assert.Assert(c, strings.Contains(autolockOutput, unlockKey), autolockOutput) 1986 return unlockKey 1987 }