github.com/Prakhar-Agarwal-byte/moby@v0.0.0-20231027092010-a14e3e8ab87e/integration-cli/docker_cli_swarm_test.go (about) 1 //go:build !windows 2 3 package main 4 5 import ( 6 "bytes" 7 "context" 8 "encoding/json" 9 "encoding/pem" 10 "fmt" 11 "net/http" 12 "net/http/httptest" 13 "os" 14 "path/filepath" 15 "runtime" 16 "strings" 17 "testing" 18 "time" 19 20 "github.com/cloudflare/cfssl/helpers" 21 "github.com/Prakhar-Agarwal-byte/moby/api/types/swarm" 22 "github.com/Prakhar-Agarwal-byte/moby/integration-cli/checker" 23 "github.com/Prakhar-Agarwal-byte/moby/integration-cli/cli" 24 "github.com/Prakhar-Agarwal-byte/moby/integration-cli/daemon" 25 "github.com/Prakhar-Agarwal-byte/moby/libnetwork/driverapi" 26 "github.com/Prakhar-Agarwal-byte/moby/libnetwork/ipamapi" 27 remoteipam "github.com/Prakhar-Agarwal-byte/moby/libnetwork/ipams/remote/api" 28 "github.com/Prakhar-Agarwal-byte/moby/pkg/plugins" 29 "github.com/Prakhar-Agarwal-byte/moby/testutil" 30 "github.com/moby/swarmkit/v2/ca/keyutils" 31 "github.com/vishvananda/netlink" 32 "gotest.tools/v3/assert" 33 "gotest.tools/v3/fs" 34 "gotest.tools/v3/icmd" 35 "gotest.tools/v3/poll" 36 ) 37 38 func (s *DockerSwarmSuite) TestSwarmUpdate(c *testing.T) { 39 ctx := testutil.GetContext(c) 40 d := s.AddDaemon(ctx, c, true, true) 41 42 getSpec := func() swarm.Spec { 43 sw := d.GetSwarm(c) 44 return sw.Spec 45 } 46 47 out, err := d.Cmd("swarm", "update", "--cert-expiry", "30h", "--dispatcher-heartbeat", "11s") 48 assert.NilError(c, err, out) 49 50 spec := getSpec() 51 assert.Equal(c, spec.CAConfig.NodeCertExpiry, 30*time.Hour) 52 assert.Equal(c, spec.Dispatcher.HeartbeatPeriod, 11*time.Second) 53 54 // setting anything under 30m for cert-expiry is not allowed 55 out, err = d.Cmd("swarm", "update", "--cert-expiry", "15m") 56 assert.ErrorContains(c, err, "") 57 assert.Assert(c, strings.Contains(out, "minimum certificate expiry time")) 58 spec = getSpec() 59 assert.Equal(c, spec.CAConfig.NodeCertExpiry, 30*time.Hour) 60 61 // passing an external CA (this is without starting a root rotation) does not fail 62 cli.Docker(cli.Args("swarm", "update", "--external-ca", "protocol=cfssl,url=https://something.org", 63 "--external-ca", "protocol=cfssl,url=https://somethingelse.org,cacert=fixtures/https/ca.pem"), 64 cli.Daemon(d)).Assert(c, icmd.Success) 65 66 expected, err := os.ReadFile("fixtures/https/ca.pem") 67 assert.NilError(c, err) 68 69 spec = getSpec() 70 assert.Equal(c, len(spec.CAConfig.ExternalCAs), 2) 71 assert.Equal(c, spec.CAConfig.ExternalCAs[0].CACert, "") 72 assert.Equal(c, spec.CAConfig.ExternalCAs[1].CACert, string(expected)) 73 74 // passing an invalid external CA fails 75 tempFile := fs.NewFile(c, "testfile", fs.WithContent("fakecert")) 76 defer tempFile.Remove() 77 78 result := cli.Docker(cli.Args("swarm", "update", 79 "--external-ca", fmt.Sprintf("protocol=cfssl,url=https://something.org,cacert=%s", tempFile.Path())), 80 cli.Daemon(d)) 81 result.Assert(c, icmd.Expected{ 82 ExitCode: 125, 83 Err: "must be in PEM format", 84 }) 85 } 86 87 func (s *DockerSwarmSuite) TestSwarmInit(c *testing.T) { 88 ctx := testutil.GetContext(c) 89 d := s.AddDaemon(ctx, c, false, false) 90 91 getSpec := func() swarm.Spec { 92 sw := d.GetSwarm(c) 93 return sw.Spec 94 } 95 96 // passing an invalid external CA fails 97 tempFile := fs.NewFile(c, "testfile", fs.WithContent("fakecert")) 98 defer tempFile.Remove() 99 100 result := cli.Docker(cli.Args("swarm", "init", "--cert-expiry", "30h", "--dispatcher-heartbeat", "11s", 101 "--external-ca", fmt.Sprintf("protocol=cfssl,url=https://somethingelse.org,cacert=%s", tempFile.Path())), 102 cli.Daemon(d)) 103 result.Assert(c, icmd.Expected{ 104 ExitCode: 125, 105 Err: "must be in PEM format", 106 }) 107 108 cli.Docker(cli.Args("swarm", "init", "--cert-expiry", "30h", "--dispatcher-heartbeat", "11s", 109 "--external-ca", "protocol=cfssl,url=https://something.org", 110 "--external-ca", "protocol=cfssl,url=https://somethingelse.org,cacert=fixtures/https/ca.pem"), 111 cli.Daemon(d)).Assert(c, icmd.Success) 112 113 expected, err := os.ReadFile("fixtures/https/ca.pem") 114 assert.NilError(c, err) 115 116 spec := getSpec() 117 assert.Equal(c, spec.CAConfig.NodeCertExpiry, 30*time.Hour) 118 assert.Equal(c, spec.Dispatcher.HeartbeatPeriod, 11*time.Second) 119 assert.Equal(c, len(spec.CAConfig.ExternalCAs), 2) 120 assert.Equal(c, spec.CAConfig.ExternalCAs[0].CACert, "") 121 assert.Equal(c, spec.CAConfig.ExternalCAs[1].CACert, string(expected)) 122 123 assert.Assert(c, d.SwarmLeave(ctx, c, true) == nil) 124 cli.Docker(cli.Args("swarm", "init"), cli.Daemon(d)).Assert(c, icmd.Success) 125 126 spec = getSpec() 127 assert.Equal(c, spec.CAConfig.NodeCertExpiry, 90*24*time.Hour) 128 assert.Equal(c, spec.Dispatcher.HeartbeatPeriod, 5*time.Second) 129 } 130 131 func (s *DockerSwarmSuite) TestSwarmInitIPv6(c *testing.T) { 132 testRequires(c, IPv6) 133 ctx := testutil.GetContext(c) 134 d1 := s.AddDaemon(ctx, c, false, false) 135 cli.Docker(cli.Args("swarm", "init", "--listen-add", "::1"), cli.Daemon(d1)).Assert(c, icmd.Success) 136 137 d2 := s.AddDaemon(ctx, c, false, false) 138 cli.Docker(cli.Args("swarm", "join", "::1"), cli.Daemon(d2)).Assert(c, icmd.Success) 139 140 out := cli.Docker(cli.Args("info"), cli.Daemon(d2)).Assert(c, icmd.Success).Combined() 141 assert.Assert(c, strings.Contains(out, "Swarm: active")) 142 } 143 144 func (s *DockerSwarmSuite) TestSwarmInitUnspecifiedAdvertiseAddr(c *testing.T) { 145 ctx := testutil.GetContext(c) 146 d := s.AddDaemon(ctx, c, false, false) 147 out, err := d.Cmd("swarm", "init", "--advertise-addr", "0.0.0.0") 148 assert.ErrorContains(c, err, "") 149 assert.Assert(c, strings.Contains(out, "advertise address must be a non-zero IP address")) 150 } 151 152 func (s *DockerSwarmSuite) TestSwarmIncompatibleDaemon(c *testing.T) { 153 ctx := testutil.GetContext(c) 154 // init swarm mode and stop a daemon 155 d := s.AddDaemon(ctx, c, true, true) 156 info := d.SwarmInfo(ctx, c) 157 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 158 d.Stop(c) 159 160 // start a daemon with --live-restore 161 err := d.StartWithError("--live-restore") 162 assert.ErrorContains(c, err, "") 163 content, err := d.ReadLogFile() 164 assert.NilError(c, err) 165 assert.Assert(c, strings.Contains(string(content), "--live-restore daemon configuration is incompatible with swarm mode")) 166 // restart for teardown 167 d.StartNode(c) 168 } 169 170 func (s *DockerSwarmSuite) TestSwarmServiceTemplatingHostname(c *testing.T) { 171 ctx := testutil.GetContext(c) 172 d := s.AddDaemon(ctx, c, true, true) 173 hostname, err := d.Cmd("node", "inspect", "--format", "{{.Description.Hostname}}", "self") 174 assert.Assert(c, err == nil, hostname) 175 176 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "test", "--hostname", "{{.Service.Name}}-{{.Task.Slot}}-{{.Node.Hostname}}", "busybox", "top") 177 assert.NilError(c, err, out) 178 179 // make sure task has been deployed. 180 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 181 182 containers := d.ActiveContainers(testutil.GetContext(c), c) 183 out, err = d.Cmd("inspect", "--type", "container", "--format", "{{.Config.Hostname}}", containers[0]) 184 assert.NilError(c, err, out) 185 assert.Equal(c, strings.Split(out, "\n")[0], "test-1-"+strings.Split(hostname, "\n")[0], "hostname with templating invalid") 186 } 187 188 // Test case for #24270 189 func (s *DockerSwarmSuite) TestSwarmServiceListFilter(c *testing.T) { 190 ctx := testutil.GetContext(c) 191 d := s.AddDaemon(ctx, c, true, true) 192 193 name1 := "redis-cluster-md5" 194 name2 := "redis-cluster" 195 name3 := "other-cluster" 196 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name1, "busybox", "top") 197 assert.NilError(c, err, out) 198 assert.Assert(c, strings.TrimSpace(out) != "") 199 200 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name2, "busybox", "top") 201 assert.NilError(c, err, out) 202 assert.Assert(c, strings.TrimSpace(out) != "") 203 204 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name3, "busybox", "top") 205 assert.NilError(c, err, out) 206 assert.Assert(c, strings.TrimSpace(out) != "") 207 208 filter1 := "name=redis-cluster-md5" 209 filter2 := "name=redis-cluster" 210 211 // We search checker.Contains with `name+" "` to prevent prefix only. 212 out, err = d.Cmd("service", "ls", "--filter", filter1) 213 assert.NilError(c, err, out) 214 assert.Assert(c, strings.Contains(out, name1+" "), out) 215 assert.Assert(c, !strings.Contains(out, name2+" "), out) 216 assert.Assert(c, !strings.Contains(out, name3+" "), out) 217 out, err = d.Cmd("service", "ls", "--filter", filter2) 218 assert.NilError(c, err, out) 219 assert.Assert(c, strings.Contains(out, name1+" "), out) 220 assert.Assert(c, strings.Contains(out, name2+" "), out) 221 assert.Assert(c, !strings.Contains(out, name3+" "), out) 222 out, err = d.Cmd("service", "ls") 223 assert.NilError(c, err, out) 224 assert.Assert(c, strings.Contains(out, name1+" "), out) 225 assert.Assert(c, strings.Contains(out, name2+" "), out) 226 assert.Assert(c, strings.Contains(out, name3+" "), out) 227 } 228 229 func (s *DockerSwarmSuite) TestSwarmNodeListFilter(c *testing.T) { 230 ctx := testutil.GetContext(c) 231 d := s.AddDaemon(ctx, c, true, true) 232 233 out, err := d.Cmd("node", "inspect", "--format", "{{ .Description.Hostname }}", "self") 234 assert.NilError(c, err, out) 235 assert.Assert(c, strings.TrimSpace(out) != "") 236 name := strings.TrimSpace(out) 237 238 filter := "name=" + name[:4] 239 240 out, err = d.Cmd("node", "ls", "--filter", filter) 241 assert.NilError(c, err, out) 242 assert.Assert(c, strings.Contains(out, name), out) 243 out, err = d.Cmd("node", "ls", "--filter", "name=none") 244 assert.NilError(c, err, out) 245 assert.Assert(c, !strings.Contains(out, name), out) 246 } 247 248 func (s *DockerSwarmSuite) TestSwarmNodeTaskListFilter(c *testing.T) { 249 ctx := testutil.GetContext(c) 250 d := s.AddDaemon(ctx, c, true, true) 251 252 name := "redis-cluster-md5" 253 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--replicas=3", "busybox", "top") 254 assert.NilError(c, err, out) 255 assert.Assert(c, strings.TrimSpace(out) != "") 256 257 // make sure task has been deployed. 258 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(3)), poll.WithTimeout(defaultReconciliationTimeout)) 259 260 filter := "name=redis-cluster" 261 262 out, err = d.Cmd("node", "ps", "--filter", filter, "self") 263 assert.NilError(c, err, out) 264 assert.Assert(c, strings.Contains(out, name+".1"), out) 265 assert.Assert(c, strings.Contains(out, name+".2"), out) 266 assert.Assert(c, strings.Contains(out, name+".3"), out) 267 out, err = d.Cmd("node", "ps", "--filter", "name=none", "self") 268 assert.NilError(c, err, out) 269 assert.Assert(c, !strings.Contains(out, name+".1"), out) 270 assert.Assert(c, !strings.Contains(out, name+".2"), out) 271 assert.Assert(c, !strings.Contains(out, name+".3"), out) 272 } 273 274 // Test case for #25375 275 func (s *DockerSwarmSuite) TestSwarmPublishAdd(c *testing.T) { 276 ctx := testutil.GetContext(c) 277 d := s.AddDaemon(ctx, c, true, true) 278 279 name := "top" 280 // this first command does not have to be retried because service creates 281 // don't return out of sequence errors. 282 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--label", "x=y", "busybox", "top") 283 assert.NilError(c, err, out) 284 assert.Assert(c, strings.TrimSpace(out) != "") 285 286 out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", name) 287 assert.NilError(c, err, out) 288 289 out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", name) 290 assert.NilError(c, err, out) 291 292 _, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", "--publish-add", "80:20", name) 293 assert.ErrorContains(c, err, "") 294 295 // this last command does not have to be retried because service inspect 296 // does not return out of sequence errors. 297 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.EndpointSpec.Ports }}", name) 298 assert.NilError(c, err, out) 299 assert.Equal(c, strings.TrimSpace(out), "[{ tcp 80 80 ingress}]") 300 } 301 302 func (s *DockerSwarmSuite) TestSwarmServiceWithGroup(c *testing.T) { 303 ctx := testutil.GetContext(c) 304 d := s.AddDaemon(ctx, c, true, true) 305 306 name := "top" 307 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--user", "root:root", "--group", "wheel", "--group", "audio", "--group", "staff", "--group", "777", "busybox", "top") 308 assert.NilError(c, err, out) 309 assert.Assert(c, strings.TrimSpace(out) != "") 310 311 // make sure task has been deployed. 312 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 313 314 out, err = d.Cmd("ps", "-q") 315 assert.NilError(c, err, out) 316 assert.Assert(c, strings.TrimSpace(out) != "") 317 318 container := strings.TrimSpace(out) 319 320 out, err = d.Cmd("exec", container, "id") 321 assert.NilError(c, err, out) 322 assert.Equal(c, strings.TrimSpace(out), "uid=0(root) gid=0(root) groups=0(root),10(wheel),29(audio),50(staff),777") 323 } 324 325 func (s *DockerSwarmSuite) TestSwarmContainerAutoStart(c *testing.T) { 326 ctx := testutil.GetContext(c) 327 d := s.AddDaemon(ctx, c, true, true) 328 329 out, err := d.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 330 assert.NilError(c, err, out) 331 assert.Assert(c, strings.TrimSpace(out) != "") 332 333 out, err = d.Cmd("run", "-id", "--restart=always", "--net=foo", "--name=test", "busybox", "top") 334 assert.NilError(c, err, out) 335 assert.Assert(c, strings.TrimSpace(out) != "") 336 337 out, err = d.Cmd("ps", "-q") 338 assert.NilError(c, err, out) 339 assert.Assert(c, strings.TrimSpace(out) != "") 340 341 d.RestartNode(c) 342 343 out, err = d.Cmd("ps", "-q") 344 assert.NilError(c, err, out) 345 assert.Assert(c, strings.TrimSpace(out) != "") 346 } 347 348 func (s *DockerSwarmSuite) TestSwarmContainerEndpointOptions(c *testing.T) { 349 ctx := testutil.GetContext(c) 350 d := s.AddDaemon(ctx, c, true, true) 351 352 out, err := d.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 353 assert.NilError(c, err, out) 354 assert.Assert(c, strings.TrimSpace(out) != "") 355 356 out, err = d.Cmd("run", "-d", "--net=foo", "--name=first", "--net-alias=first-alias", "busybox:glibc", "top") 357 assert.NilError(c, err, out) 358 359 out, err = d.Cmd("run", "-d", "--net=foo", "--name=second", "busybox:glibc", "top") 360 assert.NilError(c, err, out) 361 362 out, err = d.Cmd("run", "-d", "--net=foo", "--net-alias=third-alias", "busybox:glibc", "top") 363 assert.NilError(c, err, out) 364 365 // ping first container and its alias, also ping third and anonymous container by its alias 366 out, err = d.Cmd("exec", "second", "ping", "-c", "1", "first") 367 assert.NilError(c, err, out) 368 out, err = d.Cmd("exec", "second", "ping", "-c", "1", "first-alias") 369 assert.NilError(c, err, out) 370 out, err = d.Cmd("exec", "second", "ping", "-c", "1", "third-alias") 371 assert.NilError(c, err, out) 372 } 373 374 func (s *DockerSwarmSuite) TestSwarmContainerAttachByNetworkId(c *testing.T) { 375 ctx := testutil.GetContext(c) 376 d := s.AddDaemon(ctx, c, true, true) 377 378 out, err := d.Cmd("network", "create", "--attachable", "-d", "overlay", "testnet") 379 assert.NilError(c, err, out) 380 assert.Assert(c, strings.TrimSpace(out) != "") 381 networkID := strings.TrimSpace(out) 382 383 out, err = d.Cmd("run", "-d", "--net", networkID, "busybox", "top") 384 assert.NilError(c, err, out) 385 cID := strings.TrimSpace(out) 386 d.WaitRun(cID) 387 388 out, err = d.Cmd("rm", "-f", cID) 389 assert.NilError(c, err, out) 390 391 out, err = d.Cmd("network", "rm", "testnet") 392 assert.NilError(c, err, out) 393 394 checkNetwork := func(*testing.T) (interface{}, string) { 395 out, err := d.Cmd("network", "ls") 396 assert.NilError(c, err) 397 return out, "" 398 } 399 400 poll.WaitOn(c, pollCheck(c, checkNetwork, checker.Not(checker.Contains("testnet"))), poll.WithTimeout(3*time.Second)) 401 } 402 403 func (s *DockerSwarmSuite) TestOverlayAttachable(c *testing.T) { 404 ctx := testutil.GetContext(c) 405 d := s.AddDaemon(ctx, c, true, true) 406 407 out, err := d.Cmd("network", "create", "-d", "overlay", "--attachable", "ovnet") 408 assert.NilError(c, err, out) 409 410 // validate attachable 411 out, err = d.Cmd("network", "inspect", "--format", "{{json .Attachable}}", "ovnet") 412 assert.NilError(c, err, out) 413 assert.Equal(c, strings.TrimSpace(out), "true") 414 415 // validate containers can attach to this overlay network 416 out, err = d.Cmd("run", "-d", "--network", "ovnet", "--name", "c1", "busybox", "top") 417 assert.NilError(c, err, out) 418 419 // redo validation, there was a bug that the value of attachable changes after 420 // containers attach to the network 421 out, err = d.Cmd("network", "inspect", "--format", "{{json .Attachable}}", "ovnet") 422 assert.NilError(c, err, out) 423 assert.Equal(c, strings.TrimSpace(out), "true") 424 } 425 426 func (s *DockerSwarmSuite) TestOverlayAttachableOnSwarmLeave(c *testing.T) { 427 ctx := testutil.GetContext(c) 428 d := s.AddDaemon(ctx, c, true, true) 429 430 // Create an attachable swarm network 431 nwName := "attovl" 432 out, err := d.Cmd("network", "create", "-d", "overlay", "--attachable", nwName) 433 assert.NilError(c, err, out) 434 435 // Connect a container to the network 436 out, err = d.Cmd("run", "-d", "--network", nwName, "--name", "c1", "busybox", "top") 437 assert.NilError(c, err, out) 438 439 // Leave the swarm 440 assert.Assert(c, d.SwarmLeave(ctx, c, true) == nil) 441 442 // Check the container is disconnected 443 out, err = d.Cmd("inspect", "c1", "--format", "{{.NetworkSettings.Networks."+nwName+"}}") 444 assert.NilError(c, err, out) 445 assert.Equal(c, strings.TrimSpace(out), "<no value>") 446 447 // Check the network is gone 448 out, err = d.Cmd("network", "ls", "--format", "{{.Name}}") 449 assert.NilError(c, err, out) 450 assert.Assert(c, !strings.Contains(out, nwName), out) 451 } 452 453 func (s *DockerSwarmSuite) TestOverlayAttachableReleaseResourcesOnFailure(c *testing.T) { 454 ctx := testutil.GetContext(c) 455 d := s.AddDaemon(ctx, c, true, true) 456 457 // Create attachable network 458 out, err := d.Cmd("network", "create", "-d", "overlay", "--attachable", "--subnet", "10.10.9.0/24", "ovnet") 459 assert.NilError(c, err, out) 460 461 // Attach a container with specific IP 462 out, err = d.Cmd("run", "-d", "--network", "ovnet", "--name", "c1", "--ip", "10.10.9.33", "busybox", "top") 463 assert.NilError(c, err, out) 464 465 // Attempt to attach another container with same IP, must fail 466 out, err = d.Cmd("run", "-d", "--network", "ovnet", "--name", "c2", "--ip", "10.10.9.33", "busybox", "top") 467 assert.ErrorContains(c, err, "", out) 468 469 // Remove first container 470 out, err = d.Cmd("rm", "-f", "c1") 471 assert.NilError(c, err, out) 472 473 // Verify the network can be removed, no phantom network attachment task left over 474 out, err = d.Cmd("network", "rm", "ovnet") 475 assert.NilError(c, err, out) 476 } 477 478 func (s *DockerSwarmSuite) TestSwarmIngressNetwork(c *testing.T) { 479 ctx := testutil.GetContext(c) 480 d := s.AddDaemon(ctx, c, true, true) 481 482 // Ingress network can be removed 483 removeNetwork := func(name string) *icmd.Result { 484 return cli.Docker( 485 cli.Args("-H", d.Sock(), "network", "rm", name), 486 cli.WithStdin(strings.NewReader("Y"))) 487 } 488 489 result := removeNetwork("ingress") 490 result.Assert(c, icmd.Success) 491 492 // And recreated 493 out, err := d.Cmd("network", "create", "-d", "overlay", "--ingress", "new-ingress") 494 assert.NilError(c, err, out) 495 496 // But only one is allowed 497 out, err = d.Cmd("network", "create", "-d", "overlay", "--ingress", "another-ingress") 498 assert.ErrorContains(c, err, "") 499 assert.Assert(c, strings.Contains(strings.TrimSpace(out), "is already present"), out) 500 // It cannot be removed if it is being used 501 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "srv1", "-p", "9000:8000", "busybox", "top") 502 assert.NilError(c, err, out) 503 504 result = removeNetwork("new-ingress") 505 result.Assert(c, icmd.Expected{ 506 ExitCode: 1, 507 Err: "ingress network cannot be removed because service", 508 }) 509 510 // But it can be removed once no more services depend on it 511 out, err = d.Cmd("service", "update", "--detach", "--publish-rm", "9000:8000", "srv1") 512 assert.NilError(c, err, out) 513 514 result = removeNetwork("new-ingress") 515 result.Assert(c, icmd.Success) 516 517 // A service which needs the ingress network cannot be created if no ingress is present 518 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "srv2", "-p", "500:500", "busybox", "top") 519 assert.ErrorContains(c, err, "") 520 assert.Assert(c, strings.Contains(strings.TrimSpace(out), "no ingress network is present"), out) 521 // An existing service cannot be updated to use the ingress nw if the nw is not present 522 out, err = d.Cmd("service", "update", "--detach", "--publish-add", "9000:8000", "srv1") 523 assert.ErrorContains(c, err, "") 524 assert.Assert(c, strings.Contains(strings.TrimSpace(out), "no ingress network is present"), out) 525 // But services which do not need routing mesh can be created regardless 526 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "srv3", "--endpoint-mode", "dnsrr", "busybox", "top") 527 assert.NilError(c, err, out) 528 } 529 530 func (s *DockerSwarmSuite) TestSwarmCreateServiceWithNoIngressNetwork(c *testing.T) { 531 ctx := testutil.GetContext(c) 532 d := s.AddDaemon(ctx, c, true, true) 533 534 // Remove ingress network 535 result := cli.Docker( 536 cli.Args("-H", d.Sock(), "network", "rm", "ingress"), 537 cli.WithStdin(strings.NewReader("Y"))) 538 result.Assert(c, icmd.Success) 539 540 // Create a overlay network and launch a service on it 541 // Make sure nothing panics because ingress network is missing 542 out, err := d.Cmd("network", "create", "-d", "overlay", "another-network") 543 assert.NilError(c, err, out) 544 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "srv4", "--network", "another-network", "busybox", "top") 545 assert.NilError(c, err, out) 546 } 547 548 // Test case for #24108, also the case from: 549 // https://github.com/Prakhar-Agarwal-byte/moby/pull/24620#issuecomment-233715656 550 func (s *DockerSwarmSuite) TestSwarmTaskListFilter(c *testing.T) { 551 ctx := testutil.GetContext(c) 552 d := s.AddDaemon(ctx, c, true, true) 553 554 name := "redis-cluster-md5" 555 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--replicas=3", "busybox", "top") 556 assert.NilError(c, err, out) 557 assert.Assert(c, strings.TrimSpace(out) != "") 558 559 filter := "name=redis-cluster" 560 561 checkNumTasks := func(*testing.T) (interface{}, string) { 562 out, err := d.Cmd("service", "ps", "--filter", filter, name) 563 assert.NilError(c, err, out) 564 return len(strings.Split(out, "\n")) - 2, "" // includes header and nl in last line 565 } 566 567 // wait until all tasks have been created 568 poll.WaitOn(c, pollCheck(c, checkNumTasks, checker.Equals(3)), poll.WithTimeout(defaultReconciliationTimeout)) 569 570 out, err = d.Cmd("service", "ps", "--filter", filter, name) 571 assert.NilError(c, err, out) 572 assert.Assert(c, strings.Contains(out, name+".1"), out) 573 assert.Assert(c, strings.Contains(out, name+".2"), out) 574 assert.Assert(c, strings.Contains(out, name+".3"), out) 575 out, err = d.Cmd("service", "ps", "--filter", "name="+name+".1", name) 576 assert.NilError(c, err, out) 577 assert.Assert(c, strings.Contains(out, name+".1"), out) 578 assert.Assert(c, !strings.Contains(out, name+".2"), out) 579 assert.Assert(c, !strings.Contains(out, name+".3"), out) 580 out, err = d.Cmd("service", "ps", "--filter", "name=none", name) 581 assert.NilError(c, err, out) 582 assert.Assert(c, !strings.Contains(out, name+".1"), out) 583 assert.Assert(c, !strings.Contains(out, name+".2"), out) 584 assert.Assert(c, !strings.Contains(out, name+".3"), out) 585 name = "redis-cluster-sha1" 586 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--mode=global", "busybox", "top") 587 assert.NilError(c, err, out) 588 assert.Assert(c, strings.TrimSpace(out) != "") 589 590 poll.WaitOn(c, pollCheck(c, checkNumTasks, checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 591 592 filter = "name=redis-cluster" 593 out, err = d.Cmd("service", "ps", "--filter", filter, name) 594 assert.NilError(c, err, out) 595 assert.Assert(c, strings.Contains(out, name), out) 596 out, err = d.Cmd("service", "ps", "--filter", "name="+name, name) 597 assert.NilError(c, err, out) 598 assert.Assert(c, strings.Contains(out, name), out) 599 out, err = d.Cmd("service", "ps", "--filter", "name=none", name) 600 assert.NilError(c, err, out) 601 assert.Assert(c, !strings.Contains(out, name), out) 602 } 603 604 func (s *DockerSwarmSuite) TestPsListContainersFilterIsTask(c *testing.T) { 605 ctx := testutil.GetContext(c) 606 d := s.AddDaemon(ctx, c, true, true) 607 608 // Create a bare container 609 out, err := d.Cmd("run", "-d", "--name=bare-container", "busybox", "top") 610 assert.NilError(c, err, out) 611 bareID := strings.TrimSpace(out)[:12] 612 // Create a service 613 name := "busybox-top" 614 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "busybox", "top") 615 assert.NilError(c, err, out) 616 assert.Assert(c, strings.TrimSpace(out) != "") 617 618 // make sure task has been deployed. 619 poll.WaitOn(c, pollCheck(c, d.CheckServiceRunningTasks(ctx, name), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 620 621 // Filter non-tasks 622 out, err = d.Cmd("ps", "-a", "-q", "--filter=is-task=false") 623 assert.NilError(c, err, out) 624 psOut := strings.TrimSpace(out) 625 assert.Equal(c, psOut, bareID, fmt.Sprintf("Expected id %s, got %s for is-task label, output %q", bareID, psOut, out)) 626 627 // Filter tasks 628 out, err = d.Cmd("ps", "-a", "-q", "--filter=is-task=true") 629 assert.NilError(c, err, out) 630 lines := strings.Split(strings.Trim(out, "\n "), "\n") 631 assert.Equal(c, len(lines), 1) 632 assert.Assert(c, lines[0] != bareID, "Expected not %s, but got it for is-task label, output %q", bareID, out) 633 } 634 635 const ( 636 globalNetworkPlugin = "global-network-plugin" 637 globalIPAMPlugin = "global-ipam-plugin" 638 ) 639 640 func setupRemoteGlobalNetworkPlugin(c *testing.T, mux *http.ServeMux, url, netDrv, ipamDrv string) { 641 mux.HandleFunc("/Plugin.Activate", func(w http.ResponseWriter, r *http.Request) { 642 w.Header().Set("Content-Type", plugins.VersionMimetype) 643 fmt.Fprintf(w, `{"Implements": ["%s", "%s"]}`, driverapi.NetworkPluginEndpointType, ipamapi.PluginEndpointType) 644 }) 645 646 // Network driver implementation 647 mux.HandleFunc(fmt.Sprintf("/%s.GetCapabilities", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 648 w.Header().Set("Content-Type", plugins.VersionMimetype) 649 fmt.Fprintf(w, `{"Scope":"global"}`) 650 }) 651 652 mux.HandleFunc(fmt.Sprintf("/%s.AllocateNetwork", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 653 err := json.NewDecoder(r.Body).Decode(&remoteDriverNetworkRequest) 654 if err != nil { 655 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 656 return 657 } 658 w.Header().Set("Content-Type", plugins.VersionMimetype) 659 fmt.Fprintf(w, "null") 660 }) 661 662 mux.HandleFunc(fmt.Sprintf("/%s.FreeNetwork", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 663 w.Header().Set("Content-Type", plugins.VersionMimetype) 664 fmt.Fprintf(w, "null") 665 }) 666 667 mux.HandleFunc(fmt.Sprintf("/%s.CreateNetwork", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 668 err := json.NewDecoder(r.Body).Decode(&remoteDriverNetworkRequest) 669 if err != nil { 670 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 671 return 672 } 673 w.Header().Set("Content-Type", plugins.VersionMimetype) 674 fmt.Fprintf(w, "null") 675 }) 676 677 mux.HandleFunc(fmt.Sprintf("/%s.DeleteNetwork", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 678 w.Header().Set("Content-Type", plugins.VersionMimetype) 679 fmt.Fprintf(w, "null") 680 }) 681 682 mux.HandleFunc(fmt.Sprintf("/%s.CreateEndpoint", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 683 w.Header().Set("Content-Type", plugins.VersionMimetype) 684 fmt.Fprintf(w, `{"Interface":{"MacAddress":"a0:b1:c2:d3:e4:f5"}}`) 685 }) 686 687 mux.HandleFunc(fmt.Sprintf("/%s.Join", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 688 w.Header().Set("Content-Type", plugins.VersionMimetype) 689 690 veth := &netlink.Veth{ 691 LinkAttrs: netlink.LinkAttrs{Name: "randomIfName", TxQLen: 0}, PeerName: "cnt0", 692 } 693 if err := netlink.LinkAdd(veth); err != nil { 694 fmt.Fprintf(w, `{"Error":"failed to add veth pair: `+err.Error()+`"}`) 695 } else { 696 fmt.Fprintf(w, `{"InterfaceName":{ "SrcName":"cnt0", "DstPrefix":"veth"}}`) 697 } 698 }) 699 700 mux.HandleFunc(fmt.Sprintf("/%s.Leave", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 701 w.Header().Set("Content-Type", plugins.VersionMimetype) 702 fmt.Fprintf(w, "null") 703 }) 704 705 mux.HandleFunc(fmt.Sprintf("/%s.DeleteEndpoint", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 706 w.Header().Set("Content-Type", plugins.VersionMimetype) 707 if link, err := netlink.LinkByName("cnt0"); err == nil { 708 netlink.LinkDel(link) 709 } 710 fmt.Fprintf(w, "null") 711 }) 712 713 // IPAM Driver implementation 714 var ( 715 poolRequest remoteipam.RequestPoolRequest 716 poolReleaseReq remoteipam.ReleasePoolRequest 717 addressRequest remoteipam.RequestAddressRequest 718 addressReleaseReq remoteipam.ReleaseAddressRequest 719 lAS = "localAS" 720 gAS = "globalAS" 721 pool = "172.28.0.0/16" 722 poolID = lAS + "/" + pool 723 gw = "172.28.255.254/16" 724 ) 725 726 mux.HandleFunc(fmt.Sprintf("/%s.GetDefaultAddressSpaces", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 727 w.Header().Set("Content-Type", plugins.VersionMimetype) 728 fmt.Fprintf(w, `{"LocalDefaultAddressSpace":"`+lAS+`", "GlobalDefaultAddressSpace": "`+gAS+`"}`) 729 }) 730 731 mux.HandleFunc(fmt.Sprintf("/%s.RequestPool", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 732 err := json.NewDecoder(r.Body).Decode(&poolRequest) 733 if err != nil { 734 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 735 return 736 } 737 w.Header().Set("Content-Type", plugins.VersionMimetype) 738 if poolRequest.AddressSpace != lAS && poolRequest.AddressSpace != gAS { 739 fmt.Fprintf(w, `{"Error":"Unknown address space in pool request: `+poolRequest.AddressSpace+`"}`) 740 } else if poolRequest.Pool != "" && poolRequest.Pool != pool { 741 fmt.Fprintf(w, `{"Error":"Cannot handle explicit pool requests yet"}`) 742 } else { 743 fmt.Fprintf(w, `{"PoolID":"`+poolID+`", "Pool":"`+pool+`"}`) 744 } 745 }) 746 747 mux.HandleFunc(fmt.Sprintf("/%s.RequestAddress", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 748 err := json.NewDecoder(r.Body).Decode(&addressRequest) 749 if err != nil { 750 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 751 return 752 } 753 w.Header().Set("Content-Type", plugins.VersionMimetype) 754 // make sure libnetwork is now querying on the expected pool id 755 if addressRequest.PoolID != poolID { 756 fmt.Fprintf(w, `{"Error":"unknown pool id"}`) 757 } else if addressRequest.Address != "" { 758 fmt.Fprintf(w, `{"Error":"Cannot handle explicit address requests yet"}`) 759 } else { 760 fmt.Fprintf(w, `{"Address":"`+gw+`"}`) 761 } 762 }) 763 764 mux.HandleFunc(fmt.Sprintf("/%s.ReleaseAddress", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 765 err := json.NewDecoder(r.Body).Decode(&addressReleaseReq) 766 if err != nil { 767 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 768 return 769 } 770 w.Header().Set("Content-Type", plugins.VersionMimetype) 771 // make sure libnetwork is now asking to release the expected address from the expected poolid 772 if addressRequest.PoolID != poolID { 773 fmt.Fprintf(w, `{"Error":"unknown pool id"}`) 774 } else if addressReleaseReq.Address != gw { 775 fmt.Fprintf(w, `{"Error":"unknown address"}`) 776 } else { 777 fmt.Fprintf(w, "null") 778 } 779 }) 780 781 mux.HandleFunc(fmt.Sprintf("/%s.ReleasePool", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 782 err := json.NewDecoder(r.Body).Decode(&poolReleaseReq) 783 if err != nil { 784 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 785 return 786 } 787 w.Header().Set("Content-Type", plugins.VersionMimetype) 788 // make sure libnetwork is now asking to release the expected poolid 789 if addressRequest.PoolID != poolID { 790 fmt.Fprintf(w, `{"Error":"unknown pool id"}`) 791 } else { 792 fmt.Fprintf(w, "null") 793 } 794 }) 795 796 err := os.MkdirAll("/etc/docker/plugins", 0o755) 797 assert.NilError(c, err) 798 799 fileName := fmt.Sprintf("/etc/docker/plugins/%s.spec", netDrv) 800 err = os.WriteFile(fileName, []byte(url), 0o644) 801 assert.NilError(c, err) 802 803 ipamFileName := fmt.Sprintf("/etc/docker/plugins/%s.spec", ipamDrv) 804 err = os.WriteFile(ipamFileName, []byte(url), 0o644) 805 assert.NilError(c, err) 806 } 807 808 func (s *DockerSwarmSuite) TestSwarmNetworkPlugin(c *testing.T) { 809 ctx := testutil.GetContext(c) 810 mux := http.NewServeMux() 811 s.server = httptest.NewServer(mux) 812 assert.Assert(c, s.server != nil) // check that HTTP server has started 813 setupRemoteGlobalNetworkPlugin(c, mux, s.server.URL, globalNetworkPlugin, globalIPAMPlugin) 814 defer func() { 815 s.server.Close() 816 err := os.RemoveAll("/etc/docker/plugins") 817 assert.NilError(c, err) 818 }() 819 820 d := s.AddDaemon(ctx, c, true, true) 821 822 out, err := d.Cmd("network", "create", "-d", globalNetworkPlugin, "foo") 823 assert.ErrorContains(c, err, "", out) 824 assert.Assert(c, strings.Contains(out, "not supported in swarm mode"), out) 825 } 826 827 // Test case for #24712 828 func (s *DockerSwarmSuite) TestSwarmServiceEnvFile(c *testing.T) { 829 ctx := testutil.GetContext(c) 830 d := s.AddDaemon(ctx, c, true, true) 831 832 path := filepath.Join(d.Folder, "env.txt") 833 err := os.WriteFile(path, []byte("VAR1=A\nVAR2=A\n"), 0o644) 834 assert.NilError(c, err) 835 836 name := "worker" 837 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--env-file", path, "--env", "VAR1=B", "--env", "VAR1=C", "--env", "VAR2=", "--env", "VAR2", "--name", name, "busybox", "top") 838 assert.NilError(c, err, out) 839 assert.Assert(c, strings.TrimSpace(out) != "") 840 841 // The complete env is [VAR1=A VAR2=A VAR1=B VAR1=C VAR2= VAR2] and duplicates will be removed => [VAR1=C VAR2] 842 out, err = d.Cmd("inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.Env }}", name) 843 assert.NilError(c, err, out) 844 assert.Assert(c, strings.Contains(out, "[VAR1=C VAR2]"), out) 845 } 846 847 func (s *DockerSwarmSuite) TestSwarmServiceTTY(c *testing.T) { 848 ctx := testutil.GetContext(c) 849 d := s.AddDaemon(ctx, c, true, true) 850 851 name := "top" 852 853 ttyCheck := "if [ -t 0 ]; then echo TTY > /status; else echo none > /status; fi; exec top" 854 855 // Without --tty 856 expectedOutput := "none" 857 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "busybox", "sh", "-c", ttyCheck) 858 assert.NilError(c, err, out) 859 860 // Make sure task has been deployed. 861 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 862 863 // We need to get the container id. 864 out, err = d.Cmd("ps", "-q", "--no-trunc") 865 assert.NilError(c, err, out) 866 id := strings.TrimSpace(out) 867 868 out, err = d.Cmd("exec", id, "cat", "/status") 869 assert.NilError(c, err, out) 870 assert.Assert(c, strings.Contains(out, expectedOutput), "Expected '%s', but got %q", expectedOutput, out) 871 // Remove service 872 out, err = d.Cmd("service", "rm", name) 873 assert.NilError(c, err, out) 874 // Make sure container has been destroyed. 875 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(0)), poll.WithTimeout(defaultReconciliationTimeout)) 876 877 // With --tty 878 expectedOutput = "TTY" 879 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--tty", "busybox", "sh", "-c", ttyCheck) 880 assert.NilError(c, err, out) 881 882 // Make sure task has been deployed. 883 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 884 885 // We need to get the container id. 886 out, err = d.Cmd("ps", "-q", "--no-trunc") 887 assert.NilError(c, err, out) 888 id = strings.TrimSpace(out) 889 890 out, err = d.Cmd("exec", id, "cat", "/status") 891 assert.NilError(c, err, out) 892 assert.Assert(c, strings.Contains(out, expectedOutput), "Expected '%s', but got %q", expectedOutput, out) 893 } 894 895 func (s *DockerSwarmSuite) TestSwarmServiceTTYUpdate(c *testing.T) { 896 ctx := testutil.GetContext(c) 897 d := s.AddDaemon(ctx, c, true, true) 898 899 // Create a service 900 name := "top" 901 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "busybox", "top") 902 assert.NilError(c, err, out) 903 904 // Make sure task has been deployed. 905 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 906 907 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.TTY }}", name) 908 assert.NilError(c, err, out) 909 assert.Equal(c, strings.TrimSpace(out), "false") 910 911 out, err = d.Cmd("service", "update", "--detach", "--tty", name) 912 assert.NilError(c, err, out) 913 914 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.TTY }}", name) 915 assert.NilError(c, err, out) 916 assert.Equal(c, strings.TrimSpace(out), "true") 917 } 918 919 func (s *DockerSwarmSuite) TestSwarmServiceNetworkUpdate(c *testing.T) { 920 ctx := testutil.GetContext(c) 921 d := s.AddDaemon(ctx, c, true, true) 922 923 result := icmd.RunCmd(d.Command("network", "create", "-d", "overlay", "foo")) 924 result.Assert(c, icmd.Success) 925 fooNetwork := strings.TrimSpace(result.Combined()) 926 927 result = icmd.RunCmd(d.Command("network", "create", "-d", "overlay", "bar")) 928 result.Assert(c, icmd.Success) 929 barNetwork := strings.TrimSpace(result.Combined()) 930 931 result = icmd.RunCmd(d.Command("network", "create", "-d", "overlay", "baz")) 932 result.Assert(c, icmd.Success) 933 bazNetwork := strings.TrimSpace(result.Combined()) 934 935 // Create a service 936 name := "top" 937 result = icmd.RunCmd(d.Command("service", "create", "--detach", "--no-resolve-image", "--network", "foo", "--network", "bar", "--name", name, "busybox", "top")) 938 result.Assert(c, icmd.Success) 939 940 // Make sure task has been deployed. 941 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskNetworks(ctx), checker.DeepEquals(map[string]int{fooNetwork: 1, barNetwork: 1})), poll.WithTimeout(defaultReconciliationTimeout)) 942 943 // Remove a network 944 result = icmd.RunCmd(d.Command("service", "update", "--detach", "--network-rm", "foo", name)) 945 result.Assert(c, icmd.Success) 946 947 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskNetworks(ctx), checker.DeepEquals(map[string]int{barNetwork: 1})), poll.WithTimeout(defaultReconciliationTimeout)) 948 949 // Add a network 950 result = icmd.RunCmd(d.Command("service", "update", "--detach", "--network-add", "baz", name)) 951 result.Assert(c, icmd.Success) 952 953 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskNetworks(ctx), checker.DeepEquals(map[string]int{barNetwork: 1, bazNetwork: 1})), poll.WithTimeout(defaultReconciliationTimeout)) 954 } 955 956 func (s *DockerSwarmSuite) TestDNSConfig(c *testing.T) { 957 ctx := testutil.GetContext(c) 958 d := s.AddDaemon(ctx, c, true, true) 959 960 // Create a service 961 name := "top" 962 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--dns=1.2.3.4", "--dns-search=example.com", "--dns-option=timeout:3", "busybox", "top") 963 assert.NilError(c, err, out) 964 965 // Make sure task has been deployed. 966 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 967 968 // We need to get the container id. 969 out, err = d.Cmd("ps", "-a", "-q", "--no-trunc") 970 assert.NilError(c, err, out) 971 id := strings.TrimSpace(out) 972 973 // Compare against expected output. 974 expectedOutput1 := "nameserver 1.2.3.4" 975 expectedOutput2 := "search example.com" 976 expectedOutput3 := "options timeout:3" 977 out, err = d.Cmd("exec", id, "cat", "/etc/resolv.conf") 978 assert.NilError(c, err, out) 979 assert.Assert(c, strings.Contains(out, expectedOutput1), "Expected '%s', but got %q", expectedOutput1, out) 980 assert.Assert(c, strings.Contains(out, expectedOutput2), "Expected '%s', but got %q", expectedOutput2, out) 981 assert.Assert(c, strings.Contains(out, expectedOutput3), "Expected '%s', but got %q", expectedOutput3, out) 982 } 983 984 func (s *DockerSwarmSuite) TestDNSConfigUpdate(c *testing.T) { 985 ctx := testutil.GetContext(c) 986 d := s.AddDaemon(ctx, c, true, true) 987 988 // Create a service 989 name := "top" 990 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "busybox", "top") 991 assert.NilError(c, err, out) 992 993 // Make sure task has been deployed. 994 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 995 996 out, err = d.Cmd("service", "update", "--detach", "--dns-add=1.2.3.4", "--dns-search-add=example.com", "--dns-option-add=timeout:3", name) 997 assert.NilError(c, err, out) 998 999 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.DNSConfig }}", name) 1000 assert.NilError(c, err, out) 1001 assert.Equal(c, strings.TrimSpace(out), "{[1.2.3.4] [example.com] [timeout:3]}") 1002 } 1003 1004 func getNodeStatus(c *testing.T, d *daemon.Daemon) swarm.LocalNodeState { 1005 ctx := testutil.GetContext(c) 1006 info := d.SwarmInfo(ctx, c) 1007 return info.LocalNodeState 1008 } 1009 1010 func checkKeyIsEncrypted(d *daemon.Daemon) func(*testing.T) (interface{}, string) { 1011 return func(c *testing.T) (interface{}, string) { 1012 keyBytes, err := os.ReadFile(filepath.Join(d.Folder, "root", "swarm", "certificates", "swarm-node.key")) 1013 if err != nil { 1014 return fmt.Errorf("error reading key: %v", err), "" 1015 } 1016 1017 keyBlock, _ := pem.Decode(keyBytes) 1018 if keyBlock == nil { 1019 return fmt.Errorf("invalid PEM-encoded private key"), "" 1020 } 1021 1022 return keyutils.IsEncryptedPEMBlock(keyBlock), "" 1023 } 1024 } 1025 1026 func checkSwarmLockedToUnlocked(ctx context.Context, c *testing.T, d *daemon.Daemon) { 1027 // Wait for the PEM file to become unencrypted 1028 poll.WaitOn(c, pollCheck(c, checkKeyIsEncrypted(d), checker.Equals(false)), poll.WithTimeout(defaultReconciliationTimeout)) 1029 1030 d.RestartNode(c) 1031 poll.WaitOn(c, pollCheck(c, d.CheckLocalNodeState(ctx), checker.Equals(swarm.LocalNodeStateActive)), poll.WithTimeout(time.Second)) 1032 } 1033 1034 func checkSwarmUnlockedToLocked(ctx context.Context, c *testing.T, d *daemon.Daemon) { 1035 // Wait for the PEM file to become encrypted 1036 poll.WaitOn(c, pollCheck(c, checkKeyIsEncrypted(d), checker.Equals(true)), poll.WithTimeout(defaultReconciliationTimeout)) 1037 1038 d.RestartNode(c) 1039 poll.WaitOn(c, pollCheck(c, d.CheckLocalNodeState(ctx), checker.Equals(swarm.LocalNodeStateLocked)), poll.WithTimeout(time.Second)) 1040 } 1041 1042 func (s *DockerSwarmSuite) TestUnlockEngineAndUnlockedSwarm(c *testing.T) { 1043 ctx := testutil.GetContext(c) 1044 d := s.AddDaemon(ctx, c, false, false) 1045 1046 // unlocking a normal engine should return an error - it does not even ask for the key 1047 cmd := d.Command("swarm", "unlock") 1048 result := icmd.RunCmd(cmd) 1049 result.Assert(c, icmd.Expected{ 1050 ExitCode: 1, 1051 }) 1052 out := result.Combined() 1053 assert.Assert(c, strings.Contains(result.Combined(), "Error: This node is not part of a swarm"), out) 1054 assert.Assert(c, !strings.Contains(result.Combined(), "Please enter unlock key"), out) 1055 out, err := d.Cmd("swarm", "init") 1056 assert.NilError(c, err, out) 1057 1058 // unlocking an unlocked swarm should return an error - it does not even ask for the key 1059 cmd = d.Command("swarm", "unlock") 1060 result = icmd.RunCmd(cmd) 1061 result.Assert(c, icmd.Expected{ 1062 ExitCode: 1, 1063 }) 1064 out = result.Combined() 1065 assert.Assert(c, strings.Contains(result.Combined(), "Error: swarm is not locked"), out) 1066 assert.Assert(c, !strings.Contains(result.Combined(), "Please enter unlock key"), out) 1067 } 1068 1069 func (s *DockerSwarmSuite) TestSwarmInitLocked(c *testing.T) { 1070 ctx := testutil.GetContext(c) 1071 d := s.AddDaemon(ctx, c, false, false) 1072 1073 outs, err := d.Cmd("swarm", "init", "--autolock") 1074 assert.Assert(c, err == nil, outs) 1075 unlockKey := getUnlockKey(d, c, outs) 1076 1077 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1078 1079 // It starts off locked 1080 d.RestartNode(c) 1081 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateLocked) 1082 1083 cmd := d.Command("swarm", "unlock") 1084 cmd.Stdin = bytes.NewBufferString("wrong-secret-key") 1085 icmd.RunCmd(cmd).Assert(c, icmd.Expected{ 1086 ExitCode: 1, 1087 Err: "invalid key", 1088 }) 1089 1090 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateLocked) 1091 1092 cmd = d.Command("swarm", "unlock") 1093 cmd.Stdin = bytes.NewBufferString(unlockKey) 1094 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1095 1096 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1097 1098 outs, err = d.Cmd("node", "ls") 1099 assert.Assert(c, err == nil, outs) 1100 assert.Assert(c, !strings.Contains(outs, "Swarm is encrypted and needs to be unlocked"), outs) 1101 outs, err = d.Cmd("swarm", "update", "--autolock=false") 1102 assert.Assert(c, err == nil, outs) 1103 1104 checkSwarmLockedToUnlocked(ctx, c, d) 1105 1106 outs, err = d.Cmd("node", "ls") 1107 assert.Assert(c, err == nil, outs) 1108 assert.Assert(c, !strings.Contains(outs, "Swarm is encrypted and needs to be unlocked"), outs) 1109 } 1110 1111 func (s *DockerSwarmSuite) TestSwarmLeaveLocked(c *testing.T) { 1112 ctx := testutil.GetContext(c) 1113 d := s.AddDaemon(ctx, c, false, false) 1114 1115 outs, err := d.Cmd("swarm", "init", "--autolock") 1116 assert.Assert(c, err == nil, outs) 1117 1118 // It starts off locked 1119 d.RestartNode(c) 1120 1121 info := d.SwarmInfo(ctx, c) 1122 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateLocked) 1123 1124 outs, _ = d.Cmd("node", "ls") 1125 assert.Assert(c, strings.Contains(outs, "Swarm is encrypted and needs to be unlocked"), outs) 1126 // `docker swarm leave` a locked swarm without --force will return an error 1127 outs, _ = d.Cmd("swarm", "leave") 1128 assert.Assert(c, strings.Contains(outs, "Swarm is encrypted and locked."), outs) 1129 // It is OK for user to leave a locked swarm with --force 1130 outs, err = d.Cmd("swarm", "leave", "--force") 1131 assert.Assert(c, err == nil, outs) 1132 1133 info = d.SwarmInfo(ctx, c) 1134 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive) 1135 1136 outs, err = d.Cmd("swarm", "init") 1137 assert.Assert(c, err == nil, outs) 1138 1139 info = d.SwarmInfo(ctx, c) 1140 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 1141 } 1142 1143 func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *testing.T) { 1144 ctx := testutil.GetContext(c) 1145 d1 := s.AddDaemon(ctx, c, true, true) 1146 d2 := s.AddDaemon(ctx, c, true, true) 1147 d3 := s.AddDaemon(ctx, c, true, true) 1148 1149 // they start off unlocked 1150 d2.RestartNode(c) 1151 assert.Equal(c, getNodeStatus(c, d2), swarm.LocalNodeStateActive) 1152 1153 // stop this one so it does not get autolock info 1154 d2.Stop(c) 1155 1156 // enable autolock 1157 outs, err := d1.Cmd("swarm", "update", "--autolock") 1158 assert.Assert(c, err == nil, outs) 1159 unlockKey := getUnlockKey(d1, c, outs) 1160 1161 // The ones that got the cluster update should be set to locked 1162 for _, d := range []*daemon.Daemon{d1, d3} { 1163 checkSwarmUnlockedToLocked(ctx, c, d) 1164 1165 cmd := d.Command("swarm", "unlock") 1166 cmd.Stdin = bytes.NewBufferString(unlockKey) 1167 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1168 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1169 } 1170 1171 // d2 never got the cluster update, so it is still set to unlocked 1172 d2.StartNode(c) 1173 assert.Equal(c, getNodeStatus(c, d2), swarm.LocalNodeStateActive) 1174 1175 // d2 is now set to lock 1176 checkSwarmUnlockedToLocked(ctx, c, d2) 1177 1178 // leave it locked, and set the cluster to no longer autolock 1179 outs, err = d1.Cmd("swarm", "update", "--autolock=false") 1180 assert.Assert(c, err == nil, "out: %v", outs) 1181 1182 // the ones that got the update are now set to unlocked 1183 for _, d := range []*daemon.Daemon{d1, d3} { 1184 checkSwarmLockedToUnlocked(ctx, c, d) 1185 } 1186 1187 // d2 still locked 1188 assert.Equal(c, getNodeStatus(c, d2), swarm.LocalNodeStateLocked) 1189 1190 // unlock it 1191 cmd := d2.Command("swarm", "unlock") 1192 cmd.Stdin = bytes.NewBufferString(unlockKey) 1193 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1194 assert.Equal(c, getNodeStatus(c, d2), swarm.LocalNodeStateActive) 1195 1196 // once it's caught up, d2 is set to not be locked 1197 checkSwarmLockedToUnlocked(ctx, c, d2) 1198 1199 // managers who join now are never set to locked in the first place 1200 d4 := s.AddDaemon(ctx, c, true, true) 1201 d4.RestartNode(c) 1202 assert.Equal(c, getNodeStatus(c, d4), swarm.LocalNodeStateActive) 1203 } 1204 1205 func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *testing.T) { 1206 ctx := testutil.GetContext(c) 1207 d1 := s.AddDaemon(ctx, c, true, true) 1208 1209 // enable autolock 1210 outs, err := d1.Cmd("swarm", "update", "--autolock") 1211 assert.Assert(c, err == nil, "out: %v", outs) 1212 unlockKey := getUnlockKey(d1, c, outs) 1213 1214 // joined workers start off unlocked 1215 d2 := s.AddDaemon(ctx, c, true, false) 1216 d2.RestartNode(c) 1217 poll.WaitOn(c, pollCheck(c, d2.CheckLocalNodeState(ctx), checker.Equals(swarm.LocalNodeStateActive)), poll.WithTimeout(time.Second)) 1218 1219 // promote worker 1220 outs, err = d1.Cmd("node", "promote", d2.NodeID()) 1221 assert.NilError(c, err) 1222 assert.Assert(c, strings.Contains(outs, "promoted to a manager in the swarm"), outs) 1223 // join new manager node 1224 d3 := s.AddDaemon(ctx, c, true, true) 1225 1226 // both new nodes are locked 1227 for _, d := range []*daemon.Daemon{d2, d3} { 1228 checkSwarmUnlockedToLocked(ctx, c, d) 1229 1230 cmd := d.Command("swarm", "unlock") 1231 cmd.Stdin = bytes.NewBufferString(unlockKey) 1232 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1233 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1234 } 1235 1236 // demote manager back to worker - workers are not locked 1237 outs, err = d1.Cmd("node", "demote", d3.NodeID()) 1238 assert.NilError(c, err) 1239 assert.Assert(c, strings.Contains(outs, "demoted in the swarm"), outs) 1240 // Wait for it to actually be demoted, for the key and cert to be replaced. 1241 // Then restart and assert that the node is not locked. If we don't wait for the cert 1242 // to be replaced, then the node still has the manager TLS key which is still locked 1243 // (because we never want a manager TLS key to be on disk unencrypted if the cluster 1244 // is set to autolock) 1245 poll.WaitOn(c, pollCheck(c, d3.CheckControlAvailable(ctx), checker.False()), poll.WithTimeout(defaultReconciliationTimeout)) 1246 poll.WaitOn(c, pollCheck(c, func(c *testing.T) (interface{}, string) { 1247 certBytes, err := os.ReadFile(filepath.Join(d3.Folder, "root", "swarm", "certificates", "swarm-node.crt")) 1248 if err != nil { 1249 return "", fmt.Sprintf("error: %v", err) 1250 } 1251 certs, err := helpers.ParseCertificatesPEM(certBytes) 1252 if err == nil && len(certs) > 0 && len(certs[0].Subject.OrganizationalUnit) > 0 { 1253 return certs[0].Subject.OrganizationalUnit[0], "" 1254 } 1255 return "", "could not get organizational unit from certificate" 1256 }, checker.Equals("swarm-worker")), poll.WithTimeout(defaultReconciliationTimeout)) 1257 1258 // by now, it should *never* be locked on restart 1259 d3.RestartNode(c) 1260 poll.WaitOn(c, pollCheck(c, d3.CheckLocalNodeState(ctx), checker.Equals(swarm.LocalNodeStateActive)), poll.WithTimeout(time.Second)) 1261 } 1262 1263 func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *testing.T) { 1264 ctx := testutil.GetContext(c) 1265 d := s.AddDaemon(ctx, c, true, true) 1266 1267 outs, err := d.Cmd("swarm", "update", "--autolock") 1268 assert.Assert(c, err == nil, "out: %v", outs) 1269 unlockKey := getUnlockKey(d, c, outs) 1270 1271 // Rotate multiple times 1272 for i := 0; i != 3; i++ { 1273 outs, err = d.Cmd("swarm", "unlock-key", "-q", "--rotate") 1274 assert.Assert(c, err == nil, "out: %v", outs) 1275 // Strip \n 1276 newUnlockKey := outs[:len(outs)-1] 1277 assert.Assert(c, newUnlockKey != "") 1278 assert.Assert(c, newUnlockKey != unlockKey) 1279 1280 d.RestartNode(c) 1281 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateLocked) 1282 1283 outs, _ = d.Cmd("node", "ls") 1284 assert.Assert(c, strings.Contains(outs, "Swarm is encrypted and needs to be unlocked"), outs) 1285 cmd := d.Command("swarm", "unlock") 1286 cmd.Stdin = bytes.NewBufferString(unlockKey) 1287 result := icmd.RunCmd(cmd) 1288 1289 if result.Error == nil { 1290 // On occasion, the daemon may not have finished 1291 // rotating the KEK before restarting. The test is 1292 // intentionally written to explore this behavior. 1293 // When this happens, unlocking with the old key will 1294 // succeed. If we wait for the rotation to happen and 1295 // restart again, the new key should be required this 1296 // time. 1297 1298 time.Sleep(3 * time.Second) 1299 1300 d.RestartNode(c) 1301 1302 cmd = d.Command("swarm", "unlock") 1303 cmd.Stdin = bytes.NewBufferString(unlockKey) 1304 result = icmd.RunCmd(cmd) 1305 } 1306 result.Assert(c, icmd.Expected{ 1307 ExitCode: 1, 1308 Err: "invalid key", 1309 }) 1310 1311 outs, _ = d.Cmd("node", "ls") 1312 assert.Assert(c, strings.Contains(outs, "Swarm is encrypted and needs to be unlocked"), outs) 1313 cmd = d.Command("swarm", "unlock") 1314 cmd.Stdin = bytes.NewBufferString(newUnlockKey) 1315 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1316 1317 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1318 1319 retry := 0 1320 for { 1321 // an issue sometimes prevents leader to be available right away 1322 outs, err = d.Cmd("node", "ls") 1323 if err != nil && retry < 5 { 1324 if strings.Contains(outs, "swarm does not have a leader") { 1325 retry++ 1326 time.Sleep(3 * time.Second) 1327 continue 1328 } 1329 } 1330 assert.NilError(c, err) 1331 assert.Assert(c, !strings.Contains(outs, "Swarm is encrypted and needs to be unlocked"), outs) 1332 break 1333 } 1334 1335 unlockKey = newUnlockKey 1336 } 1337 } 1338 1339 // This differs from `TestSwarmRotateUnlockKey` because that one rotates a single node, which is the leader. 1340 // This one keeps the leader up, and asserts that other manager nodes in the cluster also have their unlock 1341 // key rotated. 1342 func (s *DockerSwarmSuite) TestSwarmClusterRotateUnlockKey(c *testing.T) { 1343 if runtime.GOARCH == "s390x" { 1344 c.Skip("Disabled on s390x") 1345 } 1346 if runtime.GOARCH == "ppc64le" { 1347 c.Skip("Disabled on ppc64le") 1348 } 1349 ctx := testutil.GetContext(c) 1350 1351 d1 := s.AddDaemon(ctx, c, true, true) // leader - don't restart this one, we don't want leader election delays 1352 d2 := s.AddDaemon(ctx, c, true, true) 1353 d3 := s.AddDaemon(ctx, c, true, true) 1354 1355 outs, err := d1.Cmd("swarm", "update", "--autolock") 1356 assert.Assert(c, err == nil, outs) 1357 unlockKey := getUnlockKey(d1, c, outs) 1358 1359 // Rotate multiple times 1360 for i := 0; i != 3; i++ { 1361 outs, err = d1.Cmd("swarm", "unlock-key", "-q", "--rotate") 1362 assert.Assert(c, err == nil, outs) 1363 // Strip \n 1364 newUnlockKey := outs[:len(outs)-1] 1365 assert.Assert(c, newUnlockKey != "") 1366 assert.Assert(c, newUnlockKey != unlockKey) 1367 1368 d2.RestartNode(c) 1369 d3.RestartNode(c) 1370 1371 for _, d := range []*daemon.Daemon{d2, d3} { 1372 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateLocked) 1373 1374 outs, _ := d.Cmd("node", "ls") 1375 assert.Assert(c, strings.Contains(outs, "Swarm is encrypted and needs to be unlocked"), outs) 1376 cmd := d.Command("swarm", "unlock") 1377 cmd.Stdin = bytes.NewBufferString(unlockKey) 1378 result := icmd.RunCmd(cmd) 1379 1380 if result.Error == nil { 1381 // On occasion, the daemon may not have finished 1382 // rotating the KEK before restarting. The test is 1383 // intentionally written to explore this behavior. 1384 // When this happens, unlocking with the old key will 1385 // succeed. If we wait for the rotation to happen and 1386 // restart again, the new key should be required this 1387 // time. 1388 1389 time.Sleep(3 * time.Second) 1390 1391 d.RestartNode(c) 1392 1393 cmd = d.Command("swarm", "unlock") 1394 cmd.Stdin = bytes.NewBufferString(unlockKey) 1395 result = icmd.RunCmd(cmd) 1396 } 1397 result.Assert(c, icmd.Expected{ 1398 ExitCode: 1, 1399 Err: "invalid key", 1400 }) 1401 1402 outs, _ = d.Cmd("node", "ls") 1403 assert.Assert(c, strings.Contains(outs, "Swarm is encrypted and needs to be unlocked"), outs) 1404 cmd = d.Command("swarm", "unlock") 1405 cmd.Stdin = bytes.NewBufferString(newUnlockKey) 1406 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1407 1408 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1409 1410 retry := 0 1411 for { 1412 // an issue sometimes prevents leader to be available right away 1413 outs, err = d.Cmd("node", "ls") 1414 if err != nil && retry < 5 { 1415 if strings.Contains(outs, "swarm does not have a leader") { 1416 retry++ 1417 c.Logf("[%s] got 'swarm does not have a leader'. retrying (attempt %d/5)", d.ID(), retry) 1418 time.Sleep(3 * time.Second) 1419 continue 1420 } else { 1421 c.Logf("[%s] gave error: '%v'. retrying (attempt %d/5): %s", d.ID(), err, retry, outs) 1422 } 1423 } 1424 assert.NilError(c, err, "[%s] failed after %d retries: %v (%s)", d.ID(), retry, err, outs) 1425 assert.Assert(c, !strings.Contains(outs, "Swarm is encrypted and needs to be unlocked"), outs) 1426 break 1427 } 1428 } 1429 1430 unlockKey = newUnlockKey 1431 } 1432 } 1433 1434 func (s *DockerSwarmSuite) TestSwarmAlternateLockUnlock(c *testing.T) { 1435 ctx := testutil.GetContext(c) 1436 d := s.AddDaemon(ctx, c, true, true) 1437 1438 for i := 0; i < 2; i++ { 1439 // set to lock 1440 outs, err := d.Cmd("swarm", "update", "--autolock") 1441 assert.Assert(c, err == nil, "out: %v", outs) 1442 assert.Assert(c, strings.Contains(outs, "docker swarm unlock"), outs) 1443 unlockKey := getUnlockKey(d, c, outs) 1444 1445 checkSwarmUnlockedToLocked(ctx, c, d) 1446 1447 cmd := d.Command("swarm", "unlock") 1448 cmd.Stdin = bytes.NewBufferString(unlockKey) 1449 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1450 1451 assert.Equal(c, getNodeStatus(c, d), swarm.LocalNodeStateActive) 1452 1453 outs, err = d.Cmd("swarm", "update", "--autolock=false") 1454 assert.Assert(c, err == nil, "out: %v", outs) 1455 1456 checkSwarmLockedToUnlocked(ctx, c, d) 1457 } 1458 } 1459 1460 func (s *DockerSwarmSuite) TestExtraHosts(c *testing.T) { 1461 ctx := testutil.GetContext(c) 1462 d := s.AddDaemon(ctx, c, true, true) 1463 1464 // Create a service 1465 name := "top" 1466 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--host=example.com:1.2.3.4", "busybox", "top") 1467 assert.NilError(c, err, out) 1468 1469 // Make sure task has been deployed. 1470 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 1471 1472 // We need to get the container id. 1473 out, err = d.Cmd("ps", "-a", "-q", "--no-trunc") 1474 assert.NilError(c, err, out) 1475 id := strings.TrimSpace(out) 1476 1477 // Compare against expected output. 1478 expectedOutput := "1.2.3.4\texample.com" 1479 out, err = d.Cmd("exec", id, "cat", "/etc/hosts") 1480 assert.NilError(c, err, out) 1481 assert.Assert(c, strings.Contains(out, expectedOutput), "Expected '%s', but got %q", expectedOutput, out) 1482 } 1483 1484 func (s *DockerSwarmSuite) TestSwarmManagerAddress(c *testing.T) { 1485 ctx := testutil.GetContext(c) 1486 d1 := s.AddDaemon(ctx, c, true, true) 1487 d2 := s.AddDaemon(ctx, c, true, false) 1488 d3 := s.AddDaemon(ctx, c, true, false) 1489 1490 // Manager Addresses will always show Node 1's address 1491 expectedOutput := fmt.Sprintf("127.0.0.1:%d", d1.SwarmPort) 1492 1493 out, err := d1.Cmd("info", "--format", "{{ (index .Swarm.RemoteManagers 0).Addr }}") 1494 assert.NilError(c, err, out) 1495 assert.Assert(c, strings.Contains(out, expectedOutput), out) 1496 1497 out, err = d2.Cmd("info", "--format", "{{ (index .Swarm.RemoteManagers 0).Addr }}") 1498 assert.NilError(c, err, out) 1499 assert.Assert(c, strings.Contains(out, expectedOutput), out) 1500 1501 out, err = d3.Cmd("info", "--format", "{{ (index .Swarm.RemoteManagers 0).Addr }}") 1502 assert.NilError(c, err, out) 1503 assert.Assert(c, strings.Contains(out, expectedOutput), out) 1504 } 1505 1506 func (s *DockerSwarmSuite) TestSwarmNetworkIPAMOptions(c *testing.T) { 1507 ctx := testutil.GetContext(c) 1508 d := s.AddDaemon(ctx, c, true, true) 1509 1510 out, err := d.Cmd("network", "create", "-d", "overlay", "--ipam-opt", "foo=bar", "foo") 1511 assert.NilError(c, err, out) 1512 assert.Assert(c, strings.TrimSpace(out) != "") 1513 1514 out, err = d.Cmd("network", "inspect", "--format", "{{.IPAM.Options}}", "foo") 1515 out = strings.TrimSpace(out) 1516 assert.NilError(c, err, out) 1517 assert.Assert(c, strings.Contains(out, "foo:bar"), out) 1518 assert.Assert(c, strings.Contains(out, "com.docker.network.ipam.serial:true"), out) 1519 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--network=foo", "--name", "top", "busybox", "top") 1520 assert.NilError(c, err, out) 1521 1522 // make sure task has been deployed. 1523 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 1524 1525 out, err = d.Cmd("network", "inspect", "--format", "{{.IPAM.Options}}", "foo") 1526 assert.NilError(c, err, out) 1527 assert.Assert(c, strings.Contains(out, "foo:bar"), out) 1528 assert.Assert(c, strings.Contains(out, "com.docker.network.ipam.serial:true"), out) 1529 } 1530 1531 // Test case for issue #27866, which did not allow NW name that is the prefix of a swarm NW ID. 1532 // e.g. if the ingress ID starts with "n1", it was impossible to create a NW named "n1". 1533 func (s *DockerSwarmSuite) TestSwarmNetworkCreateIssue27866(c *testing.T) { 1534 ctx := testutil.GetContext(c) 1535 d := s.AddDaemon(ctx, c, true, true) 1536 out, err := d.Cmd("network", "inspect", "-f", "{{.Id}}", "ingress") 1537 assert.NilError(c, err, "out: %v", out) 1538 ingressID := strings.TrimSpace(out) 1539 assert.Assert(c, ingressID != "") 1540 1541 // create a network of which name is the prefix of the ID of an overlay network 1542 // (ingressID in this case) 1543 newNetName := ingressID[0:2] 1544 out, err = d.Cmd("network", "create", "--driver", "overlay", newNetName) 1545 // In #27866, it was failing because of "network with name %s already exists" 1546 assert.NilError(c, err, "out: %v", out) 1547 out, err = d.Cmd("network", "rm", newNetName) 1548 assert.NilError(c, err, "out: %v", out) 1549 } 1550 1551 // Test case for https://github.com/Prakhar-Agarwal-byte/moby/pull/27938#issuecomment-265768303 1552 // This test creates two networks with the same name sequentially, with various drivers. 1553 // Since the operations in this test are done sequentially, the 2nd call should fail with 1554 // "network with name FOO already exists". 1555 // Note that it is to ok have multiple networks with the same name if the operations are done 1556 // in parallel. (#18864) 1557 func (s *DockerSwarmSuite) TestSwarmNetworkCreateDup(c *testing.T) { 1558 ctx := testutil.GetContext(c) 1559 d := s.AddDaemon(ctx, c, true, true) 1560 drivers := []string{"bridge", "overlay"} 1561 for i, driver1 := range drivers { 1562 for _, driver2 := range drivers { 1563 c.Run(fmt.Sprintf("driver %s then %s", driver1, driver2), func(c *testing.T) { 1564 nwName := fmt.Sprintf("network-test-%d", i) 1565 out, err := d.Cmd("network", "create", "--driver", driver1, nwName) 1566 assert.NilError(c, err, "out: %v", out) 1567 out, err = d.Cmd("network", "create", "--driver", driver2, nwName) 1568 assert.Assert(c, strings.Contains(out, fmt.Sprintf("network with name %s already exists", nwName)), out) 1569 assert.ErrorContains(c, err, "") 1570 out, err = d.Cmd("network", "rm", nwName) 1571 assert.NilError(c, err, "out: %v", out) 1572 }) 1573 } 1574 } 1575 } 1576 1577 func (s *DockerSwarmSuite) TestSwarmPublishDuplicatePorts(c *testing.T) { 1578 ctx := testutil.GetContext(c) 1579 d := s.AddDaemon(ctx, c, true, true) 1580 1581 out, err := d.Cmd("service", "create", "--no-resolve-image", "--detach=true", "--publish", "5005:80", "--publish", "5006:80", "--publish", "80", "--publish", "80", "busybox", "top") 1582 assert.NilError(c, err, out) 1583 id := strings.TrimSpace(out) 1584 1585 // make sure task has been deployed. 1586 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 1587 1588 // Total len = 4, with 2 dynamic ports and 2 non-dynamic ports 1589 // Dynamic ports are likely to be 30000 and 30001 but doesn't matter 1590 out, err = d.Cmd("service", "inspect", "--format", "{{.Endpoint.Ports}} len={{len .Endpoint.Ports}}", id) 1591 assert.NilError(c, err, out) 1592 assert.Assert(c, strings.Contains(out, "len=4"), out) 1593 assert.Assert(c, strings.Contains(out, "{ tcp 80 5005 ingress}"), out) 1594 assert.Assert(c, strings.Contains(out, "{ tcp 80 5006 ingress}"), out) 1595 } 1596 1597 func (s *DockerSwarmSuite) TestSwarmJoinWithDrain(c *testing.T) { 1598 ctx := testutil.GetContext(c) 1599 d := s.AddDaemon(ctx, c, true, true) 1600 1601 out, err := d.Cmd("node", "ls") 1602 assert.NilError(c, err) 1603 assert.Assert(c, !strings.Contains(out, "Drain"), out) 1604 out, err = d.Cmd("swarm", "join-token", "-q", "manager") 1605 assert.NilError(c, err) 1606 assert.Assert(c, strings.TrimSpace(out) != "") 1607 1608 token := strings.TrimSpace(out) 1609 1610 d1 := s.AddDaemon(ctx, c, false, false) 1611 1612 out, err = d1.Cmd("swarm", "join", "--availability=drain", "--token", token, d.SwarmListenAddr()) 1613 assert.NilError(c, err) 1614 assert.Assert(c, strings.TrimSpace(out) != "") 1615 1616 out, err = d.Cmd("node", "ls") 1617 assert.NilError(c, err) 1618 assert.Assert(c, strings.Contains(out, "Drain"), out) 1619 out, err = d1.Cmd("node", "ls") 1620 assert.NilError(c, err) 1621 assert.Assert(c, strings.Contains(out, "Drain"), out) 1622 } 1623 1624 func (s *DockerSwarmSuite) TestSwarmInitWithDrain(c *testing.T) { 1625 ctx := testutil.GetContext(c) 1626 d := s.AddDaemon(ctx, c, false, false) 1627 1628 out, err := d.Cmd("swarm", "init", "--availability", "drain") 1629 assert.NilError(c, err, "out: %v", out) 1630 1631 out, err = d.Cmd("node", "ls") 1632 assert.NilError(c, err) 1633 assert.Assert(c, strings.Contains(out, "Drain")) 1634 } 1635 1636 func (s *DockerSwarmSuite) TestSwarmReadonlyRootfs(c *testing.T) { 1637 testRequires(c, DaemonIsLinux, UserNamespaceROMount) 1638 ctx := testutil.GetContext(c) 1639 1640 d := s.AddDaemon(ctx, c, true, true) 1641 1642 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "top", "--read-only", "busybox", "top") 1643 assert.NilError(c, err, out) 1644 1645 // make sure task has been deployed. 1646 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 1647 1648 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.ReadOnly }}", "top") 1649 assert.NilError(c, err, out) 1650 assert.Equal(c, strings.TrimSpace(out), "true") 1651 1652 containers := d.ActiveContainers(testutil.GetContext(c), c) 1653 out, err = d.Cmd("inspect", "--type", "container", "--format", "{{.HostConfig.ReadonlyRootfs}}", containers[0]) 1654 assert.NilError(c, err, out) 1655 assert.Equal(c, strings.TrimSpace(out), "true") 1656 } 1657 1658 func (s *DockerSwarmSuite) TestSwarmStopSignal(c *testing.T) { 1659 ctx := testutil.GetContext(c) 1660 testRequires(c, DaemonIsLinux, UserNamespaceROMount) 1661 1662 d := s.AddDaemon(ctx, c, true, true) 1663 1664 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "top", "--stop-signal=SIGHUP", "busybox", "top") 1665 assert.NilError(c, err, out) 1666 1667 // make sure task has been deployed. 1668 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 1669 1670 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.StopSignal }}", "top") 1671 assert.NilError(c, err, out) 1672 assert.Equal(c, strings.TrimSpace(out), "SIGHUP") 1673 1674 containers := d.ActiveContainers(testutil.GetContext(c), c) 1675 out, err = d.Cmd("inspect", "--type", "container", "--format", "{{.Config.StopSignal}}", containers[0]) 1676 assert.NilError(c, err, out) 1677 assert.Equal(c, strings.TrimSpace(out), "SIGHUP") 1678 1679 out, err = d.Cmd("service", "update", "--detach", "--stop-signal=SIGUSR1", "top") 1680 assert.NilError(c, err, out) 1681 1682 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.StopSignal }}", "top") 1683 assert.NilError(c, err, out) 1684 assert.Equal(c, strings.TrimSpace(out), "SIGUSR1") 1685 } 1686 1687 func (s *DockerSwarmSuite) TestSwarmServiceLsFilterMode(c *testing.T) { 1688 ctx := testutil.GetContext(c) 1689 d := s.AddDaemon(ctx, c, true, true) 1690 1691 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "top1", "busybox", "top") 1692 assert.NilError(c, err, out) 1693 assert.Assert(c, strings.TrimSpace(out) != "") 1694 1695 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "top2", "--mode=global", "busybox", "top") 1696 assert.NilError(c, err, out) 1697 assert.Assert(c, strings.TrimSpace(out) != "") 1698 1699 // make sure task has been deployed. 1700 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(2)), poll.WithTimeout(defaultReconciliationTimeout)) 1701 1702 out, err = d.Cmd("service", "ls") 1703 assert.NilError(c, err, out) 1704 assert.Assert(c, strings.Contains(out, "top1"), out) 1705 assert.Assert(c, strings.Contains(out, "top2"), out) 1706 assert.Assert(c, !strings.Contains(out, "localnet"), out) 1707 out, err = d.Cmd("service", "ls", "--filter", "mode=global") 1708 assert.Assert(c, !strings.Contains(out, "top1"), out) 1709 assert.Assert(c, strings.Contains(out, "top2"), out) 1710 assert.NilError(c, err, out) 1711 1712 out, err = d.Cmd("service", "ls", "--filter", "mode=replicated") 1713 assert.NilError(c, err, out) 1714 assert.Assert(c, strings.Contains(out, "top1"), out) 1715 assert.Assert(c, !strings.Contains(out, "top2"), out) 1716 } 1717 1718 func (s *DockerSwarmSuite) TestSwarmInitUnspecifiedDataPathAddr(c *testing.T) { 1719 ctx := testutil.GetContext(c) 1720 d := s.AddDaemon(ctx, c, false, false) 1721 1722 out, err := d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0") 1723 assert.ErrorContains(c, err, "") 1724 assert.Assert(c, strings.Contains(out, "data path address must be a non-zero IP"), out) 1725 out, err = d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0:2000") 1726 assert.ErrorContains(c, err, "") 1727 assert.Assert(c, strings.Contains(out, "data path address must be a non-zero IP"), out) 1728 } 1729 1730 func (s *DockerSwarmSuite) TestSwarmJoinLeave(c *testing.T) { 1731 ctx := testutil.GetContext(c) 1732 d := s.AddDaemon(ctx, c, true, true) 1733 1734 out, err := d.Cmd("swarm", "join-token", "-q", "worker") 1735 assert.NilError(c, err) 1736 assert.Assert(c, strings.TrimSpace(out) != "") 1737 1738 token := strings.TrimSpace(out) 1739 1740 // Verify that back to back join/leave does not cause panics 1741 d1 := s.AddDaemon(ctx, c, false, false) 1742 for i := 0; i < 10; i++ { 1743 out, err = d1.Cmd("swarm", "join", "--token", token, d.SwarmListenAddr()) 1744 assert.NilError(c, err) 1745 assert.Assert(c, strings.TrimSpace(out) != "") 1746 1747 _, err = d1.Cmd("swarm", "leave") 1748 assert.NilError(c, err) 1749 } 1750 } 1751 1752 const defaultRetryCount = 10 1753 1754 func waitForEvent(c *testing.T, d *daemon.Daemon, since string, filter string, event string, retry int) string { 1755 if retry < 1 { 1756 c.Fatalf("retry count %d is invalid. It should be no less than 1", retry) 1757 return "" 1758 } 1759 var out string 1760 for i := 0; i < retry; i++ { 1761 until := daemonUnixTime(c) 1762 var err error 1763 if len(filter) > 0 { 1764 out, err = d.Cmd("events", "--since", since, "--until", until, filter) 1765 } else { 1766 out, err = d.Cmd("events", "--since", since, "--until", until) 1767 } 1768 assert.NilError(c, err, out) 1769 if strings.Contains(out, event) { 1770 return strings.TrimSpace(out) 1771 } 1772 // no need to sleep after last retry 1773 if i < retry-1 { 1774 time.Sleep(200 * time.Millisecond) 1775 } 1776 } 1777 c.Fatalf("docker events output '%s' doesn't contain event '%s'", out, event) 1778 return "" 1779 } 1780 1781 func (s *DockerSwarmSuite) TestSwarmClusterEventsSource(c *testing.T) { 1782 ctx := testutil.GetContext(c) 1783 d1 := s.AddDaemon(ctx, c, true, true) 1784 d2 := s.AddDaemon(ctx, c, true, true) 1785 d3 := s.AddDaemon(ctx, c, true, false) 1786 1787 // create a network 1788 out, err := d1.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 1789 assert.NilError(c, err, out) 1790 networkID := strings.TrimSpace(out) 1791 assert.Assert(c, networkID != "") 1792 1793 // d1, d2 are managers that can get swarm events 1794 waitForEvent(c, d1, "0", "-f scope=swarm", "network create "+networkID, defaultRetryCount) 1795 waitForEvent(c, d2, "0", "-f scope=swarm", "network create "+networkID, defaultRetryCount) 1796 1797 // d3 is a worker, not able to get cluster events 1798 out = waitForEvent(c, d3, "0", "-f scope=swarm", "", 1) 1799 assert.Assert(c, !strings.Contains(out, "network create "), out) 1800 } 1801 1802 func (s *DockerSwarmSuite) TestSwarmClusterEventsScope(c *testing.T) { 1803 ctx := testutil.GetContext(c) 1804 d := s.AddDaemon(ctx, c, true, true) 1805 1806 // create a service 1807 out, err := d.Cmd("service", "create", "--no-resolve-image", "--name", "test", "--detach=false", "busybox", "top") 1808 assert.NilError(c, err, out) 1809 serviceID := strings.Split(out, "\n")[0] 1810 1811 // scope swarm filters cluster events 1812 out = waitForEvent(c, d, "0", "-f scope=swarm", "service create "+serviceID, defaultRetryCount) 1813 assert.Assert(c, !strings.Contains(out, "container create "), out) 1814 // all events are returned if scope is not specified 1815 waitForEvent(c, d, "0", "", "service create "+serviceID, 1) 1816 waitForEvent(c, d, "0", "", "container create ", defaultRetryCount) 1817 1818 // scope local only shows non-cluster events 1819 out = waitForEvent(c, d, "0", "-f scope=local", "container create ", 1) 1820 assert.Assert(c, !strings.Contains(out, "service create "), out) 1821 } 1822 1823 func (s *DockerSwarmSuite) TestSwarmClusterEventsType(c *testing.T) { 1824 ctx := testutil.GetContext(c) 1825 d := s.AddDaemon(ctx, c, true, true) 1826 1827 // create a service 1828 out, err := d.Cmd("service", "create", "--no-resolve-image", "--name", "test", "--detach=false", "busybox", "top") 1829 assert.NilError(c, err, out) 1830 serviceID := strings.Split(out, "\n")[0] 1831 1832 // create a network 1833 out, err = d.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 1834 assert.NilError(c, err, out) 1835 networkID := strings.TrimSpace(out) 1836 assert.Assert(c, networkID != "") 1837 1838 // filter by service 1839 out = waitForEvent(c, d, "0", "-f type=service", "service create "+serviceID, defaultRetryCount) 1840 assert.Assert(c, !strings.Contains(out, "network create"), out) 1841 // filter by network 1842 out = waitForEvent(c, d, "0", "-f type=network", "network create "+networkID, defaultRetryCount) 1843 assert.Assert(c, !strings.Contains(out, "service create"), out) 1844 } 1845 1846 func (s *DockerSwarmSuite) TestSwarmClusterEventsService(c *testing.T) { 1847 ctx := testutil.GetContext(c) 1848 d := s.AddDaemon(ctx, c, true, true) 1849 1850 // create a service 1851 out, err := d.Cmd("service", "create", "--no-resolve-image", "--name", "test", "--detach=false", "busybox", "top") 1852 assert.NilError(c, err, out) 1853 serviceID := strings.Split(out, "\n")[0] 1854 1855 // validate service create event 1856 waitForEvent(c, d, "0", "-f scope=swarm", "service create "+serviceID, defaultRetryCount) 1857 1858 t1 := daemonUnixTime(c) 1859 out, err = d.Cmd("service", "update", "--force", "--detach=false", "test") 1860 assert.NilError(c, err, out) 1861 1862 // wait for service update start 1863 out = waitForEvent(c, d, t1, "-f scope=swarm", "service update "+serviceID, defaultRetryCount) 1864 assert.Assert(c, strings.Contains(out, "updatestate.new=updating"), out) 1865 // allow service update complete. This is a service with 1 instance 1866 time.Sleep(400 * time.Millisecond) 1867 out = waitForEvent(c, d, t1, "-f scope=swarm", "service update "+serviceID, defaultRetryCount) 1868 assert.Assert(c, strings.Contains(out, "updatestate.new=completed, updatestate.old=updating"), out) 1869 // scale service 1870 t2 := daemonUnixTime(c) 1871 out, err = d.Cmd("service", "scale", "test=3") 1872 assert.NilError(c, err, out) 1873 1874 out = waitForEvent(c, d, t2, "-f scope=swarm", "service update "+serviceID, defaultRetryCount) 1875 assert.Assert(c, strings.Contains(out, "replicas.new=3, replicas.old=1"), out) 1876 // remove service 1877 t3 := daemonUnixTime(c) 1878 out, err = d.Cmd("service", "rm", "test") 1879 assert.NilError(c, err, out) 1880 1881 waitForEvent(c, d, t3, "-f scope=swarm", "service remove "+serviceID, defaultRetryCount) 1882 } 1883 1884 func (s *DockerSwarmSuite) TestSwarmClusterEventsNode(c *testing.T) { 1885 ctx := testutil.GetContext(c) 1886 d1 := s.AddDaemon(ctx, c, true, true) 1887 s.AddDaemon(ctx, c, true, true) 1888 d3 := s.AddDaemon(ctx, c, true, true) 1889 1890 d3ID := d3.NodeID() 1891 waitForEvent(c, d1, "0", "-f scope=swarm", "node create "+d3ID, defaultRetryCount) 1892 1893 t1 := daemonUnixTime(c) 1894 out, err := d1.Cmd("node", "update", "--availability=pause", d3ID) 1895 assert.NilError(c, err, out) 1896 1897 // filter by type 1898 out = waitForEvent(c, d1, t1, "-f type=node", "node update "+d3ID, defaultRetryCount) 1899 assert.Assert(c, strings.Contains(out, "availability.new=pause, availability.old=active"), out) 1900 t2 := daemonUnixTime(c) 1901 out, err = d1.Cmd("node", "demote", d3ID) 1902 assert.NilError(c, err, out) 1903 1904 waitForEvent(c, d1, t2, "-f type=node", "node update "+d3ID, defaultRetryCount) 1905 1906 t3 := daemonUnixTime(c) 1907 out, err = d1.Cmd("node", "rm", "-f", d3ID) 1908 assert.NilError(c, err, out) 1909 1910 // filter by scope 1911 waitForEvent(c, d1, t3, "-f scope=swarm", "node remove "+d3ID, defaultRetryCount) 1912 } 1913 1914 func (s *DockerSwarmSuite) TestSwarmClusterEventsNetwork(c *testing.T) { 1915 ctx := testutil.GetContext(c) 1916 d := s.AddDaemon(ctx, c, true, true) 1917 1918 // create a network 1919 out, err := d.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 1920 assert.NilError(c, err, out) 1921 networkID := strings.TrimSpace(out) 1922 1923 waitForEvent(c, d, "0", "-f scope=swarm", "network create "+networkID, defaultRetryCount) 1924 1925 // remove network 1926 t1 := daemonUnixTime(c) 1927 out, err = d.Cmd("network", "rm", "foo") 1928 assert.NilError(c, err, out) 1929 1930 // filtered by network 1931 waitForEvent(c, d, t1, "-f type=network", "network remove "+networkID, defaultRetryCount) 1932 } 1933 1934 func (s *DockerSwarmSuite) TestSwarmClusterEventsSecret(c *testing.T) { 1935 ctx := testutil.GetContext(c) 1936 d := s.AddDaemon(ctx, c, true, true) 1937 1938 testName := "test_secret" 1939 id := d.CreateSecret(c, swarm.SecretSpec{ 1940 Annotations: swarm.Annotations{ 1941 Name: testName, 1942 }, 1943 Data: []byte("TESTINGDATA"), 1944 }) 1945 assert.Assert(c, id != "", "secrets: %s", id) 1946 1947 waitForEvent(c, d, "0", "-f scope=swarm", "secret create "+id, defaultRetryCount) 1948 1949 t1 := daemonUnixTime(c) 1950 d.DeleteSecret(c, id) 1951 // filtered by secret 1952 waitForEvent(c, d, t1, "-f type=secret", "secret remove "+id, defaultRetryCount) 1953 } 1954 1955 func (s *DockerSwarmSuite) TestSwarmClusterEventsConfig(c *testing.T) { 1956 ctx := testutil.GetContext(c) 1957 d := s.AddDaemon(ctx, c, true, true) 1958 1959 testName := "test_config" 1960 id := d.CreateConfig(c, swarm.ConfigSpec{ 1961 Annotations: swarm.Annotations{ 1962 Name: testName, 1963 }, 1964 Data: []byte("TESTINGDATA"), 1965 }) 1966 assert.Assert(c, id != "", "configs: %s", id) 1967 1968 waitForEvent(c, d, "0", "-f scope=swarm", "config create "+id, defaultRetryCount) 1969 1970 t1 := daemonUnixTime(c) 1971 d.DeleteConfig(c, id) 1972 // filtered by config 1973 waitForEvent(c, d, t1, "-f type=config", "config remove "+id, defaultRetryCount) 1974 } 1975 1976 func getUnlockKey(d *daemon.Daemon, c *testing.T, autolockOutput string) string { 1977 unlockKey, err := d.Cmd("swarm", "unlock-key", "-q") 1978 assert.Assert(c, err == nil, unlockKey) 1979 unlockKey = strings.TrimSuffix(unlockKey, "\n") 1980 1981 // Check that "docker swarm init --autolock" or "docker swarm update --autolock" 1982 // contains all the expected strings, including the unlock key 1983 assert.Assert(c, strings.Contains(autolockOutput, "docker swarm unlock"), autolockOutput) 1984 assert.Assert(c, strings.Contains(autolockOutput, unlockKey), autolockOutput) 1985 return unlockKey 1986 }