github.com/LazyboyChen7/engine@v17.12.1-ce-rc2+incompatible/integration-cli/docker_cli_swarm_test.go (about) 1 // +build !windows 2 3 package main 4 5 import ( 6 "bytes" 7 "crypto/x509" 8 "encoding/json" 9 "encoding/pem" 10 "fmt" 11 "io/ioutil" 12 "net/http" 13 "net/http/httptest" 14 "os" 15 "path/filepath" 16 "strings" 17 "time" 18 19 "github.com/cloudflare/cfssl/helpers" 20 "github.com/docker/docker/api/types" 21 "github.com/docker/docker/api/types/swarm" 22 "github.com/docker/docker/integration-cli/checker" 23 "github.com/docker/docker/integration-cli/cli" 24 "github.com/docker/docker/integration-cli/daemon" 25 "github.com/docker/libnetwork/driverapi" 26 "github.com/docker/libnetwork/ipamapi" 27 remoteipam "github.com/docker/libnetwork/ipams/remote/api" 28 "github.com/go-check/check" 29 "github.com/gotestyourself/gotestyourself/fs" 30 "github.com/gotestyourself/gotestyourself/icmd" 31 "github.com/vishvananda/netlink" 32 "golang.org/x/net/context" 33 ) 34 35 func (s *DockerSwarmSuite) TestSwarmUpdate(c *check.C) { 36 d := s.AddDaemon(c, true, true) 37 38 getSpec := func() swarm.Spec { 39 sw := d.GetSwarm(c) 40 return sw.Spec 41 } 42 43 out, err := d.Cmd("swarm", "update", "--cert-expiry", "30h", "--dispatcher-heartbeat", "11s") 44 c.Assert(err, checker.IsNil, check.Commentf("out: %v", out)) 45 46 spec := getSpec() 47 c.Assert(spec.CAConfig.NodeCertExpiry, checker.Equals, 30*time.Hour) 48 c.Assert(spec.Dispatcher.HeartbeatPeriod, checker.Equals, 11*time.Second) 49 50 // setting anything under 30m for cert-expiry is not allowed 51 out, err = d.Cmd("swarm", "update", "--cert-expiry", "15m") 52 c.Assert(err, checker.NotNil) 53 c.Assert(out, checker.Contains, "minimum certificate expiry time") 54 spec = getSpec() 55 c.Assert(spec.CAConfig.NodeCertExpiry, checker.Equals, 30*time.Hour) 56 57 // passing an external CA (this is without starting a root rotation) does not fail 58 cli.Docker(cli.Args("swarm", "update", "--external-ca", "protocol=cfssl,url=https://something.org", 59 "--external-ca", "protocol=cfssl,url=https://somethingelse.org,cacert=fixtures/https/ca.pem"), 60 cli.Daemon(d.Daemon)).Assert(c, icmd.Success) 61 62 expected, err := ioutil.ReadFile("fixtures/https/ca.pem") 63 c.Assert(err, checker.IsNil) 64 65 spec = getSpec() 66 c.Assert(spec.CAConfig.ExternalCAs, checker.HasLen, 2) 67 c.Assert(spec.CAConfig.ExternalCAs[0].CACert, checker.Equals, "") 68 c.Assert(spec.CAConfig.ExternalCAs[1].CACert, checker.Equals, string(expected)) 69 70 // passing an invalid external CA fails 71 tempFile := fs.NewFile(c, "testfile", fs.WithContent("fakecert")) 72 defer tempFile.Remove() 73 74 result := cli.Docker(cli.Args("swarm", "update", 75 "--external-ca", fmt.Sprintf("protocol=cfssl,url=https://something.org,cacert=%s", tempFile.Path())), 76 cli.Daemon(d.Daemon)) 77 result.Assert(c, icmd.Expected{ 78 ExitCode: 125, 79 Err: "must be in PEM format", 80 }) 81 } 82 83 func (s *DockerSwarmSuite) TestSwarmInit(c *check.C) { 84 d := s.AddDaemon(c, false, false) 85 86 getSpec := func() swarm.Spec { 87 sw := d.GetSwarm(c) 88 return sw.Spec 89 } 90 91 // passing an invalid external CA fails 92 tempFile := fs.NewFile(c, "testfile", fs.WithContent("fakecert")) 93 defer tempFile.Remove() 94 95 result := cli.Docker(cli.Args("swarm", "init", "--cert-expiry", "30h", "--dispatcher-heartbeat", "11s", 96 "--external-ca", fmt.Sprintf("protocol=cfssl,url=https://somethingelse.org,cacert=%s", tempFile.Path())), 97 cli.Daemon(d.Daemon)) 98 result.Assert(c, icmd.Expected{ 99 ExitCode: 125, 100 Err: "must be in PEM format", 101 }) 102 103 cli.Docker(cli.Args("swarm", "init", "--cert-expiry", "30h", "--dispatcher-heartbeat", "11s", 104 "--external-ca", "protocol=cfssl,url=https://something.org", 105 "--external-ca", "protocol=cfssl,url=https://somethingelse.org,cacert=fixtures/https/ca.pem"), 106 cli.Daemon(d.Daemon)).Assert(c, icmd.Success) 107 108 expected, err := ioutil.ReadFile("fixtures/https/ca.pem") 109 c.Assert(err, checker.IsNil) 110 111 spec := getSpec() 112 c.Assert(spec.CAConfig.NodeCertExpiry, checker.Equals, 30*time.Hour) 113 c.Assert(spec.Dispatcher.HeartbeatPeriod, checker.Equals, 11*time.Second) 114 c.Assert(spec.CAConfig.ExternalCAs, checker.HasLen, 2) 115 c.Assert(spec.CAConfig.ExternalCAs[0].CACert, checker.Equals, "") 116 c.Assert(spec.CAConfig.ExternalCAs[1].CACert, checker.Equals, string(expected)) 117 118 c.Assert(d.Leave(true), checker.IsNil) 119 cli.Docker(cli.Args("swarm", "init"), cli.Daemon(d.Daemon)).Assert(c, icmd.Success) 120 121 spec = getSpec() 122 c.Assert(spec.CAConfig.NodeCertExpiry, checker.Equals, 90*24*time.Hour) 123 c.Assert(spec.Dispatcher.HeartbeatPeriod, checker.Equals, 5*time.Second) 124 } 125 126 func (s *DockerSwarmSuite) TestSwarmInitIPv6(c *check.C) { 127 testRequires(c, IPv6) 128 d1 := s.AddDaemon(c, false, false) 129 cli.Docker(cli.Args("swarm", "init", "--listen-add", "::1"), cli.Daemon(d1.Daemon)).Assert(c, icmd.Success) 130 131 d2 := s.AddDaemon(c, false, false) 132 cli.Docker(cli.Args("swarm", "join", "::1"), cli.Daemon(d2.Daemon)).Assert(c, icmd.Success) 133 134 out := cli.Docker(cli.Args("info"), cli.Daemon(d2.Daemon)).Assert(c, icmd.Success).Combined() 135 c.Assert(out, checker.Contains, "Swarm: active") 136 } 137 138 func (s *DockerSwarmSuite) TestSwarmInitUnspecifiedAdvertiseAddr(c *check.C) { 139 d := s.AddDaemon(c, false, false) 140 out, err := d.Cmd("swarm", "init", "--advertise-addr", "0.0.0.0") 141 c.Assert(err, checker.NotNil) 142 c.Assert(out, checker.Contains, "advertise address must be a non-zero IP address") 143 } 144 145 func (s *DockerSwarmSuite) TestSwarmIncompatibleDaemon(c *check.C) { 146 // init swarm mode and stop a daemon 147 d := s.AddDaemon(c, true, true) 148 info, err := d.SwarmInfo() 149 c.Assert(err, checker.IsNil) 150 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 151 d.Stop(c) 152 153 // start a daemon with --cluster-store and --cluster-advertise 154 err = d.StartWithError("--cluster-store=consul://consuladdr:consulport/some/path", "--cluster-advertise=1.1.1.1:2375") 155 c.Assert(err, checker.NotNil) 156 content, err := d.ReadLogFile() 157 c.Assert(err, checker.IsNil) 158 c.Assert(string(content), checker.Contains, "--cluster-store and --cluster-advertise daemon configurations are incompatible with swarm mode") 159 160 // start a daemon with --live-restore 161 err = d.StartWithError("--live-restore") 162 c.Assert(err, checker.NotNil) 163 content, err = d.ReadLogFile() 164 c.Assert(err, checker.IsNil) 165 c.Assert(string(content), checker.Contains, "--live-restore daemon configuration is incompatible with swarm mode") 166 // restart for teardown 167 d.Start(c) 168 } 169 170 func (s *DockerSwarmSuite) TestSwarmServiceTemplatingHostname(c *check.C) { 171 d := s.AddDaemon(c, true, true) 172 hostname, err := d.Cmd("node", "inspect", "--format", "{{.Description.Hostname}}", "self") 173 c.Assert(err, checker.IsNil, check.Commentf(hostname)) 174 175 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "test", "--hostname", "{{.Service.Name}}-{{.Task.Slot}}-{{.Node.Hostname}}", "busybox", "top") 176 c.Assert(err, checker.IsNil, check.Commentf(out)) 177 178 // make sure task has been deployed. 179 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 180 181 containers := d.ActiveContainers() 182 out, err = d.Cmd("inspect", "--type", "container", "--format", "{{.Config.Hostname}}", containers[0]) 183 c.Assert(err, checker.IsNil, check.Commentf(out)) 184 c.Assert(strings.Split(out, "\n")[0], checker.Equals, "test-1-"+strings.Split(hostname, "\n")[0], check.Commentf("hostname with templating invalid")) 185 } 186 187 // Test case for #24270 188 func (s *DockerSwarmSuite) TestSwarmServiceListFilter(c *check.C) { 189 d := s.AddDaemon(c, true, true) 190 191 name1 := "redis-cluster-md5" 192 name2 := "redis-cluster" 193 name3 := "other-cluster" 194 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name1, "busybox", "top") 195 c.Assert(err, checker.IsNil) 196 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 197 198 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name2, "busybox", "top") 199 c.Assert(err, checker.IsNil) 200 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 201 202 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name3, "busybox", "top") 203 c.Assert(err, checker.IsNil) 204 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 205 206 filter1 := "name=redis-cluster-md5" 207 filter2 := "name=redis-cluster" 208 209 // We search checker.Contains with `name+" "` to prevent prefix only. 210 out, err = d.Cmd("service", "ls", "--filter", filter1) 211 c.Assert(err, checker.IsNil) 212 c.Assert(out, checker.Contains, name1+" ") 213 c.Assert(out, checker.Not(checker.Contains), name2+" ") 214 c.Assert(out, checker.Not(checker.Contains), name3+" ") 215 216 out, err = d.Cmd("service", "ls", "--filter", filter2) 217 c.Assert(err, checker.IsNil) 218 c.Assert(out, checker.Contains, name1+" ") 219 c.Assert(out, checker.Contains, name2+" ") 220 c.Assert(out, checker.Not(checker.Contains), name3+" ") 221 222 out, err = d.Cmd("service", "ls") 223 c.Assert(err, checker.IsNil) 224 c.Assert(out, checker.Contains, name1+" ") 225 c.Assert(out, checker.Contains, name2+" ") 226 c.Assert(out, checker.Contains, name3+" ") 227 } 228 229 func (s *DockerSwarmSuite) TestSwarmNodeListFilter(c *check.C) { 230 d := s.AddDaemon(c, true, true) 231 232 out, err := d.Cmd("node", "inspect", "--format", "{{ .Description.Hostname }}", "self") 233 c.Assert(err, checker.IsNil) 234 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 235 name := strings.TrimSpace(out) 236 237 filter := "name=" + name[:4] 238 239 out, err = d.Cmd("node", "ls", "--filter", filter) 240 c.Assert(err, checker.IsNil) 241 c.Assert(out, checker.Contains, name) 242 243 out, err = d.Cmd("node", "ls", "--filter", "name=none") 244 c.Assert(err, checker.IsNil) 245 c.Assert(out, checker.Not(checker.Contains), name) 246 } 247 248 func (s *DockerSwarmSuite) TestSwarmNodeTaskListFilter(c *check.C) { 249 d := s.AddDaemon(c, true, true) 250 251 name := "redis-cluster-md5" 252 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--replicas=3", "busybox", "top") 253 c.Assert(err, checker.IsNil) 254 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 255 256 // make sure task has been deployed. 257 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 3) 258 259 filter := "name=redis-cluster" 260 261 out, err = d.Cmd("node", "ps", "--filter", filter, "self") 262 c.Assert(err, checker.IsNil) 263 c.Assert(out, checker.Contains, name+".1") 264 c.Assert(out, checker.Contains, name+".2") 265 c.Assert(out, checker.Contains, name+".3") 266 267 out, err = d.Cmd("node", "ps", "--filter", "name=none", "self") 268 c.Assert(err, checker.IsNil) 269 c.Assert(out, checker.Not(checker.Contains), name+".1") 270 c.Assert(out, checker.Not(checker.Contains), name+".2") 271 c.Assert(out, checker.Not(checker.Contains), name+".3") 272 } 273 274 // Test case for #25375 275 func (s *DockerSwarmSuite) TestSwarmPublishAdd(c *check.C) { 276 d := s.AddDaemon(c, true, true) 277 278 name := "top" 279 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--label", "x=y", "busybox", "top") 280 c.Assert(err, checker.IsNil) 281 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 282 283 out, err = d.Cmd("service", "update", "--detach", "--publish-add", "80:80", name) 284 c.Assert(err, checker.IsNil) 285 286 out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", name) 287 c.Assert(err, checker.IsNil) 288 289 out, err = d.CmdRetryOutOfSequence("service", "update", "--detach", "--publish-add", "80:80", "--publish-add", "80:20", name) 290 c.Assert(err, checker.NotNil) 291 292 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.EndpointSpec.Ports }}", name) 293 c.Assert(err, checker.IsNil) 294 c.Assert(strings.TrimSpace(out), checker.Equals, "[{ tcp 80 80 ingress}]") 295 } 296 297 func (s *DockerSwarmSuite) TestSwarmServiceWithGroup(c *check.C) { 298 d := s.AddDaemon(c, true, true) 299 300 name := "top" 301 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--user", "root:root", "--group", "wheel", "--group", "audio", "--group", "staff", "--group", "777", "busybox", "top") 302 c.Assert(err, checker.IsNil) 303 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 304 305 // make sure task has been deployed. 306 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 307 308 out, err = d.Cmd("ps", "-q") 309 c.Assert(err, checker.IsNil) 310 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 311 312 container := strings.TrimSpace(out) 313 314 out, err = d.Cmd("exec", container, "id") 315 c.Assert(err, checker.IsNil) 316 c.Assert(strings.TrimSpace(out), checker.Equals, "uid=0(root) gid=0(root) groups=10(wheel),29(audio),50(staff),777") 317 } 318 319 func (s *DockerSwarmSuite) TestSwarmContainerAutoStart(c *check.C) { 320 d := s.AddDaemon(c, true, true) 321 322 out, err := d.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 323 c.Assert(err, checker.IsNil) 324 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 325 326 out, err = d.Cmd("run", "-id", "--restart=always", "--net=foo", "--name=test", "busybox", "top") 327 c.Assert(err, checker.IsNil, check.Commentf(out)) 328 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 329 330 out, err = d.Cmd("ps", "-q") 331 c.Assert(err, checker.IsNil, check.Commentf(out)) 332 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 333 334 d.Restart(c) 335 336 out, err = d.Cmd("ps", "-q") 337 c.Assert(err, checker.IsNil, check.Commentf(out)) 338 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 339 } 340 341 func (s *DockerSwarmSuite) TestSwarmContainerEndpointOptions(c *check.C) { 342 d := s.AddDaemon(c, true, true) 343 344 out, err := d.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 345 c.Assert(err, checker.IsNil, check.Commentf(out)) 346 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 347 348 _, err = d.Cmd("run", "-d", "--net=foo", "--name=first", "--net-alias=first-alias", "busybox", "top") 349 c.Assert(err, checker.IsNil, check.Commentf(out)) 350 351 _, err = d.Cmd("run", "-d", "--net=foo", "--name=second", "busybox", "top") 352 c.Assert(err, checker.IsNil, check.Commentf(out)) 353 354 _, err = d.Cmd("run", "-d", "--net=foo", "--net-alias=third-alias", "busybox", "top") 355 c.Assert(err, checker.IsNil, check.Commentf(out)) 356 357 // ping first container and its alias, also ping third and anonymous container by its alias 358 _, err = d.Cmd("exec", "second", "ping", "-c", "1", "first") 359 c.Assert(err, check.IsNil, check.Commentf(out)) 360 _, err = d.Cmd("exec", "second", "ping", "-c", "1", "first-alias") 361 c.Assert(err, check.IsNil, check.Commentf(out)) 362 _, err = d.Cmd("exec", "second", "ping", "-c", "1", "third-alias") 363 c.Assert(err, check.IsNil, check.Commentf(out)) 364 } 365 366 func (s *DockerSwarmSuite) TestSwarmContainerAttachByNetworkId(c *check.C) { 367 d := s.AddDaemon(c, true, true) 368 369 out, err := d.Cmd("network", "create", "--attachable", "-d", "overlay", "testnet") 370 c.Assert(err, checker.IsNil) 371 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 372 networkID := strings.TrimSpace(out) 373 374 out, err = d.Cmd("run", "-d", "--net", networkID, "busybox", "top") 375 c.Assert(err, checker.IsNil) 376 cID := strings.TrimSpace(out) 377 d.WaitRun(cID) 378 379 _, err = d.Cmd("rm", "-f", cID) 380 c.Assert(err, checker.IsNil) 381 382 _, err = d.Cmd("network", "rm", "testnet") 383 c.Assert(err, checker.IsNil) 384 385 checkNetwork := func(*check.C) (interface{}, check.CommentInterface) { 386 out, err := d.Cmd("network", "ls") 387 c.Assert(err, checker.IsNil) 388 return out, nil 389 } 390 391 waitAndAssert(c, 3*time.Second, checkNetwork, checker.Not(checker.Contains), "testnet") 392 } 393 394 func (s *DockerSwarmSuite) TestOverlayAttachable(c *check.C) { 395 d := s.AddDaemon(c, true, true) 396 397 out, err := d.Cmd("network", "create", "-d", "overlay", "--attachable", "ovnet") 398 c.Assert(err, checker.IsNil, check.Commentf(out)) 399 400 // validate attachable 401 out, err = d.Cmd("network", "inspect", "--format", "{{json .Attachable}}", "ovnet") 402 c.Assert(err, checker.IsNil, check.Commentf(out)) 403 c.Assert(strings.TrimSpace(out), checker.Equals, "true") 404 405 // validate containers can attache to this overlay network 406 out, err = d.Cmd("run", "-d", "--network", "ovnet", "--name", "c1", "busybox", "top") 407 c.Assert(err, checker.IsNil, check.Commentf(out)) 408 409 // redo validation, there was a bug that the value of attachable changes after 410 // containers attach to the network 411 out, err = d.Cmd("network", "inspect", "--format", "{{json .Attachable}}", "ovnet") 412 c.Assert(err, checker.IsNil, check.Commentf(out)) 413 c.Assert(strings.TrimSpace(out), checker.Equals, "true") 414 } 415 416 func (s *DockerSwarmSuite) TestOverlayAttachableOnSwarmLeave(c *check.C) { 417 d := s.AddDaemon(c, true, true) 418 419 // Create an attachable swarm network 420 nwName := "attovl" 421 out, err := d.Cmd("network", "create", "-d", "overlay", "--attachable", nwName) 422 c.Assert(err, checker.IsNil, check.Commentf(out)) 423 424 // Connect a container to the network 425 out, err = d.Cmd("run", "-d", "--network", nwName, "--name", "c1", "busybox", "top") 426 c.Assert(err, checker.IsNil, check.Commentf(out)) 427 428 // Leave the swarm 429 err = d.Leave(true) 430 c.Assert(err, checker.IsNil) 431 432 // Check the container is disconnected 433 out, err = d.Cmd("inspect", "c1", "--format", "{{.NetworkSettings.Networks."+nwName+"}}") 434 c.Assert(err, checker.IsNil) 435 c.Assert(strings.TrimSpace(out), checker.Equals, "<no value>") 436 437 // Check the network is gone 438 out, err = d.Cmd("network", "ls", "--format", "{{.Name}}") 439 c.Assert(err, checker.IsNil) 440 c.Assert(out, checker.Not(checker.Contains), nwName) 441 } 442 443 func (s *DockerSwarmSuite) TestOverlayAttachableReleaseResourcesOnFailure(c *check.C) { 444 d := s.AddDaemon(c, true, true) 445 446 // Create attachable network 447 out, err := d.Cmd("network", "create", "-d", "overlay", "--attachable", "--subnet", "10.10.9.0/24", "ovnet") 448 c.Assert(err, checker.IsNil, check.Commentf(out)) 449 450 // Attach a container with specific IP 451 out, err = d.Cmd("run", "-d", "--network", "ovnet", "--name", "c1", "--ip", "10.10.9.33", "busybox", "top") 452 c.Assert(err, checker.IsNil, check.Commentf(out)) 453 454 // Attempt to attach another container with same IP, must fail 455 _, err = d.Cmd("run", "-d", "--network", "ovnet", "--name", "c2", "--ip", "10.10.9.33", "busybox", "top") 456 c.Assert(err, checker.NotNil) 457 458 // Remove first container 459 out, err = d.Cmd("rm", "-f", "c1") 460 c.Assert(err, checker.IsNil, check.Commentf(out)) 461 462 // Verify the network can be removed, no phantom network attachment task left over 463 out, err = d.Cmd("network", "rm", "ovnet") 464 c.Assert(err, checker.IsNil, check.Commentf(out)) 465 } 466 467 func (s *DockerSwarmSuite) TestSwarmIngressNetwork(c *check.C) { 468 d := s.AddDaemon(c, true, true) 469 470 // Ingress network can be removed 471 removeNetwork := func(name string) *icmd.Result { 472 return cli.Docker( 473 cli.Args("-H", d.Sock(), "network", "rm", name), 474 cli.WithStdin(strings.NewReader("Y"))) 475 } 476 477 result := removeNetwork("ingress") 478 result.Assert(c, icmd.Success) 479 480 // And recreated 481 out, err := d.Cmd("network", "create", "-d", "overlay", "--ingress", "new-ingress") 482 c.Assert(err, checker.IsNil, check.Commentf(out)) 483 484 // But only one is allowed 485 out, err = d.Cmd("network", "create", "-d", "overlay", "--ingress", "another-ingress") 486 c.Assert(err, checker.NotNil) 487 c.Assert(strings.TrimSpace(out), checker.Contains, "is already present") 488 489 // It cannot be removed if it is being used 490 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "srv1", "-p", "9000:8000", "busybox", "top") 491 c.Assert(err, checker.IsNil, check.Commentf(out)) 492 493 result = removeNetwork("new-ingress") 494 result.Assert(c, icmd.Expected{ 495 ExitCode: 1, 496 Err: "ingress network cannot be removed because service", 497 }) 498 499 // But it can be removed once no more services depend on it 500 out, err = d.Cmd("service", "update", "--detach", "--publish-rm", "9000:8000", "srv1") 501 c.Assert(err, checker.IsNil, check.Commentf(out)) 502 503 result = removeNetwork("new-ingress") 504 result.Assert(c, icmd.Success) 505 506 // A service which needs the ingress network cannot be created if no ingress is present 507 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "srv2", "-p", "500:500", "busybox", "top") 508 c.Assert(err, checker.NotNil) 509 c.Assert(strings.TrimSpace(out), checker.Contains, "no ingress network is present") 510 511 // An existing service cannot be updated to use the ingress nw if the nw is not present 512 out, err = d.Cmd("service", "update", "--detach", "--publish-add", "9000:8000", "srv1") 513 c.Assert(err, checker.NotNil) 514 c.Assert(strings.TrimSpace(out), checker.Contains, "no ingress network is present") 515 516 // But services which do not need routing mesh can be created regardless 517 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "srv3", "--endpoint-mode", "dnsrr", "busybox", "top") 518 c.Assert(err, checker.IsNil, check.Commentf(out)) 519 } 520 521 func (s *DockerSwarmSuite) TestSwarmCreateServiceWithNoIngressNetwork(c *check.C) { 522 d := s.AddDaemon(c, true, true) 523 524 // Remove ingress network 525 result := cli.Docker( 526 cli.Args("-H", d.Sock(), "network", "rm", "ingress"), 527 cli.WithStdin(strings.NewReader("Y"))) 528 result.Assert(c, icmd.Success) 529 530 // Create a overlay network and launch a service on it 531 // Make sure nothing panics because ingress network is missing 532 out, err := d.Cmd("network", "create", "-d", "overlay", "another-network") 533 c.Assert(err, checker.IsNil, check.Commentf(out)) 534 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "srv4", "--network", "another-network", "busybox", "top") 535 c.Assert(err, checker.IsNil, check.Commentf(out)) 536 } 537 538 // Test case for #24108, also the case from: 539 // https://github.com/docker/docker/pull/24620#issuecomment-233715656 540 func (s *DockerSwarmSuite) TestSwarmTaskListFilter(c *check.C) { 541 d := s.AddDaemon(c, true, true) 542 543 name := "redis-cluster-md5" 544 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--replicas=3", "busybox", "top") 545 c.Assert(err, checker.IsNil) 546 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 547 548 filter := "name=redis-cluster" 549 550 checkNumTasks := func(*check.C) (interface{}, check.CommentInterface) { 551 out, err := d.Cmd("service", "ps", "--filter", filter, name) 552 c.Assert(err, checker.IsNil) 553 return len(strings.Split(out, "\n")) - 2, nil // includes header and nl in last line 554 } 555 556 // wait until all tasks have been created 557 waitAndAssert(c, defaultReconciliationTimeout, checkNumTasks, checker.Equals, 3) 558 559 out, err = d.Cmd("service", "ps", "--filter", filter, name) 560 c.Assert(err, checker.IsNil) 561 c.Assert(out, checker.Contains, name+".1") 562 c.Assert(out, checker.Contains, name+".2") 563 c.Assert(out, checker.Contains, name+".3") 564 565 out, err = d.Cmd("service", "ps", "--filter", "name="+name+".1", name) 566 c.Assert(err, checker.IsNil) 567 c.Assert(out, checker.Contains, name+".1") 568 c.Assert(out, checker.Not(checker.Contains), name+".2") 569 c.Assert(out, checker.Not(checker.Contains), name+".3") 570 571 out, err = d.Cmd("service", "ps", "--filter", "name=none", name) 572 c.Assert(err, checker.IsNil) 573 c.Assert(out, checker.Not(checker.Contains), name+".1") 574 c.Assert(out, checker.Not(checker.Contains), name+".2") 575 c.Assert(out, checker.Not(checker.Contains), name+".3") 576 577 name = "redis-cluster-sha1" 578 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--mode=global", "busybox", "top") 579 c.Assert(err, checker.IsNil) 580 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 581 582 waitAndAssert(c, defaultReconciliationTimeout, checkNumTasks, checker.Equals, 1) 583 584 filter = "name=redis-cluster" 585 out, err = d.Cmd("service", "ps", "--filter", filter, name) 586 c.Assert(err, checker.IsNil) 587 c.Assert(out, checker.Contains, name) 588 589 out, err = d.Cmd("service", "ps", "--filter", "name="+name, name) 590 c.Assert(err, checker.IsNil) 591 c.Assert(out, checker.Contains, name) 592 593 out, err = d.Cmd("service", "ps", "--filter", "name=none", name) 594 c.Assert(err, checker.IsNil) 595 c.Assert(out, checker.Not(checker.Contains), name) 596 } 597 598 func (s *DockerSwarmSuite) TestPsListContainersFilterIsTask(c *check.C) { 599 d := s.AddDaemon(c, true, true) 600 601 // Create a bare container 602 out, err := d.Cmd("run", "-d", "--name=bare-container", "busybox", "top") 603 c.Assert(err, checker.IsNil) 604 bareID := strings.TrimSpace(out)[:12] 605 // Create a service 606 name := "busybox-top" 607 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "busybox", "top") 608 c.Assert(err, checker.IsNil) 609 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 610 611 // make sure task has been deployed. 612 waitAndAssert(c, defaultReconciliationTimeout, d.CheckServiceRunningTasks(name), checker.Equals, 1) 613 614 // Filter non-tasks 615 out, err = d.Cmd("ps", "-a", "-q", "--filter=is-task=false") 616 c.Assert(err, checker.IsNil) 617 psOut := strings.TrimSpace(out) 618 c.Assert(psOut, checker.Equals, bareID, check.Commentf("Expected id %s, got %s for is-task label, output %q", bareID, psOut, out)) 619 620 // Filter tasks 621 out, err = d.Cmd("ps", "-a", "-q", "--filter=is-task=true") 622 c.Assert(err, checker.IsNil) 623 lines := strings.Split(strings.Trim(out, "\n "), "\n") 624 c.Assert(lines, checker.HasLen, 1) 625 c.Assert(lines[0], checker.Not(checker.Equals), bareID, check.Commentf("Expected not %s, but got it for is-task label, output %q", bareID, out)) 626 } 627 628 const globalNetworkPlugin = "global-network-plugin" 629 const globalIPAMPlugin = "global-ipam-plugin" 630 631 func setupRemoteGlobalNetworkPlugin(c *check.C, mux *http.ServeMux, url, netDrv, ipamDrv string) { 632 633 mux.HandleFunc("/Plugin.Activate", func(w http.ResponseWriter, r *http.Request) { 634 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 635 fmt.Fprintf(w, `{"Implements": ["%s", "%s"]}`, driverapi.NetworkPluginEndpointType, ipamapi.PluginEndpointType) 636 }) 637 638 // Network driver implementation 639 mux.HandleFunc(fmt.Sprintf("/%s.GetCapabilities", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 640 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 641 fmt.Fprintf(w, `{"Scope":"global"}`) 642 }) 643 644 mux.HandleFunc(fmt.Sprintf("/%s.AllocateNetwork", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 645 err := json.NewDecoder(r.Body).Decode(&remoteDriverNetworkRequest) 646 if err != nil { 647 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 648 return 649 } 650 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 651 fmt.Fprintf(w, "null") 652 }) 653 654 mux.HandleFunc(fmt.Sprintf("/%s.FreeNetwork", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 655 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 656 fmt.Fprintf(w, "null") 657 }) 658 659 mux.HandleFunc(fmt.Sprintf("/%s.CreateNetwork", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 660 err := json.NewDecoder(r.Body).Decode(&remoteDriverNetworkRequest) 661 if err != nil { 662 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 663 return 664 } 665 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 666 fmt.Fprintf(w, "null") 667 }) 668 669 mux.HandleFunc(fmt.Sprintf("/%s.DeleteNetwork", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 670 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 671 fmt.Fprintf(w, "null") 672 }) 673 674 mux.HandleFunc(fmt.Sprintf("/%s.CreateEndpoint", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 675 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 676 fmt.Fprintf(w, `{"Interface":{"MacAddress":"a0:b1:c2:d3:e4:f5"}}`) 677 }) 678 679 mux.HandleFunc(fmt.Sprintf("/%s.Join", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 680 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 681 682 veth := &netlink.Veth{ 683 LinkAttrs: netlink.LinkAttrs{Name: "randomIfName", TxQLen: 0}, PeerName: "cnt0"} 684 if err := netlink.LinkAdd(veth); err != nil { 685 fmt.Fprintf(w, `{"Error":"failed to add veth pair: `+err.Error()+`"}`) 686 } else { 687 fmt.Fprintf(w, `{"InterfaceName":{ "SrcName":"cnt0", "DstPrefix":"veth"}}`) 688 } 689 }) 690 691 mux.HandleFunc(fmt.Sprintf("/%s.Leave", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 692 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 693 fmt.Fprintf(w, "null") 694 }) 695 696 mux.HandleFunc(fmt.Sprintf("/%s.DeleteEndpoint", driverapi.NetworkPluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 697 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 698 if link, err := netlink.LinkByName("cnt0"); err == nil { 699 netlink.LinkDel(link) 700 } 701 fmt.Fprintf(w, "null") 702 }) 703 704 // IPAM Driver implementation 705 var ( 706 poolRequest remoteipam.RequestPoolRequest 707 poolReleaseReq remoteipam.ReleasePoolRequest 708 addressRequest remoteipam.RequestAddressRequest 709 addressReleaseReq remoteipam.ReleaseAddressRequest 710 lAS = "localAS" 711 gAS = "globalAS" 712 pool = "172.28.0.0/16" 713 poolID = lAS + "/" + pool 714 gw = "172.28.255.254/16" 715 ) 716 717 mux.HandleFunc(fmt.Sprintf("/%s.GetDefaultAddressSpaces", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 718 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 719 fmt.Fprintf(w, `{"LocalDefaultAddressSpace":"`+lAS+`", "GlobalDefaultAddressSpace": "`+gAS+`"}`) 720 }) 721 722 mux.HandleFunc(fmt.Sprintf("/%s.RequestPool", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 723 err := json.NewDecoder(r.Body).Decode(&poolRequest) 724 if err != nil { 725 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 726 return 727 } 728 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 729 if poolRequest.AddressSpace != lAS && poolRequest.AddressSpace != gAS { 730 fmt.Fprintf(w, `{"Error":"Unknown address space in pool request: `+poolRequest.AddressSpace+`"}`) 731 } else if poolRequest.Pool != "" && poolRequest.Pool != pool { 732 fmt.Fprintf(w, `{"Error":"Cannot handle explicit pool requests yet"}`) 733 } else { 734 fmt.Fprintf(w, `{"PoolID":"`+poolID+`", "Pool":"`+pool+`"}`) 735 } 736 }) 737 738 mux.HandleFunc(fmt.Sprintf("/%s.RequestAddress", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 739 err := json.NewDecoder(r.Body).Decode(&addressRequest) 740 if err != nil { 741 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 742 return 743 } 744 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 745 // make sure libnetwork is now querying on the expected pool id 746 if addressRequest.PoolID != poolID { 747 fmt.Fprintf(w, `{"Error":"unknown pool id"}`) 748 } else if addressRequest.Address != "" { 749 fmt.Fprintf(w, `{"Error":"Cannot handle explicit address requests yet"}`) 750 } else { 751 fmt.Fprintf(w, `{"Address":"`+gw+`"}`) 752 } 753 }) 754 755 mux.HandleFunc(fmt.Sprintf("/%s.ReleaseAddress", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 756 err := json.NewDecoder(r.Body).Decode(&addressReleaseReq) 757 if err != nil { 758 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 759 return 760 } 761 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 762 // make sure libnetwork is now asking to release the expected address from the expected poolid 763 if addressRequest.PoolID != poolID { 764 fmt.Fprintf(w, `{"Error":"unknown pool id"}`) 765 } else if addressReleaseReq.Address != gw { 766 fmt.Fprintf(w, `{"Error":"unknown address"}`) 767 } else { 768 fmt.Fprintf(w, "null") 769 } 770 }) 771 772 mux.HandleFunc(fmt.Sprintf("/%s.ReleasePool", ipamapi.PluginEndpointType), func(w http.ResponseWriter, r *http.Request) { 773 err := json.NewDecoder(r.Body).Decode(&poolReleaseReq) 774 if err != nil { 775 http.Error(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest) 776 return 777 } 778 w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json") 779 // make sure libnetwork is now asking to release the expected poolid 780 if addressRequest.PoolID != poolID { 781 fmt.Fprintf(w, `{"Error":"unknown pool id"}`) 782 } else { 783 fmt.Fprintf(w, "null") 784 } 785 }) 786 787 err := os.MkdirAll("/etc/docker/plugins", 0755) 788 c.Assert(err, checker.IsNil) 789 790 fileName := fmt.Sprintf("/etc/docker/plugins/%s.spec", netDrv) 791 err = ioutil.WriteFile(fileName, []byte(url), 0644) 792 c.Assert(err, checker.IsNil) 793 794 ipamFileName := fmt.Sprintf("/etc/docker/plugins/%s.spec", ipamDrv) 795 err = ioutil.WriteFile(ipamFileName, []byte(url), 0644) 796 c.Assert(err, checker.IsNil) 797 } 798 799 func (s *DockerSwarmSuite) TestSwarmNetworkPlugin(c *check.C) { 800 mux := http.NewServeMux() 801 s.server = httptest.NewServer(mux) 802 c.Assert(s.server, check.NotNil, check.Commentf("Failed to start an HTTP Server")) 803 setupRemoteGlobalNetworkPlugin(c, mux, s.server.URL, globalNetworkPlugin, globalIPAMPlugin) 804 defer func() { 805 s.server.Close() 806 err := os.RemoveAll("/etc/docker/plugins") 807 c.Assert(err, checker.IsNil) 808 }() 809 810 d := s.AddDaemon(c, true, true) 811 812 out, err := d.Cmd("network", "create", "-d", globalNetworkPlugin, "foo") 813 c.Assert(err, checker.NotNil) 814 c.Assert(out, checker.Contains, "not supported in swarm mode") 815 } 816 817 // Test case for #24712 818 func (s *DockerSwarmSuite) TestSwarmServiceEnvFile(c *check.C) { 819 d := s.AddDaemon(c, true, true) 820 821 path := filepath.Join(d.Folder, "env.txt") 822 err := ioutil.WriteFile(path, []byte("VAR1=A\nVAR2=A\n"), 0644) 823 c.Assert(err, checker.IsNil) 824 825 name := "worker" 826 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--env-file", path, "--env", "VAR1=B", "--env", "VAR1=C", "--env", "VAR2=", "--env", "VAR2", "--name", name, "busybox", "top") 827 c.Assert(err, checker.IsNil) 828 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 829 830 // The complete env is [VAR1=A VAR2=A VAR1=B VAR1=C VAR2= VAR2] and duplicates will be removed => [VAR1=C VAR2] 831 out, err = d.Cmd("inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.Env }}", name) 832 c.Assert(err, checker.IsNil) 833 c.Assert(out, checker.Contains, "[VAR1=C VAR2]") 834 } 835 836 func (s *DockerSwarmSuite) TestSwarmServiceTTY(c *check.C) { 837 d := s.AddDaemon(c, true, true) 838 839 name := "top" 840 841 ttyCheck := "if [ -t 0 ]; then echo TTY > /status && top; else echo none > /status && top; fi" 842 843 // Without --tty 844 expectedOutput := "none" 845 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "busybox", "sh", "-c", ttyCheck) 846 c.Assert(err, checker.IsNil) 847 848 // Make sure task has been deployed. 849 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 850 851 // We need to get the container id. 852 out, err = d.Cmd("ps", "-q", "--no-trunc") 853 c.Assert(err, checker.IsNil) 854 id := strings.TrimSpace(out) 855 856 out, err = d.Cmd("exec", id, "cat", "/status") 857 c.Assert(err, checker.IsNil) 858 c.Assert(out, checker.Contains, expectedOutput, check.Commentf("Expected '%s', but got %q", expectedOutput, out)) 859 860 // Remove service 861 out, err = d.Cmd("service", "rm", name) 862 c.Assert(err, checker.IsNil) 863 // Make sure container has been destroyed. 864 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 0) 865 866 // With --tty 867 expectedOutput = "TTY" 868 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--tty", "busybox", "sh", "-c", ttyCheck) 869 c.Assert(err, checker.IsNil) 870 871 // Make sure task has been deployed. 872 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 873 874 // We need to get the container id. 875 out, err = d.Cmd("ps", "-q", "--no-trunc") 876 c.Assert(err, checker.IsNil) 877 id = strings.TrimSpace(out) 878 879 out, err = d.Cmd("exec", id, "cat", "/status") 880 c.Assert(err, checker.IsNil) 881 c.Assert(out, checker.Contains, expectedOutput, check.Commentf("Expected '%s', but got %q", expectedOutput, out)) 882 } 883 884 func (s *DockerSwarmSuite) TestSwarmServiceTTYUpdate(c *check.C) { 885 d := s.AddDaemon(c, true, true) 886 887 // Create a service 888 name := "top" 889 _, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "busybox", "top") 890 c.Assert(err, checker.IsNil) 891 892 // Make sure task has been deployed. 893 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 894 895 out, err := d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.TTY }}", name) 896 c.Assert(err, checker.IsNil) 897 c.Assert(strings.TrimSpace(out), checker.Equals, "false") 898 899 _, err = d.Cmd("service", "update", "--detach", "--tty", name) 900 c.Assert(err, checker.IsNil) 901 902 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.TTY }}", name) 903 c.Assert(err, checker.IsNil) 904 c.Assert(strings.TrimSpace(out), checker.Equals, "true") 905 } 906 907 func (s *DockerSwarmSuite) TestSwarmServiceNetworkUpdate(c *check.C) { 908 d := s.AddDaemon(c, true, true) 909 910 result := icmd.RunCmd(d.Command("network", "create", "-d", "overlay", "foo")) 911 result.Assert(c, icmd.Success) 912 fooNetwork := strings.TrimSpace(string(result.Combined())) 913 914 result = icmd.RunCmd(d.Command("network", "create", "-d", "overlay", "bar")) 915 result.Assert(c, icmd.Success) 916 barNetwork := strings.TrimSpace(string(result.Combined())) 917 918 result = icmd.RunCmd(d.Command("network", "create", "-d", "overlay", "baz")) 919 result.Assert(c, icmd.Success) 920 bazNetwork := strings.TrimSpace(string(result.Combined())) 921 922 // Create a service 923 name := "top" 924 result = icmd.RunCmd(d.Command("service", "create", "--detach", "--no-resolve-image", "--network", "foo", "--network", "bar", "--name", name, "busybox", "top")) 925 result.Assert(c, icmd.Success) 926 927 // Make sure task has been deployed. 928 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskNetworks, checker.DeepEquals, 929 map[string]int{fooNetwork: 1, barNetwork: 1}) 930 931 // Remove a network 932 result = icmd.RunCmd(d.Command("service", "update", "--detach", "--network-rm", "foo", name)) 933 result.Assert(c, icmd.Success) 934 935 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskNetworks, checker.DeepEquals, 936 map[string]int{barNetwork: 1}) 937 938 // Add a network 939 result = icmd.RunCmd(d.Command("service", "update", "--detach", "--network-add", "baz", name)) 940 result.Assert(c, icmd.Success) 941 942 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskNetworks, checker.DeepEquals, 943 map[string]int{barNetwork: 1, bazNetwork: 1}) 944 } 945 946 func (s *DockerSwarmSuite) TestDNSConfig(c *check.C) { 947 d := s.AddDaemon(c, true, true) 948 949 // Create a service 950 name := "top" 951 _, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--dns=1.2.3.4", "--dns-search=example.com", "--dns-option=timeout:3", "busybox", "top") 952 c.Assert(err, checker.IsNil) 953 954 // Make sure task has been deployed. 955 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 956 957 // We need to get the container id. 958 out, err := d.Cmd("ps", "-a", "-q", "--no-trunc") 959 c.Assert(err, checker.IsNil) 960 id := strings.TrimSpace(out) 961 962 // Compare against expected output. 963 expectedOutput1 := "nameserver 1.2.3.4" 964 expectedOutput2 := "search example.com" 965 expectedOutput3 := "options timeout:3" 966 out, err = d.Cmd("exec", id, "cat", "/etc/resolv.conf") 967 c.Assert(err, checker.IsNil) 968 c.Assert(out, checker.Contains, expectedOutput1, check.Commentf("Expected '%s', but got %q", expectedOutput1, out)) 969 c.Assert(out, checker.Contains, expectedOutput2, check.Commentf("Expected '%s', but got %q", expectedOutput2, out)) 970 c.Assert(out, checker.Contains, expectedOutput3, check.Commentf("Expected '%s', but got %q", expectedOutput3, out)) 971 } 972 973 func (s *DockerSwarmSuite) TestDNSConfigUpdate(c *check.C) { 974 d := s.AddDaemon(c, true, true) 975 976 // Create a service 977 name := "top" 978 _, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "busybox", "top") 979 c.Assert(err, checker.IsNil) 980 981 // Make sure task has been deployed. 982 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 983 984 _, err = d.Cmd("service", "update", "--detach", "--dns-add=1.2.3.4", "--dns-search-add=example.com", "--dns-option-add=timeout:3", name) 985 c.Assert(err, checker.IsNil) 986 987 out, err := d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.DNSConfig }}", name) 988 c.Assert(err, checker.IsNil) 989 c.Assert(strings.TrimSpace(out), checker.Equals, "{[1.2.3.4] [example.com] [timeout:3]}") 990 } 991 992 func getNodeStatus(c *check.C, d *daemon.Swarm) swarm.LocalNodeState { 993 info, err := d.SwarmInfo() 994 c.Assert(err, checker.IsNil) 995 return info.LocalNodeState 996 } 997 998 func checkKeyIsEncrypted(d *daemon.Swarm) func(*check.C) (interface{}, check.CommentInterface) { 999 return func(c *check.C) (interface{}, check.CommentInterface) { 1000 keyBytes, err := ioutil.ReadFile(filepath.Join(d.Folder, "root", "swarm", "certificates", "swarm-node.key")) 1001 if err != nil { 1002 return fmt.Errorf("error reading key: %v", err), nil 1003 } 1004 1005 keyBlock, _ := pem.Decode(keyBytes) 1006 if keyBlock == nil { 1007 return fmt.Errorf("invalid PEM-encoded private key"), nil 1008 } 1009 1010 return x509.IsEncryptedPEMBlock(keyBlock), nil 1011 } 1012 } 1013 1014 func checkSwarmLockedToUnlocked(c *check.C, d *daemon.Swarm, unlockKey string) { 1015 // Wait for the PEM file to become unencrypted 1016 waitAndAssert(c, defaultReconciliationTimeout, checkKeyIsEncrypted(d), checker.Equals, false) 1017 1018 d.Restart(c) 1019 c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive) 1020 } 1021 1022 func checkSwarmUnlockedToLocked(c *check.C, d *daemon.Swarm) { 1023 // Wait for the PEM file to become encrypted 1024 waitAndAssert(c, defaultReconciliationTimeout, checkKeyIsEncrypted(d), checker.Equals, true) 1025 1026 d.Restart(c) 1027 c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked) 1028 } 1029 1030 func (s *DockerSwarmSuite) TestUnlockEngineAndUnlockedSwarm(c *check.C) { 1031 d := s.AddDaemon(c, false, false) 1032 1033 // unlocking a normal engine should return an error - it does not even ask for the key 1034 cmd := d.Command("swarm", "unlock") 1035 result := icmd.RunCmd(cmd) 1036 result.Assert(c, icmd.Expected{ 1037 ExitCode: 1, 1038 }) 1039 c.Assert(result.Combined(), checker.Contains, "Error: This node is not part of a swarm") 1040 c.Assert(result.Combined(), checker.Not(checker.Contains), "Please enter unlock key") 1041 1042 _, err := d.Cmd("swarm", "init") 1043 c.Assert(err, checker.IsNil) 1044 1045 // unlocking an unlocked swarm should return an error - it does not even ask for the key 1046 cmd = d.Command("swarm", "unlock") 1047 result = icmd.RunCmd(cmd) 1048 result.Assert(c, icmd.Expected{ 1049 ExitCode: 1, 1050 }) 1051 c.Assert(result.Combined(), checker.Contains, "Error: swarm is not locked") 1052 c.Assert(result.Combined(), checker.Not(checker.Contains), "Please enter unlock key") 1053 } 1054 1055 func (s *DockerSwarmSuite) TestSwarmInitLocked(c *check.C) { 1056 d := s.AddDaemon(c, false, false) 1057 1058 outs, err := d.Cmd("swarm", "init", "--autolock") 1059 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1060 1061 c.Assert(outs, checker.Contains, "docker swarm unlock") 1062 1063 var unlockKey string 1064 for _, line := range strings.Split(outs, "\n") { 1065 if strings.Contains(line, "SWMKEY") { 1066 unlockKey = strings.TrimSpace(line) 1067 break 1068 } 1069 } 1070 1071 c.Assert(unlockKey, checker.Not(checker.Equals), "") 1072 1073 outs, err = d.Cmd("swarm", "unlock-key", "-q") 1074 c.Assert(outs, checker.Equals, unlockKey+"\n") 1075 1076 c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive) 1077 1078 // It starts off locked 1079 d.Restart(c) 1080 c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked) 1081 1082 cmd := d.Command("swarm", "unlock") 1083 cmd.Stdin = bytes.NewBufferString("wrong-secret-key") 1084 icmd.RunCmd(cmd).Assert(c, icmd.Expected{ 1085 ExitCode: 1, 1086 Err: "invalid key", 1087 }) 1088 1089 c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked) 1090 1091 cmd = d.Command("swarm", "unlock") 1092 cmd.Stdin = bytes.NewBufferString(unlockKey) 1093 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1094 1095 c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive) 1096 1097 outs, err = d.Cmd("node", "ls") 1098 c.Assert(err, checker.IsNil) 1099 c.Assert(outs, checker.Not(checker.Contains), "Swarm is encrypted and needs to be unlocked") 1100 1101 outs, err = d.Cmd("swarm", "update", "--autolock=false") 1102 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1103 1104 checkSwarmLockedToUnlocked(c, d, unlockKey) 1105 1106 outs, err = d.Cmd("node", "ls") 1107 c.Assert(err, checker.IsNil) 1108 c.Assert(outs, checker.Not(checker.Contains), "Swarm is encrypted and needs to be unlocked") 1109 } 1110 1111 func (s *DockerSwarmSuite) TestSwarmLeaveLocked(c *check.C) { 1112 d := s.AddDaemon(c, false, false) 1113 1114 outs, err := d.Cmd("swarm", "init", "--autolock") 1115 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1116 1117 // It starts off locked 1118 d.Restart(c, "--swarm-default-advertise-addr=lo") 1119 1120 info, err := d.SwarmInfo() 1121 c.Assert(err, checker.IsNil) 1122 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateLocked) 1123 1124 outs, _ = d.Cmd("node", "ls") 1125 c.Assert(outs, checker.Contains, "Swarm is encrypted and needs to be unlocked") 1126 1127 // `docker swarm leave` a locked swarm without --force will return an error 1128 outs, _ = d.Cmd("swarm", "leave") 1129 c.Assert(outs, checker.Contains, "Swarm is encrypted and locked.") 1130 1131 // It is OK for user to leave a locked swarm with --force 1132 outs, err = d.Cmd("swarm", "leave", "--force") 1133 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1134 1135 info, err = d.SwarmInfo() 1136 c.Assert(err, checker.IsNil) 1137 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 1138 1139 outs, err = d.Cmd("swarm", "init") 1140 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1141 1142 info, err = d.SwarmInfo() 1143 c.Assert(err, checker.IsNil) 1144 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 1145 } 1146 1147 func (s *DockerSwarmSuite) TestSwarmLockUnlockCluster(c *check.C) { 1148 d1 := s.AddDaemon(c, true, true) 1149 d2 := s.AddDaemon(c, true, true) 1150 d3 := s.AddDaemon(c, true, true) 1151 1152 // they start off unlocked 1153 d2.Restart(c) 1154 c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive) 1155 1156 // stop this one so it does not get autolock info 1157 d2.Stop(c) 1158 1159 // enable autolock 1160 outs, err := d1.Cmd("swarm", "update", "--autolock") 1161 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1162 1163 c.Assert(outs, checker.Contains, "docker swarm unlock") 1164 1165 var unlockKey string 1166 for _, line := range strings.Split(outs, "\n") { 1167 if strings.Contains(line, "SWMKEY") { 1168 unlockKey = strings.TrimSpace(line) 1169 break 1170 } 1171 } 1172 1173 c.Assert(unlockKey, checker.Not(checker.Equals), "") 1174 1175 outs, err = d1.Cmd("swarm", "unlock-key", "-q") 1176 c.Assert(outs, checker.Equals, unlockKey+"\n") 1177 1178 // The ones that got the cluster update should be set to locked 1179 for _, d := range []*daemon.Swarm{d1, d3} { 1180 checkSwarmUnlockedToLocked(c, d) 1181 1182 cmd := d.Command("swarm", "unlock") 1183 cmd.Stdin = bytes.NewBufferString(unlockKey) 1184 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1185 c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive) 1186 } 1187 1188 // d2 never got the cluster update, so it is still set to unlocked 1189 d2.Start(c) 1190 c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive) 1191 1192 // d2 is now set to lock 1193 checkSwarmUnlockedToLocked(c, d2) 1194 1195 // leave it locked, and set the cluster to no longer autolock 1196 outs, err = d1.Cmd("swarm", "update", "--autolock=false") 1197 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1198 1199 // the ones that got the update are now set to unlocked 1200 for _, d := range []*daemon.Swarm{d1, d3} { 1201 checkSwarmLockedToUnlocked(c, d, unlockKey) 1202 } 1203 1204 // d2 still locked 1205 c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateLocked) 1206 1207 // unlock it 1208 cmd := d2.Command("swarm", "unlock") 1209 cmd.Stdin = bytes.NewBufferString(unlockKey) 1210 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1211 c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive) 1212 1213 // once it's caught up, d2 is set to not be locked 1214 checkSwarmLockedToUnlocked(c, d2, unlockKey) 1215 1216 // managers who join now are never set to locked in the first place 1217 d4 := s.AddDaemon(c, true, true) 1218 d4.Restart(c) 1219 c.Assert(getNodeStatus(c, d4), checker.Equals, swarm.LocalNodeStateActive) 1220 } 1221 1222 func (s *DockerSwarmSuite) TestSwarmJoinPromoteLocked(c *check.C) { 1223 d1 := s.AddDaemon(c, true, true) 1224 1225 // enable autolock 1226 outs, err := d1.Cmd("swarm", "update", "--autolock") 1227 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1228 1229 c.Assert(outs, checker.Contains, "docker swarm unlock") 1230 1231 var unlockKey string 1232 for _, line := range strings.Split(outs, "\n") { 1233 if strings.Contains(line, "SWMKEY") { 1234 unlockKey = strings.TrimSpace(line) 1235 break 1236 } 1237 } 1238 1239 c.Assert(unlockKey, checker.Not(checker.Equals), "") 1240 1241 outs, err = d1.Cmd("swarm", "unlock-key", "-q") 1242 c.Assert(outs, checker.Equals, unlockKey+"\n") 1243 1244 // joined workers start off unlocked 1245 d2 := s.AddDaemon(c, true, false) 1246 d2.Restart(c) 1247 c.Assert(getNodeStatus(c, d2), checker.Equals, swarm.LocalNodeStateActive) 1248 1249 // promote worker 1250 outs, err = d1.Cmd("node", "promote", d2.Info.NodeID) 1251 c.Assert(err, checker.IsNil) 1252 c.Assert(outs, checker.Contains, "promoted to a manager in the swarm") 1253 1254 // join new manager node 1255 d3 := s.AddDaemon(c, true, true) 1256 1257 // both new nodes are locked 1258 for _, d := range []*daemon.Swarm{d2, d3} { 1259 checkSwarmUnlockedToLocked(c, d) 1260 1261 cmd := d.Command("swarm", "unlock") 1262 cmd.Stdin = bytes.NewBufferString(unlockKey) 1263 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1264 c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive) 1265 } 1266 1267 // demote manager back to worker - workers are not locked 1268 outs, err = d1.Cmd("node", "demote", d3.Info.NodeID) 1269 c.Assert(err, checker.IsNil) 1270 c.Assert(outs, checker.Contains, "demoted in the swarm") 1271 1272 // Wait for it to actually be demoted, for the key and cert to be replaced. 1273 // Then restart and assert that the node is not locked. If we don't wait for the cert 1274 // to be replaced, then the node still has the manager TLS key which is still locked 1275 // (because we never want a manager TLS key to be on disk unencrypted if the cluster 1276 // is set to autolock) 1277 waitAndAssert(c, defaultReconciliationTimeout, d3.CheckControlAvailable, checker.False) 1278 waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { 1279 certBytes, err := ioutil.ReadFile(filepath.Join(d3.Folder, "root", "swarm", "certificates", "swarm-node.crt")) 1280 if err != nil { 1281 return "", check.Commentf("error: %v", err) 1282 } 1283 certs, err := helpers.ParseCertificatesPEM(certBytes) 1284 if err == nil && len(certs) > 0 && len(certs[0].Subject.OrganizationalUnit) > 0 { 1285 return certs[0].Subject.OrganizationalUnit[0], nil 1286 } 1287 return "", check.Commentf("could not get organizational unit from certificate") 1288 }, checker.Equals, "swarm-worker") 1289 1290 // by now, it should *never* be locked on restart 1291 d3.Restart(c) 1292 c.Assert(getNodeStatus(c, d3), checker.Equals, swarm.LocalNodeStateActive) 1293 } 1294 1295 func (s *DockerSwarmSuite) TestSwarmRotateUnlockKey(c *check.C) { 1296 d := s.AddDaemon(c, true, true) 1297 1298 outs, err := d.Cmd("swarm", "update", "--autolock") 1299 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1300 1301 c.Assert(outs, checker.Contains, "docker swarm unlock") 1302 1303 var unlockKey string 1304 for _, line := range strings.Split(outs, "\n") { 1305 if strings.Contains(line, "SWMKEY") { 1306 unlockKey = strings.TrimSpace(line) 1307 break 1308 } 1309 } 1310 1311 c.Assert(unlockKey, checker.Not(checker.Equals), "") 1312 1313 outs, err = d.Cmd("swarm", "unlock-key", "-q") 1314 c.Assert(outs, checker.Equals, unlockKey+"\n") 1315 1316 // Rotate multiple times 1317 for i := 0; i != 3; i++ { 1318 outs, err = d.Cmd("swarm", "unlock-key", "-q", "--rotate") 1319 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1320 // Strip \n 1321 newUnlockKey := outs[:len(outs)-1] 1322 c.Assert(newUnlockKey, checker.Not(checker.Equals), "") 1323 c.Assert(newUnlockKey, checker.Not(checker.Equals), unlockKey) 1324 1325 d.Restart(c) 1326 c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked) 1327 1328 outs, _ = d.Cmd("node", "ls") 1329 c.Assert(outs, checker.Contains, "Swarm is encrypted and needs to be unlocked") 1330 1331 cmd := d.Command("swarm", "unlock") 1332 cmd.Stdin = bytes.NewBufferString(unlockKey) 1333 result := icmd.RunCmd(cmd) 1334 1335 if result.Error == nil { 1336 // On occasion, the daemon may not have finished 1337 // rotating the KEK before restarting. The test is 1338 // intentionally written to explore this behavior. 1339 // When this happens, unlocking with the old key will 1340 // succeed. If we wait for the rotation to happen and 1341 // restart again, the new key should be required this 1342 // time. 1343 1344 time.Sleep(3 * time.Second) 1345 1346 d.Restart(c) 1347 1348 cmd = d.Command("swarm", "unlock") 1349 cmd.Stdin = bytes.NewBufferString(unlockKey) 1350 result = icmd.RunCmd(cmd) 1351 } 1352 result.Assert(c, icmd.Expected{ 1353 ExitCode: 1, 1354 Err: "invalid key", 1355 }) 1356 1357 outs, _ = d.Cmd("node", "ls") 1358 c.Assert(outs, checker.Contains, "Swarm is encrypted and needs to be unlocked") 1359 1360 cmd = d.Command("swarm", "unlock") 1361 cmd.Stdin = bytes.NewBufferString(newUnlockKey) 1362 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1363 1364 c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive) 1365 1366 outs, err = d.Cmd("node", "ls") 1367 c.Assert(err, checker.IsNil) 1368 c.Assert(outs, checker.Not(checker.Contains), "Swarm is encrypted and needs to be unlocked") 1369 1370 unlockKey = newUnlockKey 1371 } 1372 } 1373 1374 // This differs from `TestSwarmRotateUnlockKey` because that one rotates a single node, which is the leader. 1375 // This one keeps the leader up, and asserts that other manager nodes in the cluster also have their unlock 1376 // key rotated. 1377 func (s *DockerSwarmSuite) TestSwarmClusterRotateUnlockKey(c *check.C) { 1378 d1 := s.AddDaemon(c, true, true) // leader - don't restart this one, we don't want leader election delays 1379 d2 := s.AddDaemon(c, true, true) 1380 d3 := s.AddDaemon(c, true, true) 1381 1382 outs, err := d1.Cmd("swarm", "update", "--autolock") 1383 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1384 1385 c.Assert(outs, checker.Contains, "docker swarm unlock") 1386 1387 var unlockKey string 1388 for _, line := range strings.Split(outs, "\n") { 1389 if strings.Contains(line, "SWMKEY") { 1390 unlockKey = strings.TrimSpace(line) 1391 break 1392 } 1393 } 1394 1395 c.Assert(unlockKey, checker.Not(checker.Equals), "") 1396 1397 outs, err = d1.Cmd("swarm", "unlock-key", "-q") 1398 c.Assert(outs, checker.Equals, unlockKey+"\n") 1399 1400 // Rotate multiple times 1401 for i := 0; i != 3; i++ { 1402 outs, err = d1.Cmd("swarm", "unlock-key", "-q", "--rotate") 1403 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1404 // Strip \n 1405 newUnlockKey := outs[:len(outs)-1] 1406 c.Assert(newUnlockKey, checker.Not(checker.Equals), "") 1407 c.Assert(newUnlockKey, checker.Not(checker.Equals), unlockKey) 1408 1409 d2.Restart(c) 1410 d3.Restart(c) 1411 1412 for _, d := range []*daemon.Swarm{d2, d3} { 1413 c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateLocked) 1414 1415 outs, _ := d.Cmd("node", "ls") 1416 c.Assert(outs, checker.Contains, "Swarm is encrypted and needs to be unlocked") 1417 1418 cmd := d.Command("swarm", "unlock") 1419 cmd.Stdin = bytes.NewBufferString(unlockKey) 1420 result := icmd.RunCmd(cmd) 1421 1422 if result.Error == nil { 1423 // On occasion, the daemon may not have finished 1424 // rotating the KEK before restarting. The test is 1425 // intentionally written to explore this behavior. 1426 // When this happens, unlocking with the old key will 1427 // succeed. If we wait for the rotation to happen and 1428 // restart again, the new key should be required this 1429 // time. 1430 1431 time.Sleep(3 * time.Second) 1432 1433 d.Restart(c) 1434 1435 cmd = d.Command("swarm", "unlock") 1436 cmd.Stdin = bytes.NewBufferString(unlockKey) 1437 result = icmd.RunCmd(cmd) 1438 } 1439 result.Assert(c, icmd.Expected{ 1440 ExitCode: 1, 1441 Err: "invalid key", 1442 }) 1443 1444 outs, _ = d.Cmd("node", "ls") 1445 c.Assert(outs, checker.Contains, "Swarm is encrypted and needs to be unlocked") 1446 1447 cmd = d.Command("swarm", "unlock") 1448 cmd.Stdin = bytes.NewBufferString(newUnlockKey) 1449 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1450 1451 c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive) 1452 1453 outs, err = d.Cmd("node", "ls") 1454 c.Assert(err, checker.IsNil) 1455 c.Assert(outs, checker.Not(checker.Contains), "Swarm is encrypted and needs to be unlocked") 1456 } 1457 1458 unlockKey = newUnlockKey 1459 } 1460 } 1461 1462 func (s *DockerSwarmSuite) TestSwarmAlternateLockUnlock(c *check.C) { 1463 d := s.AddDaemon(c, true, true) 1464 1465 var unlockKey string 1466 for i := 0; i < 2; i++ { 1467 // set to lock 1468 outs, err := d.Cmd("swarm", "update", "--autolock") 1469 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1470 c.Assert(outs, checker.Contains, "docker swarm unlock") 1471 1472 for _, line := range strings.Split(outs, "\n") { 1473 if strings.Contains(line, "SWMKEY") { 1474 unlockKey = strings.TrimSpace(line) 1475 break 1476 } 1477 } 1478 1479 c.Assert(unlockKey, checker.Not(checker.Equals), "") 1480 checkSwarmUnlockedToLocked(c, d) 1481 1482 cmd := d.Command("swarm", "unlock") 1483 cmd.Stdin = bytes.NewBufferString(unlockKey) 1484 icmd.RunCmd(cmd).Assert(c, icmd.Success) 1485 1486 c.Assert(getNodeStatus(c, d), checker.Equals, swarm.LocalNodeStateActive) 1487 1488 outs, err = d.Cmd("swarm", "update", "--autolock=false") 1489 c.Assert(err, checker.IsNil, check.Commentf("out: %v", outs)) 1490 1491 checkSwarmLockedToUnlocked(c, d, unlockKey) 1492 } 1493 } 1494 1495 func (s *DockerSwarmSuite) TestExtraHosts(c *check.C) { 1496 d := s.AddDaemon(c, true, true) 1497 1498 // Create a service 1499 name := "top" 1500 _, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", name, "--host=example.com:1.2.3.4", "busybox", "top") 1501 c.Assert(err, checker.IsNil) 1502 1503 // Make sure task has been deployed. 1504 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 1505 1506 // We need to get the container id. 1507 out, err := d.Cmd("ps", "-a", "-q", "--no-trunc") 1508 c.Assert(err, checker.IsNil) 1509 id := strings.TrimSpace(out) 1510 1511 // Compare against expected output. 1512 expectedOutput := "1.2.3.4\texample.com" 1513 out, err = d.Cmd("exec", id, "cat", "/etc/hosts") 1514 c.Assert(err, checker.IsNil) 1515 c.Assert(out, checker.Contains, expectedOutput, check.Commentf("Expected '%s', but got %q", expectedOutput, out)) 1516 } 1517 1518 func (s *DockerSwarmSuite) TestSwarmManagerAddress(c *check.C) { 1519 d1 := s.AddDaemon(c, true, true) 1520 d2 := s.AddDaemon(c, true, false) 1521 d3 := s.AddDaemon(c, true, false) 1522 1523 // Manager Addresses will always show Node 1's address 1524 expectedOutput := fmt.Sprintf("Manager Addresses:\n 127.0.0.1:%d\n", d1.Port) 1525 1526 out, err := d1.Cmd("info") 1527 c.Assert(err, checker.IsNil) 1528 c.Assert(out, checker.Contains, expectedOutput) 1529 1530 out, err = d2.Cmd("info") 1531 c.Assert(err, checker.IsNil) 1532 c.Assert(out, checker.Contains, expectedOutput) 1533 1534 out, err = d3.Cmd("info") 1535 c.Assert(err, checker.IsNil) 1536 c.Assert(out, checker.Contains, expectedOutput) 1537 } 1538 1539 func (s *DockerSwarmSuite) TestSwarmNetworkIPAMOptions(c *check.C) { 1540 d := s.AddDaemon(c, true, true) 1541 1542 out, err := d.Cmd("network", "create", "-d", "overlay", "--ipam-opt", "foo=bar", "foo") 1543 c.Assert(err, checker.IsNil, check.Commentf(out)) 1544 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 1545 1546 out, err = d.Cmd("network", "inspect", "--format", "{{.IPAM.Options}}", "foo") 1547 c.Assert(err, checker.IsNil, check.Commentf(out)) 1548 c.Assert(strings.TrimSpace(out), checker.Contains, "foo:bar") 1549 c.Assert(strings.TrimSpace(out), checker.Contains, "com.docker.network.ipam.serial:true") 1550 1551 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--network=foo", "--name", "top", "busybox", "top") 1552 c.Assert(err, checker.IsNil, check.Commentf(out)) 1553 1554 // make sure task has been deployed. 1555 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 1556 1557 out, err = d.Cmd("network", "inspect", "--format", "{{.IPAM.Options}}", "foo") 1558 c.Assert(err, checker.IsNil, check.Commentf(out)) 1559 c.Assert(strings.TrimSpace(out), checker.Contains, "foo:bar") 1560 c.Assert(strings.TrimSpace(out), checker.Contains, "com.docker.network.ipam.serial:true") 1561 } 1562 1563 func (s *DockerTrustedSwarmSuite) TestTrustedServiceCreate(c *check.C) { 1564 d := s.swarmSuite.AddDaemon(c, true, true) 1565 1566 // Attempt creating a service from an image that is known to notary. 1567 repoName := s.trustSuite.setupTrustedImage(c, "trusted-pull") 1568 1569 name := "trusted" 1570 cli.Docker(cli.Args("-D", "service", "create", "--detach", "--no-resolve-image", "--name", name, repoName, "top"), trustedCmd, cli.Daemon(d.Daemon)).Assert(c, icmd.Expected{ 1571 Err: "resolved image tag to", 1572 }) 1573 1574 out, err := d.Cmd("service", "inspect", "--pretty", name) 1575 c.Assert(err, checker.IsNil, check.Commentf(out)) 1576 c.Assert(out, checker.Contains, repoName+"@", check.Commentf(out)) 1577 1578 // Try trusted service create on an untrusted tag. 1579 1580 repoName = fmt.Sprintf("%v/untrustedservicecreate/createtest:latest", privateRegistryURL) 1581 // tag the image and upload it to the private registry 1582 cli.DockerCmd(c, "tag", "busybox", repoName) 1583 cli.DockerCmd(c, "push", repoName) 1584 cli.DockerCmd(c, "rmi", repoName) 1585 1586 name = "untrusted" 1587 cli.Docker(cli.Args("service", "create", "--detach", "--no-resolve-image", "--name", name, repoName, "top"), trustedCmd, cli.Daemon(d.Daemon)).Assert(c, icmd.Expected{ 1588 ExitCode: 1, 1589 Err: "Error: remote trust data does not exist", 1590 }) 1591 1592 out, err = d.Cmd("service", "inspect", "--pretty", name) 1593 c.Assert(err, checker.NotNil, check.Commentf(out)) 1594 } 1595 1596 func (s *DockerTrustedSwarmSuite) TestTrustedServiceUpdate(c *check.C) { 1597 d := s.swarmSuite.AddDaemon(c, true, true) 1598 1599 // Attempt creating a service from an image that is known to notary. 1600 repoName := s.trustSuite.setupTrustedImage(c, "trusted-pull") 1601 1602 name := "myservice" 1603 1604 // Create a service without content trust 1605 cli.Docker(cli.Args("service", "create", "--detach", "--no-resolve-image", "--name", name, repoName, "top"), cli.Daemon(d.Daemon)).Assert(c, icmd.Success) 1606 1607 result := cli.Docker(cli.Args("service", "inspect", "--pretty", name), cli.Daemon(d.Daemon)) 1608 c.Assert(result.Error, checker.IsNil, check.Commentf(result.Combined())) 1609 // Daemon won't insert the digest because this is disabled by 1610 // DOCKER_SERVICE_PREFER_OFFLINE_IMAGE. 1611 c.Assert(result.Combined(), check.Not(checker.Contains), repoName+"@", check.Commentf(result.Combined())) 1612 1613 cli.Docker(cli.Args("-D", "service", "update", "--detach", "--no-resolve-image", "--image", repoName, name), trustedCmd, cli.Daemon(d.Daemon)).Assert(c, icmd.Expected{ 1614 Err: "resolved image tag to", 1615 }) 1616 1617 cli.Docker(cli.Args("service", "inspect", "--pretty", name), cli.Daemon(d.Daemon)).Assert(c, icmd.Expected{ 1618 Out: repoName + "@", 1619 }) 1620 1621 // Try trusted service update on an untrusted tag. 1622 1623 repoName = fmt.Sprintf("%v/untrustedservicecreate/createtest:latest", privateRegistryURL) 1624 // tag the image and upload it to the private registry 1625 cli.DockerCmd(c, "tag", "busybox", repoName) 1626 cli.DockerCmd(c, "push", repoName) 1627 cli.DockerCmd(c, "rmi", repoName) 1628 1629 cli.Docker(cli.Args("service", "update", "--detach", "--no-resolve-image", "--image", repoName, name), trustedCmd, cli.Daemon(d.Daemon)).Assert(c, icmd.Expected{ 1630 ExitCode: 1, 1631 Err: "Error: remote trust data does not exist", 1632 }) 1633 } 1634 1635 // Test case for issue #27866, which did not allow NW name that is the prefix of a swarm NW ID. 1636 // e.g. if the ingress ID starts with "n1", it was impossible to create a NW named "n1". 1637 func (s *DockerSwarmSuite) TestSwarmNetworkCreateIssue27866(c *check.C) { 1638 d := s.AddDaemon(c, true, true) 1639 out, err := d.Cmd("network", "inspect", "-f", "{{.Id}}", "ingress") 1640 c.Assert(err, checker.IsNil, check.Commentf("out: %v", out)) 1641 ingressID := strings.TrimSpace(out) 1642 c.Assert(ingressID, checker.Not(checker.Equals), "") 1643 1644 // create a network of which name is the prefix of the ID of an overlay network 1645 // (ingressID in this case) 1646 newNetName := ingressID[0:2] 1647 out, err = d.Cmd("network", "create", "--driver", "overlay", newNetName) 1648 // In #27866, it was failing because of "network with name %s already exists" 1649 c.Assert(err, checker.IsNil, check.Commentf("out: %v", out)) 1650 out, err = d.Cmd("network", "rm", newNetName) 1651 c.Assert(err, checker.IsNil, check.Commentf("out: %v", out)) 1652 } 1653 1654 // Test case for https://github.com/docker/docker/pull/27938#issuecomment-265768303 1655 // This test creates two networks with the same name sequentially, with various drivers. 1656 // Since the operations in this test are done sequentially, the 2nd call should fail with 1657 // "network with name FOO already exists". 1658 // Note that it is to ok have multiple networks with the same name if the operations are done 1659 // in parallel. (#18864) 1660 func (s *DockerSwarmSuite) TestSwarmNetworkCreateDup(c *check.C) { 1661 d := s.AddDaemon(c, true, true) 1662 drivers := []string{"bridge", "overlay"} 1663 for i, driver1 := range drivers { 1664 nwName := fmt.Sprintf("network-test-%d", i) 1665 for _, driver2 := range drivers { 1666 c.Logf("Creating a network named %q with %q, then %q", 1667 nwName, driver1, driver2) 1668 out, err := d.Cmd("network", "create", "--driver", driver1, nwName) 1669 c.Assert(err, checker.IsNil, check.Commentf("out: %v", out)) 1670 out, err = d.Cmd("network", "create", "--driver", driver2, nwName) 1671 c.Assert(out, checker.Contains, 1672 fmt.Sprintf("network with name %s already exists", nwName)) 1673 c.Assert(err, checker.NotNil) 1674 c.Logf("As expected, the attempt to network %q with %q failed: %s", 1675 nwName, driver2, out) 1676 out, err = d.Cmd("network", "rm", nwName) 1677 c.Assert(err, checker.IsNil, check.Commentf("out: %v", out)) 1678 } 1679 } 1680 } 1681 1682 func (s *DockerSwarmSuite) TestSwarmPublishDuplicatePorts(c *check.C) { 1683 d := s.AddDaemon(c, true, true) 1684 1685 out, err := d.Cmd("service", "create", "--no-resolve-image", "--detach=true", "--publish", "5005:80", "--publish", "5006:80", "--publish", "80", "--publish", "80", "busybox", "top") 1686 c.Assert(err, check.IsNil, check.Commentf(out)) 1687 id := strings.TrimSpace(out) 1688 1689 // make sure task has been deployed. 1690 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 1691 1692 // Total len = 4, with 2 dynamic ports and 2 non-dynamic ports 1693 // Dynamic ports are likely to be 30000 and 30001 but doesn't matter 1694 out, err = d.Cmd("service", "inspect", "--format", "{{.Endpoint.Ports}} len={{len .Endpoint.Ports}}", id) 1695 c.Assert(err, check.IsNil, check.Commentf(out)) 1696 c.Assert(out, checker.Contains, "len=4") 1697 c.Assert(out, checker.Contains, "{ tcp 80 5005 ingress}") 1698 c.Assert(out, checker.Contains, "{ tcp 80 5006 ingress}") 1699 } 1700 1701 func (s *DockerSwarmSuite) TestSwarmJoinWithDrain(c *check.C) { 1702 d := s.AddDaemon(c, true, true) 1703 1704 out, err := d.Cmd("node", "ls") 1705 c.Assert(err, checker.IsNil) 1706 c.Assert(out, checker.Not(checker.Contains), "Drain") 1707 1708 out, err = d.Cmd("swarm", "join-token", "-q", "manager") 1709 c.Assert(err, checker.IsNil) 1710 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 1711 1712 token := strings.TrimSpace(out) 1713 1714 d1 := s.AddDaemon(c, false, false) 1715 1716 out, err = d1.Cmd("swarm", "join", "--availability=drain", "--token", token, d.ListenAddr) 1717 c.Assert(err, checker.IsNil) 1718 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 1719 1720 out, err = d.Cmd("node", "ls") 1721 c.Assert(err, checker.IsNil) 1722 c.Assert(out, checker.Contains, "Drain") 1723 1724 out, err = d1.Cmd("node", "ls") 1725 c.Assert(err, checker.IsNil) 1726 c.Assert(out, checker.Contains, "Drain") 1727 } 1728 1729 func (s *DockerSwarmSuite) TestSwarmInitWithDrain(c *check.C) { 1730 d := s.AddDaemon(c, false, false) 1731 1732 out, err := d.Cmd("swarm", "init", "--availability", "drain") 1733 c.Assert(err, checker.IsNil, check.Commentf("out: %v", out)) 1734 1735 out, err = d.Cmd("node", "ls") 1736 c.Assert(err, checker.IsNil) 1737 c.Assert(out, checker.Contains, "Drain") 1738 } 1739 1740 func (s *DockerSwarmSuite) TestSwarmReadonlyRootfs(c *check.C) { 1741 testRequires(c, DaemonIsLinux, UserNamespaceROMount) 1742 1743 d := s.AddDaemon(c, true, true) 1744 1745 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "top", "--read-only", "busybox", "top") 1746 c.Assert(err, checker.IsNil, check.Commentf(out)) 1747 1748 // make sure task has been deployed. 1749 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 1750 1751 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.ReadOnly }}", "top") 1752 c.Assert(err, checker.IsNil, check.Commentf(out)) 1753 c.Assert(strings.TrimSpace(out), checker.Equals, "true") 1754 1755 containers := d.ActiveContainers() 1756 out, err = d.Cmd("inspect", "--type", "container", "--format", "{{.HostConfig.ReadonlyRootfs}}", containers[0]) 1757 c.Assert(err, checker.IsNil, check.Commentf(out)) 1758 c.Assert(strings.TrimSpace(out), checker.Equals, "true") 1759 } 1760 1761 func (s *DockerSwarmSuite) TestNetworkInspectWithDuplicateNames(c *check.C) { 1762 d := s.AddDaemon(c, true, true) 1763 1764 name := "foo" 1765 options := types.NetworkCreate{ 1766 CheckDuplicate: false, 1767 Driver: "bridge", 1768 } 1769 1770 cli, err := d.NewClient() 1771 c.Assert(err, checker.IsNil) 1772 defer cli.Close() 1773 1774 n1, err := cli.NetworkCreate(context.Background(), name, options) 1775 c.Assert(err, checker.IsNil) 1776 1777 // Full ID always works 1778 out, err := d.Cmd("network", "inspect", "--format", "{{.ID}}", n1.ID) 1779 c.Assert(err, checker.IsNil, check.Commentf(out)) 1780 c.Assert(strings.TrimSpace(out), checker.Equals, n1.ID) 1781 1782 // Name works if it is unique 1783 out, err = d.Cmd("network", "inspect", "--format", "{{.ID}}", name) 1784 c.Assert(err, checker.IsNil, check.Commentf(out)) 1785 c.Assert(strings.TrimSpace(out), checker.Equals, n1.ID) 1786 1787 n2, err := cli.NetworkCreate(context.Background(), name, options) 1788 c.Assert(err, checker.IsNil) 1789 // Full ID always works 1790 out, err = d.Cmd("network", "inspect", "--format", "{{.ID}}", n1.ID) 1791 c.Assert(err, checker.IsNil, check.Commentf(out)) 1792 c.Assert(strings.TrimSpace(out), checker.Equals, n1.ID) 1793 1794 out, err = d.Cmd("network", "inspect", "--format", "{{.ID}}", n2.ID) 1795 c.Assert(err, checker.IsNil, check.Commentf(out)) 1796 c.Assert(strings.TrimSpace(out), checker.Equals, n2.ID) 1797 1798 // Name with duplicates 1799 out, err = d.Cmd("network", "inspect", "--format", "{{.ID}}", name) 1800 c.Assert(err, checker.NotNil, check.Commentf(out)) 1801 c.Assert(out, checker.Contains, "2 matches found based on name") 1802 1803 out, err = d.Cmd("network", "rm", n2.ID) 1804 c.Assert(err, checker.IsNil, check.Commentf(out)) 1805 1806 // Dupliates with name but with different driver 1807 options.Driver = "overlay" 1808 1809 n2, err = cli.NetworkCreate(context.Background(), name, options) 1810 c.Assert(err, checker.IsNil) 1811 1812 // Full ID always works 1813 out, err = d.Cmd("network", "inspect", "--format", "{{.ID}}", n1.ID) 1814 c.Assert(err, checker.IsNil, check.Commentf(out)) 1815 c.Assert(strings.TrimSpace(out), checker.Equals, n1.ID) 1816 1817 out, err = d.Cmd("network", "inspect", "--format", "{{.ID}}", n2.ID) 1818 c.Assert(err, checker.IsNil, check.Commentf(out)) 1819 c.Assert(strings.TrimSpace(out), checker.Equals, n2.ID) 1820 1821 // Name with duplicates 1822 out, err = d.Cmd("network", "inspect", "--format", "{{.ID}}", name) 1823 c.Assert(err, checker.NotNil, check.Commentf(out)) 1824 c.Assert(out, checker.Contains, "2 matches found based on name") 1825 } 1826 1827 func (s *DockerSwarmSuite) TestSwarmStopSignal(c *check.C) { 1828 testRequires(c, DaemonIsLinux, UserNamespaceROMount) 1829 1830 d := s.AddDaemon(c, true, true) 1831 1832 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "top", "--stop-signal=SIGHUP", "busybox", "top") 1833 c.Assert(err, checker.IsNil, check.Commentf(out)) 1834 1835 // make sure task has been deployed. 1836 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 1837 1838 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.StopSignal }}", "top") 1839 c.Assert(err, checker.IsNil, check.Commentf(out)) 1840 c.Assert(strings.TrimSpace(out), checker.Equals, "SIGHUP") 1841 1842 containers := d.ActiveContainers() 1843 out, err = d.Cmd("inspect", "--type", "container", "--format", "{{.Config.StopSignal}}", containers[0]) 1844 c.Assert(err, checker.IsNil, check.Commentf(out)) 1845 c.Assert(strings.TrimSpace(out), checker.Equals, "SIGHUP") 1846 1847 out, err = d.Cmd("service", "update", "--detach", "--stop-signal=SIGUSR1", "top") 1848 c.Assert(err, checker.IsNil, check.Commentf(out)) 1849 1850 out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.TaskTemplate.ContainerSpec.StopSignal }}", "top") 1851 c.Assert(err, checker.IsNil, check.Commentf(out)) 1852 c.Assert(strings.TrimSpace(out), checker.Equals, "SIGUSR1") 1853 } 1854 1855 func (s *DockerSwarmSuite) TestSwarmServiceLsFilterMode(c *check.C) { 1856 d := s.AddDaemon(c, true, true) 1857 1858 out, err := d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "top1", "busybox", "top") 1859 c.Assert(err, checker.IsNil, check.Commentf(out)) 1860 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 1861 1862 out, err = d.Cmd("service", "create", "--detach", "--no-resolve-image", "--name", "top2", "--mode=global", "busybox", "top") 1863 c.Assert(err, checker.IsNil, check.Commentf(out)) 1864 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 1865 1866 // make sure task has been deployed. 1867 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 2) 1868 1869 out, err = d.Cmd("service", "ls") 1870 c.Assert(err, checker.IsNil, check.Commentf(out)) 1871 c.Assert(out, checker.Contains, "top1") 1872 c.Assert(out, checker.Contains, "top2") 1873 c.Assert(out, checker.Not(checker.Contains), "localnet") 1874 1875 out, err = d.Cmd("service", "ls", "--filter", "mode=global") 1876 c.Assert(out, checker.Not(checker.Contains), "top1") 1877 c.Assert(out, checker.Contains, "top2") 1878 c.Assert(err, checker.IsNil, check.Commentf(out)) 1879 1880 out, err = d.Cmd("service", "ls", "--filter", "mode=replicated") 1881 c.Assert(err, checker.IsNil, check.Commentf(out)) 1882 c.Assert(out, checker.Contains, "top1") 1883 c.Assert(out, checker.Not(checker.Contains), "top2") 1884 } 1885 1886 func (s *DockerSwarmSuite) TestSwarmInitUnspecifiedDataPathAddr(c *check.C) { 1887 d := s.AddDaemon(c, false, false) 1888 1889 out, err := d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0") 1890 c.Assert(err, checker.NotNil) 1891 c.Assert(out, checker.Contains, "data path address must be a non-zero IP") 1892 1893 out, err = d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0:2000") 1894 c.Assert(err, checker.NotNil) 1895 c.Assert(out, checker.Contains, "data path address must be a non-zero IP") 1896 } 1897 1898 func (s *DockerSwarmSuite) TestSwarmJoinLeave(c *check.C) { 1899 d := s.AddDaemon(c, true, true) 1900 1901 out, err := d.Cmd("swarm", "join-token", "-q", "worker") 1902 c.Assert(err, checker.IsNil) 1903 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 1904 1905 token := strings.TrimSpace(out) 1906 1907 // Verify that back to back join/leave does not cause panics 1908 d1 := s.AddDaemon(c, false, false) 1909 for i := 0; i < 10; i++ { 1910 out, err = d1.Cmd("swarm", "join", "--token", token, d.ListenAddr) 1911 c.Assert(err, checker.IsNil) 1912 c.Assert(strings.TrimSpace(out), checker.Not(checker.Equals), "") 1913 1914 _, err = d1.Cmd("swarm", "leave") 1915 c.Assert(err, checker.IsNil) 1916 } 1917 } 1918 1919 const defaultRetryCount = 10 1920 1921 func waitForEvent(c *check.C, d *daemon.Swarm, since string, filter string, event string, retry int) string { 1922 if retry < 1 { 1923 c.Fatalf("retry count %d is invalid. It should be no less than 1", retry) 1924 return "" 1925 } 1926 var out string 1927 for i := 0; i < retry; i++ { 1928 until := daemonUnixTime(c) 1929 var err error 1930 if len(filter) > 0 { 1931 out, err = d.Cmd("events", "--since", since, "--until", until, filter) 1932 } else { 1933 out, err = d.Cmd("events", "--since", since, "--until", until) 1934 } 1935 c.Assert(err, checker.IsNil, check.Commentf(out)) 1936 if strings.Contains(out, event) { 1937 return strings.TrimSpace(out) 1938 } 1939 // no need to sleep after last retry 1940 if i < retry-1 { 1941 time.Sleep(200 * time.Millisecond) 1942 } 1943 } 1944 c.Fatalf("docker events output '%s' doesn't contain event '%s'", out, event) 1945 return "" 1946 } 1947 1948 func (s *DockerSwarmSuite) TestSwarmClusterEventsSource(c *check.C) { 1949 d1 := s.AddDaemon(c, true, true) 1950 d2 := s.AddDaemon(c, true, true) 1951 d3 := s.AddDaemon(c, true, false) 1952 1953 // create a network 1954 out, err := d1.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 1955 c.Assert(err, checker.IsNil, check.Commentf(out)) 1956 networkID := strings.TrimSpace(out) 1957 c.Assert(networkID, checker.Not(checker.Equals), "") 1958 1959 // d1, d2 are managers that can get swarm events 1960 waitForEvent(c, d1, "0", "-f scope=swarm", "network create "+networkID, defaultRetryCount) 1961 waitForEvent(c, d2, "0", "-f scope=swarm", "network create "+networkID, defaultRetryCount) 1962 1963 // d3 is a worker, not able to get cluster events 1964 out = waitForEvent(c, d3, "0", "-f scope=swarm", "", 1) 1965 c.Assert(out, checker.Not(checker.Contains), "network create ") 1966 } 1967 1968 func (s *DockerSwarmSuite) TestSwarmClusterEventsScope(c *check.C) { 1969 d := s.AddDaemon(c, true, true) 1970 1971 // create a service 1972 out, err := d.Cmd("service", "create", "--no-resolve-image", "--name", "test", "--detach=false", "busybox", "top") 1973 c.Assert(err, checker.IsNil, check.Commentf(out)) 1974 serviceID := strings.Split(out, "\n")[0] 1975 1976 // scope swarm filters cluster events 1977 out = waitForEvent(c, d, "0", "-f scope=swarm", "service create "+serviceID, defaultRetryCount) 1978 c.Assert(out, checker.Not(checker.Contains), "container create ") 1979 1980 // all events are returned if scope is not specified 1981 waitForEvent(c, d, "0", "", "service create "+serviceID, 1) 1982 waitForEvent(c, d, "0", "", "container create ", defaultRetryCount) 1983 1984 // scope local only shows non-cluster events 1985 out = waitForEvent(c, d, "0", "-f scope=local", "container create ", 1) 1986 c.Assert(out, checker.Not(checker.Contains), "service create ") 1987 } 1988 1989 func (s *DockerSwarmSuite) TestSwarmClusterEventsType(c *check.C) { 1990 d := s.AddDaemon(c, true, true) 1991 1992 // create a service 1993 out, err := d.Cmd("service", "create", "--no-resolve-image", "--name", "test", "--detach=false", "busybox", "top") 1994 c.Assert(err, checker.IsNil, check.Commentf(out)) 1995 serviceID := strings.Split(out, "\n")[0] 1996 1997 // create a network 1998 out, err = d.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 1999 c.Assert(err, checker.IsNil, check.Commentf(out)) 2000 networkID := strings.TrimSpace(out) 2001 c.Assert(networkID, checker.Not(checker.Equals), "") 2002 2003 // filter by service 2004 out = waitForEvent(c, d, "0", "-f type=service", "service create "+serviceID, defaultRetryCount) 2005 c.Assert(out, checker.Not(checker.Contains), "network create") 2006 2007 // filter by network 2008 out = waitForEvent(c, d, "0", "-f type=network", "network create "+networkID, defaultRetryCount) 2009 c.Assert(out, checker.Not(checker.Contains), "service create") 2010 } 2011 2012 func (s *DockerSwarmSuite) TestSwarmClusterEventsService(c *check.C) { 2013 d := s.AddDaemon(c, true, true) 2014 2015 // create a service 2016 out, err := d.Cmd("service", "create", "--no-resolve-image", "--name", "test", "--detach=false", "busybox", "top") 2017 c.Assert(err, checker.IsNil, check.Commentf(out)) 2018 serviceID := strings.Split(out, "\n")[0] 2019 2020 // validate service create event 2021 waitForEvent(c, d, "0", "-f scope=swarm", "service create "+serviceID, defaultRetryCount) 2022 2023 t1 := daemonUnixTime(c) 2024 out, err = d.Cmd("service", "update", "--force", "--detach=false", "test") 2025 c.Assert(err, checker.IsNil, check.Commentf(out)) 2026 2027 // wait for service update start 2028 out = waitForEvent(c, d, t1, "-f scope=swarm", "service update "+serviceID, defaultRetryCount) 2029 c.Assert(out, checker.Contains, "updatestate.new=updating") 2030 2031 // allow service update complete. This is a service with 1 instance 2032 time.Sleep(400 * time.Millisecond) 2033 out = waitForEvent(c, d, t1, "-f scope=swarm", "service update "+serviceID, defaultRetryCount) 2034 c.Assert(out, checker.Contains, "updatestate.new=completed, updatestate.old=updating") 2035 2036 // scale service 2037 t2 := daemonUnixTime(c) 2038 out, err = d.Cmd("service", "scale", "test=3") 2039 c.Assert(err, checker.IsNil, check.Commentf(out)) 2040 2041 out = waitForEvent(c, d, t2, "-f scope=swarm", "service update "+serviceID, defaultRetryCount) 2042 c.Assert(out, checker.Contains, "replicas.new=3, replicas.old=1") 2043 2044 // remove service 2045 t3 := daemonUnixTime(c) 2046 out, err = d.Cmd("service", "rm", "test") 2047 c.Assert(err, checker.IsNil, check.Commentf(out)) 2048 2049 waitForEvent(c, d, t3, "-f scope=swarm", "service remove "+serviceID, defaultRetryCount) 2050 } 2051 2052 func (s *DockerSwarmSuite) TestSwarmClusterEventsNode(c *check.C) { 2053 d1 := s.AddDaemon(c, true, true) 2054 s.AddDaemon(c, true, true) 2055 d3 := s.AddDaemon(c, true, true) 2056 2057 d3ID := d3.NodeID 2058 waitForEvent(c, d1, "0", "-f scope=swarm", "node create "+d3ID, defaultRetryCount) 2059 2060 t1 := daemonUnixTime(c) 2061 out, err := d1.Cmd("node", "update", "--availability=pause", d3ID) 2062 c.Assert(err, checker.IsNil, check.Commentf(out)) 2063 2064 // filter by type 2065 out = waitForEvent(c, d1, t1, "-f type=node", "node update "+d3ID, defaultRetryCount) 2066 c.Assert(out, checker.Contains, "availability.new=pause, availability.old=active") 2067 2068 t2 := daemonUnixTime(c) 2069 out, err = d1.Cmd("node", "demote", d3ID) 2070 c.Assert(err, checker.IsNil, check.Commentf(out)) 2071 2072 waitForEvent(c, d1, t2, "-f type=node", "node update "+d3ID, defaultRetryCount) 2073 2074 t3 := daemonUnixTime(c) 2075 out, err = d1.Cmd("node", "rm", "-f", d3ID) 2076 c.Assert(err, checker.IsNil, check.Commentf(out)) 2077 2078 // filter by scope 2079 waitForEvent(c, d1, t3, "-f scope=swarm", "node remove "+d3ID, defaultRetryCount) 2080 } 2081 2082 func (s *DockerSwarmSuite) TestSwarmClusterEventsNetwork(c *check.C) { 2083 d := s.AddDaemon(c, true, true) 2084 2085 // create a network 2086 out, err := d.Cmd("network", "create", "--attachable", "-d", "overlay", "foo") 2087 c.Assert(err, checker.IsNil, check.Commentf(out)) 2088 networkID := strings.TrimSpace(out) 2089 2090 waitForEvent(c, d, "0", "-f scope=swarm", "network create "+networkID, defaultRetryCount) 2091 2092 // remove network 2093 t1 := daemonUnixTime(c) 2094 out, err = d.Cmd("network", "rm", "foo") 2095 c.Assert(err, checker.IsNil, check.Commentf(out)) 2096 2097 // filtered by network 2098 waitForEvent(c, d, t1, "-f type=network", "network remove "+networkID, defaultRetryCount) 2099 } 2100 2101 func (s *DockerSwarmSuite) TestSwarmClusterEventsSecret(c *check.C) { 2102 d := s.AddDaemon(c, true, true) 2103 2104 testName := "test_secret" 2105 id := d.CreateSecret(c, swarm.SecretSpec{ 2106 Annotations: swarm.Annotations{ 2107 Name: testName, 2108 }, 2109 Data: []byte("TESTINGDATA"), 2110 }) 2111 c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("secrets: %s", id)) 2112 2113 waitForEvent(c, d, "0", "-f scope=swarm", "secret create "+id, defaultRetryCount) 2114 2115 t1 := daemonUnixTime(c) 2116 d.DeleteSecret(c, id) 2117 // filtered by secret 2118 waitForEvent(c, d, t1, "-f type=secret", "secret remove "+id, defaultRetryCount) 2119 } 2120 2121 func (s *DockerSwarmSuite) TestSwarmClusterEventsConfig(c *check.C) { 2122 d := s.AddDaemon(c, true, true) 2123 2124 testName := "test_config" 2125 id := d.CreateConfig(c, swarm.ConfigSpec{ 2126 Annotations: swarm.Annotations{ 2127 Name: testName, 2128 }, 2129 Data: []byte("TESTINGDATA"), 2130 }) 2131 c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("configs: %s", id)) 2132 2133 waitForEvent(c, d, "0", "-f scope=swarm", "config create "+id, defaultRetryCount) 2134 2135 t1 := daemonUnixTime(c) 2136 d.DeleteConfig(c, id) 2137 // filtered by config 2138 waitForEvent(c, d, t1, "-f type=config", "config remove "+id, defaultRetryCount) 2139 }