github.com/brahmaroutu/docker@v1.2.1-0.20160809185609-eb28dde01f16/integration-cli/docker_api_swarm_test.go (about) 1 // +build !windows 2 3 package main 4 5 import ( 6 "fmt" 7 "net/http" 8 "os" 9 "path/filepath" 10 "strconv" 11 "strings" 12 "sync" 13 "syscall" 14 "time" 15 16 "github.com/docker/docker/pkg/integration/checker" 17 "github.com/docker/engine-api/types/swarm" 18 "github.com/go-check/check" 19 ) 20 21 var defaultReconciliationTimeout = 30 * time.Second 22 23 func (s *DockerSwarmSuite) TestApiSwarmInit(c *check.C) { 24 // todo: should find a better way to verify that components are running than /info 25 d1 := s.AddDaemon(c, true, true) 26 info, err := d1.info() 27 c.Assert(err, checker.IsNil) 28 c.Assert(info.ControlAvailable, checker.True) 29 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 30 31 d2 := s.AddDaemon(c, true, false) 32 info, err = d2.info() 33 c.Assert(err, checker.IsNil) 34 c.Assert(info.ControlAvailable, checker.False) 35 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 36 37 // Leaving cluster 38 c.Assert(d2.Leave(false), checker.IsNil) 39 40 info, err = d2.info() 41 c.Assert(err, checker.IsNil) 42 c.Assert(info.ControlAvailable, checker.False) 43 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 44 45 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: d1.joinTokens(c).Worker, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 46 47 info, err = d2.info() 48 c.Assert(err, checker.IsNil) 49 c.Assert(info.ControlAvailable, checker.False) 50 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 51 52 // Current state restoring after restarts 53 err = d1.Stop() 54 c.Assert(err, checker.IsNil) 55 err = d2.Stop() 56 c.Assert(err, checker.IsNil) 57 58 err = d1.Start() 59 c.Assert(err, checker.IsNil) 60 err = d2.Start() 61 c.Assert(err, checker.IsNil) 62 63 info, err = d1.info() 64 c.Assert(err, checker.IsNil) 65 c.Assert(info.ControlAvailable, checker.True) 66 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 67 68 info, err = d2.info() 69 c.Assert(err, checker.IsNil) 70 c.Assert(info.ControlAvailable, checker.False) 71 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 72 } 73 74 func (s *DockerSwarmSuite) TestApiSwarmJoinToken(c *check.C) { 75 d1 := s.AddDaemon(c, false, false) 76 c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) 77 78 d2 := s.AddDaemon(c, false, false) 79 err := d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.listenAddr}}) 80 c.Assert(err, checker.NotNil) 81 c.Assert(err.Error(), checker.Contains, "join token is necessary") 82 info, err := d2.info() 83 c.Assert(err, checker.IsNil) 84 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 85 86 err = d2.Join(swarm.JoinRequest{JoinToken: "foobaz", RemoteAddrs: []string{d1.listenAddr}}) 87 c.Assert(err, checker.NotNil) 88 c.Assert(err.Error(), checker.Contains, "join token is necessary") 89 info, err = d2.info() 90 c.Assert(err, checker.IsNil) 91 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 92 93 workerToken := d1.joinTokens(c).Worker 94 95 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 96 info, err = d2.info() 97 c.Assert(err, checker.IsNil) 98 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 99 c.Assert(d2.Leave(false), checker.IsNil) 100 info, err = d2.info() 101 c.Assert(err, checker.IsNil) 102 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 103 104 // change tokens 105 d1.rotateTokens(c) 106 107 err = d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}) 108 c.Assert(err, checker.NotNil) 109 c.Assert(err.Error(), checker.Contains, "join token is necessary") 110 info, err = d2.info() 111 c.Assert(err, checker.IsNil) 112 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 113 114 workerToken = d1.joinTokens(c).Worker 115 116 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 117 info, err = d2.info() 118 c.Assert(err, checker.IsNil) 119 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 120 c.Assert(d2.Leave(false), checker.IsNil) 121 info, err = d2.info() 122 c.Assert(err, checker.IsNil) 123 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 124 125 // change spec, don't change tokens 126 d1.updateSwarm(c, func(s *swarm.Spec) {}) 127 128 err = d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.listenAddr}}) 129 c.Assert(err, checker.NotNil) 130 c.Assert(err.Error(), checker.Contains, "join token is necessary") 131 info, err = d2.info() 132 c.Assert(err, checker.IsNil) 133 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 134 135 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 136 info, err = d2.info() 137 c.Assert(err, checker.IsNil) 138 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 139 c.Assert(d2.Leave(false), checker.IsNil) 140 info, err = d2.info() 141 c.Assert(err, checker.IsNil) 142 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 143 } 144 145 func (s *DockerSwarmSuite) TestApiSwarmCAHash(c *check.C) { 146 d1 := s.AddDaemon(c, true, true) 147 d2 := s.AddDaemon(c, false, false) 148 splitToken := strings.Split(d1.joinTokens(c).Worker, "-") 149 splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e" 150 replacementToken := strings.Join(splitToken, "-") 151 err := d2.Join(swarm.JoinRequest{JoinToken: replacementToken, RemoteAddrs: []string{d1.listenAddr}}) 152 c.Assert(err, checker.NotNil) 153 c.Assert(err.Error(), checker.Contains, "remote CA does not match fingerprint") 154 } 155 156 func (s *DockerSwarmSuite) TestApiSwarmPromoteDemote(c *check.C) { 157 d1 := s.AddDaemon(c, false, false) 158 c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) 159 d2 := s.AddDaemon(c, true, false) 160 161 info, err := d2.info() 162 c.Assert(err, checker.IsNil) 163 c.Assert(info.ControlAvailable, checker.False) 164 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 165 166 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 167 n.Spec.Role = swarm.NodeRoleManager 168 }) 169 170 waitAndAssert(c, defaultReconciliationTimeout, d2.checkControlAvailable, checker.True) 171 172 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 173 n.Spec.Role = swarm.NodeRoleWorker 174 }) 175 176 waitAndAssert(c, defaultReconciliationTimeout, d2.checkControlAvailable, checker.False) 177 178 // Demoting last node should fail 179 node := d1.getNode(c, d1.NodeID) 180 node.Spec.Role = swarm.NodeRoleWorker 181 url := fmt.Sprintf("/nodes/%s/update?version=%d", node.ID, node.Version.Index) 182 status, out, err := d1.SockRequest("POST", url, node.Spec) 183 c.Assert(err, checker.IsNil) 184 c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("output: %q", string(out))) 185 c.Assert(string(out), checker.Contains, "last manager of the swarm") 186 info, err = d1.info() 187 c.Assert(err, checker.IsNil) 188 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 189 c.Assert(info.ControlAvailable, checker.True) 190 191 // Promote already demoted node 192 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 193 n.Spec.Role = swarm.NodeRoleManager 194 }) 195 196 waitAndAssert(c, defaultReconciliationTimeout, d2.checkControlAvailable, checker.True) 197 } 198 199 func (s *DockerSwarmSuite) TestApiSwarmServicesEmptyList(c *check.C) { 200 d := s.AddDaemon(c, true, true) 201 202 services := d.listServices(c) 203 c.Assert(services, checker.NotNil) 204 c.Assert(len(services), checker.Equals, 0, check.Commentf("services: %#v", services)) 205 } 206 207 func (s *DockerSwarmSuite) TestApiSwarmServicesCreate(c *check.C) { 208 d := s.AddDaemon(c, true, true) 209 210 instances := 2 211 id := d.createService(c, simpleTestService, setInstances(instances)) 212 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 213 214 service := d.getService(c, id) 215 instances = 5 216 d.updateService(c, service, setInstances(instances)) 217 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 218 219 d.removeService(c, service.ID) 220 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, 0) 221 } 222 223 func (s *DockerSwarmSuite) TestApiSwarmServicesMultipleAgents(c *check.C) { 224 d1 := s.AddDaemon(c, true, true) 225 d2 := s.AddDaemon(c, true, false) 226 d3 := s.AddDaemon(c, true, false) 227 228 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks 229 230 instances := 9 231 id := d1.createService(c, simpleTestService, setInstances(instances)) 232 233 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0) 234 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0) 235 waitAndAssert(c, defaultReconciliationTimeout, d3.checkActiveContainerCount, checker.GreaterThan, 0) 236 237 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 238 239 // reconciliation on d2 node down 240 c.Assert(d2.Stop(), checker.IsNil) 241 242 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 243 244 // test downscaling 245 instances = 5 246 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 247 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 248 249 } 250 251 func (s *DockerSwarmSuite) TestApiSwarmServicesCreateGlobal(c *check.C) { 252 d1 := s.AddDaemon(c, true, true) 253 d2 := s.AddDaemon(c, true, false) 254 d3 := s.AddDaemon(c, true, false) 255 256 d1.createService(c, simpleTestService, setGlobalMode) 257 258 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, 1) 259 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 1) 260 waitAndAssert(c, defaultReconciliationTimeout, d3.checkActiveContainerCount, checker.Equals, 1) 261 262 d4 := s.AddDaemon(c, true, false) 263 d5 := s.AddDaemon(c, true, false) 264 265 waitAndAssert(c, defaultReconciliationTimeout, d4.checkActiveContainerCount, checker.Equals, 1) 266 waitAndAssert(c, defaultReconciliationTimeout, d5.checkActiveContainerCount, checker.Equals, 1) 267 } 268 269 func (s *DockerSwarmSuite) TestApiSwarmServicesUpdate(c *check.C) { 270 const nodeCount = 3 271 var daemons [nodeCount]*SwarmDaemon 272 for i := 0; i < nodeCount; i++ { 273 daemons[i] = s.AddDaemon(c, true, i == 0) 274 } 275 // wait for nodes ready 276 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 277 278 // service image at start 279 image1 := "busybox:latest" 280 // target image in update 281 image2 := "busybox:test" 282 283 // create a different tag 284 for _, d := range daemons { 285 out, err := d.Cmd("tag", image1, image2) 286 c.Assert(err, checker.IsNil, check.Commentf(out)) 287 } 288 289 // create service 290 instances := 5 291 parallelism := 2 292 id := daemons[0].createService(c, serviceForUpdate, setInstances(instances)) 293 294 // wait for tasks ready 295 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 296 map[string]int{image1: instances}) 297 298 // issue service update 299 service := daemons[0].getService(c, id) 300 daemons[0].updateService(c, service, setImage(image2)) 301 302 // first batch 303 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 304 map[string]int{image1: instances - parallelism, image2: parallelism}) 305 306 // 2nd batch 307 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 308 map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism}) 309 310 // 3nd batch 311 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 312 map[string]int{image2: instances}) 313 } 314 315 func (s *DockerSwarmSuite) TestApiSwarmServiceConstraintRole(c *check.C) { 316 const nodeCount = 3 317 var daemons [nodeCount]*SwarmDaemon 318 for i := 0; i < nodeCount; i++ { 319 daemons[i] = s.AddDaemon(c, true, i == 0) 320 } 321 // wait for nodes ready 322 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 323 324 // create service 325 constraints := []string{"node.role==worker"} 326 instances := 3 327 id := daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 328 // wait for tasks ready 329 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(c, id), checker.Equals, instances) 330 // validate tasks are running on worker nodes 331 tasks := daemons[0].getServiceTasks(c, id) 332 for _, task := range tasks { 333 node := daemons[0].getNode(c, task.NodeID) 334 c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleWorker) 335 } 336 //remove service 337 daemons[0].removeService(c, id) 338 339 // create service 340 constraints = []string{"node.role!=worker"} 341 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 342 // wait for tasks ready 343 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(c, id), checker.Equals, instances) 344 tasks = daemons[0].getServiceTasks(c, id) 345 // validate tasks are running on manager nodes 346 for _, task := range tasks { 347 node := daemons[0].getNode(c, task.NodeID) 348 c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleManager) 349 } 350 //remove service 351 daemons[0].removeService(c, id) 352 353 // create service 354 constraints = []string{"node.role==nosuchrole"} 355 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 356 // wait for tasks created 357 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceTasks(c, id), checker.Equals, instances) 358 // let scheduler try 359 time.Sleep(250 * time.Millisecond) 360 // validate tasks are not assigned to any node 361 tasks = daemons[0].getServiceTasks(c, id) 362 for _, task := range tasks { 363 c.Assert(task.NodeID, checker.Equals, "") 364 } 365 } 366 367 func (s *DockerSwarmSuite) TestApiSwarmServiceConstraintLabel(c *check.C) { 368 const nodeCount = 3 369 var daemons [nodeCount]*SwarmDaemon 370 for i := 0; i < nodeCount; i++ { 371 daemons[i] = s.AddDaemon(c, true, i == 0) 372 } 373 // wait for nodes ready 374 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 375 nodes := daemons[0].listNodes(c) 376 c.Assert(len(nodes), checker.Equals, nodeCount) 377 378 // add labels to nodes 379 daemons[0].updateNode(c, nodes[0].ID, func(n *swarm.Node) { 380 n.Spec.Annotations.Labels = map[string]string{ 381 "security": "high", 382 } 383 }) 384 for i := 1; i < nodeCount; i++ { 385 daemons[0].updateNode(c, nodes[i].ID, func(n *swarm.Node) { 386 n.Spec.Annotations.Labels = map[string]string{ 387 "security": "low", 388 } 389 }) 390 } 391 392 // create service 393 instances := 3 394 constraints := []string{"node.labels.security==high"} 395 id := daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 396 // wait for tasks ready 397 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(c, id), checker.Equals, instances) 398 tasks := daemons[0].getServiceTasks(c, id) 399 // validate all tasks are running on nodes[0] 400 for _, task := range tasks { 401 c.Assert(task.NodeID, checker.Equals, nodes[0].ID) 402 } 403 //remove service 404 daemons[0].removeService(c, id) 405 406 // create service 407 constraints = []string{"node.labels.security!=high"} 408 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 409 // wait for tasks ready 410 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(c, id), checker.Equals, instances) 411 tasks = daemons[0].getServiceTasks(c, id) 412 // validate all tasks are NOT running on nodes[0] 413 for _, task := range tasks { 414 c.Assert(task.NodeID, checker.Not(checker.Equals), nodes[0].ID) 415 } 416 //remove service 417 daemons[0].removeService(c, id) 418 419 constraints = []string{"node.labels.security==medium"} 420 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 421 // wait for tasks created 422 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceTasks(c, id), checker.Equals, instances) 423 // let scheduler try 424 time.Sleep(250 * time.Millisecond) 425 tasks = daemons[0].getServiceTasks(c, id) 426 // validate tasks are not assigned 427 for _, task := range tasks { 428 c.Assert(task.NodeID, checker.Equals, "") 429 } 430 //remove service 431 daemons[0].removeService(c, id) 432 433 // multiple constraints 434 constraints = []string{ 435 "node.labels.security==high", 436 fmt.Sprintf("node.id==%s", nodes[1].ID), 437 } 438 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 439 // wait for tasks created 440 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceTasks(c, id), checker.Equals, instances) 441 // let scheduler try 442 time.Sleep(250 * time.Millisecond) 443 tasks = daemons[0].getServiceTasks(c, id) 444 // validate tasks are not assigned 445 for _, task := range tasks { 446 c.Assert(task.NodeID, checker.Equals, "") 447 } 448 // make nodes[1] fulfills the constraints 449 daemons[0].updateNode(c, nodes[1].ID, func(n *swarm.Node) { 450 n.Spec.Annotations.Labels = map[string]string{ 451 "security": "high", 452 } 453 }) 454 // wait for tasks ready 455 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(c, id), checker.Equals, instances) 456 tasks = daemons[0].getServiceTasks(c, id) 457 for _, task := range tasks { 458 c.Assert(task.NodeID, checker.Equals, nodes[1].ID) 459 } 460 } 461 462 func (s *DockerSwarmSuite) TestApiSwarmServicesStateReporting(c *check.C) { 463 testRequires(c, SameHostDaemon) 464 testRequires(c, DaemonIsLinux) 465 466 d1 := s.AddDaemon(c, true, true) 467 d2 := s.AddDaemon(c, true, true) 468 d3 := s.AddDaemon(c, true, false) 469 470 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept 471 472 instances := 9 473 d1.createService(c, simpleTestService, setInstances(instances)) 474 475 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 476 477 getContainers := func() map[string]*SwarmDaemon { 478 m := make(map[string]*SwarmDaemon) 479 for _, d := range []*SwarmDaemon{d1, d2, d3} { 480 for _, id := range d.activeContainers() { 481 m[id] = d 482 } 483 } 484 return m 485 } 486 487 containers := getContainers() 488 c.Assert(containers, checker.HasLen, instances) 489 var toRemove string 490 for i := range containers { 491 toRemove = i 492 } 493 494 _, err := containers[toRemove].Cmd("stop", toRemove) 495 c.Assert(err, checker.IsNil) 496 497 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 498 499 containers2 := getContainers() 500 c.Assert(containers2, checker.HasLen, instances) 501 for i := range containers { 502 if i == toRemove { 503 c.Assert(containers2[i], checker.IsNil) 504 } else { 505 c.Assert(containers2[i], checker.NotNil) 506 } 507 } 508 509 containers = containers2 510 for i := range containers { 511 toRemove = i 512 } 513 514 // try with killing process outside of docker 515 pidStr, err := containers[toRemove].Cmd("inspect", "-f", "{{.State.Pid}}", toRemove) 516 c.Assert(err, checker.IsNil) 517 pid, err := strconv.Atoi(strings.TrimSpace(pidStr)) 518 c.Assert(err, checker.IsNil) 519 c.Assert(syscall.Kill(pid, syscall.SIGKILL), checker.IsNil) 520 521 time.Sleep(time.Second) // give some time to handle the signal 522 523 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 524 525 containers2 = getContainers() 526 c.Assert(containers2, checker.HasLen, instances) 527 for i := range containers { 528 if i == toRemove { 529 c.Assert(containers2[i], checker.IsNil) 530 } else { 531 c.Assert(containers2[i], checker.NotNil) 532 } 533 } 534 } 535 536 func (s *DockerSwarmSuite) TestApiSwarmLeaderProxy(c *check.C) { 537 // add three managers, one of these is leader 538 d1 := s.AddDaemon(c, true, true) 539 d2 := s.AddDaemon(c, true, true) 540 d3 := s.AddDaemon(c, true, true) 541 542 // start a service by hitting each of the 3 managers 543 d1.createService(c, simpleTestService, func(s *swarm.Service) { 544 s.Spec.Name = "test1" 545 }) 546 d2.createService(c, simpleTestService, func(s *swarm.Service) { 547 s.Spec.Name = "test2" 548 }) 549 d3.createService(c, simpleTestService, func(s *swarm.Service) { 550 s.Spec.Name = "test3" 551 }) 552 553 // 3 services should be started now, because the requests were proxied to leader 554 // query each node and make sure it returns 3 services 555 for _, d := range []*SwarmDaemon{d1, d2, d3} { 556 services := d.listServices(c) 557 c.Assert(services, checker.HasLen, 3) 558 } 559 } 560 561 func (s *DockerSwarmSuite) TestApiSwarmLeaderElection(c *check.C) { 562 // Create 3 nodes 563 d1 := s.AddDaemon(c, true, true) 564 d2 := s.AddDaemon(c, true, true) 565 d3 := s.AddDaemon(c, true, true) 566 567 // assert that the first node we made is the leader, and the other two are followers 568 c.Assert(d1.getNode(c, d1.NodeID).ManagerStatus.Leader, checker.True) 569 c.Assert(d1.getNode(c, d2.NodeID).ManagerStatus.Leader, checker.False) 570 c.Assert(d1.getNode(c, d3.NodeID).ManagerStatus.Leader, checker.False) 571 572 d1.Stop() // stop the leader 573 574 var ( 575 leader *SwarmDaemon // keep track of leader 576 followers []*SwarmDaemon // keep track of followers 577 ) 578 checkLeader := func(nodes ...*SwarmDaemon) checkF { 579 return func(c *check.C) (interface{}, check.CommentInterface) { 580 // clear these out before each run 581 leader = nil 582 followers = nil 583 for _, d := range nodes { 584 if d.getNode(c, d.NodeID).ManagerStatus.Leader { 585 leader = d 586 } else { 587 followers = append(followers, d) 588 } 589 } 590 591 if leader == nil { 592 return false, check.Commentf("no leader elected") 593 } 594 595 return true, check.Commentf("elected %v", leader.id) 596 } 597 } 598 599 // wait for an election to occur 600 waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d2, d3), checker.True) 601 602 // assert that we have a new leader 603 c.Assert(leader, checker.NotNil) 604 605 // Keep track of the current leader, since we want that to be chosen. 606 stableleader := leader 607 608 // add the d1, the initial leader, back 609 d1.Start() 610 611 // TODO(stevvooe): may need to wait for rejoin here 612 613 // wait for possible election 614 waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d1, d2, d3), checker.True) 615 // pick out the leader and the followers again 616 617 // verify that we still only have 1 leader and 2 followers 618 c.Assert(leader, checker.NotNil) 619 c.Assert(followers, checker.HasLen, 2) 620 // and that after we added d1 back, the leader hasn't changed 621 c.Assert(leader.NodeID, checker.Equals, stableleader.NodeID) 622 } 623 624 func (s *DockerSwarmSuite) TestApiSwarmRaftQuorum(c *check.C) { 625 d1 := s.AddDaemon(c, true, true) 626 d2 := s.AddDaemon(c, true, true) 627 d3 := s.AddDaemon(c, true, true) 628 629 d1.createService(c, simpleTestService) 630 631 c.Assert(d2.Stop(), checker.IsNil) 632 633 // make sure there is a leader 634 waitAndAssert(c, defaultReconciliationTimeout, d1.checkLeader, checker.IsNil) 635 636 d1.createService(c, simpleTestService, func(s *swarm.Service) { 637 s.Spec.Name = "top1" 638 }) 639 640 c.Assert(d3.Stop(), checker.IsNil) 641 642 // make sure there is a leader 643 waitAndAssert(c, defaultReconciliationTimeout, d1.checkLeader, checker.IsNil) 644 645 var service swarm.Service 646 simpleTestService(&service) 647 service.Spec.Name = "top2" 648 status, out, err := d1.SockRequest("POST", "/services/create", service.Spec) 649 c.Assert(err, checker.IsNil) 650 c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("deadline exceeded", string(out))) 651 652 c.Assert(d2.Start(), checker.IsNil) 653 654 // make sure there is a leader 655 waitAndAssert(c, defaultReconciliationTimeout, d1.checkLeader, checker.IsNil) 656 657 d1.createService(c, simpleTestService, func(s *swarm.Service) { 658 s.Spec.Name = "top3" 659 }) 660 } 661 662 func (s *DockerSwarmSuite) TestApiSwarmListNodes(c *check.C) { 663 d1 := s.AddDaemon(c, true, true) 664 d2 := s.AddDaemon(c, true, false) 665 d3 := s.AddDaemon(c, true, false) 666 667 nodes := d1.listNodes(c) 668 c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes)) 669 670 loop0: 671 for _, n := range nodes { 672 for _, d := range []*SwarmDaemon{d1, d2, d3} { 673 if n.ID == d.NodeID { 674 continue loop0 675 } 676 } 677 c.Errorf("unknown nodeID %v", n.ID) 678 } 679 } 680 681 func (s *DockerSwarmSuite) TestApiSwarmNodeUpdate(c *check.C) { 682 d := s.AddDaemon(c, true, true) 683 684 nodes := d.listNodes(c) 685 686 d.updateNode(c, nodes[0].ID, func(n *swarm.Node) { 687 n.Spec.Availability = swarm.NodeAvailabilityPause 688 }) 689 690 n := d.getNode(c, nodes[0].ID) 691 c.Assert(n.Spec.Availability, checker.Equals, swarm.NodeAvailabilityPause) 692 } 693 694 func (s *DockerSwarmSuite) TestApiSwarmNodeRemove(c *check.C) { 695 testRequires(c, Network) 696 d1 := s.AddDaemon(c, true, true) 697 d2 := s.AddDaemon(c, true, false) 698 _ = s.AddDaemon(c, true, false) 699 700 nodes := d1.listNodes(c) 701 c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes)) 702 703 // Getting the info so we can take the NodeID 704 d2Info, err := d2.info() 705 c.Assert(err, checker.IsNil) 706 707 // forceful removal of d2 should work 708 d1.removeNode(c, d2Info.NodeID, true) 709 710 nodes = d1.listNodes(c) 711 c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes)) 712 713 // Restart the node that was removed 714 err = d2.Restart() 715 c.Assert(err, checker.IsNil) 716 717 // Give some time for the node to rejoin 718 time.Sleep(1 * time.Second) 719 720 // Make sure the node didn't rejoin 721 nodes = d1.listNodes(c) 722 c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes)) 723 } 724 725 func (s *DockerSwarmSuite) TestApiSwarmNodeDrainPause(c *check.C) { 726 d1 := s.AddDaemon(c, true, true) 727 d2 := s.AddDaemon(c, true, false) 728 729 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks 730 731 // start a service, expect balanced distribution 732 instances := 8 733 id := d1.createService(c, simpleTestService, setInstances(instances)) 734 735 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0) 736 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0) 737 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 738 739 // drain d2, all containers should move to d1 740 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 741 n.Spec.Availability = swarm.NodeAvailabilityDrain 742 }) 743 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances) 744 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 0) 745 746 // set d2 back to active 747 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 748 n.Spec.Availability = swarm.NodeAvailabilityActive 749 }) 750 751 instances = 1 752 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 753 754 waitAndAssert(c, defaultReconciliationTimeout*2, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 755 756 instances = 8 757 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 758 759 // drained node first so we don't get any old containers 760 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0) 761 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0) 762 waitAndAssert(c, defaultReconciliationTimeout*2, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 763 764 d2ContainerCount := len(d2.activeContainers()) 765 766 // set d2 to paused, scale service up, only d1 gets new tasks 767 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 768 n.Spec.Availability = swarm.NodeAvailabilityPause 769 }) 770 771 instances = 14 772 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 773 774 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances-d2ContainerCount) 775 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, d2ContainerCount) 776 777 } 778 779 func (s *DockerSwarmSuite) TestApiSwarmLeaveRemovesContainer(c *check.C) { 780 d := s.AddDaemon(c, true, true) 781 782 instances := 2 783 d.createService(c, simpleTestService, setInstances(instances)) 784 785 id, err := d.Cmd("run", "-d", "busybox", "top") 786 c.Assert(err, checker.IsNil) 787 id = strings.TrimSpace(id) 788 789 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances+1) 790 791 c.Assert(d.Leave(false), checker.NotNil) 792 c.Assert(d.Leave(true), checker.IsNil) 793 794 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, 1) 795 796 id2, err := d.Cmd("ps", "-q") 797 c.Assert(err, checker.IsNil) 798 c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2)) 799 } 800 801 // #23629 802 func (s *DockerSwarmSuite) TestApiSwarmLeaveOnPendingJoin(c *check.C) { 803 s.AddDaemon(c, true, true) 804 d2 := s.AddDaemon(c, false, false) 805 806 id, err := d2.Cmd("run", "-d", "busybox", "top") 807 c.Assert(err, checker.IsNil) 808 id = strings.TrimSpace(id) 809 810 err = d2.Join(swarm.JoinRequest{ 811 RemoteAddrs: []string{"123.123.123.123:1234"}, 812 }) 813 c.Assert(err, check.NotNil) 814 c.Assert(err.Error(), checker.Contains, "Timeout was reached") 815 816 info, err := d2.info() 817 c.Assert(err, checker.IsNil) 818 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStatePending) 819 820 c.Assert(d2.Leave(true), checker.IsNil) 821 822 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 1) 823 824 id2, err := d2.Cmd("ps", "-q") 825 c.Assert(err, checker.IsNil) 826 c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2)) 827 } 828 829 // #23705 830 func (s *DockerSwarmSuite) TestApiSwarmRestoreOnPendingJoin(c *check.C) { 831 d := s.AddDaemon(c, false, false) 832 err := d.Join(swarm.JoinRequest{ 833 RemoteAddrs: []string{"123.123.123.123:1234"}, 834 }) 835 c.Assert(err, check.NotNil) 836 c.Assert(err.Error(), checker.Contains, "Timeout was reached") 837 838 waitAndAssert(c, defaultReconciliationTimeout, d.checkLocalNodeState, checker.Equals, swarm.LocalNodeStatePending) 839 840 c.Assert(d.Stop(), checker.IsNil) 841 c.Assert(d.Start(), checker.IsNil) 842 843 info, err := d.info() 844 c.Assert(err, checker.IsNil) 845 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 846 } 847 848 func (s *DockerSwarmSuite) TestApiSwarmManagerRestore(c *check.C) { 849 d1 := s.AddDaemon(c, true, true) 850 851 instances := 2 852 id := d1.createService(c, simpleTestService, setInstances(instances)) 853 854 d1.getService(c, id) 855 d1.Stop() 856 d1.Start() 857 d1.getService(c, id) 858 859 d2 := s.AddDaemon(c, true, true) 860 d2.getService(c, id) 861 d2.Stop() 862 d2.Start() 863 d2.getService(c, id) 864 865 d3 := s.AddDaemon(c, true, true) 866 d3.getService(c, id) 867 d3.Stop() 868 d3.Start() 869 d3.getService(c, id) 870 871 d3.Kill() 872 time.Sleep(1 * time.Second) // time to handle signal 873 d3.Start() 874 d3.getService(c, id) 875 } 876 877 func (s *DockerSwarmSuite) TestApiSwarmScaleNoRollingUpdate(c *check.C) { 878 d := s.AddDaemon(c, true, true) 879 880 instances := 2 881 id := d.createService(c, simpleTestService, setInstances(instances)) 882 883 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 884 containers := d.activeContainers() 885 instances = 4 886 d.updateService(c, d.getService(c, id), setInstances(instances)) 887 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 888 containers2 := d.activeContainers() 889 890 loop0: 891 for _, c1 := range containers { 892 for _, c2 := range containers2 { 893 if c1 == c2 { 894 continue loop0 895 } 896 } 897 c.Errorf("container %v not found in new set %#v", c1, containers2) 898 } 899 } 900 901 func (s *DockerSwarmSuite) TestApiSwarmInvalidAddress(c *check.C) { 902 d := s.AddDaemon(c, false, false) 903 req := swarm.InitRequest{ 904 ListenAddr: "", 905 } 906 status, _, err := d.SockRequest("POST", "/swarm/init", req) 907 c.Assert(err, checker.IsNil) 908 c.Assert(status, checker.Equals, http.StatusInternalServerError) 909 910 req2 := swarm.JoinRequest{ 911 ListenAddr: "0.0.0.0:2377", 912 RemoteAddrs: []string{""}, 913 } 914 status, _, err = d.SockRequest("POST", "/swarm/join", req2) 915 c.Assert(err, checker.IsNil) 916 c.Assert(status, checker.Equals, http.StatusInternalServerError) 917 } 918 919 func (s *DockerSwarmSuite) TestApiSwarmForceNewCluster(c *check.C) { 920 d1 := s.AddDaemon(c, true, true) 921 d2 := s.AddDaemon(c, true, true) 922 923 instances := 2 924 id := d1.createService(c, simpleTestService, setInstances(instances)) 925 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 926 927 // drain d2, all containers should move to d1 928 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 929 n.Spec.Availability = swarm.NodeAvailabilityDrain 930 }) 931 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances) 932 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 0) 933 934 c.Assert(d2.Stop(), checker.IsNil) 935 936 c.Assert(d1.Init(swarm.InitRequest{ 937 ForceNewCluster: true, 938 Spec: swarm.Spec{}, 939 }), checker.IsNil) 940 941 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances) 942 943 d3 := s.AddDaemon(c, true, true) 944 info, err := d3.info() 945 c.Assert(err, checker.IsNil) 946 c.Assert(info.ControlAvailable, checker.True) 947 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 948 949 instances = 4 950 d3.updateService(c, d3.getService(c, id), setInstances(instances)) 951 952 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 953 } 954 955 func simpleTestService(s *swarm.Service) { 956 ureplicas := uint64(1) 957 restartDelay := time.Duration(100 * time.Millisecond) 958 959 s.Spec = swarm.ServiceSpec{ 960 TaskTemplate: swarm.TaskSpec{ 961 ContainerSpec: swarm.ContainerSpec{ 962 Image: "busybox:latest", 963 Command: []string{"/bin/top"}, 964 }, 965 RestartPolicy: &swarm.RestartPolicy{ 966 Delay: &restartDelay, 967 }, 968 }, 969 Mode: swarm.ServiceMode{ 970 Replicated: &swarm.ReplicatedService{ 971 Replicas: &ureplicas, 972 }, 973 }, 974 } 975 s.Spec.Name = "top" 976 } 977 978 func serviceForUpdate(s *swarm.Service) { 979 ureplicas := uint64(1) 980 restartDelay := time.Duration(100 * time.Millisecond) 981 982 s.Spec = swarm.ServiceSpec{ 983 TaskTemplate: swarm.TaskSpec{ 984 ContainerSpec: swarm.ContainerSpec{ 985 Image: "busybox:latest", 986 Command: []string{"/bin/top"}, 987 }, 988 RestartPolicy: &swarm.RestartPolicy{ 989 Delay: &restartDelay, 990 }, 991 }, 992 Mode: swarm.ServiceMode{ 993 Replicated: &swarm.ReplicatedService{ 994 Replicas: &ureplicas, 995 }, 996 }, 997 UpdateConfig: &swarm.UpdateConfig{ 998 Parallelism: 2, 999 Delay: 4 * time.Second, 1000 FailureAction: swarm.UpdateFailureActionContinue, 1001 }, 1002 } 1003 s.Spec.Name = "updatetest" 1004 } 1005 1006 func setInstances(replicas int) serviceConstructor { 1007 ureplicas := uint64(replicas) 1008 return func(s *swarm.Service) { 1009 s.Spec.Mode = swarm.ServiceMode{ 1010 Replicated: &swarm.ReplicatedService{ 1011 Replicas: &ureplicas, 1012 }, 1013 } 1014 } 1015 } 1016 1017 func setImage(image string) serviceConstructor { 1018 return func(s *swarm.Service) { 1019 s.Spec.TaskTemplate.ContainerSpec.Image = image 1020 } 1021 } 1022 1023 func setConstraints(constraints []string) serviceConstructor { 1024 return func(s *swarm.Service) { 1025 if s.Spec.TaskTemplate.Placement == nil { 1026 s.Spec.TaskTemplate.Placement = &swarm.Placement{} 1027 } 1028 s.Spec.TaskTemplate.Placement.Constraints = constraints 1029 } 1030 } 1031 1032 func setGlobalMode(s *swarm.Service) { 1033 s.Spec.Mode = swarm.ServiceMode{ 1034 Global: &swarm.GlobalService{}, 1035 } 1036 } 1037 1038 func checkClusterHealth(c *check.C, cl []*SwarmDaemon, managerCount, workerCount int) { 1039 var totalMCount, totalWCount int 1040 1041 for _, d := range cl { 1042 var ( 1043 info swarm.Info 1044 err error 1045 ) 1046 1047 // check info in a waitAndAssert, because if the cluster doesn't have a leader, `info` will return an error 1048 checkInfo := func(c *check.C) (interface{}, check.CommentInterface) { 1049 info, err = d.info() 1050 return err, check.Commentf("cluster not ready in time") 1051 } 1052 waitAndAssert(c, defaultReconciliationTimeout, checkInfo, checker.IsNil) 1053 if !info.ControlAvailable { 1054 totalWCount++ 1055 continue 1056 } 1057 1058 var leaderFound bool 1059 totalMCount++ 1060 var mCount, wCount int 1061 1062 for _, n := range d.listNodes(c) { 1063 waitReady := func(c *check.C) (interface{}, check.CommentInterface) { 1064 if n.Status.State == swarm.NodeStateReady { 1065 return true, nil 1066 } 1067 nn := d.getNode(c, n.ID) 1068 n = *nn 1069 return n.Status.State == swarm.NodeStateReady, check.Commentf("state of node %s, reported by %s", n.ID, d.Info.NodeID) 1070 } 1071 waitAndAssert(c, defaultReconciliationTimeout, waitReady, checker.True) 1072 1073 waitActive := func(c *check.C) (interface{}, check.CommentInterface) { 1074 if n.Spec.Availability == swarm.NodeAvailabilityActive { 1075 return true, nil 1076 } 1077 nn := d.getNode(c, n.ID) 1078 n = *nn 1079 return n.Spec.Availability == swarm.NodeAvailabilityActive, check.Commentf("availability of node %s, reported by %s", n.ID, d.Info.NodeID) 1080 } 1081 waitAndAssert(c, defaultReconciliationTimeout, waitActive, checker.True) 1082 1083 if n.Spec.Role == swarm.NodeRoleManager { 1084 c.Assert(n.ManagerStatus, checker.NotNil, check.Commentf("manager status of node %s (manager), reported by %s", n.ID, d.Info.NodeID)) 1085 if n.ManagerStatus.Leader { 1086 leaderFound = true 1087 } 1088 mCount++ 1089 } else { 1090 c.Assert(n.ManagerStatus, checker.IsNil, check.Commentf("manager status of node %s (worker), reported by %s", n.ID, d.Info.NodeID)) 1091 wCount++ 1092 } 1093 } 1094 c.Assert(leaderFound, checker.True, check.Commentf("lack of leader reported by node %s", info.NodeID)) 1095 c.Assert(mCount, checker.Equals, managerCount, check.Commentf("managers count reported by node %s", info.NodeID)) 1096 c.Assert(wCount, checker.Equals, workerCount, check.Commentf("workers count reported by node %s", info.NodeID)) 1097 } 1098 c.Assert(totalMCount, checker.Equals, managerCount) 1099 c.Assert(totalWCount, checker.Equals, workerCount) 1100 } 1101 1102 func (s *DockerSwarmSuite) TestApiSwarmRestartCluster(c *check.C) { 1103 mCount, wCount := 5, 1 1104 1105 var nodes []*SwarmDaemon 1106 for i := 0; i < mCount; i++ { 1107 manager := s.AddDaemon(c, true, true) 1108 info, err := manager.info() 1109 c.Assert(err, checker.IsNil) 1110 c.Assert(info.ControlAvailable, checker.True) 1111 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 1112 nodes = append(nodes, manager) 1113 } 1114 1115 for i := 0; i < wCount; i++ { 1116 worker := s.AddDaemon(c, true, false) 1117 info, err := worker.info() 1118 c.Assert(err, checker.IsNil) 1119 c.Assert(info.ControlAvailable, checker.False) 1120 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 1121 nodes = append(nodes, worker) 1122 } 1123 1124 // stop whole cluster 1125 { 1126 var wg sync.WaitGroup 1127 wg.Add(len(nodes)) 1128 errs := make(chan error, len(nodes)) 1129 1130 for _, d := range nodes { 1131 go func(daemon *SwarmDaemon) { 1132 defer wg.Done() 1133 if err := daemon.Stop(); err != nil { 1134 errs <- err 1135 } 1136 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 1137 daemon.root = filepath.Dir(daemon.root) 1138 } 1139 }(d) 1140 } 1141 wg.Wait() 1142 close(errs) 1143 for err := range errs { 1144 c.Assert(err, check.IsNil) 1145 } 1146 } 1147 1148 // start whole cluster 1149 { 1150 var wg sync.WaitGroup 1151 wg.Add(len(nodes)) 1152 errs := make(chan error, len(nodes)) 1153 1154 for _, d := range nodes { 1155 go func(daemon *SwarmDaemon) { 1156 defer wg.Done() 1157 if err := daemon.Start("--iptables=false"); err != nil { 1158 errs <- err 1159 } 1160 }(d) 1161 } 1162 wg.Wait() 1163 close(errs) 1164 for err := range errs { 1165 c.Assert(err, check.IsNil) 1166 } 1167 } 1168 1169 checkClusterHealth(c, nodes, mCount, wCount) 1170 }