github.com/endophage/docker@v1.4.2-0.20161027011718-242853499895/integration-cli/docker_api_swarm_test.go (about) 1 // +build !windows 2 3 package main 4 5 import ( 6 "fmt" 7 "net/http" 8 "os" 9 "path/filepath" 10 "strconv" 11 "strings" 12 "sync" 13 "syscall" 14 "time" 15 16 "github.com/docker/docker/api/types/swarm" 17 "github.com/docker/docker/pkg/integration/checker" 18 "github.com/go-check/check" 19 ) 20 21 var defaultReconciliationTimeout = 30 * time.Second 22 23 func (s *DockerSwarmSuite) TestAPISwarmInit(c *check.C) { 24 // todo: should find a better way to verify that components are running than /info 25 d1 := s.AddDaemon(c, true, true) 26 info, err := d1.info() 27 c.Assert(err, checker.IsNil) 28 c.Assert(info.ControlAvailable, checker.True) 29 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 30 31 d2 := s.AddDaemon(c, true, false) 32 info, err = d2.info() 33 c.Assert(err, checker.IsNil) 34 c.Assert(info.ControlAvailable, checker.False) 35 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 36 37 // Leaving cluster 38 c.Assert(d2.Leave(false), checker.IsNil) 39 40 info, err = d2.info() 41 c.Assert(err, checker.IsNil) 42 c.Assert(info.ControlAvailable, checker.False) 43 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 44 45 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: d1.joinTokens(c).Worker, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 46 47 info, err = d2.info() 48 c.Assert(err, checker.IsNil) 49 c.Assert(info.ControlAvailable, checker.False) 50 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 51 52 // Current state restoring after restarts 53 err = d1.Stop() 54 c.Assert(err, checker.IsNil) 55 err = d2.Stop() 56 c.Assert(err, checker.IsNil) 57 58 err = d1.Start() 59 c.Assert(err, checker.IsNil) 60 err = d2.Start() 61 c.Assert(err, checker.IsNil) 62 63 info, err = d1.info() 64 c.Assert(err, checker.IsNil) 65 c.Assert(info.ControlAvailable, checker.True) 66 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 67 68 info, err = d2.info() 69 c.Assert(err, checker.IsNil) 70 c.Assert(info.ControlAvailable, checker.False) 71 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 72 } 73 74 func (s *DockerSwarmSuite) TestAPISwarmJoinToken(c *check.C) { 75 d1 := s.AddDaemon(c, false, false) 76 c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) 77 78 d2 := s.AddDaemon(c, false, false) 79 err := d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.listenAddr}}) 80 c.Assert(err, checker.NotNil) 81 c.Assert(err.Error(), checker.Contains, "join token is necessary") 82 info, err := d2.info() 83 c.Assert(err, checker.IsNil) 84 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 85 86 err = d2.Join(swarm.JoinRequest{JoinToken: "foobaz", RemoteAddrs: []string{d1.listenAddr}}) 87 c.Assert(err, checker.NotNil) 88 c.Assert(err.Error(), checker.Contains, "join token is necessary") 89 info, err = d2.info() 90 c.Assert(err, checker.IsNil) 91 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 92 93 workerToken := d1.joinTokens(c).Worker 94 95 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 96 info, err = d2.info() 97 c.Assert(err, checker.IsNil) 98 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 99 c.Assert(d2.Leave(false), checker.IsNil) 100 info, err = d2.info() 101 c.Assert(err, checker.IsNil) 102 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 103 104 // change tokens 105 d1.rotateTokens(c) 106 107 err = d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}) 108 c.Assert(err, checker.NotNil) 109 c.Assert(err.Error(), checker.Contains, "join token is necessary") 110 info, err = d2.info() 111 c.Assert(err, checker.IsNil) 112 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 113 114 workerToken = d1.joinTokens(c).Worker 115 116 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 117 info, err = d2.info() 118 c.Assert(err, checker.IsNil) 119 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 120 c.Assert(d2.Leave(false), checker.IsNil) 121 info, err = d2.info() 122 c.Assert(err, checker.IsNil) 123 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 124 125 // change spec, don't change tokens 126 d1.updateSwarm(c, func(s *swarm.Spec) {}) 127 128 err = d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.listenAddr}}) 129 c.Assert(err, checker.NotNil) 130 c.Assert(err.Error(), checker.Contains, "join token is necessary") 131 info, err = d2.info() 132 c.Assert(err, checker.IsNil) 133 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 134 135 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 136 info, err = d2.info() 137 c.Assert(err, checker.IsNil) 138 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 139 c.Assert(d2.Leave(false), checker.IsNil) 140 info, err = d2.info() 141 c.Assert(err, checker.IsNil) 142 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 143 } 144 145 func (s *DockerSwarmSuite) TestAPISwarmCAHash(c *check.C) { 146 d1 := s.AddDaemon(c, true, true) 147 d2 := s.AddDaemon(c, false, false) 148 splitToken := strings.Split(d1.joinTokens(c).Worker, "-") 149 splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e" 150 replacementToken := strings.Join(splitToken, "-") 151 err := d2.Join(swarm.JoinRequest{JoinToken: replacementToken, RemoteAddrs: []string{d1.listenAddr}}) 152 c.Assert(err, checker.NotNil) 153 c.Assert(err.Error(), checker.Contains, "remote CA does not match fingerprint") 154 } 155 156 func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) { 157 d1 := s.AddDaemon(c, false, false) 158 c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) 159 d2 := s.AddDaemon(c, true, false) 160 161 info, err := d2.info() 162 c.Assert(err, checker.IsNil) 163 c.Assert(info.ControlAvailable, checker.False) 164 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 165 166 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 167 n.Spec.Role = swarm.NodeRoleManager 168 }) 169 170 waitAndAssert(c, defaultReconciliationTimeout, d2.checkControlAvailable, checker.True) 171 172 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 173 n.Spec.Role = swarm.NodeRoleWorker 174 }) 175 176 waitAndAssert(c, defaultReconciliationTimeout, d2.checkControlAvailable, checker.False) 177 178 // Demoting last node should fail 179 node := d1.getNode(c, d1.NodeID) 180 node.Spec.Role = swarm.NodeRoleWorker 181 url := fmt.Sprintf("/nodes/%s/update?version=%d", node.ID, node.Version.Index) 182 status, out, err := d1.SockRequest("POST", url, node.Spec) 183 c.Assert(err, checker.IsNil) 184 c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("output: %q", string(out))) 185 c.Assert(string(out), checker.Contains, "last manager of the swarm") 186 info, err = d1.info() 187 c.Assert(err, checker.IsNil) 188 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 189 c.Assert(info.ControlAvailable, checker.True) 190 191 // Promote already demoted node 192 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 193 n.Spec.Role = swarm.NodeRoleManager 194 }) 195 196 waitAndAssert(c, defaultReconciliationTimeout, d2.checkControlAvailable, checker.True) 197 } 198 199 func (s *DockerSwarmSuite) TestAPISwarmServicesEmptyList(c *check.C) { 200 d := s.AddDaemon(c, true, true) 201 202 services := d.listServices(c) 203 c.Assert(services, checker.NotNil) 204 c.Assert(len(services), checker.Equals, 0, check.Commentf("services: %#v", services)) 205 } 206 207 func (s *DockerSwarmSuite) TestAPISwarmServicesCreate(c *check.C) { 208 d := s.AddDaemon(c, true, true) 209 210 instances := 2 211 id := d.createService(c, simpleTestService, setInstances(instances)) 212 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 213 214 service := d.getService(c, id) 215 instances = 5 216 d.updateService(c, service, setInstances(instances)) 217 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 218 219 d.removeService(c, service.ID) 220 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, 0) 221 } 222 223 func (s *DockerSwarmSuite) TestAPISwarmServicesMultipleAgents(c *check.C) { 224 d1 := s.AddDaemon(c, true, true) 225 d2 := s.AddDaemon(c, true, false) 226 d3 := s.AddDaemon(c, true, false) 227 228 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks 229 230 instances := 9 231 id := d1.createService(c, simpleTestService, setInstances(instances)) 232 233 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0) 234 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0) 235 waitAndAssert(c, defaultReconciliationTimeout, d3.checkActiveContainerCount, checker.GreaterThan, 0) 236 237 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 238 239 // reconciliation on d2 node down 240 c.Assert(d2.Stop(), checker.IsNil) 241 242 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 243 244 // test downscaling 245 instances = 5 246 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 247 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 248 249 } 250 251 func (s *DockerSwarmSuite) TestAPISwarmServicesCreateGlobal(c *check.C) { 252 d1 := s.AddDaemon(c, true, true) 253 d2 := s.AddDaemon(c, true, false) 254 d3 := s.AddDaemon(c, true, false) 255 256 d1.createService(c, simpleTestService, setGlobalMode) 257 258 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, 1) 259 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 1) 260 waitAndAssert(c, defaultReconciliationTimeout, d3.checkActiveContainerCount, checker.Equals, 1) 261 262 d4 := s.AddDaemon(c, true, false) 263 d5 := s.AddDaemon(c, true, false) 264 265 waitAndAssert(c, defaultReconciliationTimeout, d4.checkActiveContainerCount, checker.Equals, 1) 266 waitAndAssert(c, defaultReconciliationTimeout, d5.checkActiveContainerCount, checker.Equals, 1) 267 } 268 269 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdate(c *check.C) { 270 const nodeCount = 3 271 var daemons [nodeCount]*SwarmDaemon 272 for i := 0; i < nodeCount; i++ { 273 daemons[i] = s.AddDaemon(c, true, i == 0) 274 } 275 // wait for nodes ready 276 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 277 278 // service image at start 279 image1 := "busybox:latest" 280 // target image in update 281 image2 := "busybox:test" 282 283 // create a different tag 284 for _, d := range daemons { 285 out, err := d.Cmd("tag", image1, image2) 286 c.Assert(err, checker.IsNil, check.Commentf(out)) 287 } 288 289 // create service 290 instances := 5 291 parallelism := 2 292 id := daemons[0].createService(c, serviceForUpdate, setInstances(instances)) 293 294 // wait for tasks ready 295 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 296 map[string]int{image1: instances}) 297 298 // issue service update 299 service := daemons[0].getService(c, id) 300 daemons[0].updateService(c, service, setImage(image2)) 301 302 // first batch 303 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 304 map[string]int{image1: instances - parallelism, image2: parallelism}) 305 306 // 2nd batch 307 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 308 map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism}) 309 310 // 3nd batch 311 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 312 map[string]int{image2: instances}) 313 314 // Roll back to the previous version. This uses the CLI because 315 // rollback is a client-side operation. 316 out, err := daemons[0].Cmd("service", "update", "--rollback", id) 317 c.Assert(err, checker.IsNil, check.Commentf(out)) 318 319 // first batch 320 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 321 map[string]int{image2: instances - parallelism, image1: parallelism}) 322 323 // 2nd batch 324 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 325 map[string]int{image2: instances - 2*parallelism, image1: 2 * parallelism}) 326 327 // 3nd batch 328 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 329 map[string]int{image1: instances}) 330 } 331 332 func (s *DockerSwarmSuite) TestAPISwarmServicesFailedUpdate(c *check.C) { 333 const nodeCount = 3 334 var daemons [nodeCount]*SwarmDaemon 335 for i := 0; i < nodeCount; i++ { 336 daemons[i] = s.AddDaemon(c, true, i == 0) 337 } 338 // wait for nodes ready 339 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 340 341 // service image at start 342 image1 := "busybox:latest" 343 // target image in update 344 image2 := "busybox:badtag" 345 346 // create service 347 instances := 5 348 id := daemons[0].createService(c, serviceForUpdate, setInstances(instances)) 349 350 // wait for tasks ready 351 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 352 map[string]int{image1: instances}) 353 354 // issue service update 355 service := daemons[0].getService(c, id) 356 daemons[0].updateService(c, service, setImage(image2), setFailureAction(swarm.UpdateFailureActionPause), setMaxFailureRatio(0.25), setParallelism(1)) 357 358 // should update 2 tasks and then pause 359 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceUpdateState(id), checker.Equals, swarm.UpdateStatePaused) 360 v, _ := daemons[0].checkServiceRunningTasks(id)(c) 361 c.Assert(v, checker.Equals, instances-2) 362 363 // Roll back to the previous version. This uses the CLI because 364 // rollback is a client-side operation. 365 out, err := daemons[0].Cmd("service", "update", "--rollback", id) 366 c.Assert(err, checker.IsNil, check.Commentf(out)) 367 368 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 369 map[string]int{image1: instances}) 370 } 371 372 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintRole(c *check.C) { 373 const nodeCount = 3 374 var daemons [nodeCount]*SwarmDaemon 375 for i := 0; i < nodeCount; i++ { 376 daemons[i] = s.AddDaemon(c, true, i == 0) 377 } 378 // wait for nodes ready 379 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 380 381 // create service 382 constraints := []string{"node.role==worker"} 383 instances := 3 384 id := daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 385 // wait for tasks ready 386 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 387 // validate tasks are running on worker nodes 388 tasks := daemons[0].getServiceTasks(c, id) 389 for _, task := range tasks { 390 node := daemons[0].getNode(c, task.NodeID) 391 c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleWorker) 392 } 393 //remove service 394 daemons[0].removeService(c, id) 395 396 // create service 397 constraints = []string{"node.role!=worker"} 398 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 399 // wait for tasks ready 400 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 401 tasks = daemons[0].getServiceTasks(c, id) 402 // validate tasks are running on manager nodes 403 for _, task := range tasks { 404 node := daemons[0].getNode(c, task.NodeID) 405 c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleManager) 406 } 407 //remove service 408 daemons[0].removeService(c, id) 409 410 // create service 411 constraints = []string{"node.role==nosuchrole"} 412 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 413 // wait for tasks created 414 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceTasks(id), checker.Equals, instances) 415 // let scheduler try 416 time.Sleep(250 * time.Millisecond) 417 // validate tasks are not assigned to any node 418 tasks = daemons[0].getServiceTasks(c, id) 419 for _, task := range tasks { 420 c.Assert(task.NodeID, checker.Equals, "") 421 } 422 } 423 424 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintLabel(c *check.C) { 425 const nodeCount = 3 426 var daemons [nodeCount]*SwarmDaemon 427 for i := 0; i < nodeCount; i++ { 428 daemons[i] = s.AddDaemon(c, true, i == 0) 429 } 430 // wait for nodes ready 431 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 432 nodes := daemons[0].listNodes(c) 433 c.Assert(len(nodes), checker.Equals, nodeCount) 434 435 // add labels to nodes 436 daemons[0].updateNode(c, nodes[0].ID, func(n *swarm.Node) { 437 n.Spec.Annotations.Labels = map[string]string{ 438 "security": "high", 439 } 440 }) 441 for i := 1; i < nodeCount; i++ { 442 daemons[0].updateNode(c, nodes[i].ID, func(n *swarm.Node) { 443 n.Spec.Annotations.Labels = map[string]string{ 444 "security": "low", 445 } 446 }) 447 } 448 449 // create service 450 instances := 3 451 constraints := []string{"node.labels.security==high"} 452 id := daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 453 // wait for tasks ready 454 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 455 tasks := daemons[0].getServiceTasks(c, id) 456 // validate all tasks are running on nodes[0] 457 for _, task := range tasks { 458 c.Assert(task.NodeID, checker.Equals, nodes[0].ID) 459 } 460 //remove service 461 daemons[0].removeService(c, id) 462 463 // create service 464 constraints = []string{"node.labels.security!=high"} 465 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 466 // wait for tasks ready 467 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 468 tasks = daemons[0].getServiceTasks(c, id) 469 // validate all tasks are NOT running on nodes[0] 470 for _, task := range tasks { 471 c.Assert(task.NodeID, checker.Not(checker.Equals), nodes[0].ID) 472 } 473 //remove service 474 daemons[0].removeService(c, id) 475 476 constraints = []string{"node.labels.security==medium"} 477 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 478 // wait for tasks created 479 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceTasks(id), checker.Equals, instances) 480 // let scheduler try 481 time.Sleep(250 * time.Millisecond) 482 tasks = daemons[0].getServiceTasks(c, id) 483 // validate tasks are not assigned 484 for _, task := range tasks { 485 c.Assert(task.NodeID, checker.Equals, "") 486 } 487 //remove service 488 daemons[0].removeService(c, id) 489 490 // multiple constraints 491 constraints = []string{ 492 "node.labels.security==high", 493 fmt.Sprintf("node.id==%s", nodes[1].ID), 494 } 495 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 496 // wait for tasks created 497 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceTasks(id), checker.Equals, instances) 498 // let scheduler try 499 time.Sleep(250 * time.Millisecond) 500 tasks = daemons[0].getServiceTasks(c, id) 501 // validate tasks are not assigned 502 for _, task := range tasks { 503 c.Assert(task.NodeID, checker.Equals, "") 504 } 505 // make nodes[1] fulfills the constraints 506 daemons[0].updateNode(c, nodes[1].ID, func(n *swarm.Node) { 507 n.Spec.Annotations.Labels = map[string]string{ 508 "security": "high", 509 } 510 }) 511 // wait for tasks ready 512 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 513 tasks = daemons[0].getServiceTasks(c, id) 514 for _, task := range tasks { 515 c.Assert(task.NodeID, checker.Equals, nodes[1].ID) 516 } 517 } 518 519 func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *check.C) { 520 testRequires(c, SameHostDaemon) 521 testRequires(c, DaemonIsLinux) 522 523 d1 := s.AddDaemon(c, true, true) 524 d2 := s.AddDaemon(c, true, true) 525 d3 := s.AddDaemon(c, true, false) 526 527 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept 528 529 instances := 9 530 d1.createService(c, simpleTestService, setInstances(instances)) 531 532 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 533 534 getContainers := func() map[string]*SwarmDaemon { 535 m := make(map[string]*SwarmDaemon) 536 for _, d := range []*SwarmDaemon{d1, d2, d3} { 537 for _, id := range d.activeContainers() { 538 m[id] = d 539 } 540 } 541 return m 542 } 543 544 containers := getContainers() 545 c.Assert(containers, checker.HasLen, instances) 546 var toRemove string 547 for i := range containers { 548 toRemove = i 549 } 550 551 _, err := containers[toRemove].Cmd("stop", toRemove) 552 c.Assert(err, checker.IsNil) 553 554 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 555 556 containers2 := getContainers() 557 c.Assert(containers2, checker.HasLen, instances) 558 for i := range containers { 559 if i == toRemove { 560 c.Assert(containers2[i], checker.IsNil) 561 } else { 562 c.Assert(containers2[i], checker.NotNil) 563 } 564 } 565 566 containers = containers2 567 for i := range containers { 568 toRemove = i 569 } 570 571 // try with killing process outside of docker 572 pidStr, err := containers[toRemove].Cmd("inspect", "-f", "{{.State.Pid}}", toRemove) 573 c.Assert(err, checker.IsNil) 574 pid, err := strconv.Atoi(strings.TrimSpace(pidStr)) 575 c.Assert(err, checker.IsNil) 576 c.Assert(syscall.Kill(pid, syscall.SIGKILL), checker.IsNil) 577 578 time.Sleep(time.Second) // give some time to handle the signal 579 580 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 581 582 containers2 = getContainers() 583 c.Assert(containers2, checker.HasLen, instances) 584 for i := range containers { 585 if i == toRemove { 586 c.Assert(containers2[i], checker.IsNil) 587 } else { 588 c.Assert(containers2[i], checker.NotNil) 589 } 590 } 591 } 592 593 func (s *DockerSwarmSuite) TestAPISwarmLeaderProxy(c *check.C) { 594 // add three managers, one of these is leader 595 d1 := s.AddDaemon(c, true, true) 596 d2 := s.AddDaemon(c, true, true) 597 d3 := s.AddDaemon(c, true, true) 598 599 // start a service by hitting each of the 3 managers 600 d1.createService(c, simpleTestService, func(s *swarm.Service) { 601 s.Spec.Name = "test1" 602 }) 603 d2.createService(c, simpleTestService, func(s *swarm.Service) { 604 s.Spec.Name = "test2" 605 }) 606 d3.createService(c, simpleTestService, func(s *swarm.Service) { 607 s.Spec.Name = "test3" 608 }) 609 610 // 3 services should be started now, because the requests were proxied to leader 611 // query each node and make sure it returns 3 services 612 for _, d := range []*SwarmDaemon{d1, d2, d3} { 613 services := d.listServices(c) 614 c.Assert(services, checker.HasLen, 3) 615 } 616 } 617 618 func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) { 619 // Create 3 nodes 620 d1 := s.AddDaemon(c, true, true) 621 d2 := s.AddDaemon(c, true, true) 622 d3 := s.AddDaemon(c, true, true) 623 624 // assert that the first node we made is the leader, and the other two are followers 625 c.Assert(d1.getNode(c, d1.NodeID).ManagerStatus.Leader, checker.True) 626 c.Assert(d1.getNode(c, d2.NodeID).ManagerStatus.Leader, checker.False) 627 c.Assert(d1.getNode(c, d3.NodeID).ManagerStatus.Leader, checker.False) 628 629 d1.Stop() // stop the leader 630 631 var ( 632 leader *SwarmDaemon // keep track of leader 633 followers []*SwarmDaemon // keep track of followers 634 ) 635 checkLeader := func(nodes ...*SwarmDaemon) checkF { 636 return func(c *check.C) (interface{}, check.CommentInterface) { 637 // clear these out before each run 638 leader = nil 639 followers = nil 640 for _, d := range nodes { 641 if d.getNode(c, d.NodeID).ManagerStatus.Leader { 642 leader = d 643 } else { 644 followers = append(followers, d) 645 } 646 } 647 648 if leader == nil { 649 return false, check.Commentf("no leader elected") 650 } 651 652 return true, check.Commentf("elected %v", leader.id) 653 } 654 } 655 656 // wait for an election to occur 657 waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d2, d3), checker.True) 658 659 // assert that we have a new leader 660 c.Assert(leader, checker.NotNil) 661 662 // Keep track of the current leader, since we want that to be chosen. 663 stableleader := leader 664 665 // add the d1, the initial leader, back 666 d1.Start() 667 668 // TODO(stevvooe): may need to wait for rejoin here 669 670 // wait for possible election 671 waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d1, d2, d3), checker.True) 672 // pick out the leader and the followers again 673 674 // verify that we still only have 1 leader and 2 followers 675 c.Assert(leader, checker.NotNil) 676 c.Assert(followers, checker.HasLen, 2) 677 // and that after we added d1 back, the leader hasn't changed 678 c.Assert(leader.NodeID, checker.Equals, stableleader.NodeID) 679 } 680 681 func (s *DockerSwarmSuite) TestAPISwarmRaftQuorum(c *check.C) { 682 d1 := s.AddDaemon(c, true, true) 683 d2 := s.AddDaemon(c, true, true) 684 d3 := s.AddDaemon(c, true, true) 685 686 d1.createService(c, simpleTestService) 687 688 c.Assert(d2.Stop(), checker.IsNil) 689 690 // make sure there is a leader 691 waitAndAssert(c, defaultReconciliationTimeout, d1.checkLeader, checker.IsNil) 692 693 d1.createService(c, simpleTestService, func(s *swarm.Service) { 694 s.Spec.Name = "top1" 695 }) 696 697 c.Assert(d3.Stop(), checker.IsNil) 698 699 // make sure there is a leader 700 waitAndAssert(c, defaultReconciliationTimeout, d1.checkLeader, checker.IsNil) 701 702 var service swarm.Service 703 simpleTestService(&service) 704 service.Spec.Name = "top2" 705 status, out, err := d1.SockRequest("POST", "/services/create", service.Spec) 706 c.Assert(err, checker.IsNil) 707 c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("deadline exceeded", string(out))) 708 709 c.Assert(d2.Start(), checker.IsNil) 710 711 // make sure there is a leader 712 waitAndAssert(c, defaultReconciliationTimeout, d1.checkLeader, checker.IsNil) 713 714 d1.createService(c, simpleTestService, func(s *swarm.Service) { 715 s.Spec.Name = "top3" 716 }) 717 } 718 719 func (s *DockerSwarmSuite) TestAPISwarmListNodes(c *check.C) { 720 d1 := s.AddDaemon(c, true, true) 721 d2 := s.AddDaemon(c, true, false) 722 d3 := s.AddDaemon(c, true, false) 723 724 nodes := d1.listNodes(c) 725 c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes)) 726 727 loop0: 728 for _, n := range nodes { 729 for _, d := range []*SwarmDaemon{d1, d2, d3} { 730 if n.ID == d.NodeID { 731 continue loop0 732 } 733 } 734 c.Errorf("unknown nodeID %v", n.ID) 735 } 736 } 737 738 func (s *DockerSwarmSuite) TestAPISwarmNodeUpdate(c *check.C) { 739 d := s.AddDaemon(c, true, true) 740 741 nodes := d.listNodes(c) 742 743 d.updateNode(c, nodes[0].ID, func(n *swarm.Node) { 744 n.Spec.Availability = swarm.NodeAvailabilityPause 745 }) 746 747 n := d.getNode(c, nodes[0].ID) 748 c.Assert(n.Spec.Availability, checker.Equals, swarm.NodeAvailabilityPause) 749 } 750 751 func (s *DockerSwarmSuite) TestAPISwarmNodeRemove(c *check.C) { 752 testRequires(c, Network) 753 d1 := s.AddDaemon(c, true, true) 754 d2 := s.AddDaemon(c, true, false) 755 _ = s.AddDaemon(c, true, false) 756 757 nodes := d1.listNodes(c) 758 c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes)) 759 760 // Getting the info so we can take the NodeID 761 d2Info, err := d2.info() 762 c.Assert(err, checker.IsNil) 763 764 // forceful removal of d2 should work 765 d1.removeNode(c, d2Info.NodeID, true) 766 767 nodes = d1.listNodes(c) 768 c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes)) 769 770 // Restart the node that was removed 771 err = d2.Restart() 772 c.Assert(err, checker.IsNil) 773 774 // Give some time for the node to rejoin 775 time.Sleep(1 * time.Second) 776 777 // Make sure the node didn't rejoin 778 nodes = d1.listNodes(c) 779 c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes)) 780 } 781 782 func (s *DockerSwarmSuite) TestAPISwarmNodeDrainPause(c *check.C) { 783 d1 := s.AddDaemon(c, true, true) 784 d2 := s.AddDaemon(c, true, false) 785 786 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks 787 788 // start a service, expect balanced distribution 789 instances := 8 790 id := d1.createService(c, simpleTestService, setInstances(instances)) 791 792 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0) 793 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0) 794 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 795 796 // drain d2, all containers should move to d1 797 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 798 n.Spec.Availability = swarm.NodeAvailabilityDrain 799 }) 800 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances) 801 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 0) 802 803 // set d2 back to active 804 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 805 n.Spec.Availability = swarm.NodeAvailabilityActive 806 }) 807 808 instances = 1 809 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 810 811 waitAndAssert(c, defaultReconciliationTimeout*2, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 812 813 instances = 8 814 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 815 816 // drained node first so we don't get any old containers 817 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0) 818 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0) 819 waitAndAssert(c, defaultReconciliationTimeout*2, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 820 821 d2ContainerCount := len(d2.activeContainers()) 822 823 // set d2 to paused, scale service up, only d1 gets new tasks 824 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 825 n.Spec.Availability = swarm.NodeAvailabilityPause 826 }) 827 828 instances = 14 829 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 830 831 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances-d2ContainerCount) 832 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, d2ContainerCount) 833 834 } 835 836 func (s *DockerSwarmSuite) TestAPISwarmLeaveRemovesContainer(c *check.C) { 837 d := s.AddDaemon(c, true, true) 838 839 instances := 2 840 d.createService(c, simpleTestService, setInstances(instances)) 841 842 id, err := d.Cmd("run", "-d", "busybox", "top") 843 c.Assert(err, checker.IsNil) 844 id = strings.TrimSpace(id) 845 846 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances+1) 847 848 c.Assert(d.Leave(false), checker.NotNil) 849 c.Assert(d.Leave(true), checker.IsNil) 850 851 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, 1) 852 853 id2, err := d.Cmd("ps", "-q") 854 c.Assert(err, checker.IsNil) 855 c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2)) 856 } 857 858 // #23629 859 func (s *DockerSwarmSuite) TestAPISwarmLeaveOnPendingJoin(c *check.C) { 860 testRequires(c, Network) 861 s.AddDaemon(c, true, true) 862 d2 := s.AddDaemon(c, false, false) 863 864 id, err := d2.Cmd("run", "-d", "busybox", "top") 865 c.Assert(err, checker.IsNil) 866 id = strings.TrimSpace(id) 867 868 err = d2.Join(swarm.JoinRequest{ 869 RemoteAddrs: []string{"123.123.123.123:1234"}, 870 }) 871 c.Assert(err, check.NotNil) 872 c.Assert(err.Error(), checker.Contains, "Timeout was reached") 873 874 info, err := d2.info() 875 c.Assert(err, checker.IsNil) 876 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStatePending) 877 878 c.Assert(d2.Leave(true), checker.IsNil) 879 880 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 1) 881 882 id2, err := d2.Cmd("ps", "-q") 883 c.Assert(err, checker.IsNil) 884 c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2)) 885 } 886 887 // #23705 888 func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *check.C) { 889 testRequires(c, Network) 890 d := s.AddDaemon(c, false, false) 891 err := d.Join(swarm.JoinRequest{ 892 RemoteAddrs: []string{"123.123.123.123:1234"}, 893 }) 894 c.Assert(err, check.NotNil) 895 c.Assert(err.Error(), checker.Contains, "Timeout was reached") 896 897 waitAndAssert(c, defaultReconciliationTimeout, d.checkLocalNodeState, checker.Equals, swarm.LocalNodeStatePending) 898 899 c.Assert(d.Stop(), checker.IsNil) 900 c.Assert(d.Start(), checker.IsNil) 901 902 info, err := d.info() 903 c.Assert(err, checker.IsNil) 904 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 905 } 906 907 func (s *DockerSwarmSuite) TestAPISwarmManagerRestore(c *check.C) { 908 d1 := s.AddDaemon(c, true, true) 909 910 instances := 2 911 id := d1.createService(c, simpleTestService, setInstances(instances)) 912 913 d1.getService(c, id) 914 d1.Stop() 915 d1.Start() 916 d1.getService(c, id) 917 918 d2 := s.AddDaemon(c, true, true) 919 d2.getService(c, id) 920 d2.Stop() 921 d2.Start() 922 d2.getService(c, id) 923 924 d3 := s.AddDaemon(c, true, true) 925 d3.getService(c, id) 926 d3.Stop() 927 d3.Start() 928 d3.getService(c, id) 929 930 d3.Kill() 931 time.Sleep(1 * time.Second) // time to handle signal 932 d3.Start() 933 d3.getService(c, id) 934 } 935 936 func (s *DockerSwarmSuite) TestAPISwarmScaleNoRollingUpdate(c *check.C) { 937 d := s.AddDaemon(c, true, true) 938 939 instances := 2 940 id := d.createService(c, simpleTestService, setInstances(instances)) 941 942 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 943 containers := d.activeContainers() 944 instances = 4 945 d.updateService(c, d.getService(c, id), setInstances(instances)) 946 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 947 containers2 := d.activeContainers() 948 949 loop0: 950 for _, c1 := range containers { 951 for _, c2 := range containers2 { 952 if c1 == c2 { 953 continue loop0 954 } 955 } 956 c.Errorf("container %v not found in new set %#v", c1, containers2) 957 } 958 } 959 960 func (s *DockerSwarmSuite) TestAPISwarmInvalidAddress(c *check.C) { 961 d := s.AddDaemon(c, false, false) 962 req := swarm.InitRequest{ 963 ListenAddr: "", 964 } 965 status, _, err := d.SockRequest("POST", "/swarm/init", req) 966 c.Assert(err, checker.IsNil) 967 c.Assert(status, checker.Equals, http.StatusInternalServerError) 968 969 req2 := swarm.JoinRequest{ 970 ListenAddr: "0.0.0.0:2377", 971 RemoteAddrs: []string{""}, 972 } 973 status, _, err = d.SockRequest("POST", "/swarm/join", req2) 974 c.Assert(err, checker.IsNil) 975 c.Assert(status, checker.Equals, http.StatusInternalServerError) 976 } 977 978 func (s *DockerSwarmSuite) TestAPISwarmForceNewCluster(c *check.C) { 979 d1 := s.AddDaemon(c, true, true) 980 d2 := s.AddDaemon(c, true, true) 981 982 instances := 2 983 id := d1.createService(c, simpleTestService, setInstances(instances)) 984 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 985 986 // drain d2, all containers should move to d1 987 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 988 n.Spec.Availability = swarm.NodeAvailabilityDrain 989 }) 990 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances) 991 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 0) 992 993 c.Assert(d2.Stop(), checker.IsNil) 994 995 c.Assert(d1.Init(swarm.InitRequest{ 996 ForceNewCluster: true, 997 Spec: swarm.Spec{}, 998 }), checker.IsNil) 999 1000 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances) 1001 1002 d3 := s.AddDaemon(c, true, true) 1003 info, err := d3.info() 1004 c.Assert(err, checker.IsNil) 1005 c.Assert(info.ControlAvailable, checker.True) 1006 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 1007 1008 instances = 4 1009 d3.updateService(c, d3.getService(c, id), setInstances(instances)) 1010 1011 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 1012 } 1013 1014 func simpleTestService(s *swarm.Service) { 1015 ureplicas := uint64(1) 1016 restartDelay := time.Duration(100 * time.Millisecond) 1017 1018 s.Spec = swarm.ServiceSpec{ 1019 TaskTemplate: swarm.TaskSpec{ 1020 ContainerSpec: swarm.ContainerSpec{ 1021 Image: "busybox:latest", 1022 Command: []string{"/bin/top"}, 1023 }, 1024 RestartPolicy: &swarm.RestartPolicy{ 1025 Delay: &restartDelay, 1026 }, 1027 }, 1028 Mode: swarm.ServiceMode{ 1029 Replicated: &swarm.ReplicatedService{ 1030 Replicas: &ureplicas, 1031 }, 1032 }, 1033 } 1034 s.Spec.Name = "top" 1035 } 1036 1037 func serviceForUpdate(s *swarm.Service) { 1038 ureplicas := uint64(1) 1039 restartDelay := time.Duration(100 * time.Millisecond) 1040 1041 s.Spec = swarm.ServiceSpec{ 1042 TaskTemplate: swarm.TaskSpec{ 1043 ContainerSpec: swarm.ContainerSpec{ 1044 Image: "busybox:latest", 1045 Command: []string{"/bin/top"}, 1046 }, 1047 RestartPolicy: &swarm.RestartPolicy{ 1048 Delay: &restartDelay, 1049 }, 1050 }, 1051 Mode: swarm.ServiceMode{ 1052 Replicated: &swarm.ReplicatedService{ 1053 Replicas: &ureplicas, 1054 }, 1055 }, 1056 UpdateConfig: &swarm.UpdateConfig{ 1057 Parallelism: 2, 1058 Delay: 4 * time.Second, 1059 FailureAction: swarm.UpdateFailureActionContinue, 1060 }, 1061 } 1062 s.Spec.Name = "updatetest" 1063 } 1064 1065 func setInstances(replicas int) serviceConstructor { 1066 ureplicas := uint64(replicas) 1067 return func(s *swarm.Service) { 1068 s.Spec.Mode = swarm.ServiceMode{ 1069 Replicated: &swarm.ReplicatedService{ 1070 Replicas: &ureplicas, 1071 }, 1072 } 1073 } 1074 } 1075 1076 func setImage(image string) serviceConstructor { 1077 return func(s *swarm.Service) { 1078 s.Spec.TaskTemplate.ContainerSpec.Image = image 1079 } 1080 } 1081 1082 func setFailureAction(failureAction string) serviceConstructor { 1083 return func(s *swarm.Service) { 1084 s.Spec.UpdateConfig.FailureAction = failureAction 1085 } 1086 } 1087 1088 func setMaxFailureRatio(maxFailureRatio float32) serviceConstructor { 1089 return func(s *swarm.Service) { 1090 s.Spec.UpdateConfig.MaxFailureRatio = maxFailureRatio 1091 } 1092 } 1093 1094 func setParallelism(parallelism uint64) serviceConstructor { 1095 return func(s *swarm.Service) { 1096 s.Spec.UpdateConfig.Parallelism = parallelism 1097 } 1098 } 1099 1100 func setConstraints(constraints []string) serviceConstructor { 1101 return func(s *swarm.Service) { 1102 if s.Spec.TaskTemplate.Placement == nil { 1103 s.Spec.TaskTemplate.Placement = &swarm.Placement{} 1104 } 1105 s.Spec.TaskTemplate.Placement.Constraints = constraints 1106 } 1107 } 1108 1109 func setGlobalMode(s *swarm.Service) { 1110 s.Spec.Mode = swarm.ServiceMode{ 1111 Global: &swarm.GlobalService{}, 1112 } 1113 } 1114 1115 func checkClusterHealth(c *check.C, cl []*SwarmDaemon, managerCount, workerCount int) { 1116 var totalMCount, totalWCount int 1117 1118 for _, d := range cl { 1119 var ( 1120 info swarm.Info 1121 err error 1122 ) 1123 1124 // check info in a waitAndAssert, because if the cluster doesn't have a leader, `info` will return an error 1125 checkInfo := func(c *check.C) (interface{}, check.CommentInterface) { 1126 info, err = d.info() 1127 return err, check.Commentf("cluster not ready in time") 1128 } 1129 waitAndAssert(c, defaultReconciliationTimeout, checkInfo, checker.IsNil) 1130 if !info.ControlAvailable { 1131 totalWCount++ 1132 continue 1133 } 1134 1135 var leaderFound bool 1136 totalMCount++ 1137 var mCount, wCount int 1138 1139 for _, n := range d.listNodes(c) { 1140 waitReady := func(c *check.C) (interface{}, check.CommentInterface) { 1141 if n.Status.State == swarm.NodeStateReady { 1142 return true, nil 1143 } 1144 nn := d.getNode(c, n.ID) 1145 n = *nn 1146 return n.Status.State == swarm.NodeStateReady, check.Commentf("state of node %s, reported by %s", n.ID, d.Info.NodeID) 1147 } 1148 waitAndAssert(c, defaultReconciliationTimeout, waitReady, checker.True) 1149 1150 waitActive := func(c *check.C) (interface{}, check.CommentInterface) { 1151 if n.Spec.Availability == swarm.NodeAvailabilityActive { 1152 return true, nil 1153 } 1154 nn := d.getNode(c, n.ID) 1155 n = *nn 1156 return n.Spec.Availability == swarm.NodeAvailabilityActive, check.Commentf("availability of node %s, reported by %s", n.ID, d.Info.NodeID) 1157 } 1158 waitAndAssert(c, defaultReconciliationTimeout, waitActive, checker.True) 1159 1160 if n.Spec.Role == swarm.NodeRoleManager { 1161 c.Assert(n.ManagerStatus, checker.NotNil, check.Commentf("manager status of node %s (manager), reported by %s", n.ID, d.Info.NodeID)) 1162 if n.ManagerStatus.Leader { 1163 leaderFound = true 1164 } 1165 mCount++ 1166 } else { 1167 c.Assert(n.ManagerStatus, checker.IsNil, check.Commentf("manager status of node %s (worker), reported by %s", n.ID, d.Info.NodeID)) 1168 wCount++ 1169 } 1170 } 1171 c.Assert(leaderFound, checker.True, check.Commentf("lack of leader reported by node %s", info.NodeID)) 1172 c.Assert(mCount, checker.Equals, managerCount, check.Commentf("managers count reported by node %s", info.NodeID)) 1173 c.Assert(wCount, checker.Equals, workerCount, check.Commentf("workers count reported by node %s", info.NodeID)) 1174 } 1175 c.Assert(totalMCount, checker.Equals, managerCount) 1176 c.Assert(totalWCount, checker.Equals, workerCount) 1177 } 1178 1179 func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) { 1180 mCount, wCount := 5, 1 1181 1182 var nodes []*SwarmDaemon 1183 for i := 0; i < mCount; i++ { 1184 manager := s.AddDaemon(c, true, true) 1185 info, err := manager.info() 1186 c.Assert(err, checker.IsNil) 1187 c.Assert(info.ControlAvailable, checker.True) 1188 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 1189 nodes = append(nodes, manager) 1190 } 1191 1192 for i := 0; i < wCount; i++ { 1193 worker := s.AddDaemon(c, true, false) 1194 info, err := worker.info() 1195 c.Assert(err, checker.IsNil) 1196 c.Assert(info.ControlAvailable, checker.False) 1197 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 1198 nodes = append(nodes, worker) 1199 } 1200 1201 // stop whole cluster 1202 { 1203 var wg sync.WaitGroup 1204 wg.Add(len(nodes)) 1205 errs := make(chan error, len(nodes)) 1206 1207 for _, d := range nodes { 1208 go func(daemon *SwarmDaemon) { 1209 defer wg.Done() 1210 if err := daemon.Stop(); err != nil { 1211 errs <- err 1212 } 1213 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 1214 daemon.root = filepath.Dir(daemon.root) 1215 } 1216 }(d) 1217 } 1218 wg.Wait() 1219 close(errs) 1220 for err := range errs { 1221 c.Assert(err, check.IsNil) 1222 } 1223 } 1224 1225 // start whole cluster 1226 { 1227 var wg sync.WaitGroup 1228 wg.Add(len(nodes)) 1229 errs := make(chan error, len(nodes)) 1230 1231 for _, d := range nodes { 1232 go func(daemon *SwarmDaemon) { 1233 defer wg.Done() 1234 if err := daemon.Start("--iptables=false"); err != nil { 1235 errs <- err 1236 } 1237 }(d) 1238 } 1239 wg.Wait() 1240 close(errs) 1241 for err := range errs { 1242 c.Assert(err, check.IsNil) 1243 } 1244 } 1245 1246 checkClusterHealth(c, nodes, mCount, wCount) 1247 } 1248 1249 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateWithName(c *check.C) { 1250 d := s.AddDaemon(c, true, true) 1251 1252 instances := 2 1253 id := d.createService(c, simpleTestService, setInstances(instances)) 1254 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 1255 1256 service := d.getService(c, id) 1257 instances = 5 1258 1259 setInstances(instances)(service) 1260 url := fmt.Sprintf("/services/%s/update?version=%d", service.Spec.Name, service.Version.Index) 1261 status, out, err := d.SockRequest("POST", url, service.Spec) 1262 c.Assert(err, checker.IsNil) 1263 c.Assert(status, checker.Equals, http.StatusOK, check.Commentf("output: %q", string(out))) 1264 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 1265 }