github.com/kinvolk/docker@v1.13.1/integration-cli/docker_api_swarm_test.go (about) 1 // +build !windows 2 3 package main 4 5 import ( 6 "encoding/json" 7 "fmt" 8 "net/http" 9 "os" 10 "path/filepath" 11 "strconv" 12 "strings" 13 "sync" 14 "syscall" 15 "time" 16 17 "github.com/docker/docker/api/types" 18 "github.com/docker/docker/api/types/swarm" 19 "github.com/docker/docker/pkg/integration/checker" 20 "github.com/go-check/check" 21 ) 22 23 var defaultReconciliationTimeout = 30 * time.Second 24 25 func (s *DockerSwarmSuite) TestAPISwarmInit(c *check.C) { 26 // todo: should find a better way to verify that components are running than /info 27 d1 := s.AddDaemon(c, true, true) 28 info, err := d1.info() 29 c.Assert(err, checker.IsNil) 30 c.Assert(info.ControlAvailable, checker.True) 31 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 32 33 d2 := s.AddDaemon(c, true, false) 34 info, err = d2.info() 35 c.Assert(err, checker.IsNil) 36 c.Assert(info.ControlAvailable, checker.False) 37 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 38 39 // Leaving cluster 40 c.Assert(d2.Leave(false), checker.IsNil) 41 42 info, err = d2.info() 43 c.Assert(err, checker.IsNil) 44 c.Assert(info.ControlAvailable, checker.False) 45 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 46 47 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: d1.joinTokens(c).Worker, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 48 49 info, err = d2.info() 50 c.Assert(err, checker.IsNil) 51 c.Assert(info.ControlAvailable, checker.False) 52 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 53 54 // Current state restoring after restarts 55 err = d1.Stop() 56 c.Assert(err, checker.IsNil) 57 err = d2.Stop() 58 c.Assert(err, checker.IsNil) 59 60 err = d1.Start() 61 c.Assert(err, checker.IsNil) 62 err = d2.Start() 63 c.Assert(err, checker.IsNil) 64 65 info, err = d1.info() 66 c.Assert(err, checker.IsNil) 67 c.Assert(info.ControlAvailable, checker.True) 68 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 69 70 info, err = d2.info() 71 c.Assert(err, checker.IsNil) 72 c.Assert(info.ControlAvailable, checker.False) 73 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 74 } 75 76 func (s *DockerSwarmSuite) TestAPISwarmJoinToken(c *check.C) { 77 d1 := s.AddDaemon(c, false, false) 78 c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) 79 80 d2 := s.AddDaemon(c, false, false) 81 err := d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.listenAddr}}) 82 c.Assert(err, checker.NotNil) 83 c.Assert(err.Error(), checker.Contains, "join token is necessary") 84 info, err := d2.info() 85 c.Assert(err, checker.IsNil) 86 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 87 88 err = d2.Join(swarm.JoinRequest{JoinToken: "foobaz", RemoteAddrs: []string{d1.listenAddr}}) 89 c.Assert(err, checker.NotNil) 90 c.Assert(err.Error(), checker.Contains, "join token is necessary") 91 info, err = d2.info() 92 c.Assert(err, checker.IsNil) 93 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 94 95 workerToken := d1.joinTokens(c).Worker 96 97 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 98 info, err = d2.info() 99 c.Assert(err, checker.IsNil) 100 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 101 c.Assert(d2.Leave(false), checker.IsNil) 102 info, err = d2.info() 103 c.Assert(err, checker.IsNil) 104 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 105 106 // change tokens 107 d1.rotateTokens(c) 108 109 err = d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}) 110 c.Assert(err, checker.NotNil) 111 c.Assert(err.Error(), checker.Contains, "join token is necessary") 112 info, err = d2.info() 113 c.Assert(err, checker.IsNil) 114 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 115 116 workerToken = d1.joinTokens(c).Worker 117 118 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 119 info, err = d2.info() 120 c.Assert(err, checker.IsNil) 121 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 122 c.Assert(d2.Leave(false), checker.IsNil) 123 info, err = d2.info() 124 c.Assert(err, checker.IsNil) 125 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 126 127 // change spec, don't change tokens 128 d1.updateSwarm(c, func(s *swarm.Spec) {}) 129 130 err = d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.listenAddr}}) 131 c.Assert(err, checker.NotNil) 132 c.Assert(err.Error(), checker.Contains, "join token is necessary") 133 info, err = d2.info() 134 c.Assert(err, checker.IsNil) 135 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 136 137 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 138 info, err = d2.info() 139 c.Assert(err, checker.IsNil) 140 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 141 c.Assert(d2.Leave(false), checker.IsNil) 142 info, err = d2.info() 143 c.Assert(err, checker.IsNil) 144 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 145 } 146 147 func (s *DockerSwarmSuite) TestAPISwarmCAHash(c *check.C) { 148 d1 := s.AddDaemon(c, true, true) 149 d2 := s.AddDaemon(c, false, false) 150 splitToken := strings.Split(d1.joinTokens(c).Worker, "-") 151 splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e" 152 replacementToken := strings.Join(splitToken, "-") 153 err := d2.Join(swarm.JoinRequest{JoinToken: replacementToken, RemoteAddrs: []string{d1.listenAddr}}) 154 c.Assert(err, checker.NotNil) 155 c.Assert(err.Error(), checker.Contains, "remote CA does not match fingerprint") 156 } 157 158 func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) { 159 d1 := s.AddDaemon(c, false, false) 160 c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) 161 d2 := s.AddDaemon(c, true, false) 162 163 info, err := d2.info() 164 c.Assert(err, checker.IsNil) 165 c.Assert(info.ControlAvailable, checker.False) 166 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 167 168 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 169 n.Spec.Role = swarm.NodeRoleManager 170 }) 171 172 waitAndAssert(c, defaultReconciliationTimeout, d2.checkControlAvailable, checker.True) 173 174 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 175 n.Spec.Role = swarm.NodeRoleWorker 176 }) 177 178 waitAndAssert(c, defaultReconciliationTimeout, d2.checkControlAvailable, checker.False) 179 180 // Demoting last node should fail 181 node := d1.getNode(c, d1.NodeID) 182 node.Spec.Role = swarm.NodeRoleWorker 183 url := fmt.Sprintf("/nodes/%s/update?version=%d", node.ID, node.Version.Index) 184 status, out, err := d1.SockRequest("POST", url, node.Spec) 185 c.Assert(err, checker.IsNil) 186 c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("output: %q", string(out))) 187 c.Assert(string(out), checker.Contains, "last manager of the swarm") 188 info, err = d1.info() 189 c.Assert(err, checker.IsNil) 190 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 191 c.Assert(info.ControlAvailable, checker.True) 192 193 // Promote already demoted node 194 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 195 n.Spec.Role = swarm.NodeRoleManager 196 }) 197 198 waitAndAssert(c, defaultReconciliationTimeout, d2.checkControlAvailable, checker.True) 199 } 200 201 func (s *DockerSwarmSuite) TestAPISwarmServicesEmptyList(c *check.C) { 202 d := s.AddDaemon(c, true, true) 203 204 services := d.listServices(c) 205 c.Assert(services, checker.NotNil) 206 c.Assert(len(services), checker.Equals, 0, check.Commentf("services: %#v", services)) 207 } 208 209 func (s *DockerSwarmSuite) TestAPISwarmServicesCreate(c *check.C) { 210 d := s.AddDaemon(c, true, true) 211 212 instances := 2 213 id := d.createService(c, simpleTestService, setInstances(instances)) 214 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 215 216 service := d.getService(c, id) 217 instances = 5 218 d.updateService(c, service, setInstances(instances)) 219 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 220 221 d.removeService(c, service.ID) 222 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, 0) 223 } 224 225 func (s *DockerSwarmSuite) TestAPISwarmServicesMultipleAgents(c *check.C) { 226 d1 := s.AddDaemon(c, true, true) 227 d2 := s.AddDaemon(c, true, false) 228 d3 := s.AddDaemon(c, true, false) 229 230 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks 231 232 instances := 9 233 id := d1.createService(c, simpleTestService, setInstances(instances)) 234 235 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0) 236 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0) 237 waitAndAssert(c, defaultReconciliationTimeout, d3.checkActiveContainerCount, checker.GreaterThan, 0) 238 239 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 240 241 // reconciliation on d2 node down 242 c.Assert(d2.Stop(), checker.IsNil) 243 244 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 245 246 // test downscaling 247 instances = 5 248 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 249 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 250 251 } 252 253 func (s *DockerSwarmSuite) TestAPISwarmServicesCreateGlobal(c *check.C) { 254 d1 := s.AddDaemon(c, true, true) 255 d2 := s.AddDaemon(c, true, false) 256 d3 := s.AddDaemon(c, true, false) 257 258 d1.createService(c, simpleTestService, setGlobalMode) 259 260 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, 1) 261 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 1) 262 waitAndAssert(c, defaultReconciliationTimeout, d3.checkActiveContainerCount, checker.Equals, 1) 263 264 d4 := s.AddDaemon(c, true, false) 265 d5 := s.AddDaemon(c, true, false) 266 267 waitAndAssert(c, defaultReconciliationTimeout, d4.checkActiveContainerCount, checker.Equals, 1) 268 waitAndAssert(c, defaultReconciliationTimeout, d5.checkActiveContainerCount, checker.Equals, 1) 269 } 270 271 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdate(c *check.C) { 272 const nodeCount = 3 273 var daemons [nodeCount]*SwarmDaemon 274 for i := 0; i < nodeCount; i++ { 275 daemons[i] = s.AddDaemon(c, true, i == 0) 276 } 277 // wait for nodes ready 278 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 279 280 // service image at start 281 image1 := "busybox:latest" 282 // target image in update 283 image2 := "busybox:test" 284 285 // create a different tag 286 for _, d := range daemons { 287 out, err := d.Cmd("tag", image1, image2) 288 c.Assert(err, checker.IsNil, check.Commentf(out)) 289 } 290 291 // create service 292 instances := 5 293 parallelism := 2 294 id := daemons[0].createService(c, serviceForUpdate, setInstances(instances)) 295 296 // wait for tasks ready 297 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 298 map[string]int{image1: instances}) 299 300 // issue service update 301 service := daemons[0].getService(c, id) 302 daemons[0].updateService(c, service, setImage(image2)) 303 304 // first batch 305 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 306 map[string]int{image1: instances - parallelism, image2: parallelism}) 307 308 // 2nd batch 309 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 310 map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism}) 311 312 // 3nd batch 313 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 314 map[string]int{image2: instances}) 315 316 // Roll back to the previous version. This uses the CLI because 317 // rollback is a client-side operation. 318 out, err := daemons[0].Cmd("service", "update", "--rollback", id) 319 c.Assert(err, checker.IsNil, check.Commentf(out)) 320 321 // first batch 322 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 323 map[string]int{image2: instances - parallelism, image1: parallelism}) 324 325 // 2nd batch 326 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 327 map[string]int{image2: instances - 2*parallelism, image1: 2 * parallelism}) 328 329 // 3nd batch 330 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 331 map[string]int{image1: instances}) 332 } 333 334 func (s *DockerSwarmSuite) TestAPISwarmServicesFailedUpdate(c *check.C) { 335 const nodeCount = 3 336 var daemons [nodeCount]*SwarmDaemon 337 for i := 0; i < nodeCount; i++ { 338 daemons[i] = s.AddDaemon(c, true, i == 0) 339 } 340 // wait for nodes ready 341 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 342 343 // service image at start 344 image1 := "busybox:latest" 345 // target image in update 346 image2 := "busybox:badtag" 347 348 // create service 349 instances := 5 350 id := daemons[0].createService(c, serviceForUpdate, setInstances(instances)) 351 352 // wait for tasks ready 353 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 354 map[string]int{image1: instances}) 355 356 // issue service update 357 service := daemons[0].getService(c, id) 358 daemons[0].updateService(c, service, setImage(image2), setFailureAction(swarm.UpdateFailureActionPause), setMaxFailureRatio(0.25), setParallelism(1)) 359 360 // should update 2 tasks and then pause 361 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceUpdateState(id), checker.Equals, swarm.UpdateStatePaused) 362 v, _ := daemons[0].checkServiceRunningTasks(id)(c) 363 c.Assert(v, checker.Equals, instances-2) 364 365 // Roll back to the previous version. This uses the CLI because 366 // rollback is a client-side operation. 367 out, err := daemons[0].Cmd("service", "update", "--rollback", id) 368 c.Assert(err, checker.IsNil, check.Commentf(out)) 369 370 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 371 map[string]int{image1: instances}) 372 } 373 374 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintRole(c *check.C) { 375 const nodeCount = 3 376 var daemons [nodeCount]*SwarmDaemon 377 for i := 0; i < nodeCount; i++ { 378 daemons[i] = s.AddDaemon(c, true, i == 0) 379 } 380 // wait for nodes ready 381 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 382 383 // create service 384 constraints := []string{"node.role==worker"} 385 instances := 3 386 id := daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 387 // wait for tasks ready 388 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 389 // validate tasks are running on worker nodes 390 tasks := daemons[0].getServiceTasks(c, id) 391 for _, task := range tasks { 392 node := daemons[0].getNode(c, task.NodeID) 393 c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleWorker) 394 } 395 //remove service 396 daemons[0].removeService(c, id) 397 398 // create service 399 constraints = []string{"node.role!=worker"} 400 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 401 // wait for tasks ready 402 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 403 tasks = daemons[0].getServiceTasks(c, id) 404 // validate tasks are running on manager nodes 405 for _, task := range tasks { 406 node := daemons[0].getNode(c, task.NodeID) 407 c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleManager) 408 } 409 //remove service 410 daemons[0].removeService(c, id) 411 412 // create service 413 constraints = []string{"node.role==nosuchrole"} 414 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 415 // wait for tasks created 416 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceTasks(id), checker.Equals, instances) 417 // let scheduler try 418 time.Sleep(250 * time.Millisecond) 419 // validate tasks are not assigned to any node 420 tasks = daemons[0].getServiceTasks(c, id) 421 for _, task := range tasks { 422 c.Assert(task.NodeID, checker.Equals, "") 423 } 424 } 425 426 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintLabel(c *check.C) { 427 const nodeCount = 3 428 var daemons [nodeCount]*SwarmDaemon 429 for i := 0; i < nodeCount; i++ { 430 daemons[i] = s.AddDaemon(c, true, i == 0) 431 } 432 // wait for nodes ready 433 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 434 nodes := daemons[0].listNodes(c) 435 c.Assert(len(nodes), checker.Equals, nodeCount) 436 437 // add labels to nodes 438 daemons[0].updateNode(c, nodes[0].ID, func(n *swarm.Node) { 439 n.Spec.Annotations.Labels = map[string]string{ 440 "security": "high", 441 } 442 }) 443 for i := 1; i < nodeCount; i++ { 444 daemons[0].updateNode(c, nodes[i].ID, func(n *swarm.Node) { 445 n.Spec.Annotations.Labels = map[string]string{ 446 "security": "low", 447 } 448 }) 449 } 450 451 // create service 452 instances := 3 453 constraints := []string{"node.labels.security==high"} 454 id := daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 455 // wait for tasks ready 456 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 457 tasks := daemons[0].getServiceTasks(c, id) 458 // validate all tasks are running on nodes[0] 459 for _, task := range tasks { 460 c.Assert(task.NodeID, checker.Equals, nodes[0].ID) 461 } 462 //remove service 463 daemons[0].removeService(c, id) 464 465 // create service 466 constraints = []string{"node.labels.security!=high"} 467 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 468 // wait for tasks ready 469 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 470 tasks = daemons[0].getServiceTasks(c, id) 471 // validate all tasks are NOT running on nodes[0] 472 for _, task := range tasks { 473 c.Assert(task.NodeID, checker.Not(checker.Equals), nodes[0].ID) 474 } 475 //remove service 476 daemons[0].removeService(c, id) 477 478 constraints = []string{"node.labels.security==medium"} 479 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 480 // wait for tasks created 481 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceTasks(id), checker.Equals, instances) 482 // let scheduler try 483 time.Sleep(250 * time.Millisecond) 484 tasks = daemons[0].getServiceTasks(c, id) 485 // validate tasks are not assigned 486 for _, task := range tasks { 487 c.Assert(task.NodeID, checker.Equals, "") 488 } 489 //remove service 490 daemons[0].removeService(c, id) 491 492 // multiple constraints 493 constraints = []string{ 494 "node.labels.security==high", 495 fmt.Sprintf("node.id==%s", nodes[1].ID), 496 } 497 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 498 // wait for tasks created 499 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceTasks(id), checker.Equals, instances) 500 // let scheduler try 501 time.Sleep(250 * time.Millisecond) 502 tasks = daemons[0].getServiceTasks(c, id) 503 // validate tasks are not assigned 504 for _, task := range tasks { 505 c.Assert(task.NodeID, checker.Equals, "") 506 } 507 // make nodes[1] fulfills the constraints 508 daemons[0].updateNode(c, nodes[1].ID, func(n *swarm.Node) { 509 n.Spec.Annotations.Labels = map[string]string{ 510 "security": "high", 511 } 512 }) 513 // wait for tasks ready 514 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 515 tasks = daemons[0].getServiceTasks(c, id) 516 for _, task := range tasks { 517 c.Assert(task.NodeID, checker.Equals, nodes[1].ID) 518 } 519 } 520 521 func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *check.C) { 522 testRequires(c, SameHostDaemon) 523 testRequires(c, DaemonIsLinux) 524 525 d1 := s.AddDaemon(c, true, true) 526 d2 := s.AddDaemon(c, true, true) 527 d3 := s.AddDaemon(c, true, false) 528 529 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept 530 531 instances := 9 532 d1.createService(c, simpleTestService, setInstances(instances)) 533 534 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 535 536 getContainers := func() map[string]*SwarmDaemon { 537 m := make(map[string]*SwarmDaemon) 538 for _, d := range []*SwarmDaemon{d1, d2, d3} { 539 for _, id := range d.activeContainers() { 540 m[id] = d 541 } 542 } 543 return m 544 } 545 546 containers := getContainers() 547 c.Assert(containers, checker.HasLen, instances) 548 var toRemove string 549 for i := range containers { 550 toRemove = i 551 } 552 553 _, err := containers[toRemove].Cmd("stop", toRemove) 554 c.Assert(err, checker.IsNil) 555 556 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 557 558 containers2 := getContainers() 559 c.Assert(containers2, checker.HasLen, instances) 560 for i := range containers { 561 if i == toRemove { 562 c.Assert(containers2[i], checker.IsNil) 563 } else { 564 c.Assert(containers2[i], checker.NotNil) 565 } 566 } 567 568 containers = containers2 569 for i := range containers { 570 toRemove = i 571 } 572 573 // try with killing process outside of docker 574 pidStr, err := containers[toRemove].Cmd("inspect", "-f", "{{.State.Pid}}", toRemove) 575 c.Assert(err, checker.IsNil) 576 pid, err := strconv.Atoi(strings.TrimSpace(pidStr)) 577 c.Assert(err, checker.IsNil) 578 c.Assert(syscall.Kill(pid, syscall.SIGKILL), checker.IsNil) 579 580 time.Sleep(time.Second) // give some time to handle the signal 581 582 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 583 584 containers2 = getContainers() 585 c.Assert(containers2, checker.HasLen, instances) 586 for i := range containers { 587 if i == toRemove { 588 c.Assert(containers2[i], checker.IsNil) 589 } else { 590 c.Assert(containers2[i], checker.NotNil) 591 } 592 } 593 } 594 595 func (s *DockerSwarmSuite) TestAPISwarmLeaderProxy(c *check.C) { 596 // add three managers, one of these is leader 597 d1 := s.AddDaemon(c, true, true) 598 d2 := s.AddDaemon(c, true, true) 599 d3 := s.AddDaemon(c, true, true) 600 601 // start a service by hitting each of the 3 managers 602 d1.createService(c, simpleTestService, func(s *swarm.Service) { 603 s.Spec.Name = "test1" 604 }) 605 d2.createService(c, simpleTestService, func(s *swarm.Service) { 606 s.Spec.Name = "test2" 607 }) 608 d3.createService(c, simpleTestService, func(s *swarm.Service) { 609 s.Spec.Name = "test3" 610 }) 611 612 // 3 services should be started now, because the requests were proxied to leader 613 // query each node and make sure it returns 3 services 614 for _, d := range []*SwarmDaemon{d1, d2, d3} { 615 services := d.listServices(c) 616 c.Assert(services, checker.HasLen, 3) 617 } 618 } 619 620 func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) { 621 // Create 3 nodes 622 d1 := s.AddDaemon(c, true, true) 623 d2 := s.AddDaemon(c, true, true) 624 d3 := s.AddDaemon(c, true, true) 625 626 // assert that the first node we made is the leader, and the other two are followers 627 c.Assert(d1.getNode(c, d1.NodeID).ManagerStatus.Leader, checker.True) 628 c.Assert(d1.getNode(c, d2.NodeID).ManagerStatus.Leader, checker.False) 629 c.Assert(d1.getNode(c, d3.NodeID).ManagerStatus.Leader, checker.False) 630 631 d1.Stop() // stop the leader 632 633 var ( 634 leader *SwarmDaemon // keep track of leader 635 followers []*SwarmDaemon // keep track of followers 636 ) 637 checkLeader := func(nodes ...*SwarmDaemon) checkF { 638 return func(c *check.C) (interface{}, check.CommentInterface) { 639 // clear these out before each run 640 leader = nil 641 followers = nil 642 for _, d := range nodes { 643 if d.getNode(c, d.NodeID).ManagerStatus.Leader { 644 leader = d 645 } else { 646 followers = append(followers, d) 647 } 648 } 649 650 if leader == nil { 651 return false, check.Commentf("no leader elected") 652 } 653 654 return true, check.Commentf("elected %v", leader.id) 655 } 656 } 657 658 // wait for an election to occur 659 waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d2, d3), checker.True) 660 661 // assert that we have a new leader 662 c.Assert(leader, checker.NotNil) 663 664 // Keep track of the current leader, since we want that to be chosen. 665 stableleader := leader 666 667 // add the d1, the initial leader, back 668 d1.Start() 669 670 // TODO(stevvooe): may need to wait for rejoin here 671 672 // wait for possible election 673 waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d1, d2, d3), checker.True) 674 // pick out the leader and the followers again 675 676 // verify that we still only have 1 leader and 2 followers 677 c.Assert(leader, checker.NotNil) 678 c.Assert(followers, checker.HasLen, 2) 679 // and that after we added d1 back, the leader hasn't changed 680 c.Assert(leader.NodeID, checker.Equals, stableleader.NodeID) 681 } 682 683 func (s *DockerSwarmSuite) TestAPISwarmRaftQuorum(c *check.C) { 684 d1 := s.AddDaemon(c, true, true) 685 d2 := s.AddDaemon(c, true, true) 686 d3 := s.AddDaemon(c, true, true) 687 688 d1.createService(c, simpleTestService) 689 690 c.Assert(d2.Stop(), checker.IsNil) 691 692 // make sure there is a leader 693 waitAndAssert(c, defaultReconciliationTimeout, d1.checkLeader, checker.IsNil) 694 695 d1.createService(c, simpleTestService, func(s *swarm.Service) { 696 s.Spec.Name = "top1" 697 }) 698 699 c.Assert(d3.Stop(), checker.IsNil) 700 701 // make sure there is a leader 702 waitAndAssert(c, defaultReconciliationTimeout, d1.checkLeader, checker.IsNil) 703 704 var service swarm.Service 705 simpleTestService(&service) 706 service.Spec.Name = "top2" 707 status, out, err := d1.SockRequest("POST", "/services/create", service.Spec) 708 c.Assert(err, checker.IsNil) 709 c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("deadline exceeded", string(out))) 710 711 c.Assert(d2.Start(), checker.IsNil) 712 713 // make sure there is a leader 714 waitAndAssert(c, defaultReconciliationTimeout, d1.checkLeader, checker.IsNil) 715 716 d1.createService(c, simpleTestService, func(s *swarm.Service) { 717 s.Spec.Name = "top3" 718 }) 719 } 720 721 func (s *DockerSwarmSuite) TestAPISwarmListNodes(c *check.C) { 722 d1 := s.AddDaemon(c, true, true) 723 d2 := s.AddDaemon(c, true, false) 724 d3 := s.AddDaemon(c, true, false) 725 726 nodes := d1.listNodes(c) 727 c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes)) 728 729 loop0: 730 for _, n := range nodes { 731 for _, d := range []*SwarmDaemon{d1, d2, d3} { 732 if n.ID == d.NodeID { 733 continue loop0 734 } 735 } 736 c.Errorf("unknown nodeID %v", n.ID) 737 } 738 } 739 740 func (s *DockerSwarmSuite) TestAPISwarmNodeUpdate(c *check.C) { 741 d := s.AddDaemon(c, true, true) 742 743 nodes := d.listNodes(c) 744 745 d.updateNode(c, nodes[0].ID, func(n *swarm.Node) { 746 n.Spec.Availability = swarm.NodeAvailabilityPause 747 }) 748 749 n := d.getNode(c, nodes[0].ID) 750 c.Assert(n.Spec.Availability, checker.Equals, swarm.NodeAvailabilityPause) 751 } 752 753 func (s *DockerSwarmSuite) TestAPISwarmNodeRemove(c *check.C) { 754 testRequires(c, Network) 755 d1 := s.AddDaemon(c, true, true) 756 d2 := s.AddDaemon(c, true, false) 757 _ = s.AddDaemon(c, true, false) 758 759 nodes := d1.listNodes(c) 760 c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes)) 761 762 // Getting the info so we can take the NodeID 763 d2Info, err := d2.info() 764 c.Assert(err, checker.IsNil) 765 766 // forceful removal of d2 should work 767 d1.removeNode(c, d2Info.NodeID, true) 768 769 nodes = d1.listNodes(c) 770 c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes)) 771 772 // Restart the node that was removed 773 err = d2.Restart() 774 c.Assert(err, checker.IsNil) 775 776 // Give some time for the node to rejoin 777 time.Sleep(1 * time.Second) 778 779 // Make sure the node didn't rejoin 780 nodes = d1.listNodes(c) 781 c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes)) 782 } 783 784 func (s *DockerSwarmSuite) TestAPISwarmNodeDrainPause(c *check.C) { 785 d1 := s.AddDaemon(c, true, true) 786 d2 := s.AddDaemon(c, true, false) 787 788 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks 789 790 // start a service, expect balanced distribution 791 instances := 8 792 id := d1.createService(c, simpleTestService, setInstances(instances)) 793 794 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0) 795 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0) 796 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 797 798 // drain d2, all containers should move to d1 799 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 800 n.Spec.Availability = swarm.NodeAvailabilityDrain 801 }) 802 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances) 803 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 0) 804 805 // set d2 back to active 806 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 807 n.Spec.Availability = swarm.NodeAvailabilityActive 808 }) 809 810 instances = 1 811 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 812 813 waitAndAssert(c, defaultReconciliationTimeout*2, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 814 815 instances = 8 816 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 817 818 // drained node first so we don't get any old containers 819 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0) 820 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0) 821 waitAndAssert(c, defaultReconciliationTimeout*2, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 822 823 d2ContainerCount := len(d2.activeContainers()) 824 825 // set d2 to paused, scale service up, only d1 gets new tasks 826 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 827 n.Spec.Availability = swarm.NodeAvailabilityPause 828 }) 829 830 instances = 14 831 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 832 833 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances-d2ContainerCount) 834 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, d2ContainerCount) 835 836 } 837 838 func (s *DockerSwarmSuite) TestAPISwarmLeaveRemovesContainer(c *check.C) { 839 d := s.AddDaemon(c, true, true) 840 841 instances := 2 842 d.createService(c, simpleTestService, setInstances(instances)) 843 844 id, err := d.Cmd("run", "-d", "busybox", "top") 845 c.Assert(err, checker.IsNil) 846 id = strings.TrimSpace(id) 847 848 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances+1) 849 850 c.Assert(d.Leave(false), checker.NotNil) 851 c.Assert(d.Leave(true), checker.IsNil) 852 853 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, 1) 854 855 id2, err := d.Cmd("ps", "-q") 856 c.Assert(err, checker.IsNil) 857 c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2)) 858 } 859 860 // #23629 861 func (s *DockerSwarmSuite) TestAPISwarmLeaveOnPendingJoin(c *check.C) { 862 testRequires(c, Network) 863 s.AddDaemon(c, true, true) 864 d2 := s.AddDaemon(c, false, false) 865 866 id, err := d2.Cmd("run", "-d", "busybox", "top") 867 c.Assert(err, checker.IsNil) 868 id = strings.TrimSpace(id) 869 870 err = d2.Join(swarm.JoinRequest{ 871 RemoteAddrs: []string{"123.123.123.123:1234"}, 872 }) 873 c.Assert(err, check.NotNil) 874 c.Assert(err.Error(), checker.Contains, "Timeout was reached") 875 876 info, err := d2.info() 877 c.Assert(err, checker.IsNil) 878 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStatePending) 879 880 c.Assert(d2.Leave(true), checker.IsNil) 881 882 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 1) 883 884 id2, err := d2.Cmd("ps", "-q") 885 c.Assert(err, checker.IsNil) 886 c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2)) 887 } 888 889 // #23705 890 func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *check.C) { 891 testRequires(c, Network) 892 d := s.AddDaemon(c, false, false) 893 err := d.Join(swarm.JoinRequest{ 894 RemoteAddrs: []string{"123.123.123.123:1234"}, 895 }) 896 c.Assert(err, check.NotNil) 897 c.Assert(err.Error(), checker.Contains, "Timeout was reached") 898 899 waitAndAssert(c, defaultReconciliationTimeout, d.checkLocalNodeState, checker.Equals, swarm.LocalNodeStatePending) 900 901 c.Assert(d.Stop(), checker.IsNil) 902 c.Assert(d.Start(), checker.IsNil) 903 904 info, err := d.info() 905 c.Assert(err, checker.IsNil) 906 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 907 } 908 909 func (s *DockerSwarmSuite) TestAPISwarmManagerRestore(c *check.C) { 910 d1 := s.AddDaemon(c, true, true) 911 912 instances := 2 913 id := d1.createService(c, simpleTestService, setInstances(instances)) 914 915 d1.getService(c, id) 916 d1.Stop() 917 d1.Start() 918 d1.getService(c, id) 919 920 d2 := s.AddDaemon(c, true, true) 921 d2.getService(c, id) 922 d2.Stop() 923 d2.Start() 924 d2.getService(c, id) 925 926 d3 := s.AddDaemon(c, true, true) 927 d3.getService(c, id) 928 d3.Stop() 929 d3.Start() 930 d3.getService(c, id) 931 932 d3.Kill() 933 time.Sleep(1 * time.Second) // time to handle signal 934 d3.Start() 935 d3.getService(c, id) 936 } 937 938 func (s *DockerSwarmSuite) TestAPISwarmScaleNoRollingUpdate(c *check.C) { 939 d := s.AddDaemon(c, true, true) 940 941 instances := 2 942 id := d.createService(c, simpleTestService, setInstances(instances)) 943 944 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 945 containers := d.activeContainers() 946 instances = 4 947 d.updateService(c, d.getService(c, id), setInstances(instances)) 948 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 949 containers2 := d.activeContainers() 950 951 loop0: 952 for _, c1 := range containers { 953 for _, c2 := range containers2 { 954 if c1 == c2 { 955 continue loop0 956 } 957 } 958 c.Errorf("container %v not found in new set %#v", c1, containers2) 959 } 960 } 961 962 func (s *DockerSwarmSuite) TestAPISwarmInvalidAddress(c *check.C) { 963 d := s.AddDaemon(c, false, false) 964 req := swarm.InitRequest{ 965 ListenAddr: "", 966 } 967 status, _, err := d.SockRequest("POST", "/swarm/init", req) 968 c.Assert(err, checker.IsNil) 969 c.Assert(status, checker.Equals, http.StatusInternalServerError) 970 971 req2 := swarm.JoinRequest{ 972 ListenAddr: "0.0.0.0:2377", 973 RemoteAddrs: []string{""}, 974 } 975 status, _, err = d.SockRequest("POST", "/swarm/join", req2) 976 c.Assert(err, checker.IsNil) 977 c.Assert(status, checker.Equals, http.StatusInternalServerError) 978 } 979 980 func (s *DockerSwarmSuite) TestAPISwarmForceNewCluster(c *check.C) { 981 d1 := s.AddDaemon(c, true, true) 982 d2 := s.AddDaemon(c, true, true) 983 984 instances := 2 985 id := d1.createService(c, simpleTestService, setInstances(instances)) 986 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 987 988 // drain d2, all containers should move to d1 989 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 990 n.Spec.Availability = swarm.NodeAvailabilityDrain 991 }) 992 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances) 993 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 0) 994 995 c.Assert(d2.Stop(), checker.IsNil) 996 997 c.Assert(d1.Init(swarm.InitRequest{ 998 ForceNewCluster: true, 999 Spec: swarm.Spec{}, 1000 }), checker.IsNil) 1001 1002 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances) 1003 1004 d3 := s.AddDaemon(c, true, true) 1005 info, err := d3.info() 1006 c.Assert(err, checker.IsNil) 1007 c.Assert(info.ControlAvailable, checker.True) 1008 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 1009 1010 instances = 4 1011 d3.updateService(c, d3.getService(c, id), setInstances(instances)) 1012 1013 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 1014 } 1015 1016 func simpleTestService(s *swarm.Service) { 1017 ureplicas := uint64(1) 1018 restartDelay := time.Duration(100 * time.Millisecond) 1019 1020 s.Spec = swarm.ServiceSpec{ 1021 TaskTemplate: swarm.TaskSpec{ 1022 ContainerSpec: swarm.ContainerSpec{ 1023 Image: "busybox:latest", 1024 Command: []string{"/bin/top"}, 1025 }, 1026 RestartPolicy: &swarm.RestartPolicy{ 1027 Delay: &restartDelay, 1028 }, 1029 }, 1030 Mode: swarm.ServiceMode{ 1031 Replicated: &swarm.ReplicatedService{ 1032 Replicas: &ureplicas, 1033 }, 1034 }, 1035 } 1036 s.Spec.Name = "top" 1037 } 1038 1039 func serviceForUpdate(s *swarm.Service) { 1040 ureplicas := uint64(1) 1041 restartDelay := time.Duration(100 * time.Millisecond) 1042 1043 s.Spec = swarm.ServiceSpec{ 1044 TaskTemplate: swarm.TaskSpec{ 1045 ContainerSpec: swarm.ContainerSpec{ 1046 Image: "busybox:latest", 1047 Command: []string{"/bin/top"}, 1048 }, 1049 RestartPolicy: &swarm.RestartPolicy{ 1050 Delay: &restartDelay, 1051 }, 1052 }, 1053 Mode: swarm.ServiceMode{ 1054 Replicated: &swarm.ReplicatedService{ 1055 Replicas: &ureplicas, 1056 }, 1057 }, 1058 UpdateConfig: &swarm.UpdateConfig{ 1059 Parallelism: 2, 1060 Delay: 4 * time.Second, 1061 FailureAction: swarm.UpdateFailureActionContinue, 1062 }, 1063 } 1064 s.Spec.Name = "updatetest" 1065 } 1066 1067 func setInstances(replicas int) serviceConstructor { 1068 ureplicas := uint64(replicas) 1069 return func(s *swarm.Service) { 1070 s.Spec.Mode = swarm.ServiceMode{ 1071 Replicated: &swarm.ReplicatedService{ 1072 Replicas: &ureplicas, 1073 }, 1074 } 1075 } 1076 } 1077 1078 func setImage(image string) serviceConstructor { 1079 return func(s *swarm.Service) { 1080 s.Spec.TaskTemplate.ContainerSpec.Image = image 1081 } 1082 } 1083 1084 func setFailureAction(failureAction string) serviceConstructor { 1085 return func(s *swarm.Service) { 1086 s.Spec.UpdateConfig.FailureAction = failureAction 1087 } 1088 } 1089 1090 func setMaxFailureRatio(maxFailureRatio float32) serviceConstructor { 1091 return func(s *swarm.Service) { 1092 s.Spec.UpdateConfig.MaxFailureRatio = maxFailureRatio 1093 } 1094 } 1095 1096 func setParallelism(parallelism uint64) serviceConstructor { 1097 return func(s *swarm.Service) { 1098 s.Spec.UpdateConfig.Parallelism = parallelism 1099 } 1100 } 1101 1102 func setConstraints(constraints []string) serviceConstructor { 1103 return func(s *swarm.Service) { 1104 if s.Spec.TaskTemplate.Placement == nil { 1105 s.Spec.TaskTemplate.Placement = &swarm.Placement{} 1106 } 1107 s.Spec.TaskTemplate.Placement.Constraints = constraints 1108 } 1109 } 1110 1111 func setGlobalMode(s *swarm.Service) { 1112 s.Spec.Mode = swarm.ServiceMode{ 1113 Global: &swarm.GlobalService{}, 1114 } 1115 } 1116 1117 func checkClusterHealth(c *check.C, cl []*SwarmDaemon, managerCount, workerCount int) { 1118 var totalMCount, totalWCount int 1119 1120 for _, d := range cl { 1121 var ( 1122 info swarm.Info 1123 err error 1124 ) 1125 1126 // check info in a waitAndAssert, because if the cluster doesn't have a leader, `info` will return an error 1127 checkInfo := func(c *check.C) (interface{}, check.CommentInterface) { 1128 info, err = d.info() 1129 return err, check.Commentf("cluster not ready in time") 1130 } 1131 waitAndAssert(c, defaultReconciliationTimeout, checkInfo, checker.IsNil) 1132 if !info.ControlAvailable { 1133 totalWCount++ 1134 continue 1135 } 1136 1137 var leaderFound bool 1138 totalMCount++ 1139 var mCount, wCount int 1140 1141 for _, n := range d.listNodes(c) { 1142 waitReady := func(c *check.C) (interface{}, check.CommentInterface) { 1143 if n.Status.State == swarm.NodeStateReady { 1144 return true, nil 1145 } 1146 nn := d.getNode(c, n.ID) 1147 n = *nn 1148 return n.Status.State == swarm.NodeStateReady, check.Commentf("state of node %s, reported by %s", n.ID, d.Info.NodeID) 1149 } 1150 waitAndAssert(c, defaultReconciliationTimeout, waitReady, checker.True) 1151 1152 waitActive := func(c *check.C) (interface{}, check.CommentInterface) { 1153 if n.Spec.Availability == swarm.NodeAvailabilityActive { 1154 return true, nil 1155 } 1156 nn := d.getNode(c, n.ID) 1157 n = *nn 1158 return n.Spec.Availability == swarm.NodeAvailabilityActive, check.Commentf("availability of node %s, reported by %s", n.ID, d.Info.NodeID) 1159 } 1160 waitAndAssert(c, defaultReconciliationTimeout, waitActive, checker.True) 1161 1162 if n.Spec.Role == swarm.NodeRoleManager { 1163 c.Assert(n.ManagerStatus, checker.NotNil, check.Commentf("manager status of node %s (manager), reported by %s", n.ID, d.Info.NodeID)) 1164 if n.ManagerStatus.Leader { 1165 leaderFound = true 1166 } 1167 mCount++ 1168 } else { 1169 c.Assert(n.ManagerStatus, checker.IsNil, check.Commentf("manager status of node %s (worker), reported by %s", n.ID, d.Info.NodeID)) 1170 wCount++ 1171 } 1172 } 1173 c.Assert(leaderFound, checker.True, check.Commentf("lack of leader reported by node %s", info.NodeID)) 1174 c.Assert(mCount, checker.Equals, managerCount, check.Commentf("managers count reported by node %s", info.NodeID)) 1175 c.Assert(wCount, checker.Equals, workerCount, check.Commentf("workers count reported by node %s", info.NodeID)) 1176 } 1177 c.Assert(totalMCount, checker.Equals, managerCount) 1178 c.Assert(totalWCount, checker.Equals, workerCount) 1179 } 1180 1181 func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) { 1182 mCount, wCount := 5, 1 1183 1184 var nodes []*SwarmDaemon 1185 for i := 0; i < mCount; i++ { 1186 manager := s.AddDaemon(c, true, true) 1187 info, err := manager.info() 1188 c.Assert(err, checker.IsNil) 1189 c.Assert(info.ControlAvailable, checker.True) 1190 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 1191 nodes = append(nodes, manager) 1192 } 1193 1194 for i := 0; i < wCount; i++ { 1195 worker := s.AddDaemon(c, true, false) 1196 info, err := worker.info() 1197 c.Assert(err, checker.IsNil) 1198 c.Assert(info.ControlAvailable, checker.False) 1199 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 1200 nodes = append(nodes, worker) 1201 } 1202 1203 // stop whole cluster 1204 { 1205 var wg sync.WaitGroup 1206 wg.Add(len(nodes)) 1207 errs := make(chan error, len(nodes)) 1208 1209 for _, d := range nodes { 1210 go func(daemon *SwarmDaemon) { 1211 defer wg.Done() 1212 if err := daemon.Stop(); err != nil { 1213 errs <- err 1214 } 1215 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 1216 daemon.root = filepath.Dir(daemon.root) 1217 } 1218 }(d) 1219 } 1220 wg.Wait() 1221 close(errs) 1222 for err := range errs { 1223 c.Assert(err, check.IsNil) 1224 } 1225 } 1226 1227 // start whole cluster 1228 { 1229 var wg sync.WaitGroup 1230 wg.Add(len(nodes)) 1231 errs := make(chan error, len(nodes)) 1232 1233 for _, d := range nodes { 1234 go func(daemon *SwarmDaemon) { 1235 defer wg.Done() 1236 if err := daemon.Start("--iptables=false"); err != nil { 1237 errs <- err 1238 } 1239 }(d) 1240 } 1241 wg.Wait() 1242 close(errs) 1243 for err := range errs { 1244 c.Assert(err, check.IsNil) 1245 } 1246 } 1247 1248 checkClusterHealth(c, nodes, mCount, wCount) 1249 } 1250 1251 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateWithName(c *check.C) { 1252 d := s.AddDaemon(c, true, true) 1253 1254 instances := 2 1255 id := d.createService(c, simpleTestService, setInstances(instances)) 1256 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 1257 1258 service := d.getService(c, id) 1259 instances = 5 1260 1261 setInstances(instances)(service) 1262 url := fmt.Sprintf("/services/%s/update?version=%d", service.Spec.Name, service.Version.Index) 1263 status, out, err := d.SockRequest("POST", url, service.Spec) 1264 c.Assert(err, checker.IsNil) 1265 c.Assert(status, checker.Equals, http.StatusOK, check.Commentf("output: %q", string(out))) 1266 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 1267 } 1268 1269 func (s *DockerSwarmSuite) TestAPISwarmSecretsEmptyList(c *check.C) { 1270 d := s.AddDaemon(c, true, true) 1271 1272 secrets := d.listSecrets(c) 1273 c.Assert(secrets, checker.NotNil) 1274 c.Assert(len(secrets), checker.Equals, 0, check.Commentf("secrets: %#v", secrets)) 1275 } 1276 1277 func (s *DockerSwarmSuite) TestAPISwarmSecretsCreate(c *check.C) { 1278 d := s.AddDaemon(c, true, true) 1279 1280 testName := "test_secret" 1281 id := d.createSecret(c, swarm.SecretSpec{ 1282 swarm.Annotations{ 1283 Name: testName, 1284 }, 1285 []byte("TESTINGDATA"), 1286 }) 1287 c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("secrets: %s", id)) 1288 1289 secrets := d.listSecrets(c) 1290 c.Assert(len(secrets), checker.Equals, 1, check.Commentf("secrets: %#v", secrets)) 1291 name := secrets[0].Spec.Annotations.Name 1292 c.Assert(name, checker.Equals, testName, check.Commentf("secret: %s", name)) 1293 } 1294 1295 func (s *DockerSwarmSuite) TestAPISwarmSecretsDelete(c *check.C) { 1296 d := s.AddDaemon(c, true, true) 1297 1298 testName := "test_secret" 1299 id := d.createSecret(c, swarm.SecretSpec{ 1300 swarm.Annotations{ 1301 Name: testName, 1302 }, 1303 []byte("TESTINGDATA"), 1304 }) 1305 c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("secrets: %s", id)) 1306 1307 secret := d.getSecret(c, id) 1308 c.Assert(secret.ID, checker.Equals, id, check.Commentf("secret: %v", secret)) 1309 1310 d.deleteSecret(c, secret.ID) 1311 status, out, err := d.SockRequest("GET", "/secrets/"+id, nil) 1312 c.Assert(err, checker.IsNil) 1313 c.Assert(status, checker.Equals, http.StatusNotFound, check.Commentf("secret delete: %s", string(out))) 1314 } 1315 1316 // Test case for 30242, where duplicate networks, with different drivers `bridge` and `overlay`, 1317 // caused both scopes to be `swarm` for `docker network inspect` and `docker network ls`. 1318 // This test makes sure the fixes correctly output scopes instead. 1319 func (s *DockerSwarmSuite) TestAPIDuplicateNetworks(c *check.C) { 1320 d := s.AddDaemon(c, true, true) 1321 1322 name := "foo" 1323 networkCreateRequest := types.NetworkCreateRequest{ 1324 Name: name, 1325 NetworkCreate: types.NetworkCreate{ 1326 CheckDuplicate: false, 1327 }, 1328 } 1329 1330 var n1 types.NetworkCreateResponse 1331 networkCreateRequest.NetworkCreate.Driver = "bridge" 1332 1333 status, out, err := d.SockRequest("POST", "/networks/create", networkCreateRequest) 1334 c.Assert(err, checker.IsNil, check.Commentf(string(out))) 1335 c.Assert(status, checker.Equals, http.StatusCreated, check.Commentf(string(out))) 1336 1337 c.Assert(json.Unmarshal(out, &n1), checker.IsNil) 1338 1339 var n2 types.NetworkCreateResponse 1340 networkCreateRequest.NetworkCreate.Driver = "overlay" 1341 1342 status, out, err = d.SockRequest("POST", "/networks/create", networkCreateRequest) 1343 c.Assert(err, checker.IsNil, check.Commentf(string(out))) 1344 c.Assert(status, checker.Equals, http.StatusCreated, check.Commentf(string(out))) 1345 1346 c.Assert(json.Unmarshal(out, &n2), checker.IsNil) 1347 1348 var r1 types.NetworkResource 1349 1350 status, out, err = d.SockRequest("GET", "/networks/"+n1.ID, nil) 1351 c.Assert(err, checker.IsNil, check.Commentf(string(out))) 1352 c.Assert(status, checker.Equals, http.StatusOK, check.Commentf(string(out))) 1353 1354 c.Assert(json.Unmarshal(out, &r1), checker.IsNil) 1355 1356 c.Assert(r1.Scope, checker.Equals, "local") 1357 1358 var r2 types.NetworkResource 1359 1360 status, out, err = d.SockRequest("GET", "/networks/"+n2.ID, nil) 1361 c.Assert(err, checker.IsNil, check.Commentf(string(out))) 1362 c.Assert(status, checker.Equals, http.StatusOK, check.Commentf(string(out))) 1363 1364 c.Assert(json.Unmarshal(out, &r2), checker.IsNil) 1365 1366 c.Assert(r2.Scope, checker.Equals, "swarm") 1367 }