github.com/sld880311/docker@v0.0.0-20200524143708-d5593973a475/integration-cli/docker_api_swarm_test.go (about) 1 // +build !windows 2 3 package main 4 5 import ( 6 "encoding/json" 7 "fmt" 8 "net/http" 9 "os" 10 "path/filepath" 11 "strconv" 12 "strings" 13 "sync" 14 "syscall" 15 "time" 16 17 "github.com/docker/docker/api/types" 18 "github.com/docker/docker/api/types/container" 19 "github.com/docker/docker/api/types/swarm" 20 "github.com/docker/docker/pkg/integration/checker" 21 "github.com/go-check/check" 22 ) 23 24 var defaultReconciliationTimeout = 30 * time.Second 25 26 func (s *DockerSwarmSuite) TestAPISwarmInit(c *check.C) { 27 // todo: should find a better way to verify that components are running than /info 28 d1 := s.AddDaemon(c, true, true) 29 info, err := d1.info() 30 c.Assert(err, checker.IsNil) 31 c.Assert(info.ControlAvailable, checker.True) 32 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 33 34 d2 := s.AddDaemon(c, true, false) 35 info, err = d2.info() 36 c.Assert(err, checker.IsNil) 37 c.Assert(info.ControlAvailable, checker.False) 38 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 39 40 // Leaving cluster 41 c.Assert(d2.Leave(false), checker.IsNil) 42 43 info, err = d2.info() 44 c.Assert(err, checker.IsNil) 45 c.Assert(info.ControlAvailable, checker.False) 46 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 47 48 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: d1.joinTokens(c).Worker, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 49 50 info, err = d2.info() 51 c.Assert(err, checker.IsNil) 52 c.Assert(info.ControlAvailable, checker.False) 53 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 54 55 // Current state restoring after restarts 56 err = d1.Stop() 57 c.Assert(err, checker.IsNil) 58 err = d2.Stop() 59 c.Assert(err, checker.IsNil) 60 61 err = d1.Start() 62 c.Assert(err, checker.IsNil) 63 err = d2.Start() 64 c.Assert(err, checker.IsNil) 65 66 info, err = d1.info() 67 c.Assert(err, checker.IsNil) 68 c.Assert(info.ControlAvailable, checker.True) 69 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 70 71 info, err = d2.info() 72 c.Assert(err, checker.IsNil) 73 c.Assert(info.ControlAvailable, checker.False) 74 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 75 } 76 77 func (s *DockerSwarmSuite) TestAPISwarmJoinToken(c *check.C) { 78 d1 := s.AddDaemon(c, false, false) 79 c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) 80 81 d2 := s.AddDaemon(c, false, false) 82 err := d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.listenAddr}}) 83 c.Assert(err, checker.NotNil) 84 c.Assert(err.Error(), checker.Contains, "join token is necessary") 85 info, err := d2.info() 86 c.Assert(err, checker.IsNil) 87 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 88 89 err = d2.Join(swarm.JoinRequest{JoinToken: "foobaz", RemoteAddrs: []string{d1.listenAddr}}) 90 c.Assert(err, checker.NotNil) 91 c.Assert(err.Error(), checker.Contains, "join token is necessary") 92 info, err = d2.info() 93 c.Assert(err, checker.IsNil) 94 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 95 96 workerToken := d1.joinTokens(c).Worker 97 98 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 99 info, err = d2.info() 100 c.Assert(err, checker.IsNil) 101 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 102 c.Assert(d2.Leave(false), checker.IsNil) 103 info, err = d2.info() 104 c.Assert(err, checker.IsNil) 105 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 106 107 // change tokens 108 d1.rotateTokens(c) 109 110 err = d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}) 111 c.Assert(err, checker.NotNil) 112 c.Assert(err.Error(), checker.Contains, "join token is necessary") 113 info, err = d2.info() 114 c.Assert(err, checker.IsNil) 115 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 116 117 workerToken = d1.joinTokens(c).Worker 118 119 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 120 info, err = d2.info() 121 c.Assert(err, checker.IsNil) 122 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 123 c.Assert(d2.Leave(false), checker.IsNil) 124 info, err = d2.info() 125 c.Assert(err, checker.IsNil) 126 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 127 128 // change spec, don't change tokens 129 d1.updateSwarm(c, func(s *swarm.Spec) {}) 130 131 err = d2.Join(swarm.JoinRequest{RemoteAddrs: []string{d1.listenAddr}}) 132 c.Assert(err, checker.NotNil) 133 c.Assert(err.Error(), checker.Contains, "join token is necessary") 134 info, err = d2.info() 135 c.Assert(err, checker.IsNil) 136 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 137 138 c.Assert(d2.Join(swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.listenAddr}}), checker.IsNil) 139 info, err = d2.info() 140 c.Assert(err, checker.IsNil) 141 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 142 c.Assert(d2.Leave(false), checker.IsNil) 143 info, err = d2.info() 144 c.Assert(err, checker.IsNil) 145 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 146 } 147 148 func (s *DockerSwarmSuite) TestAPISwarmCAHash(c *check.C) { 149 d1 := s.AddDaemon(c, true, true) 150 d2 := s.AddDaemon(c, false, false) 151 splitToken := strings.Split(d1.joinTokens(c).Worker, "-") 152 splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e" 153 replacementToken := strings.Join(splitToken, "-") 154 err := d2.Join(swarm.JoinRequest{JoinToken: replacementToken, RemoteAddrs: []string{d1.listenAddr}}) 155 c.Assert(err, checker.NotNil) 156 c.Assert(err.Error(), checker.Contains, "remote CA does not match fingerprint") 157 } 158 159 func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) { 160 d1 := s.AddDaemon(c, false, false) 161 c.Assert(d1.Init(swarm.InitRequest{}), checker.IsNil) 162 d2 := s.AddDaemon(c, true, false) 163 164 info, err := d2.info() 165 c.Assert(err, checker.IsNil) 166 c.Assert(info.ControlAvailable, checker.False) 167 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 168 169 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 170 n.Spec.Role = swarm.NodeRoleManager 171 }) 172 173 waitAndAssert(c, defaultReconciliationTimeout, d2.checkControlAvailable, checker.True) 174 175 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 176 n.Spec.Role = swarm.NodeRoleWorker 177 }) 178 179 waitAndAssert(c, defaultReconciliationTimeout, d2.checkControlAvailable, checker.False) 180 181 // Demoting last node should fail 182 node := d1.getNode(c, d1.NodeID) 183 node.Spec.Role = swarm.NodeRoleWorker 184 url := fmt.Sprintf("/nodes/%s/update?version=%d", node.ID, node.Version.Index) 185 status, out, err := d1.SockRequest("POST", url, node.Spec) 186 c.Assert(err, checker.IsNil) 187 c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("output: %q", string(out))) 188 c.Assert(string(out), checker.Contains, "last manager of the swarm") 189 info, err = d1.info() 190 c.Assert(err, checker.IsNil) 191 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 192 c.Assert(info.ControlAvailable, checker.True) 193 194 // Promote already demoted node 195 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 196 n.Spec.Role = swarm.NodeRoleManager 197 }) 198 199 waitAndAssert(c, defaultReconciliationTimeout, d2.checkControlAvailable, checker.True) 200 } 201 202 func (s *DockerSwarmSuite) TestAPISwarmServicesEmptyList(c *check.C) { 203 d := s.AddDaemon(c, true, true) 204 205 services := d.listServices(c) 206 c.Assert(services, checker.NotNil) 207 c.Assert(len(services), checker.Equals, 0, check.Commentf("services: %#v", services)) 208 } 209 210 func (s *DockerSwarmSuite) TestAPISwarmServicesCreate(c *check.C) { 211 d := s.AddDaemon(c, true, true) 212 213 instances := 2 214 id := d.createService(c, simpleTestService, setInstances(instances)) 215 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 216 217 service := d.getService(c, id) 218 instances = 5 219 d.updateService(c, service, setInstances(instances)) 220 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 221 222 d.removeService(c, service.ID) 223 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, 0) 224 } 225 226 func (s *DockerSwarmSuite) TestAPISwarmServicesMultipleAgents(c *check.C) { 227 d1 := s.AddDaemon(c, true, true) 228 d2 := s.AddDaemon(c, true, false) 229 d3 := s.AddDaemon(c, true, false) 230 231 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks 232 233 instances := 9 234 id := d1.createService(c, simpleTestService, setInstances(instances)) 235 236 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0) 237 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0) 238 waitAndAssert(c, defaultReconciliationTimeout, d3.checkActiveContainerCount, checker.GreaterThan, 0) 239 240 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 241 242 // reconciliation on d2 node down 243 c.Assert(d2.Stop(), checker.IsNil) 244 245 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 246 247 // test downscaling 248 instances = 5 249 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 250 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 251 252 } 253 254 func (s *DockerSwarmSuite) TestAPISwarmServicesCreateGlobal(c *check.C) { 255 d1 := s.AddDaemon(c, true, true) 256 d2 := s.AddDaemon(c, true, false) 257 d3 := s.AddDaemon(c, true, false) 258 259 d1.createService(c, simpleTestService, setGlobalMode) 260 261 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, 1) 262 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 1) 263 waitAndAssert(c, defaultReconciliationTimeout, d3.checkActiveContainerCount, checker.Equals, 1) 264 265 d4 := s.AddDaemon(c, true, false) 266 d5 := s.AddDaemon(c, true, false) 267 268 waitAndAssert(c, defaultReconciliationTimeout, d4.checkActiveContainerCount, checker.Equals, 1) 269 waitAndAssert(c, defaultReconciliationTimeout, d5.checkActiveContainerCount, checker.Equals, 1) 270 } 271 272 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdate(c *check.C) { 273 const nodeCount = 3 274 var daemons [nodeCount]*SwarmDaemon 275 for i := 0; i < nodeCount; i++ { 276 daemons[i] = s.AddDaemon(c, true, i == 0) 277 } 278 // wait for nodes ready 279 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 280 281 // service image at start 282 image1 := "busybox:latest" 283 // target image in update 284 image2 := "busybox:test" 285 286 // create a different tag 287 for _, d := range daemons { 288 out, err := d.Cmd("tag", image1, image2) 289 c.Assert(err, checker.IsNil, check.Commentf(out)) 290 } 291 292 // create service 293 instances := 5 294 parallelism := 2 295 id := daemons[0].createService(c, serviceForUpdate, setInstances(instances)) 296 297 // wait for tasks ready 298 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 299 map[string]int{image1: instances}) 300 301 // issue service update 302 service := daemons[0].getService(c, id) 303 daemons[0].updateService(c, service, setImage(image2)) 304 305 // first batch 306 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 307 map[string]int{image1: instances - parallelism, image2: parallelism}) 308 309 // 2nd batch 310 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 311 map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism}) 312 313 // 3nd batch 314 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 315 map[string]int{image2: instances}) 316 317 // Roll back to the previous version. This uses the CLI because 318 // rollback is a client-side operation. 319 out, err := daemons[0].Cmd("service", "update", "--rollback", id) 320 c.Assert(err, checker.IsNil, check.Commentf(out)) 321 322 // first batch 323 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 324 map[string]int{image2: instances - parallelism, image1: parallelism}) 325 326 // 2nd batch 327 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 328 map[string]int{image2: instances - 2*parallelism, image1: 2 * parallelism}) 329 330 // 3nd batch 331 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 332 map[string]int{image1: instances}) 333 } 334 335 func (s *DockerSwarmSuite) TestAPISwarmServicesFailedUpdate(c *check.C) { 336 const nodeCount = 3 337 var daemons [nodeCount]*SwarmDaemon 338 for i := 0; i < nodeCount; i++ { 339 daemons[i] = s.AddDaemon(c, true, i == 0) 340 } 341 // wait for nodes ready 342 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 343 344 // service image at start 345 image1 := "busybox:latest" 346 // target image in update 347 image2 := "busybox:badtag" 348 349 // create service 350 instances := 5 351 id := daemons[0].createService(c, serviceForUpdate, setInstances(instances)) 352 353 // wait for tasks ready 354 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 355 map[string]int{image1: instances}) 356 357 // issue service update 358 service := daemons[0].getService(c, id) 359 daemons[0].updateService(c, service, setImage(image2), setFailureAction(swarm.UpdateFailureActionPause), setMaxFailureRatio(0.25), setParallelism(1)) 360 361 // should update 2 tasks and then pause 362 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceUpdateState(id), checker.Equals, swarm.UpdateStatePaused) 363 v, _ := daemons[0].checkServiceRunningTasks(id)(c) 364 c.Assert(v, checker.Equals, instances-2) 365 366 // Roll back to the previous version. This uses the CLI because 367 // rollback is a client-side operation. 368 out, err := daemons[0].Cmd("service", "update", "--rollback", id) 369 c.Assert(err, checker.IsNil, check.Commentf(out)) 370 371 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkRunningTaskImages, checker.DeepEquals, 372 map[string]int{image1: instances}) 373 } 374 375 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintRole(c *check.C) { 376 const nodeCount = 3 377 var daemons [nodeCount]*SwarmDaemon 378 for i := 0; i < nodeCount; i++ { 379 daemons[i] = s.AddDaemon(c, true, i == 0) 380 } 381 // wait for nodes ready 382 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 383 384 // create service 385 constraints := []string{"node.role==worker"} 386 instances := 3 387 id := daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 388 // wait for tasks ready 389 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 390 // validate tasks are running on worker nodes 391 tasks := daemons[0].getServiceTasks(c, id) 392 for _, task := range tasks { 393 node := daemons[0].getNode(c, task.NodeID) 394 c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleWorker) 395 } 396 //remove service 397 daemons[0].removeService(c, id) 398 399 // create service 400 constraints = []string{"node.role!=worker"} 401 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 402 // wait for tasks ready 403 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 404 tasks = daemons[0].getServiceTasks(c, id) 405 // validate tasks are running on manager nodes 406 for _, task := range tasks { 407 node := daemons[0].getNode(c, task.NodeID) 408 c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleManager) 409 } 410 //remove service 411 daemons[0].removeService(c, id) 412 413 // create service 414 constraints = []string{"node.role==nosuchrole"} 415 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 416 // wait for tasks created 417 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceTasks(id), checker.Equals, instances) 418 // let scheduler try 419 time.Sleep(250 * time.Millisecond) 420 // validate tasks are not assigned to any node 421 tasks = daemons[0].getServiceTasks(c, id) 422 for _, task := range tasks { 423 c.Assert(task.NodeID, checker.Equals, "") 424 } 425 } 426 427 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintLabel(c *check.C) { 428 const nodeCount = 3 429 var daemons [nodeCount]*SwarmDaemon 430 for i := 0; i < nodeCount; i++ { 431 daemons[i] = s.AddDaemon(c, true, i == 0) 432 } 433 // wait for nodes ready 434 waitAndAssert(c, 5*time.Second, daemons[0].checkNodeReadyCount, checker.Equals, nodeCount) 435 nodes := daemons[0].listNodes(c) 436 c.Assert(len(nodes), checker.Equals, nodeCount) 437 438 // add labels to nodes 439 daemons[0].updateNode(c, nodes[0].ID, func(n *swarm.Node) { 440 n.Spec.Annotations.Labels = map[string]string{ 441 "security": "high", 442 } 443 }) 444 for i := 1; i < nodeCount; i++ { 445 daemons[0].updateNode(c, nodes[i].ID, func(n *swarm.Node) { 446 n.Spec.Annotations.Labels = map[string]string{ 447 "security": "low", 448 } 449 }) 450 } 451 452 // create service 453 instances := 3 454 constraints := []string{"node.labels.security==high"} 455 id := daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 456 // wait for tasks ready 457 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 458 tasks := daemons[0].getServiceTasks(c, id) 459 // validate all tasks are running on nodes[0] 460 for _, task := range tasks { 461 c.Assert(task.NodeID, checker.Equals, nodes[0].ID) 462 } 463 //remove service 464 daemons[0].removeService(c, id) 465 466 // create service 467 constraints = []string{"node.labels.security!=high"} 468 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 469 // wait for tasks ready 470 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 471 tasks = daemons[0].getServiceTasks(c, id) 472 // validate all tasks are NOT running on nodes[0] 473 for _, task := range tasks { 474 c.Assert(task.NodeID, checker.Not(checker.Equals), nodes[0].ID) 475 } 476 //remove service 477 daemons[0].removeService(c, id) 478 479 constraints = []string{"node.labels.security==medium"} 480 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 481 // wait for tasks created 482 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceTasks(id), checker.Equals, instances) 483 // let scheduler try 484 time.Sleep(250 * time.Millisecond) 485 tasks = daemons[0].getServiceTasks(c, id) 486 // validate tasks are not assigned 487 for _, task := range tasks { 488 c.Assert(task.NodeID, checker.Equals, "") 489 } 490 //remove service 491 daemons[0].removeService(c, id) 492 493 // multiple constraints 494 constraints = []string{ 495 "node.labels.security==high", 496 fmt.Sprintf("node.id==%s", nodes[1].ID), 497 } 498 id = daemons[0].createService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 499 // wait for tasks created 500 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceTasks(id), checker.Equals, instances) 501 // let scheduler try 502 time.Sleep(250 * time.Millisecond) 503 tasks = daemons[0].getServiceTasks(c, id) 504 // validate tasks are not assigned 505 for _, task := range tasks { 506 c.Assert(task.NodeID, checker.Equals, "") 507 } 508 // make nodes[1] fulfills the constraints 509 daemons[0].updateNode(c, nodes[1].ID, func(n *swarm.Node) { 510 n.Spec.Annotations.Labels = map[string]string{ 511 "security": "high", 512 } 513 }) 514 // wait for tasks ready 515 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].checkServiceRunningTasks(id), checker.Equals, instances) 516 tasks = daemons[0].getServiceTasks(c, id) 517 for _, task := range tasks { 518 c.Assert(task.NodeID, checker.Equals, nodes[1].ID) 519 } 520 } 521 522 func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *check.C) { 523 testRequires(c, SameHostDaemon) 524 testRequires(c, DaemonIsLinux) 525 526 d1 := s.AddDaemon(c, true, true) 527 d2 := s.AddDaemon(c, true, true) 528 d3 := s.AddDaemon(c, true, false) 529 530 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept 531 532 instances := 9 533 d1.createService(c, simpleTestService, setInstances(instances)) 534 535 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 536 537 getContainers := func() map[string]*SwarmDaemon { 538 m := make(map[string]*SwarmDaemon) 539 for _, d := range []*SwarmDaemon{d1, d2, d3} { 540 for _, id := range d.activeContainers() { 541 m[id] = d 542 } 543 } 544 return m 545 } 546 547 containers := getContainers() 548 c.Assert(containers, checker.HasLen, instances) 549 var toRemove string 550 for i := range containers { 551 toRemove = i 552 } 553 554 _, err := containers[toRemove].Cmd("stop", toRemove) 555 c.Assert(err, checker.IsNil) 556 557 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 558 559 containers2 := getContainers() 560 c.Assert(containers2, checker.HasLen, instances) 561 for i := range containers { 562 if i == toRemove { 563 c.Assert(containers2[i], checker.IsNil) 564 } else { 565 c.Assert(containers2[i], checker.NotNil) 566 } 567 } 568 569 containers = containers2 570 for i := range containers { 571 toRemove = i 572 } 573 574 // try with killing process outside of docker 575 pidStr, err := containers[toRemove].Cmd("inspect", "-f", "{{.State.Pid}}", toRemove) 576 c.Assert(err, checker.IsNil) 577 pid, err := strconv.Atoi(strings.TrimSpace(pidStr)) 578 c.Assert(err, checker.IsNil) 579 c.Assert(syscall.Kill(pid, syscall.SIGKILL), checker.IsNil) 580 581 time.Sleep(time.Second) // give some time to handle the signal 582 583 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 584 585 containers2 = getContainers() 586 c.Assert(containers2, checker.HasLen, instances) 587 for i := range containers { 588 if i == toRemove { 589 c.Assert(containers2[i], checker.IsNil) 590 } else { 591 c.Assert(containers2[i], checker.NotNil) 592 } 593 } 594 } 595 596 func (s *DockerSwarmSuite) TestAPISwarmLeaderProxy(c *check.C) { 597 // add three managers, one of these is leader 598 d1 := s.AddDaemon(c, true, true) 599 d2 := s.AddDaemon(c, true, true) 600 d3 := s.AddDaemon(c, true, true) 601 602 // start a service by hitting each of the 3 managers 603 d1.createService(c, simpleTestService, func(s *swarm.Service) { 604 s.Spec.Name = "test1" 605 }) 606 d2.createService(c, simpleTestService, func(s *swarm.Service) { 607 s.Spec.Name = "test2" 608 }) 609 d3.createService(c, simpleTestService, func(s *swarm.Service) { 610 s.Spec.Name = "test3" 611 }) 612 613 // 3 services should be started now, because the requests were proxied to leader 614 // query each node and make sure it returns 3 services 615 for _, d := range []*SwarmDaemon{d1, d2, d3} { 616 services := d.listServices(c) 617 c.Assert(services, checker.HasLen, 3) 618 } 619 } 620 621 func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) { 622 // Create 3 nodes 623 d1 := s.AddDaemon(c, true, true) 624 d2 := s.AddDaemon(c, true, true) 625 d3 := s.AddDaemon(c, true, true) 626 627 // assert that the first node we made is the leader, and the other two are followers 628 c.Assert(d1.getNode(c, d1.NodeID).ManagerStatus.Leader, checker.True) 629 c.Assert(d1.getNode(c, d2.NodeID).ManagerStatus.Leader, checker.False) 630 c.Assert(d1.getNode(c, d3.NodeID).ManagerStatus.Leader, checker.False) 631 632 d1.Stop() // stop the leader 633 634 var ( 635 leader *SwarmDaemon // keep track of leader 636 followers []*SwarmDaemon // keep track of followers 637 ) 638 checkLeader := func(nodes ...*SwarmDaemon) checkF { 639 return func(c *check.C) (interface{}, check.CommentInterface) { 640 // clear these out before each run 641 leader = nil 642 followers = nil 643 for _, d := range nodes { 644 if d.getNode(c, d.NodeID).ManagerStatus.Leader { 645 leader = d 646 } else { 647 followers = append(followers, d) 648 } 649 } 650 651 if leader == nil { 652 return false, check.Commentf("no leader elected") 653 } 654 655 return true, check.Commentf("elected %v", leader.id) 656 } 657 } 658 659 // wait for an election to occur 660 waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d2, d3), checker.True) 661 662 // assert that we have a new leader 663 c.Assert(leader, checker.NotNil) 664 665 // Keep track of the current leader, since we want that to be chosen. 666 stableleader := leader 667 668 // add the d1, the initial leader, back 669 d1.Start() 670 671 // TODO(stevvooe): may need to wait for rejoin here 672 673 // wait for possible election 674 waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d1, d2, d3), checker.True) 675 // pick out the leader and the followers again 676 677 // verify that we still only have 1 leader and 2 followers 678 c.Assert(leader, checker.NotNil) 679 c.Assert(followers, checker.HasLen, 2) 680 // and that after we added d1 back, the leader hasn't changed 681 c.Assert(leader.NodeID, checker.Equals, stableleader.NodeID) 682 } 683 684 func (s *DockerSwarmSuite) TestAPISwarmRaftQuorum(c *check.C) { 685 d1 := s.AddDaemon(c, true, true) 686 d2 := s.AddDaemon(c, true, true) 687 d3 := s.AddDaemon(c, true, true) 688 689 d1.createService(c, simpleTestService) 690 691 c.Assert(d2.Stop(), checker.IsNil) 692 693 // make sure there is a leader 694 waitAndAssert(c, defaultReconciliationTimeout, d1.checkLeader, checker.IsNil) 695 696 d1.createService(c, simpleTestService, func(s *swarm.Service) { 697 s.Spec.Name = "top1" 698 }) 699 700 c.Assert(d3.Stop(), checker.IsNil) 701 702 // make sure there is a leader 703 waitAndAssert(c, defaultReconciliationTimeout, d1.checkLeader, checker.IsNil) 704 705 var service swarm.Service 706 simpleTestService(&service) 707 service.Spec.Name = "top2" 708 status, out, err := d1.SockRequest("POST", "/services/create", service.Spec) 709 c.Assert(err, checker.IsNil) 710 c.Assert(status, checker.Equals, http.StatusInternalServerError, check.Commentf("deadline exceeded", string(out))) 711 712 c.Assert(d2.Start(), checker.IsNil) 713 714 // make sure there is a leader 715 waitAndAssert(c, defaultReconciliationTimeout, d1.checkLeader, checker.IsNil) 716 717 d1.createService(c, simpleTestService, func(s *swarm.Service) { 718 s.Spec.Name = "top3" 719 }) 720 } 721 722 func (s *DockerSwarmSuite) TestAPISwarmListNodes(c *check.C) { 723 d1 := s.AddDaemon(c, true, true) 724 d2 := s.AddDaemon(c, true, false) 725 d3 := s.AddDaemon(c, true, false) 726 727 nodes := d1.listNodes(c) 728 c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes)) 729 730 loop0: 731 for _, n := range nodes { 732 for _, d := range []*SwarmDaemon{d1, d2, d3} { 733 if n.ID == d.NodeID { 734 continue loop0 735 } 736 } 737 c.Errorf("unknown nodeID %v", n.ID) 738 } 739 } 740 741 func (s *DockerSwarmSuite) TestAPISwarmNodeUpdate(c *check.C) { 742 d := s.AddDaemon(c, true, true) 743 744 nodes := d.listNodes(c) 745 746 d.updateNode(c, nodes[0].ID, func(n *swarm.Node) { 747 n.Spec.Availability = swarm.NodeAvailabilityPause 748 }) 749 750 n := d.getNode(c, nodes[0].ID) 751 c.Assert(n.Spec.Availability, checker.Equals, swarm.NodeAvailabilityPause) 752 } 753 754 func (s *DockerSwarmSuite) TestAPISwarmNodeRemove(c *check.C) { 755 testRequires(c, Network) 756 d1 := s.AddDaemon(c, true, true) 757 d2 := s.AddDaemon(c, true, false) 758 _ = s.AddDaemon(c, true, false) 759 760 nodes := d1.listNodes(c) 761 c.Assert(len(nodes), checker.Equals, 3, check.Commentf("nodes: %#v", nodes)) 762 763 // Getting the info so we can take the NodeID 764 d2Info, err := d2.info() 765 c.Assert(err, checker.IsNil) 766 767 // forceful removal of d2 should work 768 d1.removeNode(c, d2Info.NodeID, true) 769 770 nodes = d1.listNodes(c) 771 c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes)) 772 773 // Restart the node that was removed 774 err = d2.Restart() 775 c.Assert(err, checker.IsNil) 776 777 // Give some time for the node to rejoin 778 time.Sleep(1 * time.Second) 779 780 // Make sure the node didn't rejoin 781 nodes = d1.listNodes(c) 782 c.Assert(len(nodes), checker.Equals, 2, check.Commentf("nodes: %#v", nodes)) 783 } 784 785 func (s *DockerSwarmSuite) TestAPISwarmNodeDrainPause(c *check.C) { 786 d1 := s.AddDaemon(c, true, true) 787 d2 := s.AddDaemon(c, true, false) 788 789 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks 790 791 // start a service, expect balanced distribution 792 instances := 8 793 id := d1.createService(c, simpleTestService, setInstances(instances)) 794 795 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0) 796 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0) 797 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 798 799 // drain d2, all containers should move to d1 800 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 801 n.Spec.Availability = swarm.NodeAvailabilityDrain 802 }) 803 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances) 804 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 0) 805 806 // set d2 back to active 807 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 808 n.Spec.Availability = swarm.NodeAvailabilityActive 809 }) 810 811 instances = 1 812 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 813 814 waitAndAssert(c, defaultReconciliationTimeout*2, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 815 816 instances = 8 817 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 818 819 // drained node first so we don't get any old containers 820 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.GreaterThan, 0) 821 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.GreaterThan, 0) 822 waitAndAssert(c, defaultReconciliationTimeout*2, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 823 824 d2ContainerCount := len(d2.activeContainers()) 825 826 // set d2 to paused, scale service up, only d1 gets new tasks 827 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 828 n.Spec.Availability = swarm.NodeAvailabilityPause 829 }) 830 831 instances = 14 832 d1.updateService(c, d1.getService(c, id), setInstances(instances)) 833 834 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances-d2ContainerCount) 835 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, d2ContainerCount) 836 837 } 838 839 func (s *DockerSwarmSuite) TestAPISwarmLeaveRemovesContainer(c *check.C) { 840 d := s.AddDaemon(c, true, true) 841 842 instances := 2 843 d.createService(c, simpleTestService, setInstances(instances)) 844 845 id, err := d.Cmd("run", "-d", "busybox", "top") 846 c.Assert(err, checker.IsNil) 847 id = strings.TrimSpace(id) 848 849 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances+1) 850 851 c.Assert(d.Leave(false), checker.NotNil) 852 c.Assert(d.Leave(true), checker.IsNil) 853 854 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, 1) 855 856 id2, err := d.Cmd("ps", "-q") 857 c.Assert(err, checker.IsNil) 858 c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2)) 859 } 860 861 // #23629 862 func (s *DockerSwarmSuite) TestAPISwarmLeaveOnPendingJoin(c *check.C) { 863 testRequires(c, Network) 864 s.AddDaemon(c, true, true) 865 d2 := s.AddDaemon(c, false, false) 866 867 id, err := d2.Cmd("run", "-d", "busybox", "top") 868 c.Assert(err, checker.IsNil) 869 id = strings.TrimSpace(id) 870 871 err = d2.Join(swarm.JoinRequest{ 872 RemoteAddrs: []string{"123.123.123.123:1234"}, 873 }) 874 c.Assert(err, check.NotNil) 875 c.Assert(err.Error(), checker.Contains, "Timeout was reached") 876 877 info, err := d2.info() 878 c.Assert(err, checker.IsNil) 879 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStatePending) 880 881 c.Assert(d2.Leave(true), checker.IsNil) 882 883 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 1) 884 885 id2, err := d2.Cmd("ps", "-q") 886 c.Assert(err, checker.IsNil) 887 c.Assert(id, checker.HasPrefix, strings.TrimSpace(id2)) 888 } 889 890 // #23705 891 func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *check.C) { 892 testRequires(c, Network) 893 d := s.AddDaemon(c, false, false) 894 err := d.Join(swarm.JoinRequest{ 895 RemoteAddrs: []string{"123.123.123.123:1234"}, 896 }) 897 c.Assert(err, check.NotNil) 898 c.Assert(err.Error(), checker.Contains, "Timeout was reached") 899 900 waitAndAssert(c, defaultReconciliationTimeout, d.checkLocalNodeState, checker.Equals, swarm.LocalNodeStatePending) 901 902 c.Assert(d.Stop(), checker.IsNil) 903 c.Assert(d.Start(), checker.IsNil) 904 905 info, err := d.info() 906 c.Assert(err, checker.IsNil) 907 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateInactive) 908 } 909 910 func (s *DockerSwarmSuite) TestAPISwarmManagerRestore(c *check.C) { 911 d1 := s.AddDaemon(c, true, true) 912 913 instances := 2 914 id := d1.createService(c, simpleTestService, setInstances(instances)) 915 916 d1.getService(c, id) 917 d1.Stop() 918 d1.Start() 919 d1.getService(c, id) 920 921 d2 := s.AddDaemon(c, true, true) 922 d2.getService(c, id) 923 d2.Stop() 924 d2.Start() 925 d2.getService(c, id) 926 927 d3 := s.AddDaemon(c, true, true) 928 d3.getService(c, id) 929 d3.Stop() 930 d3.Start() 931 d3.getService(c, id) 932 933 d3.Kill() 934 time.Sleep(1 * time.Second) // time to handle signal 935 d3.Start() 936 d3.getService(c, id) 937 } 938 939 func (s *DockerSwarmSuite) TestAPISwarmScaleNoRollingUpdate(c *check.C) { 940 d := s.AddDaemon(c, true, true) 941 942 instances := 2 943 id := d.createService(c, simpleTestService, setInstances(instances)) 944 945 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 946 containers := d.activeContainers() 947 instances = 4 948 d.updateService(c, d.getService(c, id), setInstances(instances)) 949 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 950 containers2 := d.activeContainers() 951 952 loop0: 953 for _, c1 := range containers { 954 for _, c2 := range containers2 { 955 if c1 == c2 { 956 continue loop0 957 } 958 } 959 c.Errorf("container %v not found in new set %#v", c1, containers2) 960 } 961 } 962 963 func (s *DockerSwarmSuite) TestAPISwarmInvalidAddress(c *check.C) { 964 d := s.AddDaemon(c, false, false) 965 req := swarm.InitRequest{ 966 ListenAddr: "", 967 } 968 status, _, err := d.SockRequest("POST", "/swarm/init", req) 969 c.Assert(err, checker.IsNil) 970 c.Assert(status, checker.Equals, http.StatusBadRequest) 971 972 req2 := swarm.JoinRequest{ 973 ListenAddr: "0.0.0.0:2377", 974 RemoteAddrs: []string{""}, 975 } 976 status, _, err = d.SockRequest("POST", "/swarm/join", req2) 977 c.Assert(err, checker.IsNil) 978 c.Assert(status, checker.Equals, http.StatusBadRequest) 979 } 980 981 func (s *DockerSwarmSuite) TestAPISwarmForceNewCluster(c *check.C) { 982 d1 := s.AddDaemon(c, true, true) 983 d2 := s.AddDaemon(c, true, true) 984 985 instances := 2 986 id := d1.createService(c, simpleTestService, setInstances(instances)) 987 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d2.checkActiveContainerCount), checker.Equals, instances) 988 989 // drain d2, all containers should move to d1 990 d1.updateNode(c, d2.NodeID, func(n *swarm.Node) { 991 n.Spec.Availability = swarm.NodeAvailabilityDrain 992 }) 993 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances) 994 waitAndAssert(c, defaultReconciliationTimeout, d2.checkActiveContainerCount, checker.Equals, 0) 995 996 c.Assert(d2.Stop(), checker.IsNil) 997 998 c.Assert(d1.Init(swarm.InitRequest{ 999 ForceNewCluster: true, 1000 Spec: swarm.Spec{}, 1001 }), checker.IsNil) 1002 1003 waitAndAssert(c, defaultReconciliationTimeout, d1.checkActiveContainerCount, checker.Equals, instances) 1004 1005 d3 := s.AddDaemon(c, true, true) 1006 info, err := d3.info() 1007 c.Assert(err, checker.IsNil) 1008 c.Assert(info.ControlAvailable, checker.True) 1009 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 1010 1011 instances = 4 1012 d3.updateService(c, d3.getService(c, id), setInstances(instances)) 1013 1014 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.checkActiveContainerCount, d3.checkActiveContainerCount), checker.Equals, instances) 1015 } 1016 1017 func simpleTestService(s *swarm.Service) { 1018 ureplicas := uint64(1) 1019 restartDelay := time.Duration(100 * time.Millisecond) 1020 1021 s.Spec = swarm.ServiceSpec{ 1022 TaskTemplate: swarm.TaskSpec{ 1023 ContainerSpec: swarm.ContainerSpec{ 1024 Image: "busybox:latest", 1025 Command: []string{"/bin/top"}, 1026 }, 1027 RestartPolicy: &swarm.RestartPolicy{ 1028 Delay: &restartDelay, 1029 }, 1030 }, 1031 Mode: swarm.ServiceMode{ 1032 Replicated: &swarm.ReplicatedService{ 1033 Replicas: &ureplicas, 1034 }, 1035 }, 1036 } 1037 s.Spec.Name = "top" 1038 } 1039 1040 func serviceForUpdate(s *swarm.Service) { 1041 ureplicas := uint64(1) 1042 restartDelay := time.Duration(100 * time.Millisecond) 1043 1044 s.Spec = swarm.ServiceSpec{ 1045 TaskTemplate: swarm.TaskSpec{ 1046 ContainerSpec: swarm.ContainerSpec{ 1047 Image: "busybox:latest", 1048 Command: []string{"/bin/top"}, 1049 }, 1050 RestartPolicy: &swarm.RestartPolicy{ 1051 Delay: &restartDelay, 1052 }, 1053 }, 1054 Mode: swarm.ServiceMode{ 1055 Replicated: &swarm.ReplicatedService{ 1056 Replicas: &ureplicas, 1057 }, 1058 }, 1059 UpdateConfig: &swarm.UpdateConfig{ 1060 Parallelism: 2, 1061 Delay: 4 * time.Second, 1062 FailureAction: swarm.UpdateFailureActionContinue, 1063 }, 1064 } 1065 s.Spec.Name = "updatetest" 1066 } 1067 1068 func setInstances(replicas int) serviceConstructor { 1069 ureplicas := uint64(replicas) 1070 return func(s *swarm.Service) { 1071 s.Spec.Mode = swarm.ServiceMode{ 1072 Replicated: &swarm.ReplicatedService{ 1073 Replicas: &ureplicas, 1074 }, 1075 } 1076 } 1077 } 1078 1079 func setImage(image string) serviceConstructor { 1080 return func(s *swarm.Service) { 1081 s.Spec.TaskTemplate.ContainerSpec.Image = image 1082 } 1083 } 1084 1085 func setFailureAction(failureAction string) serviceConstructor { 1086 return func(s *swarm.Service) { 1087 s.Spec.UpdateConfig.FailureAction = failureAction 1088 } 1089 } 1090 1091 func setMaxFailureRatio(maxFailureRatio float32) serviceConstructor { 1092 return func(s *swarm.Service) { 1093 s.Spec.UpdateConfig.MaxFailureRatio = maxFailureRatio 1094 } 1095 } 1096 1097 func setParallelism(parallelism uint64) serviceConstructor { 1098 return func(s *swarm.Service) { 1099 s.Spec.UpdateConfig.Parallelism = parallelism 1100 } 1101 } 1102 1103 func setConstraints(constraints []string) serviceConstructor { 1104 return func(s *swarm.Service) { 1105 if s.Spec.TaskTemplate.Placement == nil { 1106 s.Spec.TaskTemplate.Placement = &swarm.Placement{} 1107 } 1108 s.Spec.TaskTemplate.Placement.Constraints = constraints 1109 } 1110 } 1111 1112 func setGlobalMode(s *swarm.Service) { 1113 s.Spec.Mode = swarm.ServiceMode{ 1114 Global: &swarm.GlobalService{}, 1115 } 1116 } 1117 1118 func checkClusterHealth(c *check.C, cl []*SwarmDaemon, managerCount, workerCount int) { 1119 var totalMCount, totalWCount int 1120 1121 for _, d := range cl { 1122 var ( 1123 info swarm.Info 1124 err error 1125 ) 1126 1127 // check info in a waitAndAssert, because if the cluster doesn't have a leader, `info` will return an error 1128 checkInfo := func(c *check.C) (interface{}, check.CommentInterface) { 1129 info, err = d.info() 1130 return err, check.Commentf("cluster not ready in time") 1131 } 1132 waitAndAssert(c, defaultReconciliationTimeout, checkInfo, checker.IsNil) 1133 if !info.ControlAvailable { 1134 totalWCount++ 1135 continue 1136 } 1137 1138 var leaderFound bool 1139 totalMCount++ 1140 var mCount, wCount int 1141 1142 for _, n := range d.listNodes(c) { 1143 waitReady := func(c *check.C) (interface{}, check.CommentInterface) { 1144 if n.Status.State == swarm.NodeStateReady { 1145 return true, nil 1146 } 1147 nn := d.getNode(c, n.ID) 1148 n = *nn 1149 return n.Status.State == swarm.NodeStateReady, check.Commentf("state of node %s, reported by %s", n.ID, d.Info.NodeID) 1150 } 1151 waitAndAssert(c, defaultReconciliationTimeout, waitReady, checker.True) 1152 1153 waitActive := func(c *check.C) (interface{}, check.CommentInterface) { 1154 if n.Spec.Availability == swarm.NodeAvailabilityActive { 1155 return true, nil 1156 } 1157 nn := d.getNode(c, n.ID) 1158 n = *nn 1159 return n.Spec.Availability == swarm.NodeAvailabilityActive, check.Commentf("availability of node %s, reported by %s", n.ID, d.Info.NodeID) 1160 } 1161 waitAndAssert(c, defaultReconciliationTimeout, waitActive, checker.True) 1162 1163 if n.Spec.Role == swarm.NodeRoleManager { 1164 c.Assert(n.ManagerStatus, checker.NotNil, check.Commentf("manager status of node %s (manager), reported by %s", n.ID, d.Info.NodeID)) 1165 if n.ManagerStatus.Leader { 1166 leaderFound = true 1167 } 1168 mCount++ 1169 } else { 1170 c.Assert(n.ManagerStatus, checker.IsNil, check.Commentf("manager status of node %s (worker), reported by %s", n.ID, d.Info.NodeID)) 1171 wCount++ 1172 } 1173 } 1174 c.Assert(leaderFound, checker.True, check.Commentf("lack of leader reported by node %s", info.NodeID)) 1175 c.Assert(mCount, checker.Equals, managerCount, check.Commentf("managers count reported by node %s", info.NodeID)) 1176 c.Assert(wCount, checker.Equals, workerCount, check.Commentf("workers count reported by node %s", info.NodeID)) 1177 } 1178 c.Assert(totalMCount, checker.Equals, managerCount) 1179 c.Assert(totalWCount, checker.Equals, workerCount) 1180 } 1181 1182 func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) { 1183 mCount, wCount := 5, 1 1184 1185 var nodes []*SwarmDaemon 1186 for i := 0; i < mCount; i++ { 1187 manager := s.AddDaemon(c, true, true) 1188 info, err := manager.info() 1189 c.Assert(err, checker.IsNil) 1190 c.Assert(info.ControlAvailable, checker.True) 1191 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 1192 nodes = append(nodes, manager) 1193 } 1194 1195 for i := 0; i < wCount; i++ { 1196 worker := s.AddDaemon(c, true, false) 1197 info, err := worker.info() 1198 c.Assert(err, checker.IsNil) 1199 c.Assert(info.ControlAvailable, checker.False) 1200 c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive) 1201 nodes = append(nodes, worker) 1202 } 1203 1204 // stop whole cluster 1205 { 1206 var wg sync.WaitGroup 1207 wg.Add(len(nodes)) 1208 errs := make(chan error, len(nodes)) 1209 1210 for _, d := range nodes { 1211 go func(daemon *SwarmDaemon) { 1212 defer wg.Done() 1213 if err := daemon.Stop(); err != nil { 1214 errs <- err 1215 } 1216 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 1217 daemon.root = filepath.Dir(daemon.root) 1218 } 1219 }(d) 1220 } 1221 wg.Wait() 1222 close(errs) 1223 for err := range errs { 1224 c.Assert(err, check.IsNil) 1225 } 1226 } 1227 1228 // start whole cluster 1229 { 1230 var wg sync.WaitGroup 1231 wg.Add(len(nodes)) 1232 errs := make(chan error, len(nodes)) 1233 1234 for _, d := range nodes { 1235 go func(daemon *SwarmDaemon) { 1236 defer wg.Done() 1237 if err := daemon.Start("--iptables=false"); err != nil { 1238 errs <- err 1239 } 1240 }(d) 1241 } 1242 wg.Wait() 1243 close(errs) 1244 for err := range errs { 1245 c.Assert(err, check.IsNil) 1246 } 1247 } 1248 1249 checkClusterHealth(c, nodes, mCount, wCount) 1250 } 1251 1252 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateWithName(c *check.C) { 1253 d := s.AddDaemon(c, true, true) 1254 1255 instances := 2 1256 id := d.createService(c, simpleTestService, setInstances(instances)) 1257 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 1258 1259 service := d.getService(c, id) 1260 instances = 5 1261 1262 setInstances(instances)(service) 1263 url := fmt.Sprintf("/services/%s/update?version=%d", service.Spec.Name, service.Version.Index) 1264 status, out, err := d.SockRequest("POST", url, service.Spec) 1265 c.Assert(err, checker.IsNil) 1266 c.Assert(status, checker.Equals, http.StatusOK, check.Commentf("output: %q", string(out))) 1267 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 1268 } 1269 1270 func (s *DockerSwarmSuite) TestAPISwarmSecretsEmptyList(c *check.C) { 1271 d := s.AddDaemon(c, true, true) 1272 1273 secrets := d.listSecrets(c) 1274 c.Assert(secrets, checker.NotNil) 1275 c.Assert(len(secrets), checker.Equals, 0, check.Commentf("secrets: %#v", secrets)) 1276 } 1277 1278 func (s *DockerSwarmSuite) TestAPISwarmSecretsCreate(c *check.C) { 1279 d := s.AddDaemon(c, true, true) 1280 1281 testName := "test_secret" 1282 id := d.createSecret(c, swarm.SecretSpec{ 1283 swarm.Annotations{ 1284 Name: testName, 1285 }, 1286 []byte("TESTINGDATA"), 1287 }) 1288 c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("secrets: %s", id)) 1289 1290 secrets := d.listSecrets(c) 1291 c.Assert(len(secrets), checker.Equals, 1, check.Commentf("secrets: %#v", secrets)) 1292 name := secrets[0].Spec.Annotations.Name 1293 c.Assert(name, checker.Equals, testName, check.Commentf("secret: %s", name)) 1294 } 1295 1296 func (s *DockerSwarmSuite) TestAPISwarmSecretsDelete(c *check.C) { 1297 d := s.AddDaemon(c, true, true) 1298 1299 testName := "test_secret" 1300 id := d.createSecret(c, swarm.SecretSpec{ 1301 swarm.Annotations{ 1302 Name: testName, 1303 }, 1304 []byte("TESTINGDATA"), 1305 }) 1306 c.Assert(id, checker.Not(checker.Equals), "", check.Commentf("secrets: %s", id)) 1307 1308 secret := d.getSecret(c, id) 1309 c.Assert(secret.ID, checker.Equals, id, check.Commentf("secret: %v", secret)) 1310 1311 d.deleteSecret(c, secret.ID) 1312 status, out, err := d.SockRequest("GET", "/secrets/"+id, nil) 1313 c.Assert(err, checker.IsNil) 1314 c.Assert(status, checker.Equals, http.StatusNotFound, check.Commentf("secret delete: %s", string(out))) 1315 } 1316 1317 // Test case for 30242, where duplicate networks, with different drivers `bridge` and `overlay`, 1318 // caused both scopes to be `swarm` for `docker network inspect` and `docker network ls`. 1319 // This test makes sure the fixes correctly output scopes instead. 1320 func (s *DockerSwarmSuite) TestAPIDuplicateNetworks(c *check.C) { 1321 d := s.AddDaemon(c, true, true) 1322 1323 name := "foo" 1324 networkCreateRequest := types.NetworkCreateRequest{ 1325 Name: name, 1326 NetworkCreate: types.NetworkCreate{ 1327 CheckDuplicate: false, 1328 }, 1329 } 1330 1331 var n1 types.NetworkCreateResponse 1332 networkCreateRequest.NetworkCreate.Driver = "bridge" 1333 1334 status, out, err := d.SockRequest("POST", "/networks/create", networkCreateRequest) 1335 c.Assert(err, checker.IsNil, check.Commentf(string(out))) 1336 c.Assert(status, checker.Equals, http.StatusCreated, check.Commentf(string(out))) 1337 1338 c.Assert(json.Unmarshal(out, &n1), checker.IsNil) 1339 1340 var n2 types.NetworkCreateResponse 1341 networkCreateRequest.NetworkCreate.Driver = "overlay" 1342 1343 status, out, err = d.SockRequest("POST", "/networks/create", networkCreateRequest) 1344 c.Assert(err, checker.IsNil, check.Commentf(string(out))) 1345 c.Assert(status, checker.Equals, http.StatusCreated, check.Commentf(string(out))) 1346 1347 c.Assert(json.Unmarshal(out, &n2), checker.IsNil) 1348 1349 var r1 types.NetworkResource 1350 1351 status, out, err = d.SockRequest("GET", "/networks/"+n1.ID, nil) 1352 c.Assert(err, checker.IsNil, check.Commentf(string(out))) 1353 c.Assert(status, checker.Equals, http.StatusOK, check.Commentf(string(out))) 1354 1355 c.Assert(json.Unmarshal(out, &r1), checker.IsNil) 1356 1357 c.Assert(r1.Scope, checker.Equals, "local") 1358 1359 var r2 types.NetworkResource 1360 1361 status, out, err = d.SockRequest("GET", "/networks/"+n2.ID, nil) 1362 c.Assert(err, checker.IsNil, check.Commentf(string(out))) 1363 c.Assert(status, checker.Equals, http.StatusOK, check.Commentf(string(out))) 1364 1365 c.Assert(json.Unmarshal(out, &r2), checker.IsNil) 1366 1367 c.Assert(r2.Scope, checker.Equals, "swarm") 1368 } 1369 1370 // Test case for 30178 1371 func (s *DockerSwarmSuite) TestAPISwarmHealthcheckNone(c *check.C) { 1372 d := s.AddDaemon(c, true, true) 1373 1374 out, err := d.Cmd("network", "create", "-d", "overlay", "lb") 1375 c.Assert(err, checker.IsNil, check.Commentf(out)) 1376 1377 instances := 1 1378 d.createService(c, simpleTestService, setInstances(instances), func(s *swarm.Service) { 1379 s.Spec.TaskTemplate.ContainerSpec.Healthcheck = &container.HealthConfig{} 1380 s.Spec.TaskTemplate.Networks = []swarm.NetworkAttachmentConfig{ 1381 {Target: "lb"}, 1382 } 1383 }) 1384 1385 waitAndAssert(c, defaultReconciliationTimeout, d.checkActiveContainerCount, checker.Equals, instances) 1386 1387 containers := d.activeContainers() 1388 1389 out, err = d.Cmd("exec", containers[0], "ping", "-c1", "-W3", "top") 1390 c.Assert(err, checker.IsNil, check.Commentf(out)) 1391 }