github.com/zhouyu0/docker-note@v0.0.0-20190722021225-b8d3825084db/integration-cli/docker_api_swarm_test.go (about) 1 // +build !windows 2 3 package main 4 5 import ( 6 "context" 7 "fmt" 8 "io/ioutil" 9 "net" 10 "net/http" 11 "path/filepath" 12 "runtime" 13 "strings" 14 "sync" 15 "time" 16 17 "github.com/cloudflare/cfssl/csr" 18 "github.com/cloudflare/cfssl/helpers" 19 "github.com/cloudflare/cfssl/initca" 20 "github.com/docker/docker/api/types" 21 "github.com/docker/docker/api/types/container" 22 "github.com/docker/docker/api/types/swarm" 23 "github.com/docker/docker/client" 24 "github.com/docker/docker/integration-cli/checker" 25 "github.com/docker/docker/integration-cli/daemon" 26 testdaemon "github.com/docker/docker/internal/test/daemon" 27 "github.com/docker/docker/internal/test/request" 28 "github.com/docker/swarmkit/ca" 29 "github.com/go-check/check" 30 "gotest.tools/assert" 31 is "gotest.tools/assert/cmp" 32 ) 33 34 var defaultReconciliationTimeout = 30 * time.Second 35 36 func (s *DockerSwarmSuite) TestAPISwarmInit(c *check.C) { 37 // todo: should find a better way to verify that components are running than /info 38 d1 := s.AddDaemon(c, true, true) 39 info := d1.SwarmInfo(c) 40 assert.Equal(c, info.ControlAvailable, true) 41 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 42 assert.Equal(c, info.Cluster.RootRotationInProgress, false) 43 44 d2 := s.AddDaemon(c, true, false) 45 info = d2.SwarmInfo(c) 46 assert.Equal(c, info.ControlAvailable, false) 47 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 48 49 // Leaving cluster 50 assert.NilError(c, d2.SwarmLeave(c, false)) 51 52 info = d2.SwarmInfo(c) 53 assert.Equal(c, info.ControlAvailable, false) 54 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive) 55 56 d2.SwarmJoin(c, swarm.JoinRequest{ 57 ListenAddr: d1.SwarmListenAddr(), 58 JoinToken: d1.JoinTokens(c).Worker, 59 RemoteAddrs: []string{d1.SwarmListenAddr()}, 60 }) 61 62 info = d2.SwarmInfo(c) 63 assert.Equal(c, info.ControlAvailable, false) 64 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 65 66 // Current state restoring after restarts 67 d1.Stop(c) 68 d2.Stop(c) 69 70 d1.StartNode(c) 71 d2.StartNode(c) 72 73 info = d1.SwarmInfo(c) 74 assert.Equal(c, info.ControlAvailable, true) 75 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 76 77 info = d2.SwarmInfo(c) 78 assert.Equal(c, info.ControlAvailable, false) 79 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 80 } 81 82 func (s *DockerSwarmSuite) TestAPISwarmJoinToken(c *check.C) { 83 d1 := s.AddDaemon(c, false, false) 84 d1.SwarmInit(c, swarm.InitRequest{}) 85 86 // todo: error message differs depending if some components of token are valid 87 88 d2 := s.AddDaemon(c, false, false) 89 c2 := d2.NewClientT(c) 90 err := c2.SwarmJoin(context.Background(), swarm.JoinRequest{ 91 ListenAddr: d2.SwarmListenAddr(), 92 RemoteAddrs: []string{d1.SwarmListenAddr()}, 93 }) 94 assert.ErrorContains(c, err, "join token is necessary") 95 info := d2.SwarmInfo(c) 96 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive) 97 98 err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{ 99 ListenAddr: d2.SwarmListenAddr(), 100 JoinToken: "foobaz", 101 RemoteAddrs: []string{d1.SwarmListenAddr()}, 102 }) 103 assert.ErrorContains(c, err, "invalid join token") 104 info = d2.SwarmInfo(c) 105 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive) 106 107 workerToken := d1.JoinTokens(c).Worker 108 109 d2.SwarmJoin(c, swarm.JoinRequest{ 110 ListenAddr: d2.SwarmListenAddr(), 111 JoinToken: workerToken, 112 RemoteAddrs: []string{d1.SwarmListenAddr()}, 113 }) 114 info = d2.SwarmInfo(c) 115 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 116 assert.NilError(c, d2.SwarmLeave(c, false)) 117 info = d2.SwarmInfo(c) 118 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive) 119 120 // change tokens 121 d1.RotateTokens(c) 122 123 err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{ 124 ListenAddr: d2.SwarmListenAddr(), 125 JoinToken: workerToken, 126 RemoteAddrs: []string{d1.SwarmListenAddr()}, 127 }) 128 assert.ErrorContains(c, err, "join token is necessary") 129 info = d2.SwarmInfo(c) 130 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive) 131 132 workerToken = d1.JoinTokens(c).Worker 133 134 d2.SwarmJoin(c, swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.SwarmListenAddr()}}) 135 info = d2.SwarmInfo(c) 136 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 137 assert.NilError(c, d2.SwarmLeave(c, false)) 138 info = d2.SwarmInfo(c) 139 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive) 140 141 // change spec, don't change tokens 142 d1.UpdateSwarm(c, func(s *swarm.Spec) {}) 143 144 err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{ 145 ListenAddr: d2.SwarmListenAddr(), 146 RemoteAddrs: []string{d1.SwarmListenAddr()}, 147 }) 148 assert.ErrorContains(c, err, "join token is necessary") 149 info = d2.SwarmInfo(c) 150 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive) 151 152 d2.SwarmJoin(c, swarm.JoinRequest{JoinToken: workerToken, RemoteAddrs: []string{d1.SwarmListenAddr()}}) 153 info = d2.SwarmInfo(c) 154 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 155 assert.NilError(c, d2.SwarmLeave(c, false)) 156 info = d2.SwarmInfo(c) 157 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive) 158 } 159 160 func (s *DockerSwarmSuite) TestUpdateSwarmAddExternalCA(c *check.C) { 161 d1 := s.AddDaemon(c, false, false) 162 d1.SwarmInit(c, swarm.InitRequest{}) 163 d1.UpdateSwarm(c, func(s *swarm.Spec) { 164 s.CAConfig.ExternalCAs = []*swarm.ExternalCA{ 165 { 166 Protocol: swarm.ExternalCAProtocolCFSSL, 167 URL: "https://thishasnoca.org", 168 }, 169 { 170 Protocol: swarm.ExternalCAProtocolCFSSL, 171 URL: "https://thishasacacert.org", 172 CACert: "cacert", 173 }, 174 } 175 }) 176 info := d1.SwarmInfo(c) 177 assert.Equal(c, len(info.Cluster.Spec.CAConfig.ExternalCAs), 2) 178 assert.Equal(c, info.Cluster.Spec.CAConfig.ExternalCAs[0].CACert, "") 179 assert.Equal(c, info.Cluster.Spec.CAConfig.ExternalCAs[1].CACert, "cacert") 180 } 181 182 func (s *DockerSwarmSuite) TestAPISwarmCAHash(c *check.C) { 183 d1 := s.AddDaemon(c, true, true) 184 d2 := s.AddDaemon(c, false, false) 185 splitToken := strings.Split(d1.JoinTokens(c).Worker, "-") 186 splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e" 187 replacementToken := strings.Join(splitToken, "-") 188 c2 := d2.NewClientT(c) 189 err := c2.SwarmJoin(context.Background(), swarm.JoinRequest{ 190 ListenAddr: d2.SwarmListenAddr(), 191 JoinToken: replacementToken, 192 RemoteAddrs: []string{d1.SwarmListenAddr()}, 193 }) 194 assert.ErrorContains(c, err, "remote CA does not match fingerprint") 195 } 196 197 func (s *DockerSwarmSuite) TestAPISwarmPromoteDemote(c *check.C) { 198 d1 := s.AddDaemon(c, false, false) 199 d1.SwarmInit(c, swarm.InitRequest{}) 200 d2 := s.AddDaemon(c, true, false) 201 202 info := d2.SwarmInfo(c) 203 assert.Equal(c, info.ControlAvailable, false) 204 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 205 206 d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { 207 n.Spec.Role = swarm.NodeRoleManager 208 }) 209 210 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.True) 211 212 d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { 213 n.Spec.Role = swarm.NodeRoleWorker 214 }) 215 216 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.False) 217 218 // Wait for the role to change to worker in the cert. This is partially 219 // done because it's something worth testing in its own right, and 220 // partially because changing the role from manager to worker and then 221 // back to manager quickly might cause the node to pause for awhile 222 // while waiting for the role to change to worker, and the test can 223 // time out during this interval. 224 waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { 225 certBytes, err := ioutil.ReadFile(filepath.Join(d2.Folder, "root", "swarm", "certificates", "swarm-node.crt")) 226 if err != nil { 227 return "", check.Commentf("error: %v", err) 228 } 229 certs, err := helpers.ParseCertificatesPEM(certBytes) 230 if err == nil && len(certs) > 0 && len(certs[0].Subject.OrganizationalUnit) > 0 { 231 return certs[0].Subject.OrganizationalUnit[0], nil 232 } 233 return "", check.Commentf("could not get organizational unit from certificate") 234 }, checker.Equals, "swarm-worker") 235 236 // Demoting last node should fail 237 node := d1.GetNode(c, d1.NodeID()) 238 node.Spec.Role = swarm.NodeRoleWorker 239 url := fmt.Sprintf("/nodes/%s/update?version=%d", node.ID, node.Version.Index) 240 res, body, err := request.Post(url, request.Host(d1.Sock()), request.JSONBody(node.Spec)) 241 assert.NilError(c, err) 242 b, err := request.ReadBody(body) 243 assert.NilError(c, err) 244 assert.Equal(c, res.StatusCode, http.StatusBadRequest, "output: %q", string(b)) 245 246 // The warning specific to demoting the last manager is best-effort and 247 // won't appear until the Role field of the demoted manager has been 248 // updated. 249 // Yes, I know this looks silly, but checker.Matches is broken, since 250 // it anchors the regexp contrary to the documentation, and this makes 251 // it impossible to match something that includes a line break. 252 if !strings.Contains(string(b), "last manager of the swarm") { 253 assert.Assert(c, strings.Contains(string(b), "this would result in a loss of quorum")) 254 } 255 info = d1.SwarmInfo(c) 256 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 257 assert.Equal(c, info.ControlAvailable, true) 258 259 // Promote already demoted node 260 d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { 261 n.Spec.Role = swarm.NodeRoleManager 262 }) 263 264 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckControlAvailable, checker.True) 265 } 266 267 func (s *DockerSwarmSuite) TestAPISwarmLeaderProxy(c *check.C) { 268 // add three managers, one of these is leader 269 d1 := s.AddDaemon(c, true, true) 270 d2 := s.AddDaemon(c, true, true) 271 d3 := s.AddDaemon(c, true, true) 272 273 // start a service by hitting each of the 3 managers 274 d1.CreateService(c, simpleTestService, func(s *swarm.Service) { 275 s.Spec.Name = "test1" 276 }) 277 d2.CreateService(c, simpleTestService, func(s *swarm.Service) { 278 s.Spec.Name = "test2" 279 }) 280 d3.CreateService(c, simpleTestService, func(s *swarm.Service) { 281 s.Spec.Name = "test3" 282 }) 283 284 // 3 services should be started now, because the requests were proxied to leader 285 // query each node and make sure it returns 3 services 286 for _, d := range []*daemon.Daemon{d1, d2, d3} { 287 services := d.ListServices(c) 288 assert.Equal(c, len(services), 3) 289 } 290 } 291 292 func (s *DockerSwarmSuite) TestAPISwarmLeaderElection(c *check.C) { 293 if runtime.GOARCH == "s390x" { 294 c.Skip("Disabled on s390x") 295 } 296 if runtime.GOARCH == "ppc64le" { 297 c.Skip("Disabled on ppc64le") 298 } 299 300 // Create 3 nodes 301 d1 := s.AddDaemon(c, true, true) 302 d2 := s.AddDaemon(c, true, true) 303 d3 := s.AddDaemon(c, true, true) 304 305 // assert that the first node we made is the leader, and the other two are followers 306 assert.Equal(c, d1.GetNode(c, d1.NodeID()).ManagerStatus.Leader, true) 307 assert.Equal(c, d1.GetNode(c, d2.NodeID()).ManagerStatus.Leader, false) 308 assert.Equal(c, d1.GetNode(c, d3.NodeID()).ManagerStatus.Leader, false) 309 310 d1.Stop(c) 311 312 var ( 313 leader *daemon.Daemon // keep track of leader 314 followers []*daemon.Daemon // keep track of followers 315 ) 316 checkLeader := func(nodes ...*daemon.Daemon) checkF { 317 return func(c *check.C) (interface{}, check.CommentInterface) { 318 // clear these out before each run 319 leader = nil 320 followers = nil 321 for _, d := range nodes { 322 if d.GetNode(c, d.NodeID()).ManagerStatus.Leader { 323 leader = d 324 } else { 325 followers = append(followers, d) 326 } 327 } 328 329 if leader == nil { 330 return false, check.Commentf("no leader elected") 331 } 332 333 return true, check.Commentf("elected %v", leader.ID()) 334 } 335 } 336 337 // wait for an election to occur 338 c.Logf("Waiting for election to occur...") 339 waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d2, d3), checker.True) 340 341 // assert that we have a new leader 342 assert.Assert(c, leader != nil) 343 344 // Keep track of the current leader, since we want that to be chosen. 345 stableleader := leader 346 347 // add the d1, the initial leader, back 348 d1.StartNode(c) 349 350 // wait for possible election 351 c.Logf("Waiting for possible election...") 352 waitAndAssert(c, defaultReconciliationTimeout, checkLeader(d1, d2, d3), checker.True) 353 // pick out the leader and the followers again 354 355 // verify that we still only have 1 leader and 2 followers 356 assert.Assert(c, leader != nil) 357 assert.Equal(c, len(followers), 2) 358 // and that after we added d1 back, the leader hasn't changed 359 assert.Equal(c, leader.NodeID(), stableleader.NodeID()) 360 } 361 362 func (s *DockerSwarmSuite) TestAPISwarmRaftQuorum(c *check.C) { 363 if runtime.GOARCH == "s390x" { 364 c.Skip("Disabled on s390x") 365 } 366 if runtime.GOARCH == "ppc64le" { 367 c.Skip("Disabled on ppc64le") 368 } 369 370 d1 := s.AddDaemon(c, true, true) 371 d2 := s.AddDaemon(c, true, true) 372 d3 := s.AddDaemon(c, true, true) 373 374 d1.CreateService(c, simpleTestService) 375 376 d2.Stop(c) 377 378 // make sure there is a leader 379 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckLeader, checker.IsNil) 380 381 d1.CreateService(c, simpleTestService, func(s *swarm.Service) { 382 s.Spec.Name = "top1" 383 }) 384 385 d3.Stop(c) 386 387 var service swarm.Service 388 simpleTestService(&service) 389 service.Spec.Name = "top2" 390 cli := d1.NewClientT(c) 391 defer cli.Close() 392 393 // d1 will eventually step down from leader because there is no longer an active quorum, wait for that to happen 394 waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { 395 _, err := cli.ServiceCreate(context.Background(), service.Spec, types.ServiceCreateOptions{}) 396 return err.Error(), nil 397 }, checker.Contains, "Make sure more than half of the managers are online.") 398 399 d2.StartNode(c) 400 401 // make sure there is a leader 402 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckLeader, checker.IsNil) 403 404 d1.CreateService(c, simpleTestService, func(s *swarm.Service) { 405 s.Spec.Name = "top3" 406 }) 407 } 408 409 func (s *DockerSwarmSuite) TestAPISwarmLeaveRemovesContainer(c *check.C) { 410 d := s.AddDaemon(c, true, true) 411 412 instances := 2 413 d.CreateService(c, simpleTestService, setInstances(instances)) 414 415 id, err := d.Cmd("run", "-d", "busybox", "top") 416 assert.NilError(c, err, id) 417 id = strings.TrimSpace(id) 418 419 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances+1) 420 421 assert.ErrorContains(c, d.SwarmLeave(c, false), "") 422 assert.NilError(c, d.SwarmLeave(c, true)) 423 424 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 425 426 id2, err := d.Cmd("ps", "-q") 427 assert.NilError(c, err, id2) 428 assert.Assert(c, strings.HasPrefix(id, strings.TrimSpace(id2))) 429 } 430 431 // #23629 432 func (s *DockerSwarmSuite) TestAPISwarmLeaveOnPendingJoin(c *check.C) { 433 testRequires(c, Network) 434 s.AddDaemon(c, true, true) 435 d2 := s.AddDaemon(c, false, false) 436 437 id, err := d2.Cmd("run", "-d", "busybox", "top") 438 assert.NilError(c, err, id) 439 id = strings.TrimSpace(id) 440 441 c2 := d2.NewClientT(c) 442 err = c2.SwarmJoin(context.Background(), swarm.JoinRequest{ 443 ListenAddr: d2.SwarmListenAddr(), 444 RemoteAddrs: []string{"123.123.123.123:1234"}, 445 }) 446 assert.ErrorContains(c, err, "Timeout was reached") 447 448 info := d2.SwarmInfo(c) 449 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStatePending) 450 451 assert.NilError(c, d2.SwarmLeave(c, true)) 452 453 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 1) 454 455 id2, err := d2.Cmd("ps", "-q") 456 assert.NilError(c, err, id2) 457 assert.Assert(c, strings.HasPrefix(id, strings.TrimSpace(id2))) 458 } 459 460 // #23705 461 func (s *DockerSwarmSuite) TestAPISwarmRestoreOnPendingJoin(c *check.C) { 462 testRequires(c, Network) 463 d := s.AddDaemon(c, false, false) 464 client := d.NewClientT(c) 465 err := client.SwarmJoin(context.Background(), swarm.JoinRequest{ 466 ListenAddr: d.SwarmListenAddr(), 467 RemoteAddrs: []string{"123.123.123.123:1234"}, 468 }) 469 assert.ErrorContains(c, err, "Timeout was reached") 470 471 waitAndAssert(c, defaultReconciliationTimeout, d.CheckLocalNodeState, checker.Equals, swarm.LocalNodeStatePending) 472 473 d.RestartNode(c) 474 475 info := d.SwarmInfo(c) 476 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateInactive) 477 } 478 479 func (s *DockerSwarmSuite) TestAPISwarmManagerRestore(c *check.C) { 480 d1 := s.AddDaemon(c, true, true) 481 482 instances := 2 483 id := d1.CreateService(c, simpleTestService, setInstances(instances)) 484 485 d1.GetService(c, id) 486 d1.RestartNode(c) 487 d1.GetService(c, id) 488 489 d2 := s.AddDaemon(c, true, true) 490 d2.GetService(c, id) 491 d2.RestartNode(c) 492 d2.GetService(c, id) 493 494 d3 := s.AddDaemon(c, true, true) 495 d3.GetService(c, id) 496 d3.RestartNode(c) 497 d3.GetService(c, id) 498 499 err := d3.Kill() 500 assert.NilError(c, err) 501 time.Sleep(1 * time.Second) // time to handle signal 502 d3.StartNode(c) 503 d3.GetService(c, id) 504 } 505 506 func (s *DockerSwarmSuite) TestAPISwarmScaleNoRollingUpdate(c *check.C) { 507 d := s.AddDaemon(c, true, true) 508 509 instances := 2 510 id := d.CreateService(c, simpleTestService, setInstances(instances)) 511 512 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances) 513 containers := d.ActiveContainers(c) 514 instances = 4 515 d.UpdateService(c, d.GetService(c, id), setInstances(instances)) 516 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances) 517 containers2 := d.ActiveContainers(c) 518 519 loop0: 520 for _, c1 := range containers { 521 for _, c2 := range containers2 { 522 if c1 == c2 { 523 continue loop0 524 } 525 } 526 c.Errorf("container %v not found in new set %#v", c1, containers2) 527 } 528 } 529 530 func (s *DockerSwarmSuite) TestAPISwarmInvalidAddress(c *check.C) { 531 d := s.AddDaemon(c, false, false) 532 req := swarm.InitRequest{ 533 ListenAddr: "", 534 } 535 res, _, err := request.Post("/swarm/init", request.Host(d.Sock()), request.JSONBody(req)) 536 assert.NilError(c, err) 537 assert.Equal(c, res.StatusCode, http.StatusBadRequest) 538 539 req2 := swarm.JoinRequest{ 540 ListenAddr: "0.0.0.0:2377", 541 RemoteAddrs: []string{""}, 542 } 543 res, _, err = request.Post("/swarm/join", request.Host(d.Sock()), request.JSONBody(req2)) 544 assert.NilError(c, err) 545 assert.Equal(c, res.StatusCode, http.StatusBadRequest) 546 } 547 548 func (s *DockerSwarmSuite) TestAPISwarmForceNewCluster(c *check.C) { 549 d1 := s.AddDaemon(c, true, true) 550 d2 := s.AddDaemon(c, true, true) 551 552 instances := 2 553 id := d1.CreateService(c, simpleTestService, setInstances(instances)) 554 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount), checker.Equals, instances) 555 556 // drain d2, all containers should move to d1 557 d1.UpdateNode(c, d2.NodeID(), func(n *swarm.Node) { 558 n.Spec.Availability = swarm.NodeAvailabilityDrain 559 }) 560 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances) 561 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 0) 562 563 d2.Stop(c) 564 565 d1.SwarmInit(c, swarm.InitRequest{ 566 ForceNewCluster: true, 567 Spec: swarm.Spec{}, 568 }) 569 570 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, instances) 571 572 d3 := s.AddDaemon(c, true, true) 573 info := d3.SwarmInfo(c) 574 assert.Equal(c, info.ControlAvailable, true) 575 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 576 577 instances = 4 578 d3.UpdateService(c, d3.GetService(c, id), setInstances(instances)) 579 580 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) 581 } 582 583 func simpleTestService(s *swarm.Service) { 584 ureplicas := uint64(1) 585 restartDelay := time.Duration(100 * time.Millisecond) 586 587 s.Spec = swarm.ServiceSpec{ 588 TaskTemplate: swarm.TaskSpec{ 589 ContainerSpec: &swarm.ContainerSpec{ 590 Image: "busybox:latest", 591 Command: []string{"/bin/top"}, 592 }, 593 RestartPolicy: &swarm.RestartPolicy{ 594 Delay: &restartDelay, 595 }, 596 }, 597 Mode: swarm.ServiceMode{ 598 Replicated: &swarm.ReplicatedService{ 599 Replicas: &ureplicas, 600 }, 601 }, 602 } 603 s.Spec.Name = "top" 604 } 605 606 func serviceForUpdate(s *swarm.Service) { 607 ureplicas := uint64(1) 608 restartDelay := time.Duration(100 * time.Millisecond) 609 610 s.Spec = swarm.ServiceSpec{ 611 TaskTemplate: swarm.TaskSpec{ 612 ContainerSpec: &swarm.ContainerSpec{ 613 Image: "busybox:latest", 614 Command: []string{"/bin/top"}, 615 }, 616 RestartPolicy: &swarm.RestartPolicy{ 617 Delay: &restartDelay, 618 }, 619 }, 620 Mode: swarm.ServiceMode{ 621 Replicated: &swarm.ReplicatedService{ 622 Replicas: &ureplicas, 623 }, 624 }, 625 UpdateConfig: &swarm.UpdateConfig{ 626 Parallelism: 2, 627 Delay: 4 * time.Second, 628 FailureAction: swarm.UpdateFailureActionContinue, 629 }, 630 RollbackConfig: &swarm.UpdateConfig{ 631 Parallelism: 3, 632 Delay: 4 * time.Second, 633 FailureAction: swarm.UpdateFailureActionContinue, 634 }, 635 } 636 s.Spec.Name = "updatetest" 637 } 638 639 func setInstances(replicas int) testdaemon.ServiceConstructor { 640 ureplicas := uint64(replicas) 641 return func(s *swarm.Service) { 642 s.Spec.Mode = swarm.ServiceMode{ 643 Replicated: &swarm.ReplicatedService{ 644 Replicas: &ureplicas, 645 }, 646 } 647 } 648 } 649 650 func setUpdateOrder(order string) testdaemon.ServiceConstructor { 651 return func(s *swarm.Service) { 652 if s.Spec.UpdateConfig == nil { 653 s.Spec.UpdateConfig = &swarm.UpdateConfig{} 654 } 655 s.Spec.UpdateConfig.Order = order 656 } 657 } 658 659 func setRollbackOrder(order string) testdaemon.ServiceConstructor { 660 return func(s *swarm.Service) { 661 if s.Spec.RollbackConfig == nil { 662 s.Spec.RollbackConfig = &swarm.UpdateConfig{} 663 } 664 s.Spec.RollbackConfig.Order = order 665 } 666 } 667 668 func setImage(image string) testdaemon.ServiceConstructor { 669 return func(s *swarm.Service) { 670 if s.Spec.TaskTemplate.ContainerSpec == nil { 671 s.Spec.TaskTemplate.ContainerSpec = &swarm.ContainerSpec{} 672 } 673 s.Spec.TaskTemplate.ContainerSpec.Image = image 674 } 675 } 676 677 func setFailureAction(failureAction string) testdaemon.ServiceConstructor { 678 return func(s *swarm.Service) { 679 s.Spec.UpdateConfig.FailureAction = failureAction 680 } 681 } 682 683 func setMaxFailureRatio(maxFailureRatio float32) testdaemon.ServiceConstructor { 684 return func(s *swarm.Service) { 685 s.Spec.UpdateConfig.MaxFailureRatio = maxFailureRatio 686 } 687 } 688 689 func setParallelism(parallelism uint64) testdaemon.ServiceConstructor { 690 return func(s *swarm.Service) { 691 s.Spec.UpdateConfig.Parallelism = parallelism 692 } 693 } 694 695 func setConstraints(constraints []string) testdaemon.ServiceConstructor { 696 return func(s *swarm.Service) { 697 if s.Spec.TaskTemplate.Placement == nil { 698 s.Spec.TaskTemplate.Placement = &swarm.Placement{} 699 } 700 s.Spec.TaskTemplate.Placement.Constraints = constraints 701 } 702 } 703 704 func setPlacementPrefs(prefs []swarm.PlacementPreference) testdaemon.ServiceConstructor { 705 return func(s *swarm.Service) { 706 if s.Spec.TaskTemplate.Placement == nil { 707 s.Spec.TaskTemplate.Placement = &swarm.Placement{} 708 } 709 s.Spec.TaskTemplate.Placement.Preferences = prefs 710 } 711 } 712 713 func setGlobalMode(s *swarm.Service) { 714 s.Spec.Mode = swarm.ServiceMode{ 715 Global: &swarm.GlobalService{}, 716 } 717 } 718 719 func checkClusterHealth(c *check.C, cl []*daemon.Daemon, managerCount, workerCount int) { 720 var totalMCount, totalWCount int 721 722 for _, d := range cl { 723 var ( 724 info swarm.Info 725 ) 726 727 // check info in a waitAndAssert, because if the cluster doesn't have a leader, `info` will return an error 728 checkInfo := func(c *check.C) (interface{}, check.CommentInterface) { 729 client := d.NewClientT(c) 730 daemonInfo, err := client.Info(context.Background()) 731 info = daemonInfo.Swarm 732 return err, check.Commentf("cluster not ready in time") 733 } 734 waitAndAssert(c, defaultReconciliationTimeout, checkInfo, checker.IsNil) 735 if !info.ControlAvailable { 736 totalWCount++ 737 continue 738 } 739 740 var leaderFound bool 741 totalMCount++ 742 var mCount, wCount int 743 744 for _, n := range d.ListNodes(c) { 745 waitReady := func(c *check.C) (interface{}, check.CommentInterface) { 746 if n.Status.State == swarm.NodeStateReady { 747 return true, nil 748 } 749 nn := d.GetNode(c, n.ID) 750 n = *nn 751 return n.Status.State == swarm.NodeStateReady, check.Commentf("state of node %s, reported by %s", n.ID, d.NodeID()) 752 } 753 waitAndAssert(c, defaultReconciliationTimeout, waitReady, checker.True) 754 755 waitActive := func(c *check.C) (interface{}, check.CommentInterface) { 756 if n.Spec.Availability == swarm.NodeAvailabilityActive { 757 return true, nil 758 } 759 nn := d.GetNode(c, n.ID) 760 n = *nn 761 return n.Spec.Availability == swarm.NodeAvailabilityActive, check.Commentf("availability of node %s, reported by %s", n.ID, d.NodeID()) 762 } 763 waitAndAssert(c, defaultReconciliationTimeout, waitActive, checker.True) 764 765 if n.Spec.Role == swarm.NodeRoleManager { 766 assert.Assert(c, n.ManagerStatus != nil, "manager status of node %s (manager), reported by %s", n.ID, d.NodeID()) 767 if n.ManagerStatus.Leader { 768 leaderFound = true 769 } 770 mCount++ 771 } else { 772 assert.Assert(c, n.ManagerStatus == nil, "manager status of node %s (worker), reported by %s", n.ID, d.NodeID()) 773 wCount++ 774 } 775 } 776 assert.Equal(c, leaderFound, true, "lack of leader reported by node %s", info.NodeID) 777 assert.Equal(c, mCount, managerCount, "managers count reported by node %s", info.NodeID) 778 assert.Equal(c, wCount, workerCount, "workers count reported by node %s", info.NodeID) 779 } 780 assert.Equal(c, totalMCount, managerCount) 781 assert.Equal(c, totalWCount, workerCount) 782 } 783 784 func (s *DockerSwarmSuite) TestAPISwarmRestartCluster(c *check.C) { 785 mCount, wCount := 5, 1 786 787 var nodes []*daemon.Daemon 788 for i := 0; i < mCount; i++ { 789 manager := s.AddDaemon(c, true, true) 790 info := manager.SwarmInfo(c) 791 assert.Equal(c, info.ControlAvailable, true) 792 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 793 nodes = append(nodes, manager) 794 } 795 796 for i := 0; i < wCount; i++ { 797 worker := s.AddDaemon(c, true, false) 798 info := worker.SwarmInfo(c) 799 assert.Equal(c, info.ControlAvailable, false) 800 assert.Equal(c, info.LocalNodeState, swarm.LocalNodeStateActive) 801 nodes = append(nodes, worker) 802 } 803 804 // stop whole cluster 805 { 806 var wg sync.WaitGroup 807 wg.Add(len(nodes)) 808 errs := make(chan error, len(nodes)) 809 810 for _, d := range nodes { 811 go func(daemon *daemon.Daemon) { 812 defer wg.Done() 813 if err := daemon.StopWithError(); err != nil { 814 errs <- err 815 } 816 }(d) 817 } 818 wg.Wait() 819 close(errs) 820 for err := range errs { 821 assert.NilError(c, err) 822 } 823 } 824 825 // start whole cluster 826 { 827 var wg sync.WaitGroup 828 wg.Add(len(nodes)) 829 errs := make(chan error, len(nodes)) 830 831 for _, d := range nodes { 832 go func(daemon *daemon.Daemon) { 833 defer wg.Done() 834 if err := daemon.StartWithError("--iptables=false"); err != nil { 835 errs <- err 836 } 837 }(d) 838 } 839 wg.Wait() 840 close(errs) 841 for err := range errs { 842 assert.NilError(c, err) 843 } 844 } 845 846 checkClusterHealth(c, nodes, mCount, wCount) 847 } 848 849 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateWithName(c *check.C) { 850 d := s.AddDaemon(c, true, true) 851 852 instances := 2 853 id := d.CreateService(c, simpleTestService, setInstances(instances)) 854 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances) 855 856 service := d.GetService(c, id) 857 instances = 5 858 859 setInstances(instances)(service) 860 cli := d.NewClientT(c) 861 defer cli.Close() 862 _, err := cli.ServiceUpdate(context.Background(), service.Spec.Name, service.Version, service.Spec, types.ServiceUpdateOptions{}) 863 assert.NilError(c, err) 864 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances) 865 } 866 867 // Unlocking an unlocked swarm results in an error 868 func (s *DockerSwarmSuite) TestAPISwarmUnlockNotLocked(c *check.C) { 869 d := s.AddDaemon(c, true, true) 870 err := d.SwarmUnlock(c, swarm.UnlockRequest{UnlockKey: "wrong-key"}) 871 assert.ErrorContains(c, err, "swarm is not locked") 872 } 873 874 // #29885 875 func (s *DockerSwarmSuite) TestAPISwarmErrorHandling(c *check.C) { 876 ln, err := net.Listen("tcp", fmt.Sprintf(":%d", defaultSwarmPort)) 877 assert.NilError(c, err) 878 defer ln.Close() 879 d := s.AddDaemon(c, false, false) 880 client := d.NewClientT(c) 881 _, err = client.SwarmInit(context.Background(), swarm.InitRequest{ 882 ListenAddr: d.SwarmListenAddr(), 883 }) 884 assert.ErrorContains(c, err, "address already in use") 885 } 886 887 // Test case for 30242, where duplicate networks, with different drivers `bridge` and `overlay`, 888 // caused both scopes to be `swarm` for `docker network inspect` and `docker network ls`. 889 // This test makes sure the fixes correctly output scopes instead. 890 func (s *DockerSwarmSuite) TestAPIDuplicateNetworks(c *check.C) { 891 d := s.AddDaemon(c, true, true) 892 cli := d.NewClientT(c) 893 defer cli.Close() 894 895 name := "foo" 896 networkCreate := types.NetworkCreate{ 897 CheckDuplicate: false, 898 } 899 900 networkCreate.Driver = "bridge" 901 902 n1, err := cli.NetworkCreate(context.Background(), name, networkCreate) 903 assert.NilError(c, err) 904 905 networkCreate.Driver = "overlay" 906 907 n2, err := cli.NetworkCreate(context.Background(), name, networkCreate) 908 assert.NilError(c, err) 909 910 r1, err := cli.NetworkInspect(context.Background(), n1.ID, types.NetworkInspectOptions{}) 911 assert.NilError(c, err) 912 assert.Equal(c, r1.Scope, "local") 913 914 r2, err := cli.NetworkInspect(context.Background(), n2.ID, types.NetworkInspectOptions{}) 915 assert.NilError(c, err) 916 assert.Equal(c, r2.Scope, "swarm") 917 } 918 919 // Test case for 30178 920 func (s *DockerSwarmSuite) TestAPISwarmHealthcheckNone(c *check.C) { 921 // Issue #36386 can be a independent one, which is worth further investigation. 922 c.Skip("Root cause of Issue #36386 is needed") 923 d := s.AddDaemon(c, true, true) 924 925 out, err := d.Cmd("network", "create", "-d", "overlay", "lb") 926 assert.NilError(c, err, out) 927 928 instances := 1 929 d.CreateService(c, simpleTestService, setInstances(instances), func(s *swarm.Service) { 930 if s.Spec.TaskTemplate.ContainerSpec == nil { 931 s.Spec.TaskTemplate.ContainerSpec = &swarm.ContainerSpec{} 932 } 933 s.Spec.TaskTemplate.ContainerSpec.Healthcheck = &container.HealthConfig{} 934 s.Spec.TaskTemplate.Networks = []swarm.NetworkAttachmentConfig{ 935 {Target: "lb"}, 936 } 937 }) 938 939 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances) 940 941 containers := d.ActiveContainers(c) 942 943 out, err = d.Cmd("exec", containers[0], "ping", "-c1", "-W3", "top") 944 assert.NilError(c, err, out) 945 } 946 947 func (s *DockerSwarmSuite) TestSwarmRepeatedRootRotation(c *check.C) { 948 m := s.AddDaemon(c, true, true) 949 w := s.AddDaemon(c, true, false) 950 951 info := m.SwarmInfo(c) 952 953 currentTrustRoot := info.Cluster.TLSInfo.TrustRoot 954 955 // rotate multiple times 956 for i := 0; i < 4; i++ { 957 var err error 958 var cert, key []byte 959 if i%2 != 0 { 960 cert, _, key, err = initca.New(&csr.CertificateRequest{ 961 CN: "newRoot", 962 KeyRequest: csr.NewBasicKeyRequest(), 963 CA: &csr.CAConfig{Expiry: ca.RootCAExpiration}, 964 }) 965 assert.NilError(c, err) 966 } 967 expectedCert := string(cert) 968 m.UpdateSwarm(c, func(s *swarm.Spec) { 969 s.CAConfig.SigningCACert = expectedCert 970 s.CAConfig.SigningCAKey = string(key) 971 s.CAConfig.ForceRotate++ 972 }) 973 974 // poll to make sure update succeeds 975 var clusterTLSInfo swarm.TLSInfo 976 for j := 0; j < 18; j++ { 977 info := m.SwarmInfo(c) 978 979 // the desired CA cert and key is always redacted 980 assert.Equal(c, info.Cluster.Spec.CAConfig.SigningCAKey, "") 981 assert.Equal(c, info.Cluster.Spec.CAConfig.SigningCACert, "") 982 983 clusterTLSInfo = info.Cluster.TLSInfo 984 985 // if root rotation is done and the trust root has changed, we don't have to poll anymore 986 if !info.Cluster.RootRotationInProgress && clusterTLSInfo.TrustRoot != currentTrustRoot { 987 break 988 } 989 990 // root rotation not done 991 time.Sleep(250 * time.Millisecond) 992 } 993 if cert != nil { 994 assert.Equal(c, clusterTLSInfo.TrustRoot, expectedCert) 995 } 996 // could take another second or two for the nodes to trust the new roots after they've all gotten 997 // new TLS certificates 998 for j := 0; j < 18; j++ { 999 mInfo := m.GetNode(c, m.NodeID()).Description.TLSInfo 1000 wInfo := m.GetNode(c, w.NodeID()).Description.TLSInfo 1001 1002 if mInfo.TrustRoot == clusterTLSInfo.TrustRoot && wInfo.TrustRoot == clusterTLSInfo.TrustRoot { 1003 break 1004 } 1005 1006 // nodes don't trust root certs yet 1007 time.Sleep(250 * time.Millisecond) 1008 } 1009 1010 assert.DeepEqual(c, m.GetNode(c, m.NodeID()).Description.TLSInfo, clusterTLSInfo) 1011 assert.DeepEqual(c, m.GetNode(c, w.NodeID()).Description.TLSInfo, clusterTLSInfo) 1012 currentTrustRoot = clusterTLSInfo.TrustRoot 1013 } 1014 } 1015 1016 func (s *DockerSwarmSuite) TestAPINetworkInspectWithScope(c *check.C) { 1017 d := s.AddDaemon(c, true, true) 1018 1019 name := "test-scoped-network" 1020 ctx := context.Background() 1021 apiclient := d.NewClientT(c) 1022 1023 resp, err := apiclient.NetworkCreate(ctx, name, types.NetworkCreate{Driver: "overlay"}) 1024 assert.NilError(c, err) 1025 1026 network, err := apiclient.NetworkInspect(ctx, name, types.NetworkInspectOptions{}) 1027 assert.NilError(c, err) 1028 assert.Check(c, is.Equal("swarm", network.Scope)) 1029 assert.Check(c, is.Equal(resp.ID, network.ID)) 1030 1031 _, err = apiclient.NetworkInspect(ctx, name, types.NetworkInspectOptions{Scope: "local"}) 1032 assert.Check(c, client.IsErrNotFound(err)) 1033 }