github.com/rita33cool1/iot-system-gateway@v0.0.0-20200911033302-e65bde238cc5/docker-engine/integration-cli/docker_api_swarm_service_test.go (about) 1 // +build !windows 2 3 package main 4 5 import ( 6 "fmt" 7 "path" 8 "strconv" 9 "strings" 10 "time" 11 12 "github.com/docker/docker/api/types" 13 "github.com/docker/docker/api/types/swarm" 14 "github.com/docker/docker/api/types/swarm/runtime" 15 "github.com/docker/docker/integration-cli/checker" 16 "github.com/docker/docker/integration-cli/daemon" 17 testdaemon "github.com/docker/docker/internal/test/daemon" 18 "github.com/docker/docker/internal/test/fixtures/plugin" 19 "github.com/docker/docker/internal/test/registry" 20 "github.com/go-check/check" 21 "golang.org/x/net/context" 22 "golang.org/x/sys/unix" 23 ) 24 25 func setPortConfig(portConfig []swarm.PortConfig) testdaemon.ServiceConstructor { 26 return func(s *swarm.Service) { 27 if s.Spec.EndpointSpec == nil { 28 s.Spec.EndpointSpec = &swarm.EndpointSpec{} 29 } 30 s.Spec.EndpointSpec.Ports = portConfig 31 } 32 } 33 34 func (s *DockerSwarmSuite) TestAPIServiceUpdatePort(c *check.C) { 35 d := s.AddDaemon(c, true, true) 36 37 // Create a service with a port mapping of 8080:8081. 38 portConfig := []swarm.PortConfig{{TargetPort: 8081, PublishedPort: 8080}} 39 serviceID := d.CreateService(c, simpleTestService, setInstances(1), setPortConfig(portConfig)) 40 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 41 42 // Update the service: changed the port mapping from 8080:8081 to 8082:8083. 43 updatedPortConfig := []swarm.PortConfig{{TargetPort: 8083, PublishedPort: 8082}} 44 remoteService := d.GetService(c, serviceID) 45 d.UpdateService(c, remoteService, setPortConfig(updatedPortConfig)) 46 47 // Inspect the service and verify port mapping. 48 updatedService := d.GetService(c, serviceID) 49 c.Assert(updatedService.Spec.EndpointSpec, check.NotNil) 50 c.Assert(len(updatedService.Spec.EndpointSpec.Ports), check.Equals, 1) 51 c.Assert(updatedService.Spec.EndpointSpec.Ports[0].TargetPort, check.Equals, uint32(8083)) 52 c.Assert(updatedService.Spec.EndpointSpec.Ports[0].PublishedPort, check.Equals, uint32(8082)) 53 } 54 55 func (s *DockerSwarmSuite) TestAPISwarmServicesEmptyList(c *check.C) { 56 d := s.AddDaemon(c, true, true) 57 58 services := d.ListServices(c) 59 c.Assert(services, checker.NotNil) 60 c.Assert(len(services), checker.Equals, 0, check.Commentf("services: %#v", services)) 61 } 62 63 func (s *DockerSwarmSuite) TestAPISwarmServicesCreate(c *check.C) { 64 d := s.AddDaemon(c, true, true) 65 66 instances := 2 67 id := d.CreateService(c, simpleTestService, setInstances(instances)) 68 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances) 69 70 cli, err := d.NewClient() 71 c.Assert(err, checker.IsNil) 72 defer cli.Close() 73 74 options := types.ServiceInspectOptions{InsertDefaults: true} 75 76 // insertDefaults inserts UpdateConfig when service is fetched by ID 77 resp, _, err := cli.ServiceInspectWithRaw(context.Background(), id, options) 78 out := fmt.Sprintf("%+v", resp) 79 c.Assert(err, checker.IsNil) 80 c.Assert(out, checker.Contains, "UpdateConfig") 81 82 // insertDefaults inserts UpdateConfig when service is fetched by ID 83 resp, _, err = cli.ServiceInspectWithRaw(context.Background(), "top", options) 84 out = fmt.Sprintf("%+v", resp) 85 c.Assert(err, checker.IsNil) 86 c.Assert(string(out), checker.Contains, "UpdateConfig") 87 88 service := d.GetService(c, id) 89 instances = 5 90 d.UpdateService(c, service, setInstances(instances)) 91 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances) 92 93 d.RemoveService(c, service.ID) 94 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 0) 95 } 96 97 func (s *DockerSwarmSuite) TestAPISwarmServicesMultipleAgents(c *check.C) { 98 d1 := s.AddDaemon(c, true, true) 99 d2 := s.AddDaemon(c, true, false) 100 d3 := s.AddDaemon(c, true, false) 101 102 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks 103 104 instances := 9 105 id := d1.CreateService(c, simpleTestService, setInstances(instances)) 106 107 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.GreaterThan, 0) 108 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.GreaterThan, 0) 109 waitAndAssert(c, defaultReconciliationTimeout, d3.CheckActiveContainerCount, checker.GreaterThan, 0) 110 111 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) 112 113 // reconciliation on d2 node down 114 d2.Stop(c) 115 116 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) 117 118 // test downscaling 119 instances = 5 120 d1.UpdateService(c, d1.GetService(c, id), setInstances(instances)) 121 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) 122 123 } 124 125 func (s *DockerSwarmSuite) TestAPISwarmServicesCreateGlobal(c *check.C) { 126 d1 := s.AddDaemon(c, true, true) 127 d2 := s.AddDaemon(c, true, false) 128 d3 := s.AddDaemon(c, true, false) 129 130 d1.CreateService(c, simpleTestService, setGlobalMode) 131 132 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, 1) 133 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 1) 134 waitAndAssert(c, defaultReconciliationTimeout, d3.CheckActiveContainerCount, checker.Equals, 1) 135 136 d4 := s.AddDaemon(c, true, false) 137 d5 := s.AddDaemon(c, true, false) 138 139 waitAndAssert(c, defaultReconciliationTimeout, d4.CheckActiveContainerCount, checker.Equals, 1) 140 waitAndAssert(c, defaultReconciliationTimeout, d5.CheckActiveContainerCount, checker.Equals, 1) 141 } 142 143 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdate(c *check.C) { 144 const nodeCount = 3 145 var daemons [nodeCount]*daemon.Daemon 146 for i := 0; i < nodeCount; i++ { 147 daemons[i] = s.AddDaemon(c, true, i == 0) 148 } 149 // wait for nodes ready 150 waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount) 151 152 // service image at start 153 image1 := "busybox:latest" 154 // target image in update 155 image2 := "busybox:test" 156 157 // create a different tag 158 for _, d := range daemons { 159 out, err := d.Cmd("tag", image1, image2) 160 c.Assert(err, checker.IsNil, check.Commentf(out)) 161 } 162 163 // create service 164 instances := 5 165 parallelism := 2 166 rollbackParallelism := 3 167 id := daemons[0].CreateService(c, serviceForUpdate, setInstances(instances)) 168 169 // wait for tasks ready 170 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 171 map[string]int{image1: instances}) 172 173 // issue service update 174 service := daemons[0].GetService(c, id) 175 daemons[0].UpdateService(c, service, setImage(image2)) 176 177 // first batch 178 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 179 map[string]int{image1: instances - parallelism, image2: parallelism}) 180 181 // 2nd batch 182 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 183 map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism}) 184 185 // 3nd batch 186 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 187 map[string]int{image2: instances}) 188 189 // Roll back to the previous version. This uses the CLI because 190 // rollback used to be a client-side operation. 191 out, err := daemons[0].Cmd("service", "update", "--detach", "--rollback", id) 192 c.Assert(err, checker.IsNil, check.Commentf(out)) 193 194 // first batch 195 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 196 map[string]int{image2: instances - rollbackParallelism, image1: rollbackParallelism}) 197 198 // 2nd batch 199 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 200 map[string]int{image1: instances}) 201 } 202 203 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateStartFirst(c *check.C) { 204 d := s.AddDaemon(c, true, true) 205 206 // service image at start 207 image1 := "busybox:latest" 208 // target image in update 209 image2 := "testhealth:latest" 210 211 // service started from this image won't pass health check 212 _, _, err := d.BuildImageWithOut(image2, 213 `FROM busybox 214 HEALTHCHECK --interval=1s --timeout=30s --retries=1024 \ 215 CMD cat /status`, 216 true) 217 c.Check(err, check.IsNil) 218 219 // create service 220 instances := 5 221 parallelism := 2 222 rollbackParallelism := 3 223 id := d.CreateService(c, serviceForUpdate, setInstances(instances), setUpdateOrder(swarm.UpdateOrderStartFirst), setRollbackOrder(swarm.UpdateOrderStartFirst)) 224 225 checkStartingTasks := func(expected int) []swarm.Task { 226 var startingTasks []swarm.Task 227 waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { 228 tasks := d.GetServiceTasks(c, id) 229 startingTasks = nil 230 for _, t := range tasks { 231 if t.Status.State == swarm.TaskStateStarting { 232 startingTasks = append(startingTasks, t) 233 } 234 } 235 return startingTasks, nil 236 }, checker.HasLen, expected) 237 238 return startingTasks 239 } 240 241 makeTasksHealthy := func(tasks []swarm.Task) { 242 for _, t := range tasks { 243 containerID := t.Status.ContainerStatus.ContainerID 244 d.Cmd("exec", containerID, "touch", "/status") 245 } 246 } 247 248 // wait for tasks ready 249 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 250 map[string]int{image1: instances}) 251 252 // issue service update 253 service := d.GetService(c, id) 254 d.UpdateService(c, service, setImage(image2)) 255 256 // first batch 257 258 // The old tasks should be running, and the new ones should be starting. 259 startingTasks := checkStartingTasks(parallelism) 260 261 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 262 map[string]int{image1: instances}) 263 264 // make it healthy 265 makeTasksHealthy(startingTasks) 266 267 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 268 map[string]int{image1: instances - parallelism, image2: parallelism}) 269 270 // 2nd batch 271 272 // The old tasks should be running, and the new ones should be starting. 273 startingTasks = checkStartingTasks(parallelism) 274 275 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 276 map[string]int{image1: instances - parallelism, image2: parallelism}) 277 278 // make it healthy 279 makeTasksHealthy(startingTasks) 280 281 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 282 map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism}) 283 284 // 3nd batch 285 286 // The old tasks should be running, and the new ones should be starting. 287 startingTasks = checkStartingTasks(1) 288 289 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 290 map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism}) 291 292 // make it healthy 293 makeTasksHealthy(startingTasks) 294 295 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 296 map[string]int{image2: instances}) 297 298 // Roll back to the previous version. This uses the CLI because 299 // rollback is a client-side operation. 300 out, err := d.Cmd("service", "update", "--detach", "--rollback", id) 301 c.Assert(err, checker.IsNil, check.Commentf(out)) 302 303 // first batch 304 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 305 map[string]int{image2: instances - rollbackParallelism, image1: rollbackParallelism}) 306 307 // 2nd batch 308 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 309 map[string]int{image1: instances}) 310 } 311 312 func (s *DockerSwarmSuite) TestAPISwarmServicesFailedUpdate(c *check.C) { 313 const nodeCount = 3 314 var daemons [nodeCount]*daemon.Daemon 315 for i := 0; i < nodeCount; i++ { 316 daemons[i] = s.AddDaemon(c, true, i == 0) 317 } 318 // wait for nodes ready 319 waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount) 320 321 // service image at start 322 image1 := "busybox:latest" 323 // target image in update 324 image2 := "busybox:badtag" 325 326 // create service 327 instances := 5 328 id := daemons[0].CreateService(c, serviceForUpdate, setInstances(instances)) 329 330 // wait for tasks ready 331 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 332 map[string]int{image1: instances}) 333 334 // issue service update 335 service := daemons[0].GetService(c, id) 336 daemons[0].UpdateService(c, service, setImage(image2), setFailureAction(swarm.UpdateFailureActionPause), setMaxFailureRatio(0.25), setParallelism(1)) 337 338 // should update 2 tasks and then pause 339 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceUpdateState(id), checker.Equals, swarm.UpdateStatePaused) 340 v, _ := daemons[0].CheckServiceRunningTasks(id)(c) 341 c.Assert(v, checker.Equals, instances-2) 342 343 // Roll back to the previous version. This uses the CLI because 344 // rollback used to be a client-side operation. 345 out, err := daemons[0].Cmd("service", "update", "--detach", "--rollback", id) 346 c.Assert(err, checker.IsNil, check.Commentf(out)) 347 348 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 349 map[string]int{image1: instances}) 350 } 351 352 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintRole(c *check.C) { 353 const nodeCount = 3 354 var daemons [nodeCount]*daemon.Daemon 355 for i := 0; i < nodeCount; i++ { 356 daemons[i] = s.AddDaemon(c, true, i == 0) 357 } 358 // wait for nodes ready 359 waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount) 360 361 // create service 362 constraints := []string{"node.role==worker"} 363 instances := 3 364 id := daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 365 // wait for tasks ready 366 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances) 367 // validate tasks are running on worker nodes 368 tasks := daemons[0].GetServiceTasks(c, id) 369 for _, task := range tasks { 370 node := daemons[0].GetNode(c, task.NodeID) 371 c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleWorker) 372 } 373 //remove service 374 daemons[0].RemoveService(c, id) 375 376 // create service 377 constraints = []string{"node.role!=worker"} 378 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 379 // wait for tasks ready 380 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances) 381 tasks = daemons[0].GetServiceTasks(c, id) 382 // validate tasks are running on manager nodes 383 for _, task := range tasks { 384 node := daemons[0].GetNode(c, task.NodeID) 385 c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleManager) 386 } 387 //remove service 388 daemons[0].RemoveService(c, id) 389 390 // create service 391 constraints = []string{"node.role==nosuchrole"} 392 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 393 // wait for tasks created 394 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceTasks(id), checker.Equals, instances) 395 // let scheduler try 396 time.Sleep(250 * time.Millisecond) 397 // validate tasks are not assigned to any node 398 tasks = daemons[0].GetServiceTasks(c, id) 399 for _, task := range tasks { 400 c.Assert(task.NodeID, checker.Equals, "") 401 } 402 } 403 404 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintLabel(c *check.C) { 405 const nodeCount = 3 406 var daemons [nodeCount]*daemon.Daemon 407 for i := 0; i < nodeCount; i++ { 408 daemons[i] = s.AddDaemon(c, true, i == 0) 409 } 410 // wait for nodes ready 411 waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount) 412 nodes := daemons[0].ListNodes(c) 413 c.Assert(len(nodes), checker.Equals, nodeCount) 414 415 // add labels to nodes 416 daemons[0].UpdateNode(c, nodes[0].ID, func(n *swarm.Node) { 417 n.Spec.Annotations.Labels = map[string]string{ 418 "security": "high", 419 } 420 }) 421 for i := 1; i < nodeCount; i++ { 422 daemons[0].UpdateNode(c, nodes[i].ID, func(n *swarm.Node) { 423 n.Spec.Annotations.Labels = map[string]string{ 424 "security": "low", 425 } 426 }) 427 } 428 429 // create service 430 instances := 3 431 constraints := []string{"node.labels.security==high"} 432 id := daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 433 // wait for tasks ready 434 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances) 435 tasks := daemons[0].GetServiceTasks(c, id) 436 // validate all tasks are running on nodes[0] 437 for _, task := range tasks { 438 c.Assert(task.NodeID, checker.Equals, nodes[0].ID) 439 } 440 //remove service 441 daemons[0].RemoveService(c, id) 442 443 // create service 444 constraints = []string{"node.labels.security!=high"} 445 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 446 // wait for tasks ready 447 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances) 448 tasks = daemons[0].GetServiceTasks(c, id) 449 // validate all tasks are NOT running on nodes[0] 450 for _, task := range tasks { 451 c.Assert(task.NodeID, checker.Not(checker.Equals), nodes[0].ID) 452 } 453 //remove service 454 daemons[0].RemoveService(c, id) 455 456 constraints = []string{"node.labels.security==medium"} 457 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 458 // wait for tasks created 459 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceTasks(id), checker.Equals, instances) 460 // let scheduler try 461 time.Sleep(250 * time.Millisecond) 462 tasks = daemons[0].GetServiceTasks(c, id) 463 // validate tasks are not assigned 464 for _, task := range tasks { 465 c.Assert(task.NodeID, checker.Equals, "") 466 } 467 //remove service 468 daemons[0].RemoveService(c, id) 469 470 // multiple constraints 471 constraints = []string{ 472 "node.labels.security==high", 473 fmt.Sprintf("node.id==%s", nodes[1].ID), 474 } 475 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 476 // wait for tasks created 477 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceTasks(id), checker.Equals, instances) 478 // let scheduler try 479 time.Sleep(250 * time.Millisecond) 480 tasks = daemons[0].GetServiceTasks(c, id) 481 // validate tasks are not assigned 482 for _, task := range tasks { 483 c.Assert(task.NodeID, checker.Equals, "") 484 } 485 // make nodes[1] fulfills the constraints 486 daemons[0].UpdateNode(c, nodes[1].ID, func(n *swarm.Node) { 487 n.Spec.Annotations.Labels = map[string]string{ 488 "security": "high", 489 } 490 }) 491 // wait for tasks ready 492 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances) 493 tasks = daemons[0].GetServiceTasks(c, id) 494 for _, task := range tasks { 495 c.Assert(task.NodeID, checker.Equals, nodes[1].ID) 496 } 497 } 498 499 func (s *DockerSwarmSuite) TestAPISwarmServicePlacementPrefs(c *check.C) { 500 const nodeCount = 3 501 var daemons [nodeCount]*daemon.Daemon 502 for i := 0; i < nodeCount; i++ { 503 daemons[i] = s.AddDaemon(c, true, i == 0) 504 } 505 // wait for nodes ready 506 waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount) 507 nodes := daemons[0].ListNodes(c) 508 c.Assert(len(nodes), checker.Equals, nodeCount) 509 510 // add labels to nodes 511 daemons[0].UpdateNode(c, nodes[0].ID, func(n *swarm.Node) { 512 n.Spec.Annotations.Labels = map[string]string{ 513 "rack": "a", 514 } 515 }) 516 for i := 1; i < nodeCount; i++ { 517 daemons[0].UpdateNode(c, nodes[i].ID, func(n *swarm.Node) { 518 n.Spec.Annotations.Labels = map[string]string{ 519 "rack": "b", 520 } 521 }) 522 } 523 524 // create service 525 instances := 4 526 prefs := []swarm.PlacementPreference{{Spread: &swarm.SpreadOver{SpreadDescriptor: "node.labels.rack"}}} 527 id := daemons[0].CreateService(c, simpleTestService, setPlacementPrefs(prefs), setInstances(instances)) 528 // wait for tasks ready 529 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances) 530 tasks := daemons[0].GetServiceTasks(c, id) 531 // validate all tasks are running on nodes[0] 532 tasksOnNode := make(map[string]int) 533 for _, task := range tasks { 534 tasksOnNode[task.NodeID]++ 535 } 536 c.Assert(tasksOnNode[nodes[0].ID], checker.Equals, 2) 537 c.Assert(tasksOnNode[nodes[1].ID], checker.Equals, 1) 538 c.Assert(tasksOnNode[nodes[2].ID], checker.Equals, 1) 539 } 540 541 func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *check.C) { 542 testRequires(c, SameHostDaemon) 543 testRequires(c, DaemonIsLinux) 544 545 d1 := s.AddDaemon(c, true, true) 546 d2 := s.AddDaemon(c, true, true) 547 d3 := s.AddDaemon(c, true, false) 548 549 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept 550 551 instances := 9 552 d1.CreateService(c, simpleTestService, setInstances(instances)) 553 554 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) 555 556 getContainers := func() map[string]*daemon.Daemon { 557 m := make(map[string]*daemon.Daemon) 558 for _, d := range []*daemon.Daemon{d1, d2, d3} { 559 for _, id := range d.ActiveContainers() { 560 m[id] = d 561 } 562 } 563 return m 564 } 565 566 containers := getContainers() 567 c.Assert(containers, checker.HasLen, instances) 568 var toRemove string 569 for i := range containers { 570 toRemove = i 571 } 572 573 _, err := containers[toRemove].Cmd("stop", toRemove) 574 c.Assert(err, checker.IsNil) 575 576 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) 577 578 containers2 := getContainers() 579 c.Assert(containers2, checker.HasLen, instances) 580 for i := range containers { 581 if i == toRemove { 582 c.Assert(containers2[i], checker.IsNil) 583 } else { 584 c.Assert(containers2[i], checker.NotNil) 585 } 586 } 587 588 containers = containers2 589 for i := range containers { 590 toRemove = i 591 } 592 593 // try with killing process outside of docker 594 pidStr, err := containers[toRemove].Cmd("inspect", "-f", "{{.State.Pid}}", toRemove) 595 c.Assert(err, checker.IsNil) 596 pid, err := strconv.Atoi(strings.TrimSpace(pidStr)) 597 c.Assert(err, checker.IsNil) 598 c.Assert(unix.Kill(pid, unix.SIGKILL), checker.IsNil) 599 600 time.Sleep(time.Second) // give some time to handle the signal 601 602 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) 603 604 containers2 = getContainers() 605 c.Assert(containers2, checker.HasLen, instances) 606 for i := range containers { 607 if i == toRemove { 608 c.Assert(containers2[i], checker.IsNil) 609 } else { 610 c.Assert(containers2[i], checker.NotNil) 611 } 612 } 613 } 614 615 // Test plugins deployed via swarm services 616 func (s *DockerSwarmSuite) TestAPISwarmServicesPlugin(c *check.C) { 617 testRequires(c, ExperimentalDaemon, DaemonIsLinux, IsAmd64) 618 619 reg := registry.NewV2(c) 620 defer reg.Close() 621 622 repo := path.Join(privateRegistryURL, "swarm", "test:v1") 623 repo2 := path.Join(privateRegistryURL, "swarm", "test:v2") 624 name := "test" 625 626 err := plugin.CreateInRegistry(context.Background(), repo, nil) 627 c.Assert(err, checker.IsNil, check.Commentf("failed to create plugin")) 628 err = plugin.CreateInRegistry(context.Background(), repo2, nil) 629 c.Assert(err, checker.IsNil, check.Commentf("failed to create plugin")) 630 631 d1 := s.AddDaemon(c, true, true) 632 d2 := s.AddDaemon(c, true, true) 633 d3 := s.AddDaemon(c, true, false) 634 635 makePlugin := func(repo, name string, constraints []string) func(*swarm.Service) { 636 return func(s *swarm.Service) { 637 s.Spec.TaskTemplate.Runtime = "plugin" 638 s.Spec.TaskTemplate.PluginSpec = &runtime.PluginSpec{ 639 Name: name, 640 Remote: repo, 641 } 642 if constraints != nil { 643 s.Spec.TaskTemplate.Placement = &swarm.Placement{ 644 Constraints: constraints, 645 } 646 } 647 } 648 } 649 650 id := d1.CreateService(c, makePlugin(repo, name, nil)) 651 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.True) 652 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.True) 653 waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.True) 654 655 service := d1.GetService(c, id) 656 d1.UpdateService(c, service, makePlugin(repo2, name, nil)) 657 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginImage(name), checker.Equals, repo2) 658 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginImage(name), checker.Equals, repo2) 659 waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginImage(name), checker.Equals, repo2) 660 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.True) 661 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.True) 662 waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.True) 663 664 d1.RemoveService(c, id) 665 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.False) 666 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.False) 667 waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.False) 668 669 // constrain to managers only 670 id = d1.CreateService(c, makePlugin(repo, name, []string{"node.role==manager"})) 671 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.True) 672 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.True) 673 waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.False) // Not a manager, not running it 674 d1.RemoveService(c, id) 675 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(name), checker.False) 676 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(name), checker.False) 677 waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(name), checker.False) 678 679 // with no name 680 id = d1.CreateService(c, makePlugin(repo, "", nil)) 681 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(repo), checker.True) 682 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(repo), checker.True) 683 waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(repo), checker.True) 684 d1.RemoveService(c, id) 685 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckPluginRunning(repo), checker.False) 686 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckPluginRunning(repo), checker.False) 687 waitAndAssert(c, defaultReconciliationTimeout, d3.CheckPluginRunning(repo), checker.False) 688 }