github.com/jiasir/docker@v1.3.3-0.20170609024000-252e610103e7/integration-cli/docker_api_swarm_service_test.go (about) 1 // +build !windows 2 3 package main 4 5 import ( 6 "fmt" 7 "strconv" 8 "strings" 9 "syscall" 10 "time" 11 12 "github.com/docker/docker/api/types/swarm" 13 "github.com/docker/docker/integration-cli/checker" 14 "github.com/docker/docker/integration-cli/daemon" 15 "github.com/go-check/check" 16 ) 17 18 func setPortConfig(portConfig []swarm.PortConfig) daemon.ServiceConstructor { 19 return func(s *swarm.Service) { 20 if s.Spec.EndpointSpec == nil { 21 s.Spec.EndpointSpec = &swarm.EndpointSpec{} 22 } 23 s.Spec.EndpointSpec.Ports = portConfig 24 } 25 } 26 27 func (s *DockerSwarmSuite) TestAPIServiceUpdatePort(c *check.C) { 28 d := s.AddDaemon(c, true, true) 29 30 // Create a service with a port mapping of 8080:8081. 31 portConfig := []swarm.PortConfig{{TargetPort: 8081, PublishedPort: 8080}} 32 serviceID := d.CreateService(c, simpleTestService, setInstances(1), setPortConfig(portConfig)) 33 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 1) 34 35 // Update the service: changed the port mapping from 8080:8081 to 8082:8083. 36 updatedPortConfig := []swarm.PortConfig{{TargetPort: 8083, PublishedPort: 8082}} 37 remoteService := d.GetService(c, serviceID) 38 d.UpdateService(c, remoteService, setPortConfig(updatedPortConfig)) 39 40 // Inspect the service and verify port mapping. 41 updatedService := d.GetService(c, serviceID) 42 c.Assert(updatedService.Spec.EndpointSpec, check.NotNil) 43 c.Assert(len(updatedService.Spec.EndpointSpec.Ports), check.Equals, 1) 44 c.Assert(updatedService.Spec.EndpointSpec.Ports[0].TargetPort, check.Equals, uint32(8083)) 45 c.Assert(updatedService.Spec.EndpointSpec.Ports[0].PublishedPort, check.Equals, uint32(8082)) 46 } 47 48 func (s *DockerSwarmSuite) TestAPISwarmServicesEmptyList(c *check.C) { 49 d := s.AddDaemon(c, true, true) 50 51 services := d.ListServices(c) 52 c.Assert(services, checker.NotNil) 53 c.Assert(len(services), checker.Equals, 0, check.Commentf("services: %#v", services)) 54 } 55 56 func (s *DockerSwarmSuite) TestAPISwarmServicesCreate(c *check.C) { 57 d := s.AddDaemon(c, true, true) 58 59 instances := 2 60 id := d.CreateService(c, simpleTestService, setInstances(instances)) 61 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances) 62 63 // insertDefaults inserts UpdateConfig when service is fetched by ID 64 _, out, err := d.SockRequest("GET", "/services/"+id+"?insertDefaults=true", nil) 65 c.Assert(err, checker.IsNil, check.Commentf("%s", out)) 66 c.Assert(string(out), checker.Contains, "UpdateConfig") 67 68 // insertDefaults inserts UpdateConfig when service is fetched by ID 69 _, out, err = d.SockRequest("GET", "/services/top?insertDefaults=true", nil) 70 c.Assert(err, checker.IsNil, check.Commentf("%s", out)) 71 c.Assert(string(out), checker.Contains, "UpdateConfig") 72 73 service := d.GetService(c, id) 74 instances = 5 75 d.UpdateService(c, service, setInstances(instances)) 76 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, instances) 77 78 d.RemoveService(c, service.ID) 79 waitAndAssert(c, defaultReconciliationTimeout, d.CheckActiveContainerCount, checker.Equals, 0) 80 } 81 82 func (s *DockerSwarmSuite) TestAPISwarmServicesMultipleAgents(c *check.C) { 83 d1 := s.AddDaemon(c, true, true) 84 d2 := s.AddDaemon(c, true, false) 85 d3 := s.AddDaemon(c, true, false) 86 87 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks 88 89 instances := 9 90 id := d1.CreateService(c, simpleTestService, setInstances(instances)) 91 92 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.GreaterThan, 0) 93 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.GreaterThan, 0) 94 waitAndAssert(c, defaultReconciliationTimeout, d3.CheckActiveContainerCount, checker.GreaterThan, 0) 95 96 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) 97 98 // reconciliation on d2 node down 99 d2.Stop(c) 100 101 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) 102 103 // test downscaling 104 instances = 5 105 d1.UpdateService(c, d1.GetService(c, id), setInstances(instances)) 106 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) 107 108 } 109 110 func (s *DockerSwarmSuite) TestAPISwarmServicesCreateGlobal(c *check.C) { 111 d1 := s.AddDaemon(c, true, true) 112 d2 := s.AddDaemon(c, true, false) 113 d3 := s.AddDaemon(c, true, false) 114 115 d1.CreateService(c, simpleTestService, setGlobalMode) 116 117 waitAndAssert(c, defaultReconciliationTimeout, d1.CheckActiveContainerCount, checker.Equals, 1) 118 waitAndAssert(c, defaultReconciliationTimeout, d2.CheckActiveContainerCount, checker.Equals, 1) 119 waitAndAssert(c, defaultReconciliationTimeout, d3.CheckActiveContainerCount, checker.Equals, 1) 120 121 d4 := s.AddDaemon(c, true, false) 122 d5 := s.AddDaemon(c, true, false) 123 124 waitAndAssert(c, defaultReconciliationTimeout, d4.CheckActiveContainerCount, checker.Equals, 1) 125 waitAndAssert(c, defaultReconciliationTimeout, d5.CheckActiveContainerCount, checker.Equals, 1) 126 } 127 128 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdate(c *check.C) { 129 const nodeCount = 3 130 var daemons [nodeCount]*daemon.Swarm 131 for i := 0; i < nodeCount; i++ { 132 daemons[i] = s.AddDaemon(c, true, i == 0) 133 } 134 // wait for nodes ready 135 waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount) 136 137 // service image at start 138 image1 := "busybox:latest" 139 // target image in update 140 image2 := "busybox:test" 141 142 // create a different tag 143 for _, d := range daemons { 144 out, err := d.Cmd("tag", image1, image2) 145 c.Assert(err, checker.IsNil, check.Commentf(out)) 146 } 147 148 // create service 149 instances := 5 150 parallelism := 2 151 rollbackParallelism := 3 152 id := daemons[0].CreateService(c, serviceForUpdate, setInstances(instances)) 153 154 // wait for tasks ready 155 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 156 map[string]int{image1: instances}) 157 158 // issue service update 159 service := daemons[0].GetService(c, id) 160 daemons[0].UpdateService(c, service, setImage(image2)) 161 162 // first batch 163 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 164 map[string]int{image1: instances - parallelism, image2: parallelism}) 165 166 // 2nd batch 167 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 168 map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism}) 169 170 // 3nd batch 171 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 172 map[string]int{image2: instances}) 173 174 // Roll back to the previous version. This uses the CLI because 175 // rollback used to be a client-side operation. 176 out, err := daemons[0].Cmd("service", "update", "--rollback", id) 177 c.Assert(err, checker.IsNil, check.Commentf(out)) 178 179 // first batch 180 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 181 map[string]int{image2: instances - rollbackParallelism, image1: rollbackParallelism}) 182 183 // 2nd batch 184 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 185 map[string]int{image1: instances}) 186 } 187 188 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateStartFirst(c *check.C) { 189 d := s.AddDaemon(c, true, true) 190 191 // service image at start 192 image1 := "busybox:latest" 193 // target image in update 194 image2 := "testhealth" 195 196 // service started from this image won't pass health check 197 _, _, err := d.BuildImageWithOut(image2, 198 `FROM busybox 199 HEALTHCHECK --interval=1s --timeout=30s --retries=1024 \ 200 CMD cat /status`, 201 true) 202 c.Check(err, check.IsNil) 203 204 // create service 205 instances := 5 206 parallelism := 2 207 rollbackParallelism := 3 208 id := d.CreateService(c, serviceForUpdate, setInstances(instances), setUpdateOrder(swarm.UpdateOrderStartFirst), setRollbackOrder(swarm.UpdateOrderStartFirst)) 209 210 checkStartingTasks := func(expected int) []swarm.Task { 211 var startingTasks []swarm.Task 212 waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { 213 tasks := d.GetServiceTasks(c, id) 214 startingTasks = nil 215 for _, t := range tasks { 216 if t.Status.State == swarm.TaskStateStarting { 217 startingTasks = append(startingTasks, t) 218 } 219 } 220 return startingTasks, nil 221 }, checker.HasLen, expected) 222 223 return startingTasks 224 } 225 226 makeTasksHealthy := func(tasks []swarm.Task) { 227 for _, t := range tasks { 228 containerID := t.Status.ContainerStatus.ContainerID 229 d.Cmd("exec", containerID, "touch", "/status") 230 } 231 } 232 233 // wait for tasks ready 234 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 235 map[string]int{image1: instances}) 236 237 // issue service update 238 service := d.GetService(c, id) 239 d.UpdateService(c, service, setImage(image2)) 240 241 // first batch 242 243 // The old tasks should be running, and the new ones should be starting. 244 startingTasks := checkStartingTasks(parallelism) 245 246 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 247 map[string]int{image1: instances}) 248 249 // make it healthy 250 makeTasksHealthy(startingTasks) 251 252 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 253 map[string]int{image1: instances - parallelism, image2: parallelism}) 254 255 // 2nd batch 256 257 // The old tasks should be running, and the new ones should be starting. 258 startingTasks = checkStartingTasks(parallelism) 259 260 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 261 map[string]int{image1: instances - parallelism, image2: parallelism}) 262 263 // make it healthy 264 makeTasksHealthy(startingTasks) 265 266 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 267 map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism}) 268 269 // 3nd batch 270 271 // The old tasks should be running, and the new ones should be starting. 272 startingTasks = checkStartingTasks(1) 273 274 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 275 map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism}) 276 277 // make it healthy 278 makeTasksHealthy(startingTasks) 279 280 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 281 map[string]int{image2: instances}) 282 283 // Roll back to the previous version. This uses the CLI because 284 // rollback is a client-side operation. 285 out, err := d.Cmd("service", "update", "--rollback", id) 286 c.Assert(err, checker.IsNil, check.Commentf(out)) 287 288 // first batch 289 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 290 map[string]int{image2: instances - rollbackParallelism, image1: rollbackParallelism}) 291 292 // 2nd batch 293 waitAndAssert(c, defaultReconciliationTimeout, d.CheckRunningTaskImages, checker.DeepEquals, 294 map[string]int{image1: instances}) 295 } 296 297 func (s *DockerSwarmSuite) TestAPISwarmServicesFailedUpdate(c *check.C) { 298 const nodeCount = 3 299 var daemons [nodeCount]*daemon.Swarm 300 for i := 0; i < nodeCount; i++ { 301 daemons[i] = s.AddDaemon(c, true, i == 0) 302 } 303 // wait for nodes ready 304 waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount) 305 306 // service image at start 307 image1 := "busybox:latest" 308 // target image in update 309 image2 := "busybox:badtag" 310 311 // create service 312 instances := 5 313 id := daemons[0].CreateService(c, serviceForUpdate, setInstances(instances)) 314 315 // wait for tasks ready 316 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 317 map[string]int{image1: instances}) 318 319 // issue service update 320 service := daemons[0].GetService(c, id) 321 daemons[0].UpdateService(c, service, setImage(image2), setFailureAction(swarm.UpdateFailureActionPause), setMaxFailureRatio(0.25), setParallelism(1)) 322 323 // should update 2 tasks and then pause 324 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceUpdateState(id), checker.Equals, swarm.UpdateStatePaused) 325 v, _ := daemons[0].CheckServiceRunningTasks(id)(c) 326 c.Assert(v, checker.Equals, instances-2) 327 328 // Roll back to the previous version. This uses the CLI because 329 // rollback used to be a client-side operation. 330 out, err := daemons[0].Cmd("service", "update", "--rollback", id) 331 c.Assert(err, checker.IsNil, check.Commentf(out)) 332 333 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckRunningTaskImages, checker.DeepEquals, 334 map[string]int{image1: instances}) 335 } 336 337 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintRole(c *check.C) { 338 const nodeCount = 3 339 var daemons [nodeCount]*daemon.Swarm 340 for i := 0; i < nodeCount; i++ { 341 daemons[i] = s.AddDaemon(c, true, i == 0) 342 } 343 // wait for nodes ready 344 waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount) 345 346 // create service 347 constraints := []string{"node.role==worker"} 348 instances := 3 349 id := daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 350 // wait for tasks ready 351 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances) 352 // validate tasks are running on worker nodes 353 tasks := daemons[0].GetServiceTasks(c, id) 354 for _, task := range tasks { 355 node := daemons[0].GetNode(c, task.NodeID) 356 c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleWorker) 357 } 358 //remove service 359 daemons[0].RemoveService(c, id) 360 361 // create service 362 constraints = []string{"node.role!=worker"} 363 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 364 // wait for tasks ready 365 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances) 366 tasks = daemons[0].GetServiceTasks(c, id) 367 // validate tasks are running on manager nodes 368 for _, task := range tasks { 369 node := daemons[0].GetNode(c, task.NodeID) 370 c.Assert(node.Spec.Role, checker.Equals, swarm.NodeRoleManager) 371 } 372 //remove service 373 daemons[0].RemoveService(c, id) 374 375 // create service 376 constraints = []string{"node.role==nosuchrole"} 377 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 378 // wait for tasks created 379 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceTasks(id), checker.Equals, instances) 380 // let scheduler try 381 time.Sleep(250 * time.Millisecond) 382 // validate tasks are not assigned to any node 383 tasks = daemons[0].GetServiceTasks(c, id) 384 for _, task := range tasks { 385 c.Assert(task.NodeID, checker.Equals, "") 386 } 387 } 388 389 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintLabel(c *check.C) { 390 const nodeCount = 3 391 var daemons [nodeCount]*daemon.Swarm 392 for i := 0; i < nodeCount; i++ { 393 daemons[i] = s.AddDaemon(c, true, i == 0) 394 } 395 // wait for nodes ready 396 waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount) 397 nodes := daemons[0].ListNodes(c) 398 c.Assert(len(nodes), checker.Equals, nodeCount) 399 400 // add labels to nodes 401 daemons[0].UpdateNode(c, nodes[0].ID, func(n *swarm.Node) { 402 n.Spec.Annotations.Labels = map[string]string{ 403 "security": "high", 404 } 405 }) 406 for i := 1; i < nodeCount; i++ { 407 daemons[0].UpdateNode(c, nodes[i].ID, func(n *swarm.Node) { 408 n.Spec.Annotations.Labels = map[string]string{ 409 "security": "low", 410 } 411 }) 412 } 413 414 // create service 415 instances := 3 416 constraints := []string{"node.labels.security==high"} 417 id := daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 418 // wait for tasks ready 419 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances) 420 tasks := daemons[0].GetServiceTasks(c, id) 421 // validate all tasks are running on nodes[0] 422 for _, task := range tasks { 423 c.Assert(task.NodeID, checker.Equals, nodes[0].ID) 424 } 425 //remove service 426 daemons[0].RemoveService(c, id) 427 428 // create service 429 constraints = []string{"node.labels.security!=high"} 430 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 431 // wait for tasks ready 432 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances) 433 tasks = daemons[0].GetServiceTasks(c, id) 434 // validate all tasks are NOT running on nodes[0] 435 for _, task := range tasks { 436 c.Assert(task.NodeID, checker.Not(checker.Equals), nodes[0].ID) 437 } 438 //remove service 439 daemons[0].RemoveService(c, id) 440 441 constraints = []string{"node.labels.security==medium"} 442 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 443 // wait for tasks created 444 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceTasks(id), checker.Equals, instances) 445 // let scheduler try 446 time.Sleep(250 * time.Millisecond) 447 tasks = daemons[0].GetServiceTasks(c, id) 448 // validate tasks are not assigned 449 for _, task := range tasks { 450 c.Assert(task.NodeID, checker.Equals, "") 451 } 452 //remove service 453 daemons[0].RemoveService(c, id) 454 455 // multiple constraints 456 constraints = []string{ 457 "node.labels.security==high", 458 fmt.Sprintf("node.id==%s", nodes[1].ID), 459 } 460 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 461 // wait for tasks created 462 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceTasks(id), checker.Equals, instances) 463 // let scheduler try 464 time.Sleep(250 * time.Millisecond) 465 tasks = daemons[0].GetServiceTasks(c, id) 466 // validate tasks are not assigned 467 for _, task := range tasks { 468 c.Assert(task.NodeID, checker.Equals, "") 469 } 470 // make nodes[1] fulfills the constraints 471 daemons[0].UpdateNode(c, nodes[1].ID, func(n *swarm.Node) { 472 n.Spec.Annotations.Labels = map[string]string{ 473 "security": "high", 474 } 475 }) 476 // wait for tasks ready 477 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances) 478 tasks = daemons[0].GetServiceTasks(c, id) 479 for _, task := range tasks { 480 c.Assert(task.NodeID, checker.Equals, nodes[1].ID) 481 } 482 } 483 484 func (s *DockerSwarmSuite) TestAPISwarmServicePlacementPrefs(c *check.C) { 485 const nodeCount = 3 486 var daemons [nodeCount]*daemon.Swarm 487 for i := 0; i < nodeCount; i++ { 488 daemons[i] = s.AddDaemon(c, true, i == 0) 489 } 490 // wait for nodes ready 491 waitAndAssert(c, 5*time.Second, daemons[0].CheckNodeReadyCount, checker.Equals, nodeCount) 492 nodes := daemons[0].ListNodes(c) 493 c.Assert(len(nodes), checker.Equals, nodeCount) 494 495 // add labels to nodes 496 daemons[0].UpdateNode(c, nodes[0].ID, func(n *swarm.Node) { 497 n.Spec.Annotations.Labels = map[string]string{ 498 "rack": "a", 499 } 500 }) 501 for i := 1; i < nodeCount; i++ { 502 daemons[0].UpdateNode(c, nodes[i].ID, func(n *swarm.Node) { 503 n.Spec.Annotations.Labels = map[string]string{ 504 "rack": "b", 505 } 506 }) 507 } 508 509 // create service 510 instances := 4 511 prefs := []swarm.PlacementPreference{{Spread: &swarm.SpreadOver{SpreadDescriptor: "node.labels.rack"}}} 512 id := daemons[0].CreateService(c, simpleTestService, setPlacementPrefs(prefs), setInstances(instances)) 513 // wait for tasks ready 514 waitAndAssert(c, defaultReconciliationTimeout, daemons[0].CheckServiceRunningTasks(id), checker.Equals, instances) 515 tasks := daemons[0].GetServiceTasks(c, id) 516 // validate all tasks are running on nodes[0] 517 tasksOnNode := make(map[string]int) 518 for _, task := range tasks { 519 tasksOnNode[task.NodeID]++ 520 } 521 c.Assert(tasksOnNode[nodes[0].ID], checker.Equals, 2) 522 c.Assert(tasksOnNode[nodes[1].ID], checker.Equals, 1) 523 c.Assert(tasksOnNode[nodes[2].ID], checker.Equals, 1) 524 } 525 526 func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *check.C) { 527 testRequires(c, SameHostDaemon) 528 testRequires(c, DaemonIsLinux) 529 530 d1 := s.AddDaemon(c, true, true) 531 d2 := s.AddDaemon(c, true, true) 532 d3 := s.AddDaemon(c, true, false) 533 534 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept 535 536 instances := 9 537 d1.CreateService(c, simpleTestService, setInstances(instances)) 538 539 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) 540 541 getContainers := func() map[string]*daemon.Swarm { 542 m := make(map[string]*daemon.Swarm) 543 for _, d := range []*daemon.Swarm{d1, d2, d3} { 544 for _, id := range d.ActiveContainers() { 545 m[id] = d 546 } 547 } 548 return m 549 } 550 551 containers := getContainers() 552 c.Assert(containers, checker.HasLen, instances) 553 var toRemove string 554 for i := range containers { 555 toRemove = i 556 } 557 558 _, err := containers[toRemove].Cmd("stop", toRemove) 559 c.Assert(err, checker.IsNil) 560 561 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) 562 563 containers2 := getContainers() 564 c.Assert(containers2, checker.HasLen, instances) 565 for i := range containers { 566 if i == toRemove { 567 c.Assert(containers2[i], checker.IsNil) 568 } else { 569 c.Assert(containers2[i], checker.NotNil) 570 } 571 } 572 573 containers = containers2 574 for i := range containers { 575 toRemove = i 576 } 577 578 // try with killing process outside of docker 579 pidStr, err := containers[toRemove].Cmd("inspect", "-f", "{{.State.Pid}}", toRemove) 580 c.Assert(err, checker.IsNil) 581 pid, err := strconv.Atoi(strings.TrimSpace(pidStr)) 582 c.Assert(err, checker.IsNil) 583 c.Assert(syscall.Kill(pid, syscall.SIGKILL), checker.IsNil) 584 585 time.Sleep(time.Second) // give some time to handle the signal 586 587 waitAndAssert(c, defaultReconciliationTimeout, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals, instances) 588 589 containers2 = getContainers() 590 c.Assert(containers2, checker.HasLen, instances) 591 for i := range containers { 592 if i == toRemove { 593 c.Assert(containers2[i], checker.IsNil) 594 } else { 595 c.Assert(containers2[i], checker.NotNil) 596 } 597 } 598 }