github.com/kaisenlinux/docker@v0.0.0-20230510090727-ea55db55fac7/engine/integration-cli/docker_api_swarm_service_test.go (about) 1 //go:build !windows 2 // +build !windows 3 4 package main 5 6 import ( 7 "context" 8 "fmt" 9 "strconv" 10 "strings" 11 "testing" 12 "time" 13 14 "github.com/docker/docker/api/types" 15 "github.com/docker/docker/api/types/swarm" 16 "github.com/docker/docker/integration-cli/checker" 17 "github.com/docker/docker/integration-cli/cli" 18 "github.com/docker/docker/integration-cli/cli/build" 19 "github.com/docker/docker/integration-cli/daemon" 20 testdaemon "github.com/docker/docker/testutil/daemon" 21 "golang.org/x/sys/unix" 22 "gotest.tools/v3/assert" 23 "gotest.tools/v3/icmd" 24 "gotest.tools/v3/poll" 25 ) 26 27 func setPortConfig(portConfig []swarm.PortConfig) testdaemon.ServiceConstructor { 28 return func(s *swarm.Service) { 29 if s.Spec.EndpointSpec == nil { 30 s.Spec.EndpointSpec = &swarm.EndpointSpec{} 31 } 32 s.Spec.EndpointSpec.Ports = portConfig 33 } 34 } 35 36 func (s *DockerSwarmSuite) TestAPIServiceUpdatePort(c *testing.T) { 37 d := s.AddDaemon(c, true, true) 38 39 // Create a service with a port mapping of 8080:8081. 40 portConfig := []swarm.PortConfig{{TargetPort: 8081, PublishedPort: 8080}} 41 serviceID := d.CreateService(c, simpleTestService, setInstances(1), setPortConfig(portConfig)) 42 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount, checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 43 44 // Update the service: changed the port mapping from 8080:8081 to 8082:8083. 45 updatedPortConfig := []swarm.PortConfig{{TargetPort: 8083, PublishedPort: 8082}} 46 remoteService := d.GetService(c, serviceID) 47 d.UpdateService(c, remoteService, setPortConfig(updatedPortConfig)) 48 49 // Inspect the service and verify port mapping. 50 updatedService := d.GetService(c, serviceID) 51 assert.Assert(c, updatedService.Spec.EndpointSpec != nil) 52 assert.Equal(c, len(updatedService.Spec.EndpointSpec.Ports), 1) 53 assert.Equal(c, updatedService.Spec.EndpointSpec.Ports[0].TargetPort, uint32(8083)) 54 assert.Equal(c, updatedService.Spec.EndpointSpec.Ports[0].PublishedPort, uint32(8082)) 55 } 56 57 func (s *DockerSwarmSuite) TestAPISwarmServicesEmptyList(c *testing.T) { 58 d := s.AddDaemon(c, true, true) 59 60 services := d.ListServices(c) 61 assert.Assert(c, services != nil) 62 assert.Assert(c, len(services) == 0, "services: %#v", services) 63 } 64 65 func (s *DockerSwarmSuite) TestAPISwarmServicesCreate(c *testing.T) { 66 d := s.AddDaemon(c, true, true) 67 68 instances := 2 69 id := d.CreateService(c, simpleTestService, setInstances(instances)) 70 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount, checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 71 72 client := d.NewClientT(c) 73 defer client.Close() 74 75 options := types.ServiceInspectOptions{InsertDefaults: true} 76 77 // insertDefaults inserts UpdateConfig when service is fetched by ID 78 resp, _, err := client.ServiceInspectWithRaw(context.Background(), id, options) 79 out := fmt.Sprintf("%+v", resp) 80 assert.NilError(c, err) 81 assert.Assert(c, strings.Contains(out, "UpdateConfig")) 82 83 // insertDefaults inserts UpdateConfig when service is fetched by ID 84 resp, _, err = client.ServiceInspectWithRaw(context.Background(), "top", options) 85 out = fmt.Sprintf("%+v", resp) 86 assert.NilError(c, err) 87 assert.Assert(c, strings.Contains(out, "UpdateConfig")) 88 89 service := d.GetService(c, id) 90 instances = 5 91 d.UpdateService(c, service, setInstances(instances)) 92 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount, checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 93 94 d.RemoveService(c, service.ID) 95 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount, checker.Equals(0)), poll.WithTimeout(defaultReconciliationTimeout)) 96 } 97 98 func (s *DockerSwarmSuite) TestAPISwarmServicesMultipleAgents(c *testing.T) { 99 d1 := s.AddDaemon(c, true, true) 100 d2 := s.AddDaemon(c, true, false) 101 d3 := s.AddDaemon(c, true, false) 102 103 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks 104 105 instances := 9 106 id := d1.CreateService(c, simpleTestService, setInstances(instances)) 107 108 poll.WaitOn(c, pollCheck(c, d1.CheckActiveContainerCount, checker.GreaterThan(0)), poll.WithTimeout(defaultReconciliationTimeout)) 109 poll.WaitOn(c, pollCheck(c, d2.CheckActiveContainerCount, checker.GreaterThan(0)), poll.WithTimeout(defaultReconciliationTimeout)) 110 poll.WaitOn(c, pollCheck(c, d3.CheckActiveContainerCount, checker.GreaterThan(0)), poll.WithTimeout(defaultReconciliationTimeout)) 111 112 poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 113 114 // reconciliation on d2 node down 115 d2.Stop(c) 116 117 poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 118 119 // test downscaling 120 instances = 5 121 d1.UpdateService(c, d1.GetService(c, id), setInstances(instances)) 122 poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 123 124 } 125 126 func (s *DockerSwarmSuite) TestAPISwarmServicesCreateGlobal(c *testing.T) { 127 d1 := s.AddDaemon(c, true, true) 128 d2 := s.AddDaemon(c, true, false) 129 d3 := s.AddDaemon(c, true, false) 130 131 d1.CreateService(c, simpleTestService, setGlobalMode) 132 133 poll.WaitOn(c, pollCheck(c, d1.CheckActiveContainerCount, checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 134 poll.WaitOn(c, pollCheck(c, d2.CheckActiveContainerCount, checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 135 poll.WaitOn(c, pollCheck(c, d3.CheckActiveContainerCount, checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 136 137 d4 := s.AddDaemon(c, true, false) 138 d5 := s.AddDaemon(c, true, false) 139 140 poll.WaitOn(c, pollCheck(c, d4.CheckActiveContainerCount, checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 141 poll.WaitOn(c, pollCheck(c, d5.CheckActiveContainerCount, checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 142 } 143 144 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdate(c *testing.T) { 145 const nodeCount = 3 146 var daemons [nodeCount]*daemon.Daemon 147 for i := 0; i < nodeCount; i++ { 148 daemons[i] = s.AddDaemon(c, true, i == 0) 149 } 150 // wait for nodes ready 151 poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount, checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second)) 152 153 // service image at start 154 image1 := "busybox:latest" 155 // target image in update 156 image2 := "busybox:test" 157 158 // create a different tag 159 for _, d := range daemons { 160 out, err := d.Cmd("tag", image1, image2) 161 assert.NilError(c, err, out) 162 } 163 164 // create service 165 instances := 5 166 parallelism := 2 167 rollbackParallelism := 3 168 id := daemons[0].CreateService(c, serviceForUpdate, setInstances(instances)) 169 170 // wait for tasks ready 171 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages, checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 172 173 // issue service update 174 service := daemons[0].GetService(c, id) 175 daemons[0].UpdateService(c, service, setImage(image2)) 176 177 // first batch 178 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages, checker.DeepEquals(map[string]int{image1: instances - parallelism, image2: parallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 179 180 // 2nd batch 181 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages, checker.DeepEquals(map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 182 183 // 3nd batch 184 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages, checker.DeepEquals(map[string]int{image2: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 185 186 // Roll back to the previous version. This uses the CLI because 187 // rollback used to be a client-side operation. 188 out, err := daemons[0].Cmd("service", "update", "--detach", "--rollback", id) 189 assert.NilError(c, err, out) 190 191 // first batch 192 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages, checker.DeepEquals(map[string]int{image2: instances - rollbackParallelism, image1: rollbackParallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 193 194 // 2nd batch 195 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages, checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 196 197 } 198 199 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateStartFirst(c *testing.T) { 200 d := s.AddDaemon(c, true, true) 201 202 // service image at start 203 image1 := "busybox:latest" 204 // target image in update 205 image2 := "testhealth:latest" 206 207 // service started from this image won't pass health check 208 result := cli.BuildCmd(c, image2, cli.Daemon(d), 209 build.WithDockerfile(`FROM busybox 210 HEALTHCHECK --interval=1s --timeout=30s --retries=1024 \ 211 CMD cat /status`), 212 ) 213 result.Assert(c, icmd.Success) 214 215 // create service 216 instances := 5 217 parallelism := 2 218 rollbackParallelism := 3 219 id := d.CreateService(c, serviceForUpdate, setInstances(instances), setUpdateOrder(swarm.UpdateOrderStartFirst), setRollbackOrder(swarm.UpdateOrderStartFirst)) 220 221 checkStartingTasks := func(expected int) []swarm.Task { 222 var startingTasks []swarm.Task 223 poll.WaitOn(c, pollCheck(c, func(c *testing.T) (interface{}, string) { 224 tasks := d.GetServiceTasks(c, id) 225 startingTasks = nil 226 for _, t := range tasks { 227 if t.Status.State == swarm.TaskStateStarting { 228 startingTasks = append(startingTasks, t) 229 } 230 } 231 return startingTasks, "" 232 }, checker.HasLen(expected)), poll.WithTimeout(defaultReconciliationTimeout)) 233 234 return startingTasks 235 } 236 237 makeTasksHealthy := func(tasks []swarm.Task) { 238 for _, t := range tasks { 239 containerID := t.Status.ContainerStatus.ContainerID 240 d.Cmd("exec", containerID, "touch", "/status") 241 } 242 } 243 244 // wait for tasks ready 245 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages, checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 246 247 // issue service update 248 service := d.GetService(c, id) 249 d.UpdateService(c, service, setImage(image2)) 250 251 // first batch 252 253 // The old tasks should be running, and the new ones should be starting. 254 startingTasks := checkStartingTasks(parallelism) 255 256 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages, checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 257 258 // make it healthy 259 makeTasksHealthy(startingTasks) 260 261 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages, checker.DeepEquals(map[string]int{image1: instances - parallelism, image2: parallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 262 263 // 2nd batch 264 265 // The old tasks should be running, and the new ones should be starting. 266 startingTasks = checkStartingTasks(parallelism) 267 268 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages, checker.DeepEquals(map[string]int{image1: instances - parallelism, image2: parallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 269 270 // make it healthy 271 makeTasksHealthy(startingTasks) 272 273 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages, checker.DeepEquals(map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 274 275 // 3nd batch 276 277 // The old tasks should be running, and the new ones should be starting. 278 startingTasks = checkStartingTasks(1) 279 280 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages, checker.DeepEquals(map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 281 282 // make it healthy 283 makeTasksHealthy(startingTasks) 284 285 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages, checker.DeepEquals(map[string]int{image2: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 286 287 // Roll back to the previous version. This uses the CLI because 288 // rollback is a client-side operation. 289 out, err := d.Cmd("service", "update", "--detach", "--rollback", id) 290 assert.NilError(c, err, out) 291 292 // first batch 293 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages, checker.DeepEquals(map[string]int{image2: instances - rollbackParallelism, image1: rollbackParallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 294 295 // 2nd batch 296 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages, checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 297 298 } 299 300 func (s *DockerSwarmSuite) TestAPISwarmServicesFailedUpdate(c *testing.T) { 301 const nodeCount = 3 302 var daemons [nodeCount]*daemon.Daemon 303 for i := 0; i < nodeCount; i++ { 304 daemons[i] = s.AddDaemon(c, true, i == 0) 305 } 306 // wait for nodes ready 307 poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount, checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second)) 308 309 // service image at start 310 image1 := "busybox:latest" 311 // target image in update 312 image2 := "busybox:badtag" 313 314 // create service 315 instances := 5 316 id := daemons[0].CreateService(c, serviceForUpdate, setInstances(instances)) 317 318 // wait for tasks ready 319 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages, checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 320 321 // issue service update 322 service := daemons[0].GetService(c, id) 323 daemons[0].UpdateService(c, service, setImage(image2), setFailureAction(swarm.UpdateFailureActionPause), setMaxFailureRatio(0.25), setParallelism(1)) 324 325 // should update 2 tasks and then pause 326 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceUpdateState(id), checker.Equals(swarm.UpdateStatePaused)), poll.WithTimeout(defaultReconciliationTimeout)) 327 v, _ := daemons[0].CheckServiceRunningTasks(id)(c) 328 assert.Assert(c, v == instances-2) 329 330 // Roll back to the previous version. This uses the CLI because 331 // rollback used to be a client-side operation. 332 out, err := daemons[0].Cmd("service", "update", "--detach", "--rollback", id) 333 assert.NilError(c, err, out) 334 335 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages, checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 336 337 } 338 339 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintRole(c *testing.T) { 340 const nodeCount = 3 341 var daemons [nodeCount]*daemon.Daemon 342 for i := 0; i < nodeCount; i++ { 343 daemons[i] = s.AddDaemon(c, true, i == 0) 344 } 345 // wait for nodes ready 346 poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount, checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second)) 347 348 // create service 349 constraints := []string{"node.role==worker"} 350 instances := 3 351 id := daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 352 // wait for tasks ready 353 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 354 // validate tasks are running on worker nodes 355 tasks := daemons[0].GetServiceTasks(c, id) 356 for _, task := range tasks { 357 node := daemons[0].GetNode(c, task.NodeID) 358 assert.Equal(c, node.Spec.Role, swarm.NodeRoleWorker) 359 } 360 // remove service 361 daemons[0].RemoveService(c, id) 362 363 // create service 364 constraints = []string{"node.role!=worker"} 365 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 366 // wait for tasks ready 367 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 368 tasks = daemons[0].GetServiceTasks(c, id) 369 // validate tasks are running on manager nodes 370 for _, task := range tasks { 371 node := daemons[0].GetNode(c, task.NodeID) 372 assert.Equal(c, node.Spec.Role, swarm.NodeRoleManager) 373 } 374 // remove service 375 daemons[0].RemoveService(c, id) 376 377 // create service 378 constraints = []string{"node.role==nosuchrole"} 379 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 380 // wait for tasks created 381 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceTasks(id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 382 // let scheduler try 383 time.Sleep(250 * time.Millisecond) 384 // validate tasks are not assigned to any node 385 tasks = daemons[0].GetServiceTasks(c, id) 386 for _, task := range tasks { 387 assert.Equal(c, task.NodeID, "") 388 } 389 } 390 391 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintLabel(c *testing.T) { 392 const nodeCount = 3 393 var daemons [nodeCount]*daemon.Daemon 394 for i := 0; i < nodeCount; i++ { 395 daemons[i] = s.AddDaemon(c, true, i == 0) 396 } 397 // wait for nodes ready 398 poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount, checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second)) 399 nodes := daemons[0].ListNodes(c) 400 assert.Equal(c, len(nodes), nodeCount) 401 402 // add labels to nodes 403 daemons[0].UpdateNode(c, nodes[0].ID, func(n *swarm.Node) { 404 n.Spec.Annotations.Labels = map[string]string{ 405 "security": "high", 406 } 407 }) 408 for i := 1; i < nodeCount; i++ { 409 daemons[0].UpdateNode(c, nodes[i].ID, func(n *swarm.Node) { 410 n.Spec.Annotations.Labels = map[string]string{ 411 "security": "low", 412 } 413 }) 414 } 415 416 // create service 417 instances := 3 418 constraints := []string{"node.labels.security==high"} 419 id := daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 420 // wait for tasks ready 421 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 422 tasks := daemons[0].GetServiceTasks(c, id) 423 // validate all tasks are running on nodes[0] 424 for _, task := range tasks { 425 assert.Assert(c, task.NodeID == nodes[0].ID) 426 } 427 // remove service 428 daemons[0].RemoveService(c, id) 429 430 // create service 431 constraints = []string{"node.labels.security!=high"} 432 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 433 // wait for tasks ready 434 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 435 tasks = daemons[0].GetServiceTasks(c, id) 436 // validate all tasks are NOT running on nodes[0] 437 for _, task := range tasks { 438 assert.Assert(c, task.NodeID != nodes[0].ID) 439 } 440 // remove service 441 daemons[0].RemoveService(c, id) 442 443 constraints = []string{"node.labels.security==medium"} 444 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 445 // wait for tasks created 446 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceTasks(id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 447 // let scheduler try 448 time.Sleep(250 * time.Millisecond) 449 tasks = daemons[0].GetServiceTasks(c, id) 450 // validate tasks are not assigned 451 for _, task := range tasks { 452 assert.Assert(c, task.NodeID == "") 453 } 454 // remove service 455 daemons[0].RemoveService(c, id) 456 457 // multiple constraints 458 constraints = []string{ 459 "node.labels.security==high", 460 fmt.Sprintf("node.id==%s", nodes[1].ID), 461 } 462 id = daemons[0].CreateService(c, simpleTestService, setConstraints(constraints), setInstances(instances)) 463 // wait for tasks created 464 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceTasks(id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 465 // let scheduler try 466 time.Sleep(250 * time.Millisecond) 467 tasks = daemons[0].GetServiceTasks(c, id) 468 // validate tasks are not assigned 469 for _, task := range tasks { 470 assert.Assert(c, task.NodeID == "") 471 } 472 // make nodes[1] fulfills the constraints 473 daemons[0].UpdateNode(c, nodes[1].ID, func(n *swarm.Node) { 474 n.Spec.Annotations.Labels = map[string]string{ 475 "security": "high", 476 } 477 }) 478 // wait for tasks ready 479 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 480 tasks = daemons[0].GetServiceTasks(c, id) 481 for _, task := range tasks { 482 assert.Assert(c, task.NodeID == nodes[1].ID) 483 } 484 } 485 486 func (s *DockerSwarmSuite) TestAPISwarmServicePlacementPrefs(c *testing.T) { 487 const nodeCount = 3 488 var daemons [nodeCount]*daemon.Daemon 489 for i := 0; i < nodeCount; i++ { 490 daemons[i] = s.AddDaemon(c, true, i == 0) 491 } 492 // wait for nodes ready 493 poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount, checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second)) 494 nodes := daemons[0].ListNodes(c) 495 assert.Equal(c, len(nodes), nodeCount) 496 497 // add labels to nodes 498 daemons[0].UpdateNode(c, nodes[0].ID, func(n *swarm.Node) { 499 n.Spec.Annotations.Labels = map[string]string{ 500 "rack": "a", 501 } 502 }) 503 for i := 1; i < nodeCount; i++ { 504 daemons[0].UpdateNode(c, nodes[i].ID, func(n *swarm.Node) { 505 n.Spec.Annotations.Labels = map[string]string{ 506 "rack": "b", 507 } 508 }) 509 } 510 511 // create service 512 instances := 4 513 prefs := []swarm.PlacementPreference{{Spread: &swarm.SpreadOver{SpreadDescriptor: "node.labels.rack"}}} 514 id := daemons[0].CreateService(c, simpleTestService, setPlacementPrefs(prefs), setInstances(instances)) 515 // wait for tasks ready 516 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 517 tasks := daemons[0].GetServiceTasks(c, id) 518 // validate all tasks are running on nodes[0] 519 tasksOnNode := make(map[string]int) 520 for _, task := range tasks { 521 tasksOnNode[task.NodeID]++ 522 } 523 assert.Assert(c, tasksOnNode[nodes[0].ID] == 2) 524 assert.Assert(c, tasksOnNode[nodes[1].ID] == 1) 525 assert.Assert(c, tasksOnNode[nodes[2].ID] == 1) 526 } 527 528 func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *testing.T) { 529 testRequires(c, testEnv.IsLocalDaemon) 530 testRequires(c, DaemonIsLinux) 531 532 d1 := s.AddDaemon(c, true, true) 533 d2 := s.AddDaemon(c, true, true) 534 d3 := s.AddDaemon(c, true, false) 535 536 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept 537 538 instances := 9 539 d1.CreateService(c, simpleTestService, setInstances(instances)) 540 541 poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 542 543 getContainers := func() map[string]*daemon.Daemon { 544 m := make(map[string]*daemon.Daemon) 545 for _, d := range []*daemon.Daemon{d1, d2, d3} { 546 for _, id := range d.ActiveContainers(c) { 547 m[id] = d 548 } 549 } 550 return m 551 } 552 553 containers := getContainers() 554 assert.Assert(c, len(containers) == instances) 555 var toRemove string 556 for i := range containers { 557 toRemove = i 558 } 559 560 _, err := containers[toRemove].Cmd("stop", toRemove) 561 assert.NilError(c, err) 562 563 poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 564 565 containers2 := getContainers() 566 assert.Assert(c, len(containers2) == instances) 567 for i := range containers { 568 if i == toRemove { 569 assert.Assert(c, containers2[i] == nil) 570 } else { 571 assert.Assert(c, containers2[i] != nil) 572 } 573 } 574 575 containers = containers2 576 for i := range containers { 577 toRemove = i 578 } 579 580 // try with killing process outside of docker 581 pidStr, err := containers[toRemove].Cmd("inspect", "-f", "{{.State.Pid}}", toRemove) 582 assert.NilError(c, err) 583 pid, err := strconv.Atoi(strings.TrimSpace(pidStr)) 584 assert.NilError(c, err) 585 assert.NilError(c, unix.Kill(pid, unix.SIGKILL)) 586 587 time.Sleep(time.Second) // give some time to handle the signal 588 589 poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount, d2.CheckActiveContainerCount, d3.CheckActiveContainerCount), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 590 591 containers2 = getContainers() 592 assert.Assert(c, len(containers2) == instances) 593 for i := range containers { 594 if i == toRemove { 595 assert.Assert(c, containers2[i] == nil) 596 } else { 597 assert.Assert(c, containers2[i] != nil) 598 } 599 } 600 }