github.com/khulnasoft-lab/khulnasoft@v26.0.1-0.20240328202558-330a6f959fe0+incompatible/integration-cli/docker_api_swarm_service_test.go (about) 1 //go:build !windows 2 3 package main 4 5 import ( 6 "fmt" 7 "strconv" 8 "strings" 9 "testing" 10 "time" 11 12 "github.com/docker/docker/api/types" 13 "github.com/docker/docker/api/types/swarm" 14 "github.com/docker/docker/integration-cli/checker" 15 "github.com/docker/docker/integration-cli/cli" 16 "github.com/docker/docker/integration-cli/cli/build" 17 "github.com/docker/docker/integration-cli/daemon" 18 "github.com/docker/docker/testutil" 19 testdaemon "github.com/docker/docker/testutil/daemon" 20 "golang.org/x/sys/unix" 21 "gotest.tools/v3/assert" 22 "gotest.tools/v3/icmd" 23 "gotest.tools/v3/poll" 24 ) 25 26 func setPortConfig(portConfig []swarm.PortConfig) testdaemon.ServiceConstructor { 27 return func(s *swarm.Service) { 28 if s.Spec.EndpointSpec == nil { 29 s.Spec.EndpointSpec = &swarm.EndpointSpec{} 30 } 31 s.Spec.EndpointSpec.Ports = portConfig 32 } 33 } 34 35 func (s *DockerSwarmSuite) TestAPIServiceUpdatePort(c *testing.T) { 36 ctx := testutil.GetContext(c) 37 d := s.AddDaemon(ctx, c, true, true) 38 39 // Create a service with a port mapping of 8080:8081. 40 portConfig := []swarm.PortConfig{{TargetPort: 8081, PublishedPort: 8080}} 41 serviceID := d.CreateService(ctx, c, simpleTestService, setInstances(1), setPortConfig(portConfig)) 42 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 43 44 // Update the service: changed the port mapping from 8080:8081 to 8082:8083. 45 updatedPortConfig := []swarm.PortConfig{{TargetPort: 8083, PublishedPort: 8082}} 46 remoteService := d.GetService(ctx, c, serviceID) 47 d.UpdateService(ctx, c, remoteService, setPortConfig(updatedPortConfig)) 48 49 // Inspect the service and verify port mapping. 50 updatedService := d.GetService(ctx, c, serviceID) 51 assert.Assert(c, updatedService.Spec.EndpointSpec != nil) 52 assert.Equal(c, len(updatedService.Spec.EndpointSpec.Ports), 1) 53 assert.Equal(c, updatedService.Spec.EndpointSpec.Ports[0].TargetPort, uint32(8083)) 54 assert.Equal(c, updatedService.Spec.EndpointSpec.Ports[0].PublishedPort, uint32(8082)) 55 } 56 57 func (s *DockerSwarmSuite) TestAPISwarmServicesEmptyList(c *testing.T) { 58 ctx := testutil.GetContext(c) 59 d := s.AddDaemon(ctx, c, true, true) 60 61 services := d.ListServices(ctx, c) 62 assert.Assert(c, services != nil) 63 assert.Assert(c, len(services) == 0, "services: %#v", services) 64 } 65 66 func (s *DockerSwarmSuite) TestAPISwarmServicesCreate(c *testing.T) { 67 ctx := testutil.GetContext(c) 68 d := s.AddDaemon(ctx, c, true, true) 69 70 instances := 2 71 id := d.CreateService(ctx, c, simpleTestService, setInstances(instances)) 72 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 73 74 client := d.NewClientT(c) 75 defer client.Close() 76 77 options := types.ServiceInspectOptions{InsertDefaults: true} 78 79 // insertDefaults inserts UpdateConfig when service is fetched by ID 80 resp, _, err := client.ServiceInspectWithRaw(ctx, id, options) 81 out := fmt.Sprintf("%+v", resp) 82 assert.NilError(c, err) 83 assert.Assert(c, strings.Contains(out, "UpdateConfig")) 84 85 // insertDefaults inserts UpdateConfig when service is fetched by ID 86 resp, _, err = client.ServiceInspectWithRaw(ctx, "top", options) 87 out = fmt.Sprintf("%+v", resp) 88 assert.NilError(c, err) 89 assert.Assert(c, strings.Contains(out, "UpdateConfig")) 90 91 service := d.GetService(ctx, c, id) 92 instances = 5 93 d.UpdateService(ctx, c, service, setInstances(instances)) 94 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 95 96 d.RemoveService(ctx, c, service.ID) 97 poll.WaitOn(c, pollCheck(c, d.CheckActiveContainerCount(ctx), checker.Equals(0)), poll.WithTimeout(defaultReconciliationTimeout)) 98 } 99 100 func (s *DockerSwarmSuite) TestAPISwarmServicesMultipleAgents(c *testing.T) { 101 ctx := testutil.GetContext(c) 102 d1 := s.AddDaemon(ctx, c, true, true) 103 d2 := s.AddDaemon(ctx, c, true, false) 104 d3 := s.AddDaemon(ctx, c, true, false) 105 106 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept tasks 107 108 instances := 9 109 id := d1.CreateService(ctx, c, simpleTestService, setInstances(instances)) 110 111 poll.WaitOn(c, pollCheck(c, d1.CheckActiveContainerCount(ctx), checker.GreaterThan(0)), poll.WithTimeout(defaultReconciliationTimeout)) 112 poll.WaitOn(c, pollCheck(c, d2.CheckActiveContainerCount(ctx), checker.GreaterThan(0)), poll.WithTimeout(defaultReconciliationTimeout)) 113 poll.WaitOn(c, pollCheck(c, d3.CheckActiveContainerCount(ctx), checker.GreaterThan(0)), poll.WithTimeout(defaultReconciliationTimeout)) 114 115 poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d2.CheckActiveContainerCount(ctx), d3.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 116 117 // reconciliation on d2 node down 118 d2.Stop(c) 119 120 poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d3.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 121 122 // test downscaling 123 instances = 5 124 d1.UpdateService(ctx, c, d1.GetService(ctx, c, id), setInstances(instances)) 125 poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d3.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 126 } 127 128 func (s *DockerSwarmSuite) TestAPISwarmServicesCreateGlobal(c *testing.T) { 129 ctx := testutil.GetContext(c) 130 d1 := s.AddDaemon(ctx, c, true, true) 131 d2 := s.AddDaemon(ctx, c, true, false) 132 d3 := s.AddDaemon(ctx, c, true, false) 133 134 d1.CreateService(ctx, c, simpleTestService, setGlobalMode) 135 136 poll.WaitOn(c, pollCheck(c, d1.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 137 poll.WaitOn(c, pollCheck(c, d2.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 138 poll.WaitOn(c, pollCheck(c, d3.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 139 140 d4 := s.AddDaemon(ctx, c, true, false) 141 d5 := s.AddDaemon(ctx, c, true, false) 142 143 poll.WaitOn(c, pollCheck(c, d4.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 144 poll.WaitOn(c, pollCheck(c, d5.CheckActiveContainerCount(ctx), checker.Equals(1)), poll.WithTimeout(defaultReconciliationTimeout)) 145 } 146 147 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdate(c *testing.T) { 148 ctx := testutil.GetContext(c) 149 const nodeCount = 3 150 var daemons [nodeCount]*daemon.Daemon 151 for i := 0; i < nodeCount; i++ { 152 daemons[i] = s.AddDaemon(ctx, c, true, i == 0) 153 } 154 // wait for nodes ready 155 poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount(ctx), checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second)) 156 157 // service image at start 158 image1 := "busybox:latest" 159 // target image in update 160 image2 := "busybox:test" 161 162 // create a different tag 163 for _, d := range daemons { 164 out, err := d.Cmd("tag", image1, image2) 165 assert.NilError(c, err, out) 166 } 167 168 // create service 169 instances := 5 170 parallelism := 2 171 rollbackParallelism := 3 172 id := daemons[0].CreateService(ctx, c, serviceForUpdate, setInstances(instances)) 173 174 // wait for tasks ready 175 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 176 177 // issue service update 178 service := daemons[0].GetService(ctx, c, id) 179 daemons[0].UpdateService(ctx, c, service, setImage(image2)) 180 181 // first batch 182 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances - parallelism, image2: parallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 183 184 // 2nd batch 185 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 186 187 // 3nd batch 188 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image2: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 189 190 // Roll back to the previous version. This uses the CLI because 191 // rollback used to be a client-side operation. 192 out, err := daemons[0].Cmd("service", "update", "--detach", "--rollback", id) 193 assert.NilError(c, err, out) 194 195 // first batch 196 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image2: instances - rollbackParallelism, image1: rollbackParallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 197 198 // 2nd batch 199 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 200 } 201 202 func (s *DockerSwarmSuite) TestAPISwarmServicesUpdateStartFirst(c *testing.T) { 203 ctx := testutil.GetContext(c) 204 d := s.AddDaemon(ctx, c, true, true) 205 206 // service image at start 207 image1 := "busybox:latest" 208 // target image in update 209 image2 := "testhealth:latest" 210 211 // service started from this image won't pass health check 212 result := cli.BuildCmd(c, image2, cli.Daemon(d), 213 build.WithDockerfile(`FROM busybox 214 HEALTHCHECK --interval=1s --timeout=30s --retries=1024 \ 215 CMD cat /status`), 216 ) 217 result.Assert(c, icmd.Success) 218 219 // create service 220 instances := 5 221 parallelism := 2 222 rollbackParallelism := 3 223 id := d.CreateService(ctx, c, serviceForUpdate, setInstances(instances), setUpdateOrder(swarm.UpdateOrderStartFirst), setRollbackOrder(swarm.UpdateOrderStartFirst)) 224 225 checkStartingTasks := func(expected int) []swarm.Task { 226 var startingTasks []swarm.Task 227 poll.WaitOn(c, pollCheck(c, func(c *testing.T) (interface{}, string) { 228 tasks := d.GetServiceTasks(ctx, c, id) 229 startingTasks = nil 230 for _, t := range tasks { 231 if t.Status.State == swarm.TaskStateStarting { 232 startingTasks = append(startingTasks, t) 233 } 234 } 235 return startingTasks, "" 236 }, checker.HasLen(expected)), poll.WithTimeout(defaultReconciliationTimeout)) 237 238 return startingTasks 239 } 240 241 makeTasksHealthy := func(tasks []swarm.Task) { 242 for _, t := range tasks { 243 containerID := t.Status.ContainerStatus.ContainerID 244 d.Cmd("exec", containerID, "touch", "/status") 245 } 246 } 247 248 // wait for tasks ready 249 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 250 251 // issue service update 252 service := d.GetService(ctx, c, id) 253 d.UpdateService(ctx, c, service, setImage(image2)) 254 255 // first batch 256 257 // The old tasks should be running, and the new ones should be starting. 258 startingTasks := checkStartingTasks(parallelism) 259 260 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 261 262 // make it healthy 263 makeTasksHealthy(startingTasks) 264 265 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances - parallelism, image2: parallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 266 267 // 2nd batch 268 269 // The old tasks should be running, and the new ones should be starting. 270 startingTasks = checkStartingTasks(parallelism) 271 272 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances - parallelism, image2: parallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 273 274 // make it healthy 275 makeTasksHealthy(startingTasks) 276 277 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 278 279 // 3nd batch 280 281 // The old tasks should be running, and the new ones should be starting. 282 startingTasks = checkStartingTasks(1) 283 284 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances - 2*parallelism, image2: 2 * parallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 285 286 // make it healthy 287 makeTasksHealthy(startingTasks) 288 289 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image2: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 290 291 // Roll back to the previous version. This uses the CLI because 292 // rollback is a client-side operation. 293 out, err := d.Cmd("service", "update", "--detach", "--rollback", id) 294 assert.NilError(c, err, out) 295 296 // first batch 297 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image2: instances - rollbackParallelism, image1: rollbackParallelism})), poll.WithTimeout(defaultReconciliationTimeout)) 298 299 // 2nd batch 300 poll.WaitOn(c, pollCheck(c, d.CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 301 } 302 303 func (s *DockerSwarmSuite) TestAPISwarmServicesFailedUpdate(c *testing.T) { 304 ctx := testutil.GetContext(c) 305 const nodeCount = 3 306 var daemons [nodeCount]*daemon.Daemon 307 for i := 0; i < nodeCount; i++ { 308 daemons[i] = s.AddDaemon(ctx, c, true, i == 0) 309 } 310 // wait for nodes ready 311 poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount(ctx), checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second)) 312 313 // service image at start 314 image1 := "busybox:latest" 315 // target image in update 316 image2 := "busybox:badtag" 317 318 // create service 319 instances := 5 320 id := daemons[0].CreateService(ctx, c, serviceForUpdate, setInstances(instances)) 321 322 // wait for tasks ready 323 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 324 325 // issue service update 326 service := daemons[0].GetService(ctx, c, id) 327 daemons[0].UpdateService(ctx, c, service, setImage(image2), setFailureAction(swarm.UpdateFailureActionPause), setMaxFailureRatio(0.25), setParallelism(1)) 328 329 // should update 2 tasks and then pause 330 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceUpdateState(ctx, id), checker.Equals(swarm.UpdateStatePaused)), poll.WithTimeout(defaultReconciliationTimeout)) 331 v, _ := daemons[0].CheckServiceRunningTasks(ctx, id)(c) 332 assert.Assert(c, v == instances-2) 333 334 // Roll back to the previous version. This uses the CLI because 335 // rollback used to be a client-side operation. 336 out, err := daemons[0].Cmd("service", "update", "--detach", "--rollback", id) 337 assert.NilError(c, err, out) 338 339 poll.WaitOn(c, pollCheck(c, daemons[0].CheckRunningTaskImages(ctx), checker.DeepEquals(map[string]int{image1: instances})), poll.WithTimeout(defaultReconciliationTimeout)) 340 } 341 342 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintRole(c *testing.T) { 343 ctx := testutil.GetContext(c) 344 const nodeCount = 3 345 var daemons [nodeCount]*daemon.Daemon 346 for i := 0; i < nodeCount; i++ { 347 daemons[i] = s.AddDaemon(ctx, c, true, i == 0) 348 } 349 // wait for nodes ready 350 poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount(ctx), checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second)) 351 352 // create service 353 constraints := []string{"node.role==worker"} 354 instances := 3 355 id := daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances)) 356 // wait for tasks ready 357 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 358 // validate tasks are running on worker nodes 359 tasks := daemons[0].GetServiceTasks(ctx, c, id) 360 for _, task := range tasks { 361 node := daemons[0].GetNode(ctx, c, task.NodeID) 362 assert.Equal(c, node.Spec.Role, swarm.NodeRoleWorker) 363 } 364 // remove service 365 daemons[0].RemoveService(ctx, c, id) 366 367 // create service 368 constraints = []string{"node.role!=worker"} 369 id = daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances)) 370 // wait for tasks ready 371 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 372 tasks = daemons[0].GetServiceTasks(ctx, c, id) 373 // validate tasks are running on manager nodes 374 for _, task := range tasks { 375 node := daemons[0].GetNode(ctx, c, task.NodeID) 376 assert.Equal(c, node.Spec.Role, swarm.NodeRoleManager) 377 } 378 // remove service 379 daemons[0].RemoveService(ctx, c, id) 380 381 // create service 382 constraints = []string{"node.role==nosuchrole"} 383 id = daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances)) 384 // wait for tasks created 385 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 386 // let scheduler try 387 time.Sleep(250 * time.Millisecond) 388 // validate tasks are not assigned to any node 389 tasks = daemons[0].GetServiceTasks(ctx, c, id) 390 for _, task := range tasks { 391 assert.Equal(c, task.NodeID, "") 392 } 393 } 394 395 func (s *DockerSwarmSuite) TestAPISwarmServiceConstraintLabel(c *testing.T) { 396 ctx := testutil.GetContext(c) 397 const nodeCount = 3 398 var daemons [nodeCount]*daemon.Daemon 399 for i := 0; i < nodeCount; i++ { 400 daemons[i] = s.AddDaemon(ctx, c, true, i == 0) 401 } 402 // wait for nodes ready 403 poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount(ctx), checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second)) 404 nodes := daemons[0].ListNodes(ctx, c) 405 assert.Equal(c, len(nodes), nodeCount) 406 407 // add labels to nodes 408 daemons[0].UpdateNode(ctx, c, nodes[0].ID, func(n *swarm.Node) { 409 n.Spec.Annotations.Labels = map[string]string{ 410 "security": "high", 411 } 412 }) 413 for i := 1; i < nodeCount; i++ { 414 daemons[0].UpdateNode(ctx, c, nodes[i].ID, func(n *swarm.Node) { 415 n.Spec.Annotations.Labels = map[string]string{ 416 "security": "low", 417 } 418 }) 419 } 420 421 // create service 422 instances := 3 423 constraints := []string{"node.labels.security==high"} 424 id := daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances)) 425 // wait for tasks ready 426 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 427 tasks := daemons[0].GetServiceTasks(ctx, c, id) 428 // validate all tasks are running on nodes[0] 429 for _, task := range tasks { 430 assert.Assert(c, task.NodeID == nodes[0].ID) 431 } 432 // remove service 433 daemons[0].RemoveService(ctx, c, id) 434 435 // create service 436 constraints = []string{"node.labels.security!=high"} 437 id = daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances)) 438 // wait for tasks ready 439 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 440 tasks = daemons[0].GetServiceTasks(ctx, c, id) 441 // validate all tasks are NOT running on nodes[0] 442 for _, task := range tasks { 443 assert.Assert(c, task.NodeID != nodes[0].ID) 444 } 445 // remove service 446 daemons[0].RemoveService(ctx, c, id) 447 448 constraints = []string{"node.labels.security==medium"} 449 id = daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances)) 450 // wait for tasks created 451 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 452 // let scheduler try 453 time.Sleep(250 * time.Millisecond) 454 tasks = daemons[0].GetServiceTasks(ctx, c, id) 455 // validate tasks are not assigned 456 for _, task := range tasks { 457 assert.Assert(c, task.NodeID == "") 458 } 459 // remove service 460 daemons[0].RemoveService(ctx, c, id) 461 462 // multiple constraints 463 constraints = []string{ 464 "node.labels.security==high", 465 fmt.Sprintf("node.id==%s", nodes[1].ID), 466 } 467 id = daemons[0].CreateService(ctx, c, simpleTestService, setConstraints(constraints), setInstances(instances)) 468 // wait for tasks created 469 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 470 // let scheduler try 471 time.Sleep(250 * time.Millisecond) 472 tasks = daemons[0].GetServiceTasks(ctx, c, id) 473 // validate tasks are not assigned 474 for _, task := range tasks { 475 assert.Assert(c, task.NodeID == "") 476 } 477 // make nodes[1] fulfills the constraints 478 daemons[0].UpdateNode(ctx, c, nodes[1].ID, func(n *swarm.Node) { 479 n.Spec.Annotations.Labels = map[string]string{ 480 "security": "high", 481 } 482 }) 483 // wait for tasks ready 484 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 485 tasks = daemons[0].GetServiceTasks(ctx, c, id) 486 for _, task := range tasks { 487 assert.Assert(c, task.NodeID == nodes[1].ID) 488 } 489 } 490 491 func (s *DockerSwarmSuite) TestAPISwarmServicePlacementPrefs(c *testing.T) { 492 ctx := testutil.GetContext(c) 493 494 const nodeCount = 3 495 var daemons [nodeCount]*daemon.Daemon 496 for i := 0; i < nodeCount; i++ { 497 daemons[i] = s.AddDaemon(ctx, c, true, i == 0) 498 } 499 // wait for nodes ready 500 poll.WaitOn(c, pollCheck(c, daemons[0].CheckNodeReadyCount(ctx), checker.Equals(nodeCount)), poll.WithTimeout(5*time.Second)) 501 nodes := daemons[0].ListNodes(ctx, c) 502 assert.Equal(c, len(nodes), nodeCount) 503 504 // add labels to nodes 505 daemons[0].UpdateNode(ctx, c, nodes[0].ID, func(n *swarm.Node) { 506 n.Spec.Annotations.Labels = map[string]string{ 507 "rack": "a", 508 } 509 }) 510 for i := 1; i < nodeCount; i++ { 511 daemons[0].UpdateNode(ctx, c, nodes[i].ID, func(n *swarm.Node) { 512 n.Spec.Annotations.Labels = map[string]string{ 513 "rack": "b", 514 } 515 }) 516 } 517 518 // create service 519 instances := 4 520 prefs := []swarm.PlacementPreference{{Spread: &swarm.SpreadOver{SpreadDescriptor: "node.labels.rack"}}} 521 id := daemons[0].CreateService(ctx, c, simpleTestService, setPlacementPrefs(prefs), setInstances(instances)) 522 // wait for tasks ready 523 poll.WaitOn(c, pollCheck(c, daemons[0].CheckServiceRunningTasks(ctx, id), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 524 tasks := daemons[0].GetServiceTasks(ctx, c, id) 525 // validate all tasks are running on nodes[0] 526 tasksOnNode := make(map[string]int) 527 for _, task := range tasks { 528 tasksOnNode[task.NodeID]++ 529 } 530 assert.Assert(c, tasksOnNode[nodes[0].ID] == 2) 531 assert.Assert(c, tasksOnNode[nodes[1].ID] == 1) 532 assert.Assert(c, tasksOnNode[nodes[2].ID] == 1) 533 } 534 535 func (s *DockerSwarmSuite) TestAPISwarmServicesStateReporting(c *testing.T) { 536 testRequires(c, testEnv.IsLocalDaemon) 537 testRequires(c, DaemonIsLinux) 538 ctx := testutil.GetContext(c) 539 540 d1 := s.AddDaemon(ctx, c, true, true) 541 d2 := s.AddDaemon(ctx, c, true, true) 542 d3 := s.AddDaemon(ctx, c, true, false) 543 544 time.Sleep(1 * time.Second) // make sure all daemons are ready to accept 545 546 instances := 9 547 d1.CreateService(ctx, c, simpleTestService, setInstances(instances)) 548 549 poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d2.CheckActiveContainerCount(ctx), d3.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 550 551 getContainers := func() map[string]*daemon.Daemon { 552 m := make(map[string]*daemon.Daemon) 553 for _, d := range []*daemon.Daemon{d1, d2, d3} { 554 for _, id := range d.ActiveContainers(testutil.GetContext(c), c) { 555 m[id] = d 556 } 557 } 558 return m 559 } 560 561 containers := getContainers() 562 assert.Assert(c, len(containers) == instances) 563 var toRemove string 564 for i := range containers { 565 toRemove = i 566 } 567 568 _, err := containers[toRemove].Cmd("stop", toRemove) 569 assert.NilError(c, err) 570 571 poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d2.CheckActiveContainerCount(ctx), d3.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 572 573 containers2 := getContainers() 574 assert.Assert(c, len(containers2) == instances) 575 for i := range containers { 576 if i == toRemove { 577 assert.Assert(c, containers2[i] == nil) 578 } else { 579 assert.Assert(c, containers2[i] != nil) 580 } 581 } 582 583 containers = containers2 584 for i := range containers { 585 toRemove = i 586 } 587 588 // try with killing process outside of docker 589 pidStr, err := containers[toRemove].Cmd("inspect", "-f", "{{.State.Pid}}", toRemove) 590 assert.NilError(c, err) 591 pid, err := strconv.Atoi(strings.TrimSpace(pidStr)) 592 assert.NilError(c, err) 593 assert.NilError(c, unix.Kill(pid, unix.SIGKILL)) 594 595 time.Sleep(time.Second) // give some time to handle the signal 596 597 poll.WaitOn(c, pollCheck(c, reducedCheck(sumAsIntegers, d1.CheckActiveContainerCount(ctx), d2.CheckActiveContainerCount(ctx), d3.CheckActiveContainerCount(ctx)), checker.Equals(instances)), poll.WithTimeout(defaultReconciliationTimeout)) 598 599 containers2 = getContainers() 600 assert.Assert(c, len(containers2) == instances) 601 for i := range containers { 602 if i == toRemove { 603 assert.Assert(c, containers2[i] == nil) 604 } else { 605 assert.Assert(c, containers2[i] != nil) 606 } 607 } 608 }