github.com/zhouyu0/docker-note@v0.0.0-20190722021225-b8d3825084db/daemon/cluster/executor/container/adapter.go (about) 1 package container // import "github.com/docker/docker/daemon/cluster/executor/container" 2 3 import ( 4 "context" 5 "encoding/base64" 6 "encoding/json" 7 "fmt" 8 "io" 9 "os" 10 "strings" 11 "syscall" 12 "time" 13 14 "github.com/docker/distribution/reference" 15 "github.com/docker/docker/api/types" 16 "github.com/docker/docker/api/types/backend" 17 containertypes "github.com/docker/docker/api/types/container" 18 "github.com/docker/docker/api/types/events" 19 containerpkg "github.com/docker/docker/container" 20 "github.com/docker/docker/daemon" 21 "github.com/docker/docker/daemon/cluster/convert" 22 executorpkg "github.com/docker/docker/daemon/cluster/executor" 23 volumeopts "github.com/docker/docker/volume/service/opts" 24 "github.com/docker/libnetwork" 25 "github.com/docker/swarmkit/agent/exec" 26 "github.com/docker/swarmkit/api" 27 "github.com/docker/swarmkit/log" 28 gogotypes "github.com/gogo/protobuf/types" 29 "github.com/opencontainers/go-digest" 30 "github.com/pkg/errors" 31 "github.com/sirupsen/logrus" 32 "golang.org/x/time/rate" 33 ) 34 35 // nodeAttachmentReadyInterval is the interval to poll 36 const nodeAttachmentReadyInterval = 100 * time.Millisecond 37 38 // containerAdapter conducts remote operations for a container. All calls 39 // are mostly naked calls to the client API, seeded with information from 40 // containerConfig. 41 type containerAdapter struct { 42 backend executorpkg.Backend 43 imageBackend executorpkg.ImageBackend 44 volumeBackend executorpkg.VolumeBackend 45 container *containerConfig 46 dependencies exec.DependencyGetter 47 } 48 49 func newContainerAdapter(b executorpkg.Backend, i executorpkg.ImageBackend, v executorpkg.VolumeBackend, task *api.Task, node *api.NodeDescription, dependencies exec.DependencyGetter) (*containerAdapter, error) { 50 ctnr, err := newContainerConfig(task, node) 51 if err != nil { 52 return nil, err 53 } 54 55 return &containerAdapter{ 56 container: ctnr, 57 backend: b, 58 imageBackend: i, 59 volumeBackend: v, 60 dependencies: dependencies, 61 }, nil 62 } 63 64 func (c *containerAdapter) pullImage(ctx context.Context) error { 65 spec := c.container.spec() 66 67 // Skip pulling if the image is referenced by image ID. 68 if _, err := digest.Parse(spec.Image); err == nil { 69 return nil 70 } 71 72 // Skip pulling if the image is referenced by digest and already 73 // exists locally. 74 named, err := reference.ParseNormalizedNamed(spec.Image) 75 if err == nil { 76 if _, ok := named.(reference.Canonical); ok { 77 _, err := c.imageBackend.LookupImage(spec.Image) 78 if err == nil { 79 return nil 80 } 81 } 82 } 83 84 // if the image needs to be pulled, the auth config will be retrieved and updated 85 var encodedAuthConfig string 86 if spec.PullOptions != nil { 87 encodedAuthConfig = spec.PullOptions.RegistryAuth 88 } 89 90 authConfig := &types.AuthConfig{} 91 if encodedAuthConfig != "" { 92 if err := json.NewDecoder(base64.NewDecoder(base64.URLEncoding, strings.NewReader(encodedAuthConfig))).Decode(authConfig); err != nil { 93 logrus.Warnf("invalid authconfig: %v", err) 94 } 95 } 96 97 pr, pw := io.Pipe() 98 metaHeaders := map[string][]string{} 99 go func() { 100 // TODO @jhowardmsft LCOW Support: This will need revisiting as 101 // the stack is built up to include LCOW support for swarm. 102 err := c.imageBackend.PullImage(ctx, c.container.image(), "", nil, metaHeaders, authConfig, pw) 103 pw.CloseWithError(err) 104 }() 105 106 dec := json.NewDecoder(pr) 107 dec.UseNumber() 108 m := map[string]interface{}{} 109 spamLimiter := rate.NewLimiter(rate.Every(time.Second), 1) 110 111 lastStatus := "" 112 for { 113 if err := dec.Decode(&m); err != nil { 114 if err == io.EOF { 115 break 116 } 117 return err 118 } 119 l := log.G(ctx) 120 // limit pull progress logs unless the status changes 121 if spamLimiter.Allow() || lastStatus != m["status"] { 122 // if we have progress details, we have everything we need 123 if progress, ok := m["progressDetail"].(map[string]interface{}); ok { 124 // first, log the image and status 125 l = l.WithFields(logrus.Fields{ 126 "image": c.container.image(), 127 "status": m["status"], 128 }) 129 // then, if we have progress, log the progress 130 if progress["current"] != nil && progress["total"] != nil { 131 l = l.WithFields(logrus.Fields{ 132 "current": progress["current"], 133 "total": progress["total"], 134 }) 135 } 136 } 137 l.Debug("pull in progress") 138 } 139 // sometimes, we get no useful information at all, and add no fields 140 if status, ok := m["status"].(string); ok { 141 lastStatus = status 142 } 143 } 144 145 // if the final stream object contained an error, return it 146 if errMsg, ok := m["error"]; ok { 147 return fmt.Errorf("%v", errMsg) 148 } 149 return nil 150 } 151 152 // waitNodeAttachments validates that NetworkAttachments exist on this node 153 // for every network in use by this task. It blocks until the network 154 // attachments are ready, or the context times out. If it returns nil, then the 155 // node's network attachments are all there. 156 func (c *containerAdapter) waitNodeAttachments(ctx context.Context) error { 157 // to do this, we're going to get the attachment store and try getting the 158 // IP address for each network. if any network comes back not existing, 159 // we'll wait and try again. 160 attachmentStore := c.backend.GetAttachmentStore() 161 if attachmentStore == nil { 162 return fmt.Errorf("error getting attachment store") 163 } 164 165 // essentially, we're long-polling here. this is really sub-optimal, but a 166 // better solution based off signaling channels would require a more 167 // substantial rearchitecture and probably not be worth our time in terms 168 // of performance gains. 169 poll := time.NewTicker(nodeAttachmentReadyInterval) 170 defer poll.Stop() 171 for { 172 // set a flag ready to true. if we try to get a network IP that doesn't 173 // exist yet, we will set this flag to "false" 174 ready := true 175 for _, attachment := range c.container.networksAttachments { 176 // we only need node attachments (IP address) for overlay networks 177 // TODO(dperny): unsure if this will work with other network 178 // drivers, but i also don't think other network drivers use the 179 // node attachment IP address. 180 if attachment.Network.DriverState.Name == "overlay" { 181 if _, exists := attachmentStore.GetIPForNetwork(attachment.Network.ID); !exists { 182 ready = false 183 } 184 } 185 } 186 187 // if everything is ready here, then we can just return no error 188 if ready { 189 return nil 190 } 191 192 // otherwise, try polling again, or wait for context canceled. 193 select { 194 case <-ctx.Done(): 195 return fmt.Errorf("node is missing network attachments, ip addresses may be exhausted") 196 case <-poll.C: 197 } 198 } 199 } 200 201 func (c *containerAdapter) createNetworks(ctx context.Context) error { 202 for name := range c.container.networksAttachments { 203 ncr, err := c.container.networkCreateRequest(name) 204 if err != nil { 205 return err 206 } 207 208 if err := c.backend.CreateManagedNetwork(ncr); err != nil { // todo name missing 209 if _, ok := err.(libnetwork.NetworkNameError); ok { 210 continue 211 } 212 // We will continue if CreateManagedNetwork returns PredefinedNetworkError error. 213 // Other callers still can treat it as Error. 214 if _, ok := err.(daemon.PredefinedNetworkError); ok { 215 continue 216 } 217 return err 218 } 219 } 220 221 return nil 222 } 223 224 func (c *containerAdapter) removeNetworks(ctx context.Context) error { 225 for name, v := range c.container.networksAttachments { 226 if err := c.backend.DeleteManagedNetwork(v.Network.ID); err != nil { 227 switch errors.Cause(err).(type) { 228 case *libnetwork.ActiveEndpointsError: 229 continue 230 case libnetwork.ErrNoSuchNetwork: 231 continue 232 default: 233 log.G(ctx).Errorf("network %s remove failed: %v", name, err) 234 return err 235 } 236 } 237 } 238 239 return nil 240 } 241 242 func (c *containerAdapter) networkAttach(ctx context.Context) error { 243 config := c.container.createNetworkingConfig(c.backend) 244 245 var ( 246 networkName string 247 networkID string 248 ) 249 250 if config != nil { 251 for n, epConfig := range config.EndpointsConfig { 252 networkName = n 253 networkID = epConfig.NetworkID 254 break 255 } 256 } 257 258 return c.backend.UpdateAttachment(networkName, networkID, c.container.networkAttachmentContainerID(), config) 259 } 260 261 func (c *containerAdapter) waitForDetach(ctx context.Context) error { 262 config := c.container.createNetworkingConfig(c.backend) 263 264 var ( 265 networkName string 266 networkID string 267 ) 268 269 if config != nil { 270 for n, epConfig := range config.EndpointsConfig { 271 networkName = n 272 networkID = epConfig.NetworkID 273 break 274 } 275 } 276 277 return c.backend.WaitForDetachment(ctx, networkName, networkID, c.container.taskID(), c.container.networkAttachmentContainerID()) 278 } 279 280 func (c *containerAdapter) create(ctx context.Context) error { 281 var cr containertypes.ContainerCreateCreatedBody 282 var err error 283 if cr, err = c.backend.CreateManagedContainer(types.ContainerCreateConfig{ 284 Name: c.container.name(), 285 Config: c.container.config(), 286 HostConfig: c.container.hostConfig(), 287 // Use the first network in container create 288 NetworkingConfig: c.container.createNetworkingConfig(c.backend), 289 }); err != nil { 290 return err 291 } 292 293 // Docker daemon currently doesn't support multiple networks in container create 294 // Connect to all other networks 295 nc := c.container.connectNetworkingConfig(c.backend) 296 297 if nc != nil { 298 for n, ep := range nc.EndpointsConfig { 299 if err := c.backend.ConnectContainerToNetwork(cr.ID, n, ep); err != nil { 300 return err 301 } 302 } 303 } 304 305 container := c.container.task.Spec.GetContainer() 306 if container == nil { 307 return errors.New("unable to get container from task spec") 308 } 309 310 if err := c.backend.SetContainerDependencyStore(cr.ID, c.dependencies); err != nil { 311 return err 312 } 313 314 // configure secrets 315 secretRefs := convert.SecretReferencesFromGRPC(container.Secrets) 316 if err := c.backend.SetContainerSecretReferences(cr.ID, secretRefs); err != nil { 317 return err 318 } 319 320 configRefs := convert.ConfigReferencesFromGRPC(container.Configs) 321 if err := c.backend.SetContainerConfigReferences(cr.ID, configRefs); err != nil { 322 return err 323 } 324 325 return c.backend.UpdateContainerServiceConfig(cr.ID, c.container.serviceConfig()) 326 } 327 328 // checkMounts ensures that the provided mounts won't have any host-specific 329 // problems at start up. For example, we disallow bind mounts without an 330 // existing path, which slightly different from the container API. 331 func (c *containerAdapter) checkMounts() error { 332 spec := c.container.spec() 333 for _, mount := range spec.Mounts { 334 switch mount.Type { 335 case api.MountTypeBind: 336 if _, err := os.Stat(mount.Source); os.IsNotExist(err) { 337 return fmt.Errorf("invalid bind mount source, source path not found: %s", mount.Source) 338 } 339 } 340 } 341 342 return nil 343 } 344 345 func (c *containerAdapter) start(ctx context.Context) error { 346 if err := c.checkMounts(); err != nil { 347 return err 348 } 349 350 return c.backend.ContainerStart(c.container.name(), nil, "", "") 351 } 352 353 func (c *containerAdapter) inspect(ctx context.Context) (types.ContainerJSON, error) { 354 cs, err := c.backend.ContainerInspectCurrent(c.container.name(), false) 355 if ctx.Err() != nil { 356 return types.ContainerJSON{}, ctx.Err() 357 } 358 if err != nil { 359 return types.ContainerJSON{}, err 360 } 361 return *cs, nil 362 } 363 364 // events issues a call to the events API and returns a channel with all 365 // events. The stream of events can be shutdown by cancelling the context. 366 func (c *containerAdapter) events(ctx context.Context) <-chan events.Message { 367 log.G(ctx).Debugf("waiting on events") 368 buffer, l := c.backend.SubscribeToEvents(time.Time{}, time.Time{}, c.container.eventFilter()) 369 eventsq := make(chan events.Message, len(buffer)) 370 371 for _, event := range buffer { 372 eventsq <- event 373 } 374 375 go func() { 376 defer c.backend.UnsubscribeFromEvents(l) 377 378 for { 379 select { 380 case ev := <-l: 381 jev, ok := ev.(events.Message) 382 if !ok { 383 log.G(ctx).Warnf("unexpected event message: %q", ev) 384 continue 385 } 386 select { 387 case eventsq <- jev: 388 case <-ctx.Done(): 389 return 390 } 391 case <-ctx.Done(): 392 return 393 } 394 } 395 }() 396 397 return eventsq 398 } 399 400 func (c *containerAdapter) wait(ctx context.Context) (<-chan containerpkg.StateStatus, error) { 401 return c.backend.ContainerWait(ctx, c.container.nameOrID(), containerpkg.WaitConditionNotRunning) 402 } 403 404 func (c *containerAdapter) shutdown(ctx context.Context) error { 405 // Default stop grace period to nil (daemon will use the stopTimeout of the container) 406 var stopgrace *int 407 spec := c.container.spec() 408 if spec.StopGracePeriod != nil { 409 stopgraceValue := int(spec.StopGracePeriod.Seconds) 410 stopgrace = &stopgraceValue 411 } 412 return c.backend.ContainerStop(c.container.name(), stopgrace) 413 } 414 415 func (c *containerAdapter) terminate(ctx context.Context) error { 416 return c.backend.ContainerKill(c.container.name(), uint64(syscall.SIGKILL)) 417 } 418 419 func (c *containerAdapter) remove(ctx context.Context) error { 420 return c.backend.ContainerRm(c.container.name(), &types.ContainerRmConfig{ 421 RemoveVolume: true, 422 ForceRemove: true, 423 }) 424 } 425 426 func (c *containerAdapter) createVolumes(ctx context.Context) error { 427 // Create plugin volumes that are embedded inside a Mount 428 for _, mount := range c.container.task.Spec.GetContainer().Mounts { 429 if mount.Type != api.MountTypeVolume { 430 continue 431 } 432 433 if mount.VolumeOptions == nil { 434 continue 435 } 436 437 if mount.VolumeOptions.DriverConfig == nil { 438 continue 439 } 440 441 req := c.container.volumeCreateRequest(&mount) 442 443 // Check if this volume exists on the engine 444 if _, err := c.volumeBackend.Create(ctx, req.Name, req.Driver, 445 volumeopts.WithCreateOptions(req.DriverOpts), 446 volumeopts.WithCreateLabels(req.Labels), 447 ); err != nil { 448 // TODO(amitshukla): Today, volume create through the engine api does not return an error 449 // when the named volume with the same parameters already exists. 450 // It returns an error if the driver name is different - that is a valid error 451 return err 452 } 453 454 } 455 456 return nil 457 } 458 459 func (c *containerAdapter) activateServiceBinding() error { 460 return c.backend.ActivateContainerServiceBinding(c.container.name()) 461 } 462 463 func (c *containerAdapter) deactivateServiceBinding() error { 464 return c.backend.DeactivateContainerServiceBinding(c.container.name()) 465 } 466 467 func (c *containerAdapter) logs(ctx context.Context, options api.LogSubscriptionOptions) (<-chan *backend.LogMessage, error) { 468 apiOptions := &types.ContainerLogsOptions{ 469 Follow: options.Follow, 470 471 // Always say yes to Timestamps and Details. we make the decision 472 // of whether to return these to the user or not way higher up the 473 // stack. 474 Timestamps: true, 475 Details: true, 476 } 477 478 if options.Since != nil { 479 since, err := gogotypes.TimestampFromProto(options.Since) 480 if err != nil { 481 return nil, err 482 } 483 // print since as this formatted string because the docker container 484 // logs interface expects it like this. 485 // see github.com/docker/docker/api/types/time.ParseTimestamps 486 apiOptions.Since = fmt.Sprintf("%d.%09d", since.Unix(), int64(since.Nanosecond())) 487 } 488 489 if options.Tail < 0 { 490 // See protobuf documentation for details of how this works. 491 apiOptions.Tail = fmt.Sprint(-options.Tail - 1) 492 } else if options.Tail > 0 { 493 return nil, errors.New("tail relative to start of logs not supported via docker API") 494 } 495 496 if len(options.Streams) == 0 { 497 // empty == all 498 apiOptions.ShowStdout, apiOptions.ShowStderr = true, true 499 } else { 500 for _, stream := range options.Streams { 501 switch stream { 502 case api.LogStreamStdout: 503 apiOptions.ShowStdout = true 504 case api.LogStreamStderr: 505 apiOptions.ShowStderr = true 506 } 507 } 508 } 509 msgs, _, err := c.backend.ContainerLogs(ctx, c.container.name(), apiOptions) 510 if err != nil { 511 return nil, err 512 } 513 return msgs, nil 514 } 515 516 // todo: typed/wrapped errors 517 func isContainerCreateNameConflict(err error) bool { 518 return strings.Contains(err.Error(), "Conflict. The name") 519 } 520 521 func isUnknownContainer(err error) bool { 522 return strings.Contains(err.Error(), "No such container:") 523 } 524 525 func isStoppedContainer(err error) bool { 526 return strings.Contains(err.Error(), "is already stopped") 527 }