github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/libcontainerd/remote/client.go (about)

     1  package remote // import "github.com/demonoid81/moby/libcontainerd/remote"
     2  
     3  import (
     4  	"context"
     5  	"encoding/json"
     6  	"io"
     7  	"os"
     8  	"path/filepath"
     9  	"reflect"
    10  	"runtime"
    11  	"strings"
    12  	"sync"
    13  	"syscall"
    14  	"time"
    15  
    16  	"github.com/containerd/containerd"
    17  	apievents "github.com/containerd/containerd/api/events"
    18  	"github.com/containerd/containerd/api/types"
    19  	"github.com/containerd/containerd/archive"
    20  	"github.com/containerd/containerd/cio"
    21  	"github.com/containerd/containerd/content"
    22  	containerderrors "github.com/containerd/containerd/errdefs"
    23  	"github.com/containerd/containerd/events"
    24  	"github.com/containerd/containerd/images"
    25  	"github.com/containerd/containerd/runtime/linux/runctypes"
    26  	v2runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
    27  	"github.com/containerd/typeurl"
    28  	"github.com/demonoid81/moby/errdefs"
    29  	"github.com/demonoid81/moby/libcontainerd/queue"
    30  	libcontainerdtypes "github.com/demonoid81/moby/libcontainerd/types"
    31  	"github.com/demonoid81/moby/pkg/ioutils"
    32  	v1 "github.com/opencontainers/image-spec/specs-go/v1"
    33  	specs "github.com/opencontainers/runtime-spec/specs-go"
    34  	"github.com/pkg/errors"
    35  	"github.com/sirupsen/logrus"
    36  	"google.golang.org/grpc/codes"
    37  	"google.golang.org/grpc/status"
    38  )
    39  
    40  // DockerContainerBundlePath is the label key pointing to the container's bundle path
    41  const DockerContainerBundlePath = "com.docker/engine.bundle.path"
    42  
    43  type client struct {
    44  	client   *containerd.Client
    45  	stateDir string
    46  	logger   *logrus.Entry
    47  	ns       string
    48  
    49  	backend         libcontainerdtypes.Backend
    50  	eventQ          queue.Queue
    51  	oomMu           sync.Mutex
    52  	oom             map[string]bool
    53  	useShimV2       bool
    54  	v2runcoptionsMu sync.Mutex
    55  	// v2runcoptions is used for copying options specified on Create() to Start()
    56  	v2runcoptions map[string]v2runcoptions.Options
    57  }
    58  
    59  // NewClient creates a new libcontainerd client from a containerd client
    60  func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend, useShimV2 bool) (libcontainerdtypes.Client, error) {
    61  	c := &client{
    62  		client:        cli,
    63  		stateDir:      stateDir,
    64  		logger:        logrus.WithField("module", "libcontainerd").WithField("namespace", ns),
    65  		ns:            ns,
    66  		backend:       b,
    67  		oom:           make(map[string]bool),
    68  		useShimV2:     useShimV2,
    69  		v2runcoptions: make(map[string]v2runcoptions.Options),
    70  	}
    71  
    72  	go c.processEventStream(ctx, ns)
    73  
    74  	return c, nil
    75  }
    76  
    77  func (c *client) Version(ctx context.Context) (containerd.Version, error) {
    78  	return c.client.Version(ctx)
    79  }
    80  
    81  // Restore loads the containerd container.
    82  // It should not be called concurrently with any other operation for the given ID.
    83  func (c *client) Restore(ctx context.Context, id string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, p libcontainerdtypes.Process, err error) {
    84  	var dio *cio.DirectIO
    85  	defer func() {
    86  		if err != nil && dio != nil {
    87  			dio.Cancel()
    88  			dio.Close()
    89  		}
    90  		err = wrapError(err)
    91  	}()
    92  
    93  	ctr, err := c.client.LoadContainer(ctx, id)
    94  	if err != nil {
    95  		return false, -1, nil, errors.WithStack(wrapError(err))
    96  	}
    97  
    98  	attachIO := func(fifos *cio.FIFOSet) (cio.IO, error) {
    99  		// dio must be assigned to the previously defined dio for the defer above
   100  		// to handle cleanup
   101  		dio, err = c.newDirectIO(ctx, fifos)
   102  		if err != nil {
   103  			return nil, err
   104  		}
   105  		return attachStdio(dio)
   106  	}
   107  	t, err := ctr.Task(ctx, attachIO)
   108  	if err != nil && !containerderrors.IsNotFound(err) {
   109  		return false, -1, nil, errors.Wrap(wrapError(err), "error getting containerd task for container")
   110  	}
   111  
   112  	if t != nil {
   113  		s, err := t.Status(ctx)
   114  		if err != nil {
   115  			return false, -1, nil, errors.Wrap(wrapError(err), "error getting task status")
   116  		}
   117  		alive = s.Status != containerd.Stopped
   118  		pid = int(t.Pid())
   119  	}
   120  
   121  	c.logger.WithFields(logrus.Fields{
   122  		"container": id,
   123  		"alive":     alive,
   124  		"pid":       pid,
   125  	}).Debug("restored container")
   126  
   127  	return alive, pid, &restoredProcess{
   128  		p: t,
   129  	}, nil
   130  }
   131  
   132  func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, runtimeOptions interface{}, opts ...containerd.NewContainerOpts) error {
   133  	bdir := c.bundleDir(id)
   134  	c.logger.WithField("bundle", bdir).WithField("root", ociSpec.Root.Path).Debug("bundle dir created")
   135  
   136  	rt := runtimeName
   137  	if c.useShimV2 {
   138  		rt = shimV2RuntimeName
   139  	}
   140  	newOpts := []containerd.NewContainerOpts{
   141  		containerd.WithSpec(ociSpec),
   142  		containerd.WithRuntime(rt, runtimeOptions),
   143  		WithBundle(bdir, ociSpec),
   144  	}
   145  	opts = append(opts, newOpts...)
   146  
   147  	_, err := c.client.NewContainer(ctx, id, opts...)
   148  	if err != nil {
   149  		if containerderrors.IsAlreadyExists(err) {
   150  			return errors.WithStack(errdefs.Conflict(errors.New("id already in use")))
   151  		}
   152  		return wrapError(err)
   153  	}
   154  	if c.useShimV2 {
   155  		if x, ok := runtimeOptions.(*v2runcoptions.Options); ok {
   156  			c.v2runcoptionsMu.Lock()
   157  			c.v2runcoptions[id] = *x
   158  			c.v2runcoptionsMu.Unlock()
   159  		}
   160  	}
   161  	return nil
   162  }
   163  
   164  // Start create and start a task for the specified containerd id
   165  func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) {
   166  	ctr, err := c.getContainer(ctx, id)
   167  	if err != nil {
   168  		return -1, err
   169  	}
   170  	var (
   171  		cp             *types.Descriptor
   172  		t              containerd.Task
   173  		rio            cio.IO
   174  		stdinCloseSync = make(chan struct{})
   175  	)
   176  
   177  	if checkpointDir != "" {
   178  		// write checkpoint to the content store
   179  		tar := archive.Diff(ctx, "", checkpointDir)
   180  		cp, err = c.writeContent(ctx, images.MediaTypeContainerd1Checkpoint, checkpointDir, tar)
   181  		// remove the checkpoint when we're done
   182  		defer func() {
   183  			if cp != nil {
   184  				err := c.client.ContentStore().Delete(context.Background(), cp.Digest)
   185  				if err != nil {
   186  					c.logger.WithError(err).WithFields(logrus.Fields{
   187  						"ref":    checkpointDir,
   188  						"digest": cp.Digest,
   189  					}).Warnf("failed to delete temporary checkpoint entry")
   190  				}
   191  			}
   192  		}()
   193  		if err := tar.Close(); err != nil {
   194  			return -1, errors.Wrap(err, "failed to close checkpoint tar stream")
   195  		}
   196  		if err != nil {
   197  			return -1, errors.Wrapf(err, "failed to upload checkpoint to containerd")
   198  		}
   199  	}
   200  
   201  	spec, err := ctr.Spec(ctx)
   202  	if err != nil {
   203  		return -1, errors.Wrap(err, "failed to retrieve spec")
   204  	}
   205  	labels, err := ctr.Labels(ctx)
   206  	if err != nil {
   207  		return -1, errors.Wrap(err, "failed to retrieve labels")
   208  	}
   209  	bundle := labels[DockerContainerBundlePath]
   210  	uid, gid := getSpecUser(spec)
   211  
   212  	taskOpts := []containerd.NewTaskOpts{
   213  		func(_ context.Context, _ *containerd.Client, info *containerd.TaskInfo) error {
   214  			info.Checkpoint = cp
   215  			return nil
   216  		},
   217  	}
   218  
   219  	if runtime.GOOS != "windows" {
   220  		taskOpts = append(taskOpts, func(_ context.Context, _ *containerd.Client, info *containerd.TaskInfo) error {
   221  			if c.useShimV2 {
   222  				// For v2, we need to inherit options specified on Create
   223  				c.v2runcoptionsMu.Lock()
   224  				opts, ok := c.v2runcoptions[id]
   225  				c.v2runcoptionsMu.Unlock()
   226  				if !ok {
   227  					opts = v2runcoptions.Options{}
   228  				}
   229  				opts.IoUid = uint32(uid)
   230  				opts.IoGid = uint32(gid)
   231  				opts.NoPivotRoot = os.Getenv("DOCKER_RAMDISK") != ""
   232  				info.Options = &opts
   233  			} else {
   234  				info.Options = &runctypes.CreateOptions{
   235  					IoUid:       uint32(uid),
   236  					IoGid:       uint32(gid),
   237  					NoPivotRoot: os.Getenv("DOCKER_RAMDISK") != "",
   238  				}
   239  			}
   240  
   241  			return nil
   242  		})
   243  	} else {
   244  		taskOpts = append(taskOpts, withLogLevel(c.logger.Level))
   245  	}
   246  
   247  	t, err = ctr.NewTask(ctx,
   248  		func(id string) (cio.IO, error) {
   249  			fifos := newFIFOSet(bundle, libcontainerdtypes.InitProcessName, withStdin, spec.Process.Terminal)
   250  
   251  			rio, err = c.createIO(fifos, id, libcontainerdtypes.InitProcessName, stdinCloseSync, attachStdio)
   252  			return rio, err
   253  		},
   254  		taskOpts...,
   255  	)
   256  	if err != nil {
   257  		close(stdinCloseSync)
   258  		if rio != nil {
   259  			rio.Cancel()
   260  			rio.Close()
   261  		}
   262  		return -1, wrapError(err)
   263  	}
   264  
   265  	// Signal c.createIO that it can call CloseIO
   266  	close(stdinCloseSync)
   267  
   268  	if err := t.Start(ctx); err != nil {
   269  		if _, err := t.Delete(ctx); err != nil {
   270  			c.logger.WithError(err).WithField("container", id).
   271  				Error("failed to delete task after fail start")
   272  		}
   273  		return -1, wrapError(err)
   274  	}
   275  
   276  	return int(t.Pid()), nil
   277  }
   278  
   279  // Exec creates exec process.
   280  //
   281  // The containerd client calls Exec to register the exec config in the shim side.
   282  // When the client calls Start, the shim will create stdin fifo if needs. But
   283  // for the container main process, the stdin fifo will be created in Create not
   284  // the Start call. stdinCloseSync channel should be closed after Start exec
   285  // process.
   286  func (c *client) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) {
   287  	ctr, err := c.getContainer(ctx, containerID)
   288  	if err != nil {
   289  		return -1, err
   290  	}
   291  	t, err := ctr.Task(ctx, nil)
   292  	if err != nil {
   293  		if containerderrors.IsNotFound(err) {
   294  			return -1, errors.WithStack(errdefs.InvalidParameter(errors.New("container is not running")))
   295  		}
   296  		return -1, wrapError(err)
   297  	}
   298  
   299  	var (
   300  		p              containerd.Process
   301  		rio            cio.IO
   302  		stdinCloseSync = make(chan struct{})
   303  	)
   304  
   305  	labels, err := ctr.Labels(ctx)
   306  	if err != nil {
   307  		return -1, wrapError(err)
   308  	}
   309  
   310  	fifos := newFIFOSet(labels[DockerContainerBundlePath], processID, withStdin, spec.Terminal)
   311  
   312  	defer func() {
   313  		if err != nil {
   314  			if rio != nil {
   315  				rio.Cancel()
   316  				rio.Close()
   317  			}
   318  		}
   319  	}()
   320  
   321  	p, err = t.Exec(ctx, processID, spec, func(id string) (cio.IO, error) {
   322  		rio, err = c.createIO(fifos, containerID, processID, stdinCloseSync, attachStdio)
   323  		return rio, err
   324  	})
   325  	if err != nil {
   326  		close(stdinCloseSync)
   327  		if containerderrors.IsAlreadyExists(err) {
   328  			return -1, errors.WithStack(errdefs.Conflict(errors.New("id already in use")))
   329  		}
   330  		return -1, wrapError(err)
   331  	}
   332  
   333  	// Signal c.createIO that it can call CloseIO
   334  	//
   335  	// the stdin of exec process will be created after p.Start in containerd
   336  	defer close(stdinCloseSync)
   337  
   338  	if err = p.Start(ctx); err != nil {
   339  		// use new context for cleanup because old one may be cancelled by user, but leave a timeout to make sure
   340  		// we are not waiting forever if containerd is unresponsive or to work around fifo cancelling issues in
   341  		// older containerd-shim
   342  		ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
   343  		defer cancel()
   344  		p.Delete(ctx)
   345  		return -1, wrapError(err)
   346  	}
   347  	return int(p.Pid()), nil
   348  }
   349  
   350  func (c *client) SignalProcess(ctx context.Context, containerID, processID string, signal int) error {
   351  	p, err := c.getProcess(ctx, containerID, processID)
   352  	if err != nil {
   353  		return err
   354  	}
   355  	return wrapError(p.Kill(ctx, syscall.Signal(signal)))
   356  }
   357  
   358  func (c *client) ResizeTerminal(ctx context.Context, containerID, processID string, width, height int) error {
   359  	p, err := c.getProcess(ctx, containerID, processID)
   360  	if err != nil {
   361  		return err
   362  	}
   363  
   364  	return p.Resize(ctx, uint32(width), uint32(height))
   365  }
   366  
   367  func (c *client) CloseStdin(ctx context.Context, containerID, processID string) error {
   368  	p, err := c.getProcess(ctx, containerID, processID)
   369  	if err != nil {
   370  		return err
   371  	}
   372  
   373  	return p.CloseIO(ctx, containerd.WithStdinCloser)
   374  }
   375  
   376  func (c *client) Pause(ctx context.Context, containerID string) error {
   377  	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
   378  	if err != nil {
   379  		return err
   380  	}
   381  
   382  	return wrapError(p.(containerd.Task).Pause(ctx))
   383  }
   384  
   385  func (c *client) Resume(ctx context.Context, containerID string) error {
   386  	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
   387  	if err != nil {
   388  		return err
   389  	}
   390  
   391  	return p.(containerd.Task).Resume(ctx)
   392  }
   393  
   394  func (c *client) Stats(ctx context.Context, containerID string) (*libcontainerdtypes.Stats, error) {
   395  	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
   396  	if err != nil {
   397  		return nil, err
   398  	}
   399  
   400  	m, err := p.(containerd.Task).Metrics(ctx)
   401  	if err != nil {
   402  		return nil, err
   403  	}
   404  
   405  	v, err := typeurl.UnmarshalAny(m.Data)
   406  	if err != nil {
   407  		return nil, err
   408  	}
   409  	return libcontainerdtypes.InterfaceToStats(m.Timestamp, v), nil
   410  }
   411  
   412  func (c *client) ListPids(ctx context.Context, containerID string) ([]uint32, error) {
   413  	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
   414  	if err != nil {
   415  		return nil, err
   416  	}
   417  
   418  	pis, err := p.(containerd.Task).Pids(ctx)
   419  	if err != nil {
   420  		return nil, err
   421  	}
   422  
   423  	var pids []uint32
   424  	for _, i := range pis {
   425  		pids = append(pids, i.Pid)
   426  	}
   427  
   428  	return pids, nil
   429  }
   430  
   431  func (c *client) Summary(ctx context.Context, containerID string) ([]libcontainerdtypes.Summary, error) {
   432  	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
   433  	if err != nil {
   434  		return nil, err
   435  	}
   436  
   437  	pis, err := p.(containerd.Task).Pids(ctx)
   438  	if err != nil {
   439  		return nil, err
   440  	}
   441  
   442  	var infos []libcontainerdtypes.Summary
   443  	for _, pi := range pis {
   444  		i, err := typeurl.UnmarshalAny(pi.Info)
   445  		if err != nil {
   446  			return nil, errors.Wrap(err, "unable to decode process details")
   447  		}
   448  		s, err := summaryFromInterface(i)
   449  		if err != nil {
   450  			return nil, err
   451  		}
   452  		infos = append(infos, *s)
   453  	}
   454  
   455  	return infos, nil
   456  }
   457  
   458  type restoredProcess struct {
   459  	p containerd.Process
   460  }
   461  
   462  func (p *restoredProcess) Delete(ctx context.Context) (uint32, time.Time, error) {
   463  	if p.p == nil {
   464  		return 255, time.Now(), nil
   465  	}
   466  	status, err := p.p.Delete(ctx)
   467  	if err != nil {
   468  		return 255, time.Now(), nil
   469  	}
   470  	return status.ExitCode(), status.ExitTime(), nil
   471  }
   472  
   473  func (c *client) DeleteTask(ctx context.Context, containerID string) (uint32, time.Time, error) {
   474  	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
   475  	if err != nil {
   476  		return 255, time.Now(), nil
   477  	}
   478  
   479  	status, err := p.Delete(ctx)
   480  	if err != nil {
   481  		return 255, time.Now(), nil
   482  	}
   483  	return status.ExitCode(), status.ExitTime(), nil
   484  }
   485  
   486  func (c *client) Delete(ctx context.Context, containerID string) error {
   487  	ctr, err := c.getContainer(ctx, containerID)
   488  	if err != nil {
   489  		return err
   490  	}
   491  	labels, err := ctr.Labels(ctx)
   492  	if err != nil {
   493  		return err
   494  	}
   495  	bundle := labels[DockerContainerBundlePath]
   496  	if err := ctr.Delete(ctx); err != nil {
   497  		return wrapError(err)
   498  	}
   499  	c.oomMu.Lock()
   500  	delete(c.oom, containerID)
   501  	c.oomMu.Unlock()
   502  	c.v2runcoptionsMu.Lock()
   503  	delete(c.v2runcoptions, containerID)
   504  	c.v2runcoptionsMu.Unlock()
   505  	if os.Getenv("LIBCONTAINERD_NOCLEAN") != "1" {
   506  		if err := os.RemoveAll(bundle); err != nil {
   507  			c.logger.WithError(err).WithFields(logrus.Fields{
   508  				"container": containerID,
   509  				"bundle":    bundle,
   510  			}).Error("failed to remove state dir")
   511  		}
   512  	}
   513  	return nil
   514  }
   515  
   516  func (c *client) Status(ctx context.Context, containerID string) (containerd.ProcessStatus, error) {
   517  	t, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
   518  	if err != nil {
   519  		return containerd.Unknown, err
   520  	}
   521  	s, err := t.Status(ctx)
   522  	if err != nil {
   523  		return containerd.Unknown, wrapError(err)
   524  	}
   525  	return s.Status, nil
   526  }
   527  
   528  func (c *client) CreateCheckpoint(ctx context.Context, containerID, checkpointDir string, exit bool) error {
   529  	p, err := c.getProcess(ctx, containerID, libcontainerdtypes.InitProcessName)
   530  	if err != nil {
   531  		return err
   532  	}
   533  
   534  	opts := []containerd.CheckpointTaskOpts{}
   535  	if exit {
   536  		opts = append(opts, func(r *containerd.CheckpointTaskInfo) error {
   537  			if r.Options == nil {
   538  				r.Options = &runctypes.CheckpointOptions{
   539  					Exit: true,
   540  				}
   541  			} else {
   542  				opts, _ := r.Options.(*runctypes.CheckpointOptions)
   543  				opts.Exit = true
   544  			}
   545  			return nil
   546  		})
   547  	}
   548  	img, err := p.(containerd.Task).Checkpoint(ctx, opts...)
   549  	if err != nil {
   550  		return wrapError(err)
   551  	}
   552  	// Whatever happens, delete the checkpoint from containerd
   553  	defer func() {
   554  		err := c.client.ImageService().Delete(context.Background(), img.Name())
   555  		if err != nil {
   556  			c.logger.WithError(err).WithField("digest", img.Target().Digest).
   557  				Warnf("failed to delete checkpoint image")
   558  		}
   559  	}()
   560  
   561  	b, err := content.ReadBlob(ctx, c.client.ContentStore(), img.Target())
   562  	if err != nil {
   563  		return errdefs.System(errors.Wrapf(err, "failed to retrieve checkpoint data"))
   564  	}
   565  	var index v1.Index
   566  	if err := json.Unmarshal(b, &index); err != nil {
   567  		return errdefs.System(errors.Wrapf(err, "failed to decode checkpoint data"))
   568  	}
   569  
   570  	var cpDesc *v1.Descriptor
   571  	for _, m := range index.Manifests {
   572  		if m.MediaType == images.MediaTypeContainerd1Checkpoint {
   573  			cpDesc = &m
   574  			break
   575  		}
   576  	}
   577  	if cpDesc == nil {
   578  		return errdefs.System(errors.Wrapf(err, "invalid checkpoint"))
   579  	}
   580  
   581  	rat, err := c.client.ContentStore().ReaderAt(ctx, *cpDesc)
   582  	if err != nil {
   583  		return errdefs.System(errors.Wrapf(err, "failed to get checkpoint reader"))
   584  	}
   585  	defer rat.Close()
   586  	_, err = archive.Apply(ctx, checkpointDir, content.NewReader(rat))
   587  	if err != nil {
   588  		return errdefs.System(errors.Wrapf(err, "failed to read checkpoint reader"))
   589  	}
   590  
   591  	return err
   592  }
   593  
   594  func (c *client) getContainer(ctx context.Context, id string) (containerd.Container, error) {
   595  	ctr, err := c.client.LoadContainer(ctx, id)
   596  	if err != nil {
   597  		if containerderrors.IsNotFound(err) {
   598  			return nil, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
   599  		}
   600  		return nil, wrapError(err)
   601  	}
   602  	return ctr, nil
   603  }
   604  
   605  func (c *client) getProcess(ctx context.Context, containerID, processID string) (containerd.Process, error) {
   606  	ctr, err := c.getContainer(ctx, containerID)
   607  	if err != nil {
   608  		return nil, err
   609  	}
   610  	t, err := ctr.Task(ctx, nil)
   611  	if err != nil {
   612  		if containerderrors.IsNotFound(err) {
   613  			return nil, errors.WithStack(errdefs.NotFound(errors.New("container is not running")))
   614  		}
   615  		return nil, wrapError(err)
   616  	}
   617  	if processID == libcontainerdtypes.InitProcessName {
   618  		return t, nil
   619  	}
   620  	p, err := t.LoadProcess(ctx, processID, nil)
   621  	if err != nil {
   622  		if containerderrors.IsNotFound(err) {
   623  			return nil, errors.WithStack(errdefs.NotFound(errors.New("no such exec")))
   624  		}
   625  		return nil, wrapError(err)
   626  	}
   627  	return p, nil
   628  }
   629  
   630  // createIO creates the io to be used by a process
   631  // This needs to get a pointer to interface as upon closure the process may not have yet been registered
   632  func (c *client) createIO(fifos *cio.FIFOSet, containerID, processID string, stdinCloseSync chan struct{}, attachStdio libcontainerdtypes.StdioCallback) (cio.IO, error) {
   633  	var (
   634  		io  *cio.DirectIO
   635  		err error
   636  	)
   637  	io, err = c.newDirectIO(context.Background(), fifos)
   638  	if err != nil {
   639  		return nil, err
   640  	}
   641  
   642  	if io.Stdin != nil {
   643  		var (
   644  			err       error
   645  			stdinOnce sync.Once
   646  		)
   647  		pipe := io.Stdin
   648  		io.Stdin = ioutils.NewWriteCloserWrapper(pipe, func() error {
   649  			stdinOnce.Do(func() {
   650  				err = pipe.Close()
   651  				// Do the rest in a new routine to avoid a deadlock if the
   652  				// Exec/Start call failed.
   653  				go func() {
   654  					<-stdinCloseSync
   655  					p, err := c.getProcess(context.Background(), containerID, processID)
   656  					if err == nil {
   657  						err = p.CloseIO(context.Background(), containerd.WithStdinCloser)
   658  						if err != nil && strings.Contains(err.Error(), "transport is closing") {
   659  							err = nil
   660  						}
   661  					}
   662  				}()
   663  			})
   664  			return err
   665  		})
   666  	}
   667  
   668  	rio, err := attachStdio(io)
   669  	if err != nil {
   670  		io.Cancel()
   671  		io.Close()
   672  	}
   673  	return rio, err
   674  }
   675  
   676  func (c *client) processEvent(ctx context.Context, et libcontainerdtypes.EventType, ei libcontainerdtypes.EventInfo) {
   677  	c.eventQ.Append(ei.ContainerID, func() {
   678  		err := c.backend.ProcessEvent(ei.ContainerID, et, ei)
   679  		if err != nil {
   680  			c.logger.WithError(err).WithFields(logrus.Fields{
   681  				"container":  ei.ContainerID,
   682  				"event":      et,
   683  				"event-info": ei,
   684  			}).Error("failed to process event")
   685  		}
   686  
   687  		if et == libcontainerdtypes.EventExit && ei.ProcessID != ei.ContainerID {
   688  			p, err := c.getProcess(ctx, ei.ContainerID, ei.ProcessID)
   689  			if err != nil {
   690  
   691  				c.logger.WithError(errors.New("no such process")).
   692  					WithFields(logrus.Fields{
   693  						"error":     err,
   694  						"container": ei.ContainerID,
   695  						"process":   ei.ProcessID,
   696  					}).Error("exit event")
   697  				return
   698  			}
   699  
   700  			ctr, err := c.getContainer(ctx, ei.ContainerID)
   701  			if err != nil {
   702  				c.logger.WithFields(logrus.Fields{
   703  					"container": ei.ContainerID,
   704  					"error":     err,
   705  				}).Error("failed to find container")
   706  			} else {
   707  				labels, err := ctr.Labels(ctx)
   708  				if err != nil {
   709  					c.logger.WithFields(logrus.Fields{
   710  						"container": ei.ContainerID,
   711  						"error":     err,
   712  					}).Error("failed to get container labels")
   713  					return
   714  				}
   715  				newFIFOSet(labels[DockerContainerBundlePath], ei.ProcessID, true, false).Close()
   716  			}
   717  			_, err = p.Delete(context.Background())
   718  			if err != nil {
   719  				c.logger.WithError(err).WithFields(logrus.Fields{
   720  					"container": ei.ContainerID,
   721  					"process":   ei.ProcessID,
   722  				}).Warn("failed to delete process")
   723  			}
   724  		}
   725  	})
   726  }
   727  
   728  func (c *client) processEventStream(ctx context.Context, ns string) {
   729  	var (
   730  		err error
   731  		ev  *events.Envelope
   732  		et  libcontainerdtypes.EventType
   733  		ei  libcontainerdtypes.EventInfo
   734  	)
   735  
   736  	// Filter on both namespace *and* topic. To create an "and" filter,
   737  	// this must be a single, comma-separated string
   738  	eventStream, errC := c.client.EventService().Subscribe(ctx, "namespace=="+ns+",topic~=|^/tasks/|")
   739  
   740  	c.logger.Debug("processing event stream")
   741  
   742  	for {
   743  		var oomKilled bool
   744  		select {
   745  		case err = <-errC:
   746  			if err != nil {
   747  				errStatus, ok := status.FromError(err)
   748  				if !ok || errStatus.Code() != codes.Canceled {
   749  					c.logger.WithError(err).Error("failed to get event")
   750  
   751  					// rate limit
   752  					select {
   753  					case <-time.After(time.Second):
   754  						go c.processEventStream(ctx, ns)
   755  						return
   756  					case <-ctx.Done():
   757  					}
   758  				}
   759  				c.logger.WithError(ctx.Err()).Info("stopping event stream following graceful shutdown")
   760  			}
   761  			return
   762  		case ev = <-eventStream:
   763  			if ev.Event == nil {
   764  				c.logger.WithField("event", ev).Warn("invalid event")
   765  				continue
   766  			}
   767  
   768  			v, err := typeurl.UnmarshalAny(ev.Event)
   769  			if err != nil {
   770  				c.logger.WithError(err).WithField("event", ev).Warn("failed to unmarshal event")
   771  				continue
   772  			}
   773  
   774  			c.logger.WithField("topic", ev.Topic).Debug("event")
   775  
   776  			switch t := v.(type) {
   777  			case *apievents.TaskCreate:
   778  				et = libcontainerdtypes.EventCreate
   779  				ei = libcontainerdtypes.EventInfo{
   780  					ContainerID: t.ContainerID,
   781  					ProcessID:   t.ContainerID,
   782  					Pid:         t.Pid,
   783  				}
   784  			case *apievents.TaskStart:
   785  				et = libcontainerdtypes.EventStart
   786  				ei = libcontainerdtypes.EventInfo{
   787  					ContainerID: t.ContainerID,
   788  					ProcessID:   t.ContainerID,
   789  					Pid:         t.Pid,
   790  				}
   791  			case *apievents.TaskExit:
   792  				et = libcontainerdtypes.EventExit
   793  				ei = libcontainerdtypes.EventInfo{
   794  					ContainerID: t.ContainerID,
   795  					ProcessID:   t.ID,
   796  					Pid:         t.Pid,
   797  					ExitCode:    t.ExitStatus,
   798  					ExitedAt:    t.ExitedAt,
   799  				}
   800  			case *apievents.TaskOOM:
   801  				et = libcontainerdtypes.EventOOM
   802  				ei = libcontainerdtypes.EventInfo{
   803  					ContainerID: t.ContainerID,
   804  					OOMKilled:   true,
   805  				}
   806  				oomKilled = true
   807  			case *apievents.TaskExecAdded:
   808  				et = libcontainerdtypes.EventExecAdded
   809  				ei = libcontainerdtypes.EventInfo{
   810  					ContainerID: t.ContainerID,
   811  					ProcessID:   t.ExecID,
   812  				}
   813  			case *apievents.TaskExecStarted:
   814  				et = libcontainerdtypes.EventExecStarted
   815  				ei = libcontainerdtypes.EventInfo{
   816  					ContainerID: t.ContainerID,
   817  					ProcessID:   t.ExecID,
   818  					Pid:         t.Pid,
   819  				}
   820  			case *apievents.TaskPaused:
   821  				et = libcontainerdtypes.EventPaused
   822  				ei = libcontainerdtypes.EventInfo{
   823  					ContainerID: t.ContainerID,
   824  				}
   825  			case *apievents.TaskResumed:
   826  				et = libcontainerdtypes.EventResumed
   827  				ei = libcontainerdtypes.EventInfo{
   828  					ContainerID: t.ContainerID,
   829  				}
   830  			default:
   831  				c.logger.WithFields(logrus.Fields{
   832  					"topic": ev.Topic,
   833  					"type":  reflect.TypeOf(t)},
   834  				).Info("ignoring event")
   835  				continue
   836  			}
   837  
   838  			c.oomMu.Lock()
   839  			if oomKilled {
   840  				c.oom[ei.ContainerID] = true
   841  			}
   842  			ei.OOMKilled = c.oom[ei.ContainerID]
   843  			c.oomMu.Unlock()
   844  
   845  			c.processEvent(ctx, et, ei)
   846  		}
   847  	}
   848  }
   849  
   850  func (c *client) writeContent(ctx context.Context, mediaType, ref string, r io.Reader) (*types.Descriptor, error) {
   851  	writer, err := c.client.ContentStore().Writer(ctx, content.WithRef(ref))
   852  	if err != nil {
   853  		return nil, err
   854  	}
   855  	defer writer.Close()
   856  	size, err := io.Copy(writer, r)
   857  	if err != nil {
   858  		return nil, err
   859  	}
   860  	labels := map[string]string{
   861  		"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339),
   862  	}
   863  	if err := writer.Commit(ctx, 0, "", content.WithLabels(labels)); err != nil {
   864  		return nil, err
   865  	}
   866  	return &types.Descriptor{
   867  		MediaType: mediaType,
   868  		Digest:    writer.Digest(),
   869  		Size_:     size,
   870  	}, nil
   871  }
   872  
   873  func (c *client) bundleDir(id string) string {
   874  	return filepath.Join(c.stateDir, id)
   875  }
   876  
   877  func wrapError(err error) error {
   878  	switch {
   879  	case err == nil:
   880  		return nil
   881  	case containerderrors.IsNotFound(err):
   882  		return errdefs.NotFound(err)
   883  	}
   884  
   885  	msg := err.Error()
   886  	for _, s := range []string{"container does not exist", "not found", "no such container"} {
   887  		if strings.Contains(msg, s) {
   888  			return errdefs.NotFound(err)
   889  		}
   890  	}
   891  	return err
   892  }