github.com/bigcommerce/nomad@v0.9.3-bc/drivers/shared/executor/executor.go (about)

     1  package executor
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io"
     7  	"io/ioutil"
     8  	"os"
     9  	"os/exec"
    10  	"path/filepath"
    11  	"runtime"
    12  	"strings"
    13  	"syscall"
    14  	"time"
    15  
    16  	"github.com/armon/circbuf"
    17  	"github.com/hashicorp/consul-template/signals"
    18  	hclog "github.com/hashicorp/go-hclog"
    19  	multierror "github.com/hashicorp/go-multierror"
    20  	"github.com/hashicorp/nomad/client/allocdir"
    21  	"github.com/hashicorp/nomad/client/lib/fifo"
    22  	"github.com/hashicorp/nomad/client/stats"
    23  	cstructs "github.com/hashicorp/nomad/client/structs"
    24  	"github.com/hashicorp/nomad/plugins/drivers"
    25  	"github.com/kr/pty"
    26  
    27  	shelpers "github.com/hashicorp/nomad/helper/stats"
    28  )
    29  
    30  const (
    31  	// ExecutorVersionLatest is the current and latest version of the executor
    32  	ExecutorVersionLatest = "2.0.0"
    33  
    34  	// ExecutorVersionPre0_9 is the version of executor use prior to the release
    35  	// of 0.9.x
    36  	ExecutorVersionPre0_9 = "1.1.0"
    37  )
    38  
    39  var (
    40  	// The statistics the basic executor exposes
    41  	ExecutorBasicMeasuredMemStats = []string{"RSS", "Swap"}
    42  	ExecutorBasicMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"}
    43  )
    44  
    45  // Executor is the interface which allows a driver to launch and supervise
    46  // a process
    47  type Executor interface {
    48  	// Launch a user process configured by the given ExecCommand
    49  	Launch(launchCmd *ExecCommand) (*ProcessState, error)
    50  
    51  	// Wait blocks until the process exits or an error occures
    52  	Wait(ctx context.Context) (*ProcessState, error)
    53  
    54  	// Shutdown will shutdown the executor by stopping the user process,
    55  	// cleaning up and resources created by the executor. The shutdown sequence
    56  	// will first send the given signal to the process. This defaults to "SIGINT"
    57  	// if not specified. The executor will then wait for the process to exit
    58  	// before cleaning up other resources. If the executor waits longer than the
    59  	// given grace period, the process is forcefully killed.
    60  	//
    61  	// To force kill the user process, gracePeriod can be set to 0.
    62  	Shutdown(signal string, gracePeriod time.Duration) error
    63  
    64  	// UpdateResources updates any resource isolation enforcement with new
    65  	// constraints if supported.
    66  	UpdateResources(*drivers.Resources) error
    67  
    68  	// Version returns the executor API version
    69  	Version() (*ExecutorVersion, error)
    70  
    71  	// Returns a channel of stats. Stats are collected and
    72  	// pushed to the channel on the given interval
    73  	Stats(context.Context, time.Duration) (<-chan *cstructs.TaskResourceUsage, error)
    74  
    75  	// Signal sends the given signal to the user process
    76  	Signal(os.Signal) error
    77  
    78  	// Exec executes the given command and args inside the executor context
    79  	// and returns the output and exit code.
    80  	Exec(deadline time.Time, cmd string, args []string) ([]byte, int, error)
    81  
    82  	ExecStreaming(ctx context.Context, cmd []string, tty bool,
    83  		stream drivers.ExecTaskStream) error
    84  }
    85  
    86  // ExecCommand holds the user command, args, and other isolation related
    87  // settings.
    88  type ExecCommand struct {
    89  	// Cmd is the command that the user wants to run.
    90  	Cmd string
    91  
    92  	// Args is the args of the command that the user wants to run.
    93  	Args []string
    94  
    95  	// Resources defined by the task
    96  	Resources *drivers.Resources
    97  
    98  	// StdoutPath is the path the process stdout should be written to
    99  	StdoutPath string
   100  	stdout     io.WriteCloser
   101  
   102  	// StderrPath is the path the process stderr should be written to
   103  	StderrPath string
   104  	stderr     io.WriteCloser
   105  
   106  	// Env is the list of KEY=val pairs of environment variables to be set
   107  	Env []string
   108  
   109  	// User is the user which the executor uses to run the command.
   110  	User string
   111  
   112  	// TaskDir is the directory path on the host where for the task
   113  	TaskDir string
   114  
   115  	// ResourceLimits determines whether resource limits are enforced by the
   116  	// executor.
   117  	ResourceLimits bool
   118  
   119  	// Cgroup marks whether we put the process in a cgroup. Setting this field
   120  	// doesn't enforce resource limits. To enforce limits, set ResourceLimits.
   121  	// Using the cgroup does allow more precise cleanup of processes.
   122  	BasicProcessCgroup bool
   123  
   124  	// Mounts are the host paths to be be made available inside rootfs
   125  	Mounts []*drivers.MountConfig
   126  
   127  	// Devices are the the device nodes to be created in isolation environment
   128  	Devices []*drivers.DeviceConfig
   129  }
   130  
   131  // SetWriters sets the writer for the process stdout and stderr. This should
   132  // not be used if writing to a file path such as a fifo file. SetStdoutWriter
   133  // is mainly used for unit testing purposes.
   134  func (c *ExecCommand) SetWriters(out io.WriteCloser, err io.WriteCloser) {
   135  	c.stdout = out
   136  	c.stderr = err
   137  }
   138  
   139  // GetWriters returns the unexported io.WriteCloser for the stdout and stderr
   140  // handles. This is mainly used for unit testing purposes.
   141  func (c *ExecCommand) GetWriters() (stdout io.WriteCloser, stderr io.WriteCloser) {
   142  	return c.stdout, c.stderr
   143  }
   144  
   145  type nopCloser struct {
   146  	io.Writer
   147  }
   148  
   149  func (nopCloser) Close() error { return nil }
   150  
   151  // Stdout returns a writer for the configured file descriptor
   152  func (c *ExecCommand) Stdout() (io.WriteCloser, error) {
   153  	if c.stdout == nil {
   154  		if c.StdoutPath != "" {
   155  			f, err := fifo.OpenWriter(c.StdoutPath)
   156  			if err != nil {
   157  				return nil, fmt.Errorf("failed to create stdout: %v", err)
   158  			}
   159  			c.stdout = f
   160  		} else {
   161  			c.stdout = nopCloser{ioutil.Discard}
   162  		}
   163  	}
   164  	return c.stdout, nil
   165  }
   166  
   167  // Stderr returns a writer for the configured file descriptor
   168  func (c *ExecCommand) Stderr() (io.WriteCloser, error) {
   169  	if c.stderr == nil {
   170  		if c.StderrPath != "" {
   171  			f, err := fifo.OpenWriter(c.StderrPath)
   172  			if err != nil {
   173  				return nil, fmt.Errorf("failed to create stderr: %v", err)
   174  			}
   175  			c.stderr = f
   176  		} else {
   177  			c.stderr = nopCloser{ioutil.Discard}
   178  		}
   179  	}
   180  	return c.stderr, nil
   181  }
   182  
   183  func (c *ExecCommand) Close() {
   184  	if c.stdout != nil {
   185  		c.stdout.Close()
   186  	}
   187  	if c.stderr != nil {
   188  		c.stderr.Close()
   189  	}
   190  }
   191  
   192  // ProcessState holds information about the state of a user process.
   193  type ProcessState struct {
   194  	Pid      int
   195  	ExitCode int
   196  	Signal   int
   197  	Time     time.Time
   198  }
   199  
   200  // ExecutorVersion is the version of the executor
   201  type ExecutorVersion struct {
   202  	Version string
   203  }
   204  
   205  func (v *ExecutorVersion) GoString() string {
   206  	return v.Version
   207  }
   208  
   209  // UniversalExecutor is an implementation of the Executor which launches and
   210  // supervises processes. In addition to process supervision it provides resource
   211  // and file system isolation
   212  type UniversalExecutor struct {
   213  	childCmd   exec.Cmd
   214  	commandCfg *ExecCommand
   215  
   216  	exitState     *ProcessState
   217  	processExited chan interface{}
   218  
   219  	// resConCtx is used to track and cleanup additional resources created by
   220  	// the executor. Currently this is only used for cgroups.
   221  	resConCtx resourceContainerContext
   222  
   223  	totalCpuStats  *stats.CpuStats
   224  	userCpuStats   *stats.CpuStats
   225  	systemCpuStats *stats.CpuStats
   226  	pidCollector   *pidCollector
   227  
   228  	logger hclog.Logger
   229  }
   230  
   231  // NewExecutor returns an Executor
   232  func NewExecutor(logger hclog.Logger) Executor {
   233  	logger = logger.Named("executor")
   234  	if err := shelpers.Init(); err != nil {
   235  		logger.Error("unable to initialize stats", "error", err)
   236  	}
   237  	return &UniversalExecutor{
   238  		logger:         logger,
   239  		processExited:  make(chan interface{}),
   240  		totalCpuStats:  stats.NewCpuStats(),
   241  		userCpuStats:   stats.NewCpuStats(),
   242  		systemCpuStats: stats.NewCpuStats(),
   243  		pidCollector:   newPidCollector(logger),
   244  	}
   245  }
   246  
   247  // Version returns the api version of the executor
   248  func (e *UniversalExecutor) Version() (*ExecutorVersion, error) {
   249  	return &ExecutorVersion{Version: ExecutorVersionLatest}, nil
   250  }
   251  
   252  // Launch launches the main process and returns its state. It also
   253  // configures an applies isolation on certain platforms.
   254  func (e *UniversalExecutor) Launch(command *ExecCommand) (*ProcessState, error) {
   255  	e.logger.Trace("preparing to launch command", "command", command.Cmd, "args", strings.Join(command.Args, " "))
   256  
   257  	e.commandCfg = command
   258  
   259  	// setting the user of the process
   260  	if command.User != "" {
   261  		e.logger.Debug("running command as user", "user", command.User)
   262  		if err := e.runAs(command.User); err != nil {
   263  			return nil, err
   264  		}
   265  	}
   266  
   267  	// set the task dir as the working directory for the command
   268  	e.childCmd.Dir = e.commandCfg.TaskDir
   269  
   270  	// start command in separate process group
   271  	if err := e.setNewProcessGroup(); err != nil {
   272  		return nil, err
   273  	}
   274  
   275  	// Setup cgroups on linux
   276  	if err := e.configureResourceContainer(os.Getpid()); err != nil {
   277  		return nil, err
   278  	}
   279  
   280  	stdout, err := e.commandCfg.Stdout()
   281  	if err != nil {
   282  		return nil, err
   283  	}
   284  	stderr, err := e.commandCfg.Stderr()
   285  	if err != nil {
   286  		return nil, err
   287  	}
   288  
   289  	e.childCmd.Stdout = stdout
   290  	e.childCmd.Stderr = stderr
   291  
   292  	// Look up the binary path and make it executable
   293  	absPath, err := lookupBin(command.TaskDir, command.Cmd)
   294  	if err != nil {
   295  		return nil, err
   296  	}
   297  
   298  	if err := makeExecutable(absPath); err != nil {
   299  		return nil, err
   300  	}
   301  
   302  	path := absPath
   303  
   304  	// Set the commands arguments
   305  	e.childCmd.Path = path
   306  	e.childCmd.Args = append([]string{e.childCmd.Path}, command.Args...)
   307  	e.childCmd.Env = e.commandCfg.Env
   308  
   309  	// Start the process
   310  	e.logger.Debug("launching", "command", command.Cmd, "args", strings.Join(command.Args, " "))
   311  	if err := e.childCmd.Start(); err != nil {
   312  		return nil, fmt.Errorf("failed to start command path=%q --- args=%q: %v", path, e.childCmd.Args, err)
   313  	}
   314  
   315  	go e.pidCollector.collectPids(e.processExited, getAllPids)
   316  	go e.wait()
   317  	return &ProcessState{Pid: e.childCmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil
   318  }
   319  
   320  // Exec a command inside a container for exec and java drivers.
   321  func (e *UniversalExecutor) Exec(deadline time.Time, name string, args []string) ([]byte, int, error) {
   322  	ctx, cancel := context.WithDeadline(context.Background(), deadline)
   323  	defer cancel()
   324  	return ExecScript(ctx, e.childCmd.Dir, e.commandCfg.Env, e.childCmd.SysProcAttr, name, args)
   325  }
   326  
   327  // ExecScript executes cmd with args and returns the output, exit code, and
   328  // error. Output is truncated to drivers/shared/structs.CheckBufSize
   329  func ExecScript(ctx context.Context, dir string, env []string, attrs *syscall.SysProcAttr,
   330  	name string, args []string) ([]byte, int, error) {
   331  	cmd := exec.CommandContext(ctx, name, args...)
   332  
   333  	// Copy runtime environment from the main command
   334  	cmd.SysProcAttr = attrs
   335  	cmd.Dir = dir
   336  	cmd.Env = env
   337  
   338  	// Capture output
   339  	buf, _ := circbuf.NewBuffer(int64(drivers.CheckBufSize))
   340  	cmd.Stdout = buf
   341  	cmd.Stderr = buf
   342  
   343  	if err := cmd.Run(); err != nil {
   344  		exitErr, ok := err.(*exec.ExitError)
   345  		if !ok {
   346  			// Non-exit error, return it and let the caller treat
   347  			// it as a critical failure
   348  			return nil, 0, err
   349  		}
   350  
   351  		// Some kind of error happened; default to critical
   352  		exitCode := 2
   353  		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
   354  			exitCode = status.ExitStatus()
   355  		}
   356  
   357  		// Don't return the exitError as the caller only needs the
   358  		// output and code.
   359  		return buf.Bytes(), exitCode, nil
   360  	}
   361  	return buf.Bytes(), 0, nil
   362  }
   363  
   364  func (e *UniversalExecutor) ExecStreaming(ctx context.Context, command []string, tty bool,
   365  	stream drivers.ExecTaskStream) error {
   366  
   367  	if len(command) == 0 {
   368  		return fmt.Errorf("command is required")
   369  	}
   370  
   371  	cmd := exec.CommandContext(ctx, command[0], command[1:]...)
   372  
   373  	cmd.Dir = "/"
   374  	cmd.Env = e.childCmd.Env
   375  
   376  	execHelper := &execHelper{
   377  		logger: e.logger,
   378  
   379  		newTerminal: func() (func() (*os.File, error), *os.File, error) {
   380  			pty, tty, err := pty.Open()
   381  			if err != nil {
   382  				return nil, nil, err
   383  			}
   384  
   385  			return func() (*os.File, error) { return pty, nil }, tty, err
   386  		},
   387  		setTTY: func(tty *os.File) error {
   388  			cmd.SysProcAttr = sessionCmdAttr(tty)
   389  
   390  			cmd.Stdin = tty
   391  			cmd.Stdout = tty
   392  			cmd.Stderr = tty
   393  			return nil
   394  		},
   395  		setIO: func(stdin io.Reader, stdout, stderr io.Writer) error {
   396  			cmd.Stdin = stdin
   397  			cmd.Stdout = stdout
   398  			cmd.Stderr = stderr
   399  			return nil
   400  		},
   401  		processStart: cmd.Start,
   402  		processWait: func() (*os.ProcessState, error) {
   403  			err := cmd.Wait()
   404  			return cmd.ProcessState, err
   405  		},
   406  	}
   407  
   408  	return execHelper.run(ctx, tty, stream)
   409  }
   410  
   411  // Wait waits until a process has exited and returns it's exitcode and errors
   412  func (e *UniversalExecutor) Wait(ctx context.Context) (*ProcessState, error) {
   413  	select {
   414  	case <-ctx.Done():
   415  		return nil, ctx.Err()
   416  	case <-e.processExited:
   417  		return e.exitState, nil
   418  	}
   419  }
   420  
   421  func (e *UniversalExecutor) UpdateResources(resources *drivers.Resources) error {
   422  	return nil
   423  }
   424  
   425  func (e *UniversalExecutor) wait() {
   426  	defer close(e.processExited)
   427  	defer e.commandCfg.Close()
   428  	pid := e.childCmd.Process.Pid
   429  	err := e.childCmd.Wait()
   430  	if err == nil {
   431  		e.exitState = &ProcessState{Pid: pid, ExitCode: 0, Time: time.Now()}
   432  		return
   433  	}
   434  
   435  	exitCode := 1
   436  	var signal int
   437  	if exitErr, ok := err.(*exec.ExitError); ok {
   438  		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
   439  			exitCode = status.ExitStatus()
   440  			if status.Signaled() {
   441  				// bash(1) uses the lower 7 bits of a uint8
   442  				// to indicate normal program failure (see
   443  				// <sysexits.h>). If a process terminates due
   444  				// to a signal, encode the signal number to
   445  				// indicate which signal caused the process
   446  				// to terminate.  Mirror this exit code
   447  				// encoding scheme.
   448  				const exitSignalBase = 128
   449  				signal = int(status.Signal())
   450  				exitCode = exitSignalBase + signal
   451  			}
   452  		}
   453  	} else {
   454  		e.logger.Warn("unexpected Cmd.Wait() error type", "error", err)
   455  	}
   456  
   457  	e.exitState = &ProcessState{Pid: pid, ExitCode: exitCode, Signal: signal, Time: time.Now()}
   458  }
   459  
   460  var (
   461  	// finishedErr is the error message received when trying to kill and already
   462  	// exited process.
   463  	finishedErr = "os: process already finished"
   464  
   465  	// noSuchProcessErr is the error message received when trying to kill a non
   466  	// existing process (e.g. when killing a process group).
   467  	noSuchProcessErr = "no such process"
   468  )
   469  
   470  // Exit cleans up the alloc directory, destroys resource container and kills the
   471  // user process
   472  func (e *UniversalExecutor) Shutdown(signal string, grace time.Duration) error {
   473  	e.logger.Debug("shutdown requested", "signal", signal, "grace_period_ms", grace.Round(time.Millisecond))
   474  	var merr multierror.Error
   475  
   476  	// If the executor did not launch a process, return.
   477  	if e.commandCfg == nil {
   478  		return nil
   479  	}
   480  
   481  	// If there is no process we can't shutdown
   482  	if e.childCmd.Process == nil {
   483  		e.logger.Warn("failed to shutdown", "error", "no process found")
   484  		return fmt.Errorf("executor failed to shutdown error: no process found")
   485  	}
   486  
   487  	proc, err := os.FindProcess(e.childCmd.Process.Pid)
   488  	if err != nil {
   489  		err = fmt.Errorf("executor failed to find process: %v", err)
   490  		e.logger.Warn("failed to shutdown", "error", err)
   491  		return err
   492  	}
   493  
   494  	// If grace is 0 then skip shutdown logic
   495  	if grace > 0 {
   496  		// Default signal to SIGINT if not set
   497  		if signal == "" {
   498  			signal = "SIGINT"
   499  		}
   500  
   501  		sig, ok := signals.SignalLookup[signal]
   502  		if !ok {
   503  			err = fmt.Errorf("error unknown signal given for shutdown: %s", signal)
   504  			e.logger.Warn("failed to shutdown", "error", err)
   505  			return err
   506  		}
   507  
   508  		if err := e.shutdownProcess(sig, proc); err != nil {
   509  			e.logger.Warn("failed to shutdown", "error", err)
   510  			return err
   511  		}
   512  
   513  		select {
   514  		case <-e.processExited:
   515  		case <-time.After(grace):
   516  			proc.Kill()
   517  		}
   518  	} else {
   519  		proc.Kill()
   520  	}
   521  
   522  	// Wait for process to exit
   523  	select {
   524  	case <-e.processExited:
   525  	case <-time.After(time.Second * 15):
   526  		e.logger.Warn("process did not exit after 15 seconds")
   527  		merr.Errors = append(merr.Errors, fmt.Errorf("process did not exit after 15 seconds"))
   528  	}
   529  
   530  	// Prefer killing the process via the resource container.
   531  	if !(e.commandCfg.ResourceLimits || e.commandCfg.BasicProcessCgroup) {
   532  		if err := e.cleanupChildProcesses(proc); err != nil && err.Error() != finishedErr {
   533  			merr.Errors = append(merr.Errors,
   534  				fmt.Errorf("can't kill process with pid %d: %v", e.childCmd.Process.Pid, err))
   535  		}
   536  	}
   537  
   538  	if e.commandCfg.ResourceLimits || e.commandCfg.BasicProcessCgroup {
   539  		if err := e.resConCtx.executorCleanup(); err != nil {
   540  			merr.Errors = append(merr.Errors, err)
   541  		}
   542  	}
   543  
   544  	if err := merr.ErrorOrNil(); err != nil {
   545  		e.logger.Warn("failed to shutdown", "error", err)
   546  		return err
   547  	}
   548  
   549  	return nil
   550  }
   551  
   552  // Signal sends the passed signal to the task
   553  func (e *UniversalExecutor) Signal(s os.Signal) error {
   554  	if e.childCmd.Process == nil {
   555  		return fmt.Errorf("Task not yet run")
   556  	}
   557  
   558  	e.logger.Debug("sending signal to PID", "signal", s, "pid", e.childCmd.Process.Pid)
   559  	err := e.childCmd.Process.Signal(s)
   560  	if err != nil {
   561  		e.logger.Error("sending signal failed", "signal", s, "error", err)
   562  		return err
   563  	}
   564  
   565  	return nil
   566  }
   567  
   568  func (e *UniversalExecutor) Stats(ctx context.Context, interval time.Duration) (<-chan *cstructs.TaskResourceUsage, error) {
   569  	ch := make(chan *cstructs.TaskResourceUsage)
   570  	go e.handleStats(ch, ctx, interval)
   571  	return ch, nil
   572  }
   573  
   574  func (e *UniversalExecutor) handleStats(ch chan *cstructs.TaskResourceUsage, ctx context.Context, interval time.Duration) {
   575  	defer close(ch)
   576  	timer := time.NewTimer(0)
   577  	for {
   578  		select {
   579  		case <-ctx.Done():
   580  			return
   581  
   582  		case <-timer.C:
   583  			timer.Reset(interval)
   584  		}
   585  
   586  		pidStats, err := e.pidCollector.pidStats()
   587  		if err != nil {
   588  			e.logger.Warn("error collecting stats", "error", err)
   589  			return
   590  		}
   591  
   592  		select {
   593  		case <-ctx.Done():
   594  			return
   595  		case ch <- aggregatedResourceUsage(e.systemCpuStats, pidStats):
   596  		}
   597  	}
   598  }
   599  
   600  // lookupBin looks for path to the binary to run by looking for the binary in
   601  // the following locations, in-order:
   602  // task/local/, task/, on the host file system, in host $PATH
   603  // The return path is absolute.
   604  func lookupBin(taskDir string, bin string) (string, error) {
   605  	// Check in the local directory
   606  	local := filepath.Join(taskDir, allocdir.TaskLocal, bin)
   607  	if _, err := os.Stat(local); err == nil {
   608  		return local, nil
   609  	}
   610  
   611  	// Check at the root of the task's directory
   612  	root := filepath.Join(taskDir, bin)
   613  	if _, err := os.Stat(root); err == nil {
   614  		return root, nil
   615  	}
   616  
   617  	// when checking host paths, check with Stat first if path is absolute
   618  	// as exec.LookPath only considers files already marked as executable
   619  	// and only consider this for absolute paths to avoid depending on
   620  	// current directory of nomad which may cause unexpected behavior
   621  	if _, err := os.Stat(bin); err == nil && filepath.IsAbs(bin) {
   622  		return bin, nil
   623  	}
   624  
   625  	// Check the $PATH
   626  	if host, err := exec.LookPath(bin); err == nil {
   627  		return host, nil
   628  	}
   629  
   630  	return "", fmt.Errorf("binary %q could not be found", bin)
   631  }
   632  
   633  // makeExecutable makes the given file executable for root,group,others.
   634  func makeExecutable(binPath string) error {
   635  	if runtime.GOOS == "windows" {
   636  		return nil
   637  	}
   638  
   639  	fi, err := os.Stat(binPath)
   640  	if err != nil {
   641  		if os.IsNotExist(err) {
   642  			return fmt.Errorf("binary %q does not exist", binPath)
   643  		}
   644  		return fmt.Errorf("specified binary is invalid: %v", err)
   645  	}
   646  
   647  	// If it is not executable, make it so.
   648  	perm := fi.Mode().Perm()
   649  	req := os.FileMode(0555)
   650  	if perm&req != req {
   651  		if err := os.Chmod(binPath, perm|req); err != nil {
   652  			return fmt.Errorf("error making %q executable: %s", binPath, err)
   653  		}
   654  	}
   655  	return nil
   656  }