github.com/Prakhar-Agarwal-byte/moby@v0.0.0-20231027092010-a14e3e8ab87e/testutil/daemon/daemon.go (about)

     1  package daemon // import "github.com/Prakhar-Agarwal-byte/moby/testutil/daemon"
     2  
     3  import (
     4  	"bufio"
     5  	"context"
     6  	"encoding/json"
     7  	"io"
     8  	"net/http"
     9  	"os"
    10  	"os/exec"
    11  	"os/user"
    12  	"path/filepath"
    13  	"strconv"
    14  	"strings"
    15  	"testing"
    16  	"time"
    17  
    18  	"github.com/Prakhar-Agarwal-byte/moby/api/types/events"
    19  	"github.com/Prakhar-Agarwal-byte/moby/api/types/system"
    20  	"github.com/Prakhar-Agarwal-byte/moby/client"
    21  	"github.com/Prakhar-Agarwal-byte/moby/container"
    22  	"github.com/Prakhar-Agarwal-byte/moby/pkg/ioutils"
    23  	"github.com/Prakhar-Agarwal-byte/moby/pkg/stringid"
    24  	"github.com/Prakhar-Agarwal-byte/moby/pkg/tailfile"
    25  	"github.com/Prakhar-Agarwal-byte/moby/testutil/request"
    26  	"github.com/docker/go-connections/sockets"
    27  	"github.com/docker/go-connections/tlsconfig"
    28  	"github.com/pkg/errors"
    29  	"gotest.tools/v3/assert"
    30  	"gotest.tools/v3/poll"
    31  )
    32  
    33  // LogT is the subset of the testing.TB interface used by the daemon.
    34  type LogT interface {
    35  	Logf(string, ...interface{})
    36  }
    37  
    38  // nopLog is a no-op implementation of LogT that is used in daemons created by
    39  // NewDaemon (where no testing.TB is available).
    40  type nopLog struct{}
    41  
    42  func (nopLog) Logf(string, ...interface{}) {}
    43  
    44  const (
    45  	defaultDockerdBinary         = "dockerd"
    46  	defaultContainerdSocket      = "/var/run/docker/containerd/containerd.sock"
    47  	defaultDockerdRootlessBinary = "dockerd-rootless.sh"
    48  	defaultUnixSocket            = "/var/run/docker.sock"
    49  	defaultTLSHost               = "localhost:2376"
    50  )
    51  
    52  var errDaemonNotStarted = errors.New("daemon not started")
    53  
    54  // SockRoot holds the path of the default docker integration daemon socket
    55  var SockRoot = filepath.Join(os.TempDir(), "docker-integration")
    56  
    57  type clientConfig struct {
    58  	transport *http.Transport
    59  	scheme    string
    60  	addr      string
    61  }
    62  
    63  // Daemon represents a Docker daemon for the testing framework
    64  type Daemon struct {
    65  	Root              string
    66  	Folder            string
    67  	Wait              chan error
    68  	UseDefaultHost    bool
    69  	UseDefaultTLSHost bool
    70  
    71  	id                         string
    72  	logFile                    *os.File
    73  	cmd                        *exec.Cmd
    74  	storageDriver              string
    75  	userlandProxy              bool
    76  	defaultCgroupNamespaceMode string
    77  	execRoot                   string
    78  	experimental               bool
    79  	init                       bool
    80  	dockerdBinary              string
    81  	log                        LogT
    82  	pidFile                    string
    83  	args                       []string
    84  	extraEnv                   []string
    85  	containerdSocket           string
    86  	rootlessUser               *user.User
    87  	rootlessXDGRuntimeDir      string
    88  
    89  	// swarm related field
    90  	swarmListenAddr string
    91  	SwarmPort       int // FIXME(vdemeester) should probably not be exported
    92  	DefaultAddrPool []string
    93  	SubnetSize      uint32
    94  	DataPathPort    uint32
    95  	OOMScoreAdjust  int
    96  	// cached information
    97  	CachedInfo system.Info
    98  }
    99  
   100  // NewDaemon returns a Daemon instance to be used for testing.
   101  // The daemon will not automatically start.
   102  // The daemon will modify and create files under workingDir.
   103  func NewDaemon(workingDir string, ops ...Option) (*Daemon, error) {
   104  	storageDriver := os.Getenv("DOCKER_GRAPHDRIVER")
   105  
   106  	if err := os.MkdirAll(SockRoot, 0o700); err != nil {
   107  		return nil, errors.Wrapf(err, "failed to create daemon socket root %q", SockRoot)
   108  	}
   109  
   110  	id := "d" + stringid.TruncateID(stringid.GenerateRandomID())
   111  	dir := filepath.Join(workingDir, id)
   112  	daemonFolder, err := filepath.Abs(dir)
   113  	if err != nil {
   114  		return nil, err
   115  	}
   116  	daemonRoot := filepath.Join(daemonFolder, "root")
   117  	if err := os.MkdirAll(daemonRoot, 0o755); err != nil {
   118  		return nil, errors.Wrapf(err, "failed to create daemon root %q", daemonRoot)
   119  	}
   120  
   121  	userlandProxy := true
   122  	if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" {
   123  		if val, err := strconv.ParseBool(env); err != nil {
   124  			userlandProxy = val
   125  		}
   126  	}
   127  	d := &Daemon{
   128  		id:            id,
   129  		Folder:        daemonFolder,
   130  		Root:          daemonRoot,
   131  		storageDriver: storageDriver,
   132  		userlandProxy: userlandProxy,
   133  		// dxr stands for docker-execroot (shortened for avoiding unix(7) path length limitation)
   134  		execRoot:         filepath.Join(os.TempDir(), "dxr", id),
   135  		dockerdBinary:    defaultDockerdBinary,
   136  		swarmListenAddr:  defaultSwarmListenAddr,
   137  		SwarmPort:        DefaultSwarmPort,
   138  		log:              nopLog{},
   139  		containerdSocket: defaultContainerdSocket,
   140  	}
   141  
   142  	for _, op := range ops {
   143  		op(d)
   144  	}
   145  
   146  	if d.rootlessUser != nil {
   147  		if err := os.Chmod(SockRoot, 0o777); err != nil {
   148  			return nil, err
   149  		}
   150  		uid, err := strconv.Atoi(d.rootlessUser.Uid)
   151  		if err != nil {
   152  			return nil, err
   153  		}
   154  		gid, err := strconv.Atoi(d.rootlessUser.Gid)
   155  		if err != nil {
   156  			return nil, err
   157  		}
   158  		if err := os.Chown(d.Folder, uid, gid); err != nil {
   159  			return nil, err
   160  		}
   161  		if err := os.Chown(d.Root, uid, gid); err != nil {
   162  			return nil, err
   163  		}
   164  		if err := os.MkdirAll(filepath.Dir(d.execRoot), 0o700); err != nil {
   165  			return nil, err
   166  		}
   167  		if err := os.Chown(filepath.Dir(d.execRoot), uid, gid); err != nil {
   168  			return nil, err
   169  		}
   170  		if err := os.MkdirAll(d.execRoot, 0o700); err != nil {
   171  			return nil, err
   172  		}
   173  		if err := os.Chown(d.execRoot, uid, gid); err != nil {
   174  			return nil, err
   175  		}
   176  		d.rootlessXDGRuntimeDir = filepath.Join(d.Folder, "xdgrun")
   177  		if err := os.MkdirAll(d.rootlessXDGRuntimeDir, 0o700); err != nil {
   178  			return nil, err
   179  		}
   180  		if err := os.Chown(d.rootlessXDGRuntimeDir, uid, gid); err != nil {
   181  			return nil, err
   182  		}
   183  		d.containerdSocket = ""
   184  	}
   185  
   186  	return d, nil
   187  }
   188  
   189  // New returns a Daemon instance to be used for testing.
   190  // This will create a directory such as d123456789 in the folder specified by
   191  // $DOCKER_INTEGRATION_DAEMON_DEST or $DEST.
   192  // The daemon will not automatically start.
   193  func New(t testing.TB, ops ...Option) *Daemon {
   194  	t.Helper()
   195  	dest := os.Getenv("DOCKER_INTEGRATION_DAEMON_DEST")
   196  	if dest == "" {
   197  		dest = os.Getenv("DEST")
   198  	}
   199  	dest = filepath.Join(dest, t.Name())
   200  
   201  	assert.Check(t, dest != "", "Please set the DOCKER_INTEGRATION_DAEMON_DEST or the DEST environment variable")
   202  
   203  	if os.Getenv("DOCKER_ROOTLESS") != "" {
   204  		if os.Getenv("DOCKER_REMAP_ROOT") != "" {
   205  			t.Skip("DOCKER_ROOTLESS doesn't support DOCKER_REMAP_ROOT currently")
   206  		}
   207  		if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" {
   208  			if val, err := strconv.ParseBool(env); err == nil && !val {
   209  				t.Skip("DOCKER_ROOTLESS doesn't support DOCKER_USERLANDPROXY=false")
   210  			}
   211  		}
   212  		ops = append(ops, WithRootlessUser("unprivilegeduser"))
   213  	}
   214  	ops = append(ops, WithOOMScoreAdjust(-500))
   215  
   216  	d, err := NewDaemon(dest, ops...)
   217  	assert.NilError(t, err, "could not create daemon at %q", dest)
   218  	if d.rootlessUser != nil && d.dockerdBinary != defaultDockerdBinary {
   219  		t.Skipf("DOCKER_ROOTLESS doesn't support specifying non-default dockerd binary path %q", d.dockerdBinary)
   220  	}
   221  
   222  	return d
   223  }
   224  
   225  // BinaryPath returns the binary and its arguments.
   226  func (d *Daemon) BinaryPath() (string, error) {
   227  	dockerdBinary, err := exec.LookPath(d.dockerdBinary)
   228  	if err != nil {
   229  		return "", errors.Wrapf(err, "[%s] could not find docker binary in $PATH", d.id)
   230  	}
   231  	return dockerdBinary, nil
   232  }
   233  
   234  // ContainersNamespace returns the containerd namespace used for containers.
   235  func (d *Daemon) ContainersNamespace() string {
   236  	return d.id
   237  }
   238  
   239  // RootDir returns the root directory of the daemon.
   240  func (d *Daemon) RootDir() string {
   241  	return d.Root
   242  }
   243  
   244  // ID returns the generated id of the daemon
   245  func (d *Daemon) ID() string {
   246  	return d.id
   247  }
   248  
   249  // StorageDriver returns the configured storage driver of the daemon
   250  func (d *Daemon) StorageDriver() string {
   251  	return d.storageDriver
   252  }
   253  
   254  // Sock returns the socket path of the daemon
   255  func (d *Daemon) Sock() string {
   256  	return "unix://" + d.sockPath()
   257  }
   258  
   259  func (d *Daemon) sockPath() string {
   260  	return filepath.Join(SockRoot, d.id+".sock")
   261  }
   262  
   263  // LogFileName returns the path the daemon's log file
   264  func (d *Daemon) LogFileName() string {
   265  	return d.logFile.Name()
   266  }
   267  
   268  // ReadLogFile returns the content of the daemon log file
   269  func (d *Daemon) ReadLogFile() ([]byte, error) {
   270  	_ = d.logFile.Sync()
   271  	return os.ReadFile(d.logFile.Name())
   272  }
   273  
   274  // NewClientT creates new client based on daemon's socket path
   275  func (d *Daemon) NewClientT(t testing.TB, extraOpts ...client.Opt) *client.Client {
   276  	t.Helper()
   277  
   278  	c, err := d.NewClient(extraOpts...)
   279  	assert.NilError(t, err, "[%s] could not create daemon client", d.id)
   280  	t.Cleanup(func() { c.Close() })
   281  	return c
   282  }
   283  
   284  // NewClient creates new client based on daemon's socket path
   285  func (d *Daemon) NewClient(extraOpts ...client.Opt) (*client.Client, error) {
   286  	clientOpts := []client.Opt{
   287  		client.FromEnv,
   288  		client.WithHost(d.Sock()),
   289  	}
   290  	clientOpts = append(clientOpts, extraOpts...)
   291  
   292  	return client.NewClientWithOpts(clientOpts...)
   293  }
   294  
   295  // Cleanup cleans the daemon files : exec root (network namespaces, ...), swarmkit files
   296  func (d *Daemon) Cleanup(t testing.TB) {
   297  	t.Helper()
   298  	cleanupMount(t, d)
   299  	cleanupRaftDir(t, d)
   300  	cleanupDaemonStorage(t, d)
   301  	cleanupNetworkNamespace(t, d)
   302  }
   303  
   304  // TailLogsT attempts to tail N lines from the daemon logs.
   305  // If there is an error the error is only logged, it does not cause an error with the test.
   306  func (d *Daemon) TailLogsT(t LogT, n int) {
   307  	lines, err := d.TailLogs(n)
   308  	if err != nil {
   309  		t.Logf("[%s] %v", d.id, err)
   310  		return
   311  	}
   312  	for _, l := range lines {
   313  		t.Logf("[%s] %s", d.id, string(l))
   314  	}
   315  }
   316  
   317  // PollCheckLogs is a poll.Check that checks the daemon logs using the passed in match function.
   318  func (d *Daemon) PollCheckLogs(ctx context.Context, match func(s string) bool) poll.Check {
   319  	return func(t poll.LogT) poll.Result {
   320  		ok, _, err := d.ScanLogs(ctx, match)
   321  		if err != nil {
   322  			return poll.Error(err)
   323  		}
   324  		if !ok {
   325  			return poll.Continue("waiting for daemon logs match")
   326  		}
   327  		return poll.Success()
   328  	}
   329  }
   330  
   331  // ScanLogsMatchString returns a function that can be used to scan the daemon logs for the passed in string (`contains`).
   332  func ScanLogsMatchString(contains string) func(string) bool {
   333  	return func(line string) bool {
   334  		return strings.Contains(line, contains)
   335  	}
   336  }
   337  
   338  // ScanLogsMatchAll returns a function that can be used to scan the daemon logs until *all* the passed in strings are matched
   339  func ScanLogsMatchAll(contains ...string) func(string) bool {
   340  	matched := make(map[string]bool)
   341  	return func(line string) bool {
   342  		for _, c := range contains {
   343  			if strings.Contains(line, c) {
   344  				matched[c] = true
   345  			}
   346  		}
   347  		return len(matched) == len(contains)
   348  	}
   349  }
   350  
   351  // ScanLogsT uses `ScanLogs` to match the daemon logs using the passed in match function.
   352  // If there is an error or the match fails, the test will fail.
   353  func (d *Daemon) ScanLogsT(ctx context.Context, t testing.TB, match func(s string) bool) (bool, string) {
   354  	t.Helper()
   355  	ok, line, err := d.ScanLogs(ctx, match)
   356  	assert.NilError(t, err)
   357  	return ok, line
   358  }
   359  
   360  // ScanLogs scans the daemon logs and passes each line to the match function.
   361  func (d *Daemon) ScanLogs(ctx context.Context, match func(s string) bool) (bool, string, error) {
   362  	stat, err := d.logFile.Stat()
   363  	if err != nil {
   364  		return false, "", err
   365  	}
   366  	rdr := io.NewSectionReader(d.logFile, 0, stat.Size())
   367  
   368  	scanner := bufio.NewScanner(rdr)
   369  	for scanner.Scan() {
   370  		if match(scanner.Text()) {
   371  			return true, scanner.Text(), nil
   372  		}
   373  		select {
   374  		case <-ctx.Done():
   375  			return false, "", ctx.Err()
   376  		default:
   377  		}
   378  	}
   379  	return false, "", scanner.Err()
   380  }
   381  
   382  // TailLogs tails N lines from the daemon logs
   383  func (d *Daemon) TailLogs(n int) ([][]byte, error) {
   384  	logF, err := os.Open(d.logFile.Name())
   385  	if err != nil {
   386  		return nil, errors.Wrap(err, "error opening daemon log file after failed start")
   387  	}
   388  
   389  	defer logF.Close()
   390  	lines, err := tailfile.TailFile(logF, n)
   391  	if err != nil {
   392  		return nil, errors.Wrap(err, "error tailing log daemon logs")
   393  	}
   394  
   395  	return lines, nil
   396  }
   397  
   398  // Start starts the daemon and return once it is ready to receive requests.
   399  func (d *Daemon) Start(t testing.TB, args ...string) {
   400  	t.Helper()
   401  	if err := d.StartWithError(args...); err != nil {
   402  		d.TailLogsT(t, 20)
   403  		d.DumpStackAndQuit() // in case the daemon is stuck
   404  		t.Fatalf("[%s] failed to start daemon with arguments %v : %v", d.id, d.args, err)
   405  	}
   406  }
   407  
   408  // StartWithError starts the daemon and return once it is ready to receive requests.
   409  // It returns an error in case it couldn't start.
   410  func (d *Daemon) StartWithError(args ...string) error {
   411  	logFile, err := os.OpenFile(filepath.Join(d.Folder, "docker.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o600)
   412  	if err != nil {
   413  		return errors.Wrapf(err, "[%s] failed to create logfile", d.id)
   414  	}
   415  
   416  	return d.StartWithLogFile(logFile, args...)
   417  }
   418  
   419  // StartWithLogFile will start the daemon and attach its streams to a given file.
   420  func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error {
   421  	d.handleUserns()
   422  	dockerdBinary, err := d.BinaryPath()
   423  	if err != nil {
   424  		return err
   425  	}
   426  
   427  	if d.pidFile == "" {
   428  		d.pidFile = filepath.Join(d.Folder, "docker.pid")
   429  	}
   430  
   431  	d.args = []string{}
   432  	if d.rootlessUser != nil {
   433  		if d.dockerdBinary != defaultDockerdBinary {
   434  			return errors.Errorf("[%s] DOCKER_ROOTLESS doesn't support non-default dockerd binary path %q", d.id, d.dockerdBinary)
   435  		}
   436  		dockerdBinary = "sudo"
   437  		d.args = append(d.args,
   438  			"-u", d.rootlessUser.Username,
   439  			"--preserve-env",
   440  			"--preserve-env=PATH", // Pass through PATH, overriding secure_path.
   441  			"XDG_RUNTIME_DIR="+d.rootlessXDGRuntimeDir,
   442  			"HOME="+d.rootlessUser.HomeDir,
   443  			"--",
   444  			defaultDockerdRootlessBinary,
   445  		)
   446  	}
   447  
   448  	d.args = append(d.args,
   449  		"--data-root", d.Root,
   450  		"--exec-root", d.execRoot,
   451  		"--pidfile", d.pidFile,
   452  		"--userland-proxy="+strconv.FormatBool(d.userlandProxy),
   453  		"--containerd-namespace", d.id,
   454  		"--containerd-plugins-namespace", d.id+"p",
   455  	)
   456  	if d.containerdSocket != "" {
   457  		d.args = append(d.args, "--containerd", d.containerdSocket)
   458  	}
   459  
   460  	if d.defaultCgroupNamespaceMode != "" {
   461  		d.args = append(d.args, "--default-cgroupns-mode", d.defaultCgroupNamespaceMode)
   462  	}
   463  	if d.experimental {
   464  		d.args = append(d.args, "--experimental")
   465  	}
   466  	if d.init {
   467  		d.args = append(d.args, "--init")
   468  	}
   469  	if !(d.UseDefaultHost || d.UseDefaultTLSHost) {
   470  		d.args = append(d.args, "--host", d.Sock())
   471  	}
   472  	if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" {
   473  		d.args = append(d.args, "--userns-remap", root)
   474  	}
   475  
   476  	// If we don't explicitly set the log-level or debug flag(-D) then
   477  	// turn on debug mode
   478  	foundLog := false
   479  	foundSd := false
   480  	for _, a := range providedArgs {
   481  		if strings.Contains(a, "--log-level") || strings.Contains(a, "-D") || strings.Contains(a, "--debug") {
   482  			foundLog = true
   483  		}
   484  		if strings.Contains(a, "--storage-driver") {
   485  			foundSd = true
   486  		}
   487  	}
   488  	if !foundLog {
   489  		d.args = append(d.args, "--debug")
   490  	}
   491  	if d.storageDriver != "" && !foundSd {
   492  		d.args = append(d.args, "--storage-driver", d.storageDriver)
   493  	}
   494  
   495  	d.args = append(d.args, providedArgs...)
   496  	cmd := exec.Command(dockerdBinary, d.args...)
   497  	cmd.Env = append(os.Environ(), "DOCKER_SERVICE_PREFER_OFFLINE_IMAGE=1")
   498  	cmd.Env = append(cmd.Env, d.extraEnv...)
   499  	cmd.Env = append(cmd.Env, "OTEL_SERVICE_NAME=dockerd-"+d.id)
   500  	cmd.Stdout = out
   501  	cmd.Stderr = out
   502  	d.logFile = out
   503  	if d.rootlessUser != nil {
   504  		// sudo requires this for propagating signals
   505  		setsid(cmd)
   506  	}
   507  
   508  	if err := cmd.Start(); err != nil {
   509  		return errors.Wrapf(err, "[%s] could not start daemon container", d.id)
   510  	}
   511  
   512  	wait := make(chan error, 1)
   513  	d.cmd = cmd
   514  	d.Wait = wait
   515  
   516  	go func() {
   517  		ret := cmd.Wait()
   518  		d.log.Logf("[%s] exiting daemon", d.id)
   519  		// If we send before logging, we might accidentally log _after_ the test is done.
   520  		// As of Go 1.12, this incurs a panic instead of silently being dropped.
   521  		wait <- ret
   522  		close(wait)
   523  	}()
   524  
   525  	clientConfig, err := d.getClientConfig()
   526  	if err != nil {
   527  		return err
   528  	}
   529  	client := &http.Client{
   530  		Transport: clientConfig.transport,
   531  	}
   532  
   533  	req, err := http.NewRequest(http.MethodGet, "/_ping", nil)
   534  	if err != nil {
   535  		return errors.Wrapf(err, "[%s] could not create new request", d.id)
   536  	}
   537  	req.URL.Host = clientConfig.addr
   538  	req.URL.Scheme = clientConfig.scheme
   539  
   540  	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
   541  	defer cancel()
   542  
   543  	// make sure daemon is ready to receive requests
   544  	for i := 0; ; i++ {
   545  		d.log.Logf("[%s] waiting for daemon to start", d.id)
   546  
   547  		select {
   548  		case <-ctx.Done():
   549  			return errors.Wrapf(ctx.Err(), "[%s] daemon exited and never started", d.id)
   550  		case err := <-d.Wait:
   551  			return errors.Wrapf(err, "[%s] daemon exited during startup", d.id)
   552  		default:
   553  			rctx, rcancel := context.WithTimeout(context.TODO(), 2*time.Second)
   554  			defer rcancel()
   555  
   556  			resp, err := client.Do(req.WithContext(rctx))
   557  			if err != nil {
   558  				if i > 2 { // don't log the first couple, this ends up just being noise
   559  					d.log.Logf("[%s] error pinging daemon on start: %v", d.id, err)
   560  				}
   561  
   562  				select {
   563  				case <-ctx.Done():
   564  				case <-time.After(500 * time.Millisecond):
   565  				}
   566  				continue
   567  			}
   568  
   569  			resp.Body.Close()
   570  			if resp.StatusCode != http.StatusOK {
   571  				d.log.Logf("[%s] received status != 200 OK: %s\n", d.id, resp.Status)
   572  			}
   573  			d.log.Logf("[%s] daemon started\n", d.id)
   574  			d.Root, err = d.queryRootDir()
   575  			if err != nil {
   576  				return errors.Wrapf(err, "[%s] error querying daemon for root directory", d.id)
   577  			}
   578  			return nil
   579  		}
   580  	}
   581  }
   582  
   583  // StartWithBusybox will first start the daemon with Daemon.Start()
   584  // then save the busybox image from the main daemon and load it into this Daemon instance.
   585  func (d *Daemon) StartWithBusybox(ctx context.Context, t testing.TB, arg ...string) {
   586  	t.Helper()
   587  	d.Start(t, arg...)
   588  	d.LoadBusybox(ctx, t)
   589  }
   590  
   591  // Kill will send a SIGKILL to the daemon
   592  func (d *Daemon) Kill() error {
   593  	if d.cmd == nil || d.Wait == nil {
   594  		return errDaemonNotStarted
   595  	}
   596  
   597  	defer func() {
   598  		d.logFile.Close()
   599  		d.cmd = nil
   600  	}()
   601  
   602  	if err := d.cmd.Process.Kill(); err != nil {
   603  		return err
   604  	}
   605  
   606  	if d.pidFile != "" {
   607  		_ = os.Remove(d.pidFile)
   608  	}
   609  	return nil
   610  }
   611  
   612  // Pid returns the pid of the daemon
   613  func (d *Daemon) Pid() int {
   614  	return d.cmd.Process.Pid
   615  }
   616  
   617  // Interrupt stops the daemon by sending it an Interrupt signal
   618  func (d *Daemon) Interrupt() error {
   619  	return d.Signal(os.Interrupt)
   620  }
   621  
   622  // Signal sends the specified signal to the daemon if running
   623  func (d *Daemon) Signal(signal os.Signal) error {
   624  	if d.cmd == nil || d.Wait == nil {
   625  		return errDaemonNotStarted
   626  	}
   627  	return d.cmd.Process.Signal(signal)
   628  }
   629  
   630  // DumpStackAndQuit sends SIGQUIT to the daemon, which triggers it to dump its
   631  // stack to its log file and exit
   632  // This is used primarily for gathering debug information on test timeout
   633  func (d *Daemon) DumpStackAndQuit() {
   634  	if d.cmd == nil || d.cmd.Process == nil {
   635  		return
   636  	}
   637  	SignalDaemonDump(d.cmd.Process.Pid)
   638  }
   639  
   640  // Stop will send a SIGINT every second and wait for the daemon to stop.
   641  // If it times out, a SIGKILL is sent.
   642  // Stop will not delete the daemon directory. If a purged daemon is needed,
   643  // instantiate a new one with NewDaemon.
   644  // If an error occurs while starting the daemon, the test will fail.
   645  func (d *Daemon) Stop(t testing.TB) {
   646  	t.Helper()
   647  	err := d.StopWithError()
   648  	if err != nil {
   649  		if err != errDaemonNotStarted {
   650  			t.Fatalf("[%s] error while stopping the daemon: %v", d.id, err)
   651  		} else {
   652  			t.Logf("[%s] daemon is not started", d.id)
   653  		}
   654  	}
   655  }
   656  
   657  // StopWithError will send a SIGINT every second and wait for the daemon to stop.
   658  // If it timeouts, a SIGKILL is sent.
   659  // Stop will not delete the daemon directory. If a purged daemon is needed,
   660  // instantiate a new one with NewDaemon.
   661  func (d *Daemon) StopWithError() (err error) {
   662  	if d.cmd == nil || d.Wait == nil {
   663  		return errDaemonNotStarted
   664  	}
   665  	defer func() {
   666  		if err != nil {
   667  			d.log.Logf("[%s] error while stopping daemon: %v", d.id, err)
   668  		} else {
   669  			d.log.Logf("[%s] daemon stopped", d.id)
   670  			if d.pidFile != "" {
   671  				_ = os.Remove(d.pidFile)
   672  			}
   673  		}
   674  		if err := d.logFile.Close(); err != nil {
   675  			d.log.Logf("[%s] failed to close daemon logfile: %v", d.id, err)
   676  		}
   677  		d.cmd = nil
   678  	}()
   679  
   680  	i := 1
   681  	ticker := time.NewTicker(time.Second)
   682  	defer ticker.Stop()
   683  	tick := ticker.C
   684  
   685  	d.log.Logf("[%s] stopping daemon", d.id)
   686  
   687  	if err := d.cmd.Process.Signal(os.Interrupt); err != nil {
   688  		if strings.Contains(err.Error(), "os: process already finished") {
   689  			return errDaemonNotStarted
   690  		}
   691  		return errors.Wrapf(err, "[%s] could not send signal", d.id)
   692  	}
   693  
   694  out1:
   695  	for {
   696  		select {
   697  		case err := <-d.Wait:
   698  			return err
   699  		case <-time.After(20 * time.Second):
   700  			// time for stopping jobs and run onShutdown hooks
   701  			d.log.Logf("[%s] daemon stop timed out after 20 seconds", d.id)
   702  			break out1
   703  		}
   704  	}
   705  
   706  out2:
   707  	for {
   708  		select {
   709  		case err := <-d.Wait:
   710  			return err
   711  		case <-tick:
   712  			i++
   713  			if i > 5 {
   714  				d.log.Logf("[%s] tried to interrupt daemon for %d times, now try to kill it", d.id, i)
   715  				break out2
   716  			}
   717  			d.log.Logf("[%d] attempt #%d/5: daemon is still running with pid %d", i, d.cmd.Process.Pid)
   718  			if err := d.cmd.Process.Signal(os.Interrupt); err != nil {
   719  				return errors.Wrapf(err, "[%s] attempt #%d/5 could not send signal", d.id, i)
   720  			}
   721  		}
   722  	}
   723  
   724  	if err := d.cmd.Process.Kill(); err != nil {
   725  		d.log.Logf("[%s] failed to kill daemon: %v", d.id, err)
   726  		return err
   727  	}
   728  
   729  	return nil
   730  }
   731  
   732  // Restart will restart the daemon by first stopping it and the starting it.
   733  // If an error occurs while starting the daemon, the test will fail.
   734  func (d *Daemon) Restart(t testing.TB, args ...string) {
   735  	t.Helper()
   736  	d.Stop(t)
   737  	d.Start(t, args...)
   738  }
   739  
   740  // RestartWithError will restart the daemon by first stopping it and then starting it.
   741  func (d *Daemon) RestartWithError(arg ...string) error {
   742  	if err := d.StopWithError(); err != nil {
   743  		return err
   744  	}
   745  	return d.StartWithError(arg...)
   746  }
   747  
   748  func (d *Daemon) handleUserns() {
   749  	// in the case of tests running a user namespace-enabled daemon, we have resolved
   750  	// d.Root to be the actual final path of the graph dir after the "uid.gid" of
   751  	// remapped root is added--we need to subtract it from the path before calling
   752  	// start or else we will continue making subdirectories rather than truly restarting
   753  	// with the same location/root:
   754  	if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" {
   755  		d.Root = filepath.Dir(d.Root)
   756  	}
   757  }
   758  
   759  // ReloadConfig asks the daemon to reload its configuration
   760  func (d *Daemon) ReloadConfig() error {
   761  	if d.cmd == nil || d.cmd.Process == nil {
   762  		return errors.New("daemon is not running")
   763  	}
   764  
   765  	errCh := make(chan error, 1)
   766  	started := make(chan struct{})
   767  	go func() {
   768  		_, body, err := request.Get(context.TODO(), "/events", request.Host(d.Sock()))
   769  		close(started)
   770  		if err != nil {
   771  			errCh <- err
   772  			return
   773  		}
   774  		defer body.Close()
   775  		dec := json.NewDecoder(body)
   776  		for {
   777  			var e events.Message
   778  			if err := dec.Decode(&e); err != nil {
   779  				errCh <- err
   780  				return
   781  			}
   782  			if e.Type != events.DaemonEventType {
   783  				continue
   784  			}
   785  			if e.Action != events.ActionReload {
   786  				continue
   787  			}
   788  			close(errCh) // notify that we are done
   789  			return
   790  		}
   791  	}()
   792  
   793  	<-started
   794  	if err := signalDaemonReload(d.cmd.Process.Pid); err != nil {
   795  		return errors.Wrapf(err, "[%s] error signaling daemon reload", d.id)
   796  	}
   797  	select {
   798  	case err := <-errCh:
   799  		if err != nil {
   800  			return errors.Wrapf(err, "[%s] error waiting for daemon reload event", d.id)
   801  		}
   802  	case <-time.After(30 * time.Second):
   803  		return errors.Errorf("[%s] daemon reload event timed out after 30 seconds", d.id)
   804  	}
   805  	return nil
   806  }
   807  
   808  // LoadBusybox image into the daemon
   809  func (d *Daemon) LoadBusybox(ctx context.Context, t testing.TB) {
   810  	t.Helper()
   811  	clientHost, err := client.NewClientWithOpts(client.FromEnv)
   812  	assert.NilError(t, err, "[%s] failed to create client", d.id)
   813  	defer clientHost.Close()
   814  
   815  	reader, err := clientHost.ImageSave(ctx, []string{"busybox:latest"})
   816  	assert.NilError(t, err, "[%s] failed to download busybox", d.id)
   817  	defer reader.Close()
   818  
   819  	c := d.NewClientT(t)
   820  	defer c.Close()
   821  
   822  	resp, err := c.ImageLoad(ctx, reader, true)
   823  	assert.NilError(t, err, "[%s] failed to load busybox", d.id)
   824  	defer resp.Body.Close()
   825  }
   826  
   827  func (d *Daemon) getClientConfig() (*clientConfig, error) {
   828  	var (
   829  		transport *http.Transport
   830  		scheme    string
   831  		addr      string
   832  		proto     string
   833  	)
   834  	if d.UseDefaultTLSHost {
   835  		option := &tlsconfig.Options{
   836  			CAFile:   "fixtures/https/ca.pem",
   837  			CertFile: "fixtures/https/client-cert.pem",
   838  			KeyFile:  "fixtures/https/client-key.pem",
   839  		}
   840  		tlsConfig, err := tlsconfig.Client(*option)
   841  		if err != nil {
   842  			return nil, err
   843  		}
   844  		transport = &http.Transport{
   845  			TLSClientConfig: tlsConfig,
   846  		}
   847  		addr = defaultTLSHost
   848  		scheme = "https"
   849  		proto = "tcp"
   850  	} else if d.UseDefaultHost {
   851  		addr = defaultUnixSocket
   852  		proto = "unix"
   853  		scheme = "http"
   854  		transport = &http.Transport{}
   855  	} else {
   856  		addr = d.sockPath()
   857  		proto = "unix"
   858  		scheme = "http"
   859  		transport = &http.Transport{}
   860  	}
   861  
   862  	if err := sockets.ConfigureTransport(transport, proto, addr); err != nil {
   863  		return nil, err
   864  	}
   865  	transport.DisableKeepAlives = true
   866  	if proto == "unix" {
   867  		addr = filepath.Base(addr)
   868  	}
   869  	return &clientConfig{
   870  		transport: transport,
   871  		scheme:    scheme,
   872  		addr:      addr,
   873  	}, nil
   874  }
   875  
   876  func (d *Daemon) queryRootDir() (string, error) {
   877  	// update daemon root by asking /info endpoint (to support user
   878  	// namespaced daemon with root remapped uid.gid directory)
   879  	clientConfig, err := d.getClientConfig()
   880  	if err != nil {
   881  		return "", err
   882  	}
   883  
   884  	c := &http.Client{
   885  		Transport: clientConfig.transport,
   886  	}
   887  
   888  	req, err := http.NewRequest(http.MethodGet, "/info", nil)
   889  	if err != nil {
   890  		return "", err
   891  	}
   892  	req.Header.Set("Content-Type", "application/json")
   893  	req.URL.Host = clientConfig.addr
   894  	req.URL.Scheme = clientConfig.scheme
   895  
   896  	resp, err := c.Do(req)
   897  	if err != nil {
   898  		return "", err
   899  	}
   900  	body := ioutils.NewReadCloserWrapper(resp.Body, func() error {
   901  		return resp.Body.Close()
   902  	})
   903  
   904  	type Info struct {
   905  		DockerRootDir string
   906  	}
   907  	var b []byte
   908  	var i Info
   909  	b, err = request.ReadBody(body)
   910  	if err == nil && resp.StatusCode == http.StatusOK {
   911  		// read the docker root dir
   912  		if err = json.Unmarshal(b, &i); err == nil {
   913  			return i.DockerRootDir, nil
   914  		}
   915  	}
   916  	return "", err
   917  }
   918  
   919  // Info returns the info struct for this daemon
   920  func (d *Daemon) Info(t testing.TB) system.Info {
   921  	t.Helper()
   922  	c := d.NewClientT(t)
   923  	info, err := c.Info(context.Background())
   924  	assert.NilError(t, err)
   925  	assert.NilError(t, c.Close())
   926  	return info
   927  }
   928  
   929  // TamperWithContainerConfig modifies the on-disk config of a container.
   930  func (d *Daemon) TamperWithContainerConfig(t testing.TB, containerID string, tamper func(*container.Container)) {
   931  	t.Helper()
   932  
   933  	configPath := filepath.Join(d.Root, "containers", containerID, "config.v2.json")
   934  	configBytes, err := os.ReadFile(configPath)
   935  	assert.NilError(t, err)
   936  
   937  	var c container.Container
   938  	assert.NilError(t, json.Unmarshal(configBytes, &c))
   939  	c.State = container.NewState()
   940  	tamper(&c)
   941  	configBytes, err = json.Marshal(&c)
   942  	assert.NilError(t, err)
   943  	assert.NilError(t, os.WriteFile(configPath, configBytes, 0o600))
   944  }
   945  
   946  // cleanupRaftDir removes swarmkit wal files if present
   947  func cleanupRaftDir(t testing.TB, d *Daemon) {
   948  	t.Helper()
   949  	for _, p := range []string{"wal", "wal-v3-encrypted", "snap-v3-encrypted"} {
   950  		dir := filepath.Join(d.Root, "swarm/raft", p)
   951  		if err := os.RemoveAll(dir); err != nil {
   952  			t.Logf("[%s] error removing %v: %v", d.id, dir, err)
   953  		}
   954  	}
   955  }
   956  
   957  // cleanupDaemonStorage removes the daemon's storage directory.
   958  //
   959  // Note that we don't delete the whole directory, as some files (e.g. daemon
   960  // logs) are collected for inclusion in the "bundles" that are stored as Jenkins
   961  // artifacts.
   962  //
   963  // We currently do not include container logs in the bundles, so this also
   964  // removes the "containers" sub-directory.
   965  func cleanupDaemonStorage(t testing.TB, d *Daemon) {
   966  	t.Helper()
   967  	dirs := []string{
   968  		"builder",
   969  		"buildkit",
   970  		"containers",
   971  		"image",
   972  		"network",
   973  		"plugins",
   974  		"tmp",
   975  		"trust",
   976  		"volumes",
   977  		// note: this assumes storage-driver name matches the subdirectory,
   978  		// which is currently true, but not guaranteed.
   979  		d.storageDriver,
   980  	}
   981  
   982  	for _, p := range dirs {
   983  		dir := filepath.Join(d.Root, p)
   984  		if err := os.RemoveAll(dir); err != nil {
   985  			t.Logf("[%s] error removing %v: %v", d.id, dir, err)
   986  		}
   987  	}
   988  }