github.com/Prakhar-Agarwal-byte/moby@v0.0.0-20231027092010-a14e3e8ab87e/testutil/daemon/daemon.go (about) 1 package daemon // import "github.com/Prakhar-Agarwal-byte/moby/testutil/daemon" 2 3 import ( 4 "bufio" 5 "context" 6 "encoding/json" 7 "io" 8 "net/http" 9 "os" 10 "os/exec" 11 "os/user" 12 "path/filepath" 13 "strconv" 14 "strings" 15 "testing" 16 "time" 17 18 "github.com/Prakhar-Agarwal-byte/moby/api/types/events" 19 "github.com/Prakhar-Agarwal-byte/moby/api/types/system" 20 "github.com/Prakhar-Agarwal-byte/moby/client" 21 "github.com/Prakhar-Agarwal-byte/moby/container" 22 "github.com/Prakhar-Agarwal-byte/moby/pkg/ioutils" 23 "github.com/Prakhar-Agarwal-byte/moby/pkg/stringid" 24 "github.com/Prakhar-Agarwal-byte/moby/pkg/tailfile" 25 "github.com/Prakhar-Agarwal-byte/moby/testutil/request" 26 "github.com/docker/go-connections/sockets" 27 "github.com/docker/go-connections/tlsconfig" 28 "github.com/pkg/errors" 29 "gotest.tools/v3/assert" 30 "gotest.tools/v3/poll" 31 ) 32 33 // LogT is the subset of the testing.TB interface used by the daemon. 34 type LogT interface { 35 Logf(string, ...interface{}) 36 } 37 38 // nopLog is a no-op implementation of LogT that is used in daemons created by 39 // NewDaemon (where no testing.TB is available). 40 type nopLog struct{} 41 42 func (nopLog) Logf(string, ...interface{}) {} 43 44 const ( 45 defaultDockerdBinary = "dockerd" 46 defaultContainerdSocket = "/var/run/docker/containerd/containerd.sock" 47 defaultDockerdRootlessBinary = "dockerd-rootless.sh" 48 defaultUnixSocket = "/var/run/docker.sock" 49 defaultTLSHost = "localhost:2376" 50 ) 51 52 var errDaemonNotStarted = errors.New("daemon not started") 53 54 // SockRoot holds the path of the default docker integration daemon socket 55 var SockRoot = filepath.Join(os.TempDir(), "docker-integration") 56 57 type clientConfig struct { 58 transport *http.Transport 59 scheme string 60 addr string 61 } 62 63 // Daemon represents a Docker daemon for the testing framework 64 type Daemon struct { 65 Root string 66 Folder string 67 Wait chan error 68 UseDefaultHost bool 69 UseDefaultTLSHost bool 70 71 id string 72 logFile *os.File 73 cmd *exec.Cmd 74 storageDriver string 75 userlandProxy bool 76 defaultCgroupNamespaceMode string 77 execRoot string 78 experimental bool 79 init bool 80 dockerdBinary string 81 log LogT 82 pidFile string 83 args []string 84 extraEnv []string 85 containerdSocket string 86 rootlessUser *user.User 87 rootlessXDGRuntimeDir string 88 89 // swarm related field 90 swarmListenAddr string 91 SwarmPort int // FIXME(vdemeester) should probably not be exported 92 DefaultAddrPool []string 93 SubnetSize uint32 94 DataPathPort uint32 95 OOMScoreAdjust int 96 // cached information 97 CachedInfo system.Info 98 } 99 100 // NewDaemon returns a Daemon instance to be used for testing. 101 // The daemon will not automatically start. 102 // The daemon will modify and create files under workingDir. 103 func NewDaemon(workingDir string, ops ...Option) (*Daemon, error) { 104 storageDriver := os.Getenv("DOCKER_GRAPHDRIVER") 105 106 if err := os.MkdirAll(SockRoot, 0o700); err != nil { 107 return nil, errors.Wrapf(err, "failed to create daemon socket root %q", SockRoot) 108 } 109 110 id := "d" + stringid.TruncateID(stringid.GenerateRandomID()) 111 dir := filepath.Join(workingDir, id) 112 daemonFolder, err := filepath.Abs(dir) 113 if err != nil { 114 return nil, err 115 } 116 daemonRoot := filepath.Join(daemonFolder, "root") 117 if err := os.MkdirAll(daemonRoot, 0o755); err != nil { 118 return nil, errors.Wrapf(err, "failed to create daemon root %q", daemonRoot) 119 } 120 121 userlandProxy := true 122 if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { 123 if val, err := strconv.ParseBool(env); err != nil { 124 userlandProxy = val 125 } 126 } 127 d := &Daemon{ 128 id: id, 129 Folder: daemonFolder, 130 Root: daemonRoot, 131 storageDriver: storageDriver, 132 userlandProxy: userlandProxy, 133 // dxr stands for docker-execroot (shortened for avoiding unix(7) path length limitation) 134 execRoot: filepath.Join(os.TempDir(), "dxr", id), 135 dockerdBinary: defaultDockerdBinary, 136 swarmListenAddr: defaultSwarmListenAddr, 137 SwarmPort: DefaultSwarmPort, 138 log: nopLog{}, 139 containerdSocket: defaultContainerdSocket, 140 } 141 142 for _, op := range ops { 143 op(d) 144 } 145 146 if d.rootlessUser != nil { 147 if err := os.Chmod(SockRoot, 0o777); err != nil { 148 return nil, err 149 } 150 uid, err := strconv.Atoi(d.rootlessUser.Uid) 151 if err != nil { 152 return nil, err 153 } 154 gid, err := strconv.Atoi(d.rootlessUser.Gid) 155 if err != nil { 156 return nil, err 157 } 158 if err := os.Chown(d.Folder, uid, gid); err != nil { 159 return nil, err 160 } 161 if err := os.Chown(d.Root, uid, gid); err != nil { 162 return nil, err 163 } 164 if err := os.MkdirAll(filepath.Dir(d.execRoot), 0o700); err != nil { 165 return nil, err 166 } 167 if err := os.Chown(filepath.Dir(d.execRoot), uid, gid); err != nil { 168 return nil, err 169 } 170 if err := os.MkdirAll(d.execRoot, 0o700); err != nil { 171 return nil, err 172 } 173 if err := os.Chown(d.execRoot, uid, gid); err != nil { 174 return nil, err 175 } 176 d.rootlessXDGRuntimeDir = filepath.Join(d.Folder, "xdgrun") 177 if err := os.MkdirAll(d.rootlessXDGRuntimeDir, 0o700); err != nil { 178 return nil, err 179 } 180 if err := os.Chown(d.rootlessXDGRuntimeDir, uid, gid); err != nil { 181 return nil, err 182 } 183 d.containerdSocket = "" 184 } 185 186 return d, nil 187 } 188 189 // New returns a Daemon instance to be used for testing. 190 // This will create a directory such as d123456789 in the folder specified by 191 // $DOCKER_INTEGRATION_DAEMON_DEST or $DEST. 192 // The daemon will not automatically start. 193 func New(t testing.TB, ops ...Option) *Daemon { 194 t.Helper() 195 dest := os.Getenv("DOCKER_INTEGRATION_DAEMON_DEST") 196 if dest == "" { 197 dest = os.Getenv("DEST") 198 } 199 dest = filepath.Join(dest, t.Name()) 200 201 assert.Check(t, dest != "", "Please set the DOCKER_INTEGRATION_DAEMON_DEST or the DEST environment variable") 202 203 if os.Getenv("DOCKER_ROOTLESS") != "" { 204 if os.Getenv("DOCKER_REMAP_ROOT") != "" { 205 t.Skip("DOCKER_ROOTLESS doesn't support DOCKER_REMAP_ROOT currently") 206 } 207 if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { 208 if val, err := strconv.ParseBool(env); err == nil && !val { 209 t.Skip("DOCKER_ROOTLESS doesn't support DOCKER_USERLANDPROXY=false") 210 } 211 } 212 ops = append(ops, WithRootlessUser("unprivilegeduser")) 213 } 214 ops = append(ops, WithOOMScoreAdjust(-500)) 215 216 d, err := NewDaemon(dest, ops...) 217 assert.NilError(t, err, "could not create daemon at %q", dest) 218 if d.rootlessUser != nil && d.dockerdBinary != defaultDockerdBinary { 219 t.Skipf("DOCKER_ROOTLESS doesn't support specifying non-default dockerd binary path %q", d.dockerdBinary) 220 } 221 222 return d 223 } 224 225 // BinaryPath returns the binary and its arguments. 226 func (d *Daemon) BinaryPath() (string, error) { 227 dockerdBinary, err := exec.LookPath(d.dockerdBinary) 228 if err != nil { 229 return "", errors.Wrapf(err, "[%s] could not find docker binary in $PATH", d.id) 230 } 231 return dockerdBinary, nil 232 } 233 234 // ContainersNamespace returns the containerd namespace used for containers. 235 func (d *Daemon) ContainersNamespace() string { 236 return d.id 237 } 238 239 // RootDir returns the root directory of the daemon. 240 func (d *Daemon) RootDir() string { 241 return d.Root 242 } 243 244 // ID returns the generated id of the daemon 245 func (d *Daemon) ID() string { 246 return d.id 247 } 248 249 // StorageDriver returns the configured storage driver of the daemon 250 func (d *Daemon) StorageDriver() string { 251 return d.storageDriver 252 } 253 254 // Sock returns the socket path of the daemon 255 func (d *Daemon) Sock() string { 256 return "unix://" + d.sockPath() 257 } 258 259 func (d *Daemon) sockPath() string { 260 return filepath.Join(SockRoot, d.id+".sock") 261 } 262 263 // LogFileName returns the path the daemon's log file 264 func (d *Daemon) LogFileName() string { 265 return d.logFile.Name() 266 } 267 268 // ReadLogFile returns the content of the daemon log file 269 func (d *Daemon) ReadLogFile() ([]byte, error) { 270 _ = d.logFile.Sync() 271 return os.ReadFile(d.logFile.Name()) 272 } 273 274 // NewClientT creates new client based on daemon's socket path 275 func (d *Daemon) NewClientT(t testing.TB, extraOpts ...client.Opt) *client.Client { 276 t.Helper() 277 278 c, err := d.NewClient(extraOpts...) 279 assert.NilError(t, err, "[%s] could not create daemon client", d.id) 280 t.Cleanup(func() { c.Close() }) 281 return c 282 } 283 284 // NewClient creates new client based on daemon's socket path 285 func (d *Daemon) NewClient(extraOpts ...client.Opt) (*client.Client, error) { 286 clientOpts := []client.Opt{ 287 client.FromEnv, 288 client.WithHost(d.Sock()), 289 } 290 clientOpts = append(clientOpts, extraOpts...) 291 292 return client.NewClientWithOpts(clientOpts...) 293 } 294 295 // Cleanup cleans the daemon files : exec root (network namespaces, ...), swarmkit files 296 func (d *Daemon) Cleanup(t testing.TB) { 297 t.Helper() 298 cleanupMount(t, d) 299 cleanupRaftDir(t, d) 300 cleanupDaemonStorage(t, d) 301 cleanupNetworkNamespace(t, d) 302 } 303 304 // TailLogsT attempts to tail N lines from the daemon logs. 305 // If there is an error the error is only logged, it does not cause an error with the test. 306 func (d *Daemon) TailLogsT(t LogT, n int) { 307 lines, err := d.TailLogs(n) 308 if err != nil { 309 t.Logf("[%s] %v", d.id, err) 310 return 311 } 312 for _, l := range lines { 313 t.Logf("[%s] %s", d.id, string(l)) 314 } 315 } 316 317 // PollCheckLogs is a poll.Check that checks the daemon logs using the passed in match function. 318 func (d *Daemon) PollCheckLogs(ctx context.Context, match func(s string) bool) poll.Check { 319 return func(t poll.LogT) poll.Result { 320 ok, _, err := d.ScanLogs(ctx, match) 321 if err != nil { 322 return poll.Error(err) 323 } 324 if !ok { 325 return poll.Continue("waiting for daemon logs match") 326 } 327 return poll.Success() 328 } 329 } 330 331 // ScanLogsMatchString returns a function that can be used to scan the daemon logs for the passed in string (`contains`). 332 func ScanLogsMatchString(contains string) func(string) bool { 333 return func(line string) bool { 334 return strings.Contains(line, contains) 335 } 336 } 337 338 // ScanLogsMatchAll returns a function that can be used to scan the daemon logs until *all* the passed in strings are matched 339 func ScanLogsMatchAll(contains ...string) func(string) bool { 340 matched := make(map[string]bool) 341 return func(line string) bool { 342 for _, c := range contains { 343 if strings.Contains(line, c) { 344 matched[c] = true 345 } 346 } 347 return len(matched) == len(contains) 348 } 349 } 350 351 // ScanLogsT uses `ScanLogs` to match the daemon logs using the passed in match function. 352 // If there is an error or the match fails, the test will fail. 353 func (d *Daemon) ScanLogsT(ctx context.Context, t testing.TB, match func(s string) bool) (bool, string) { 354 t.Helper() 355 ok, line, err := d.ScanLogs(ctx, match) 356 assert.NilError(t, err) 357 return ok, line 358 } 359 360 // ScanLogs scans the daemon logs and passes each line to the match function. 361 func (d *Daemon) ScanLogs(ctx context.Context, match func(s string) bool) (bool, string, error) { 362 stat, err := d.logFile.Stat() 363 if err != nil { 364 return false, "", err 365 } 366 rdr := io.NewSectionReader(d.logFile, 0, stat.Size()) 367 368 scanner := bufio.NewScanner(rdr) 369 for scanner.Scan() { 370 if match(scanner.Text()) { 371 return true, scanner.Text(), nil 372 } 373 select { 374 case <-ctx.Done(): 375 return false, "", ctx.Err() 376 default: 377 } 378 } 379 return false, "", scanner.Err() 380 } 381 382 // TailLogs tails N lines from the daemon logs 383 func (d *Daemon) TailLogs(n int) ([][]byte, error) { 384 logF, err := os.Open(d.logFile.Name()) 385 if err != nil { 386 return nil, errors.Wrap(err, "error opening daemon log file after failed start") 387 } 388 389 defer logF.Close() 390 lines, err := tailfile.TailFile(logF, n) 391 if err != nil { 392 return nil, errors.Wrap(err, "error tailing log daemon logs") 393 } 394 395 return lines, nil 396 } 397 398 // Start starts the daemon and return once it is ready to receive requests. 399 func (d *Daemon) Start(t testing.TB, args ...string) { 400 t.Helper() 401 if err := d.StartWithError(args...); err != nil { 402 d.TailLogsT(t, 20) 403 d.DumpStackAndQuit() // in case the daemon is stuck 404 t.Fatalf("[%s] failed to start daemon with arguments %v : %v", d.id, d.args, err) 405 } 406 } 407 408 // StartWithError starts the daemon and return once it is ready to receive requests. 409 // It returns an error in case it couldn't start. 410 func (d *Daemon) StartWithError(args ...string) error { 411 logFile, err := os.OpenFile(filepath.Join(d.Folder, "docker.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0o600) 412 if err != nil { 413 return errors.Wrapf(err, "[%s] failed to create logfile", d.id) 414 } 415 416 return d.StartWithLogFile(logFile, args...) 417 } 418 419 // StartWithLogFile will start the daemon and attach its streams to a given file. 420 func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error { 421 d.handleUserns() 422 dockerdBinary, err := d.BinaryPath() 423 if err != nil { 424 return err 425 } 426 427 if d.pidFile == "" { 428 d.pidFile = filepath.Join(d.Folder, "docker.pid") 429 } 430 431 d.args = []string{} 432 if d.rootlessUser != nil { 433 if d.dockerdBinary != defaultDockerdBinary { 434 return errors.Errorf("[%s] DOCKER_ROOTLESS doesn't support non-default dockerd binary path %q", d.id, d.dockerdBinary) 435 } 436 dockerdBinary = "sudo" 437 d.args = append(d.args, 438 "-u", d.rootlessUser.Username, 439 "--preserve-env", 440 "--preserve-env=PATH", // Pass through PATH, overriding secure_path. 441 "XDG_RUNTIME_DIR="+d.rootlessXDGRuntimeDir, 442 "HOME="+d.rootlessUser.HomeDir, 443 "--", 444 defaultDockerdRootlessBinary, 445 ) 446 } 447 448 d.args = append(d.args, 449 "--data-root", d.Root, 450 "--exec-root", d.execRoot, 451 "--pidfile", d.pidFile, 452 "--userland-proxy="+strconv.FormatBool(d.userlandProxy), 453 "--containerd-namespace", d.id, 454 "--containerd-plugins-namespace", d.id+"p", 455 ) 456 if d.containerdSocket != "" { 457 d.args = append(d.args, "--containerd", d.containerdSocket) 458 } 459 460 if d.defaultCgroupNamespaceMode != "" { 461 d.args = append(d.args, "--default-cgroupns-mode", d.defaultCgroupNamespaceMode) 462 } 463 if d.experimental { 464 d.args = append(d.args, "--experimental") 465 } 466 if d.init { 467 d.args = append(d.args, "--init") 468 } 469 if !(d.UseDefaultHost || d.UseDefaultTLSHost) { 470 d.args = append(d.args, "--host", d.Sock()) 471 } 472 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 473 d.args = append(d.args, "--userns-remap", root) 474 } 475 476 // If we don't explicitly set the log-level or debug flag(-D) then 477 // turn on debug mode 478 foundLog := false 479 foundSd := false 480 for _, a := range providedArgs { 481 if strings.Contains(a, "--log-level") || strings.Contains(a, "-D") || strings.Contains(a, "--debug") { 482 foundLog = true 483 } 484 if strings.Contains(a, "--storage-driver") { 485 foundSd = true 486 } 487 } 488 if !foundLog { 489 d.args = append(d.args, "--debug") 490 } 491 if d.storageDriver != "" && !foundSd { 492 d.args = append(d.args, "--storage-driver", d.storageDriver) 493 } 494 495 d.args = append(d.args, providedArgs...) 496 cmd := exec.Command(dockerdBinary, d.args...) 497 cmd.Env = append(os.Environ(), "DOCKER_SERVICE_PREFER_OFFLINE_IMAGE=1") 498 cmd.Env = append(cmd.Env, d.extraEnv...) 499 cmd.Env = append(cmd.Env, "OTEL_SERVICE_NAME=dockerd-"+d.id) 500 cmd.Stdout = out 501 cmd.Stderr = out 502 d.logFile = out 503 if d.rootlessUser != nil { 504 // sudo requires this for propagating signals 505 setsid(cmd) 506 } 507 508 if err := cmd.Start(); err != nil { 509 return errors.Wrapf(err, "[%s] could not start daemon container", d.id) 510 } 511 512 wait := make(chan error, 1) 513 d.cmd = cmd 514 d.Wait = wait 515 516 go func() { 517 ret := cmd.Wait() 518 d.log.Logf("[%s] exiting daemon", d.id) 519 // If we send before logging, we might accidentally log _after_ the test is done. 520 // As of Go 1.12, this incurs a panic instead of silently being dropped. 521 wait <- ret 522 close(wait) 523 }() 524 525 clientConfig, err := d.getClientConfig() 526 if err != nil { 527 return err 528 } 529 client := &http.Client{ 530 Transport: clientConfig.transport, 531 } 532 533 req, err := http.NewRequest(http.MethodGet, "/_ping", nil) 534 if err != nil { 535 return errors.Wrapf(err, "[%s] could not create new request", d.id) 536 } 537 req.URL.Host = clientConfig.addr 538 req.URL.Scheme = clientConfig.scheme 539 540 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) 541 defer cancel() 542 543 // make sure daemon is ready to receive requests 544 for i := 0; ; i++ { 545 d.log.Logf("[%s] waiting for daemon to start", d.id) 546 547 select { 548 case <-ctx.Done(): 549 return errors.Wrapf(ctx.Err(), "[%s] daemon exited and never started", d.id) 550 case err := <-d.Wait: 551 return errors.Wrapf(err, "[%s] daemon exited during startup", d.id) 552 default: 553 rctx, rcancel := context.WithTimeout(context.TODO(), 2*time.Second) 554 defer rcancel() 555 556 resp, err := client.Do(req.WithContext(rctx)) 557 if err != nil { 558 if i > 2 { // don't log the first couple, this ends up just being noise 559 d.log.Logf("[%s] error pinging daemon on start: %v", d.id, err) 560 } 561 562 select { 563 case <-ctx.Done(): 564 case <-time.After(500 * time.Millisecond): 565 } 566 continue 567 } 568 569 resp.Body.Close() 570 if resp.StatusCode != http.StatusOK { 571 d.log.Logf("[%s] received status != 200 OK: %s\n", d.id, resp.Status) 572 } 573 d.log.Logf("[%s] daemon started\n", d.id) 574 d.Root, err = d.queryRootDir() 575 if err != nil { 576 return errors.Wrapf(err, "[%s] error querying daemon for root directory", d.id) 577 } 578 return nil 579 } 580 } 581 } 582 583 // StartWithBusybox will first start the daemon with Daemon.Start() 584 // then save the busybox image from the main daemon and load it into this Daemon instance. 585 func (d *Daemon) StartWithBusybox(ctx context.Context, t testing.TB, arg ...string) { 586 t.Helper() 587 d.Start(t, arg...) 588 d.LoadBusybox(ctx, t) 589 } 590 591 // Kill will send a SIGKILL to the daemon 592 func (d *Daemon) Kill() error { 593 if d.cmd == nil || d.Wait == nil { 594 return errDaemonNotStarted 595 } 596 597 defer func() { 598 d.logFile.Close() 599 d.cmd = nil 600 }() 601 602 if err := d.cmd.Process.Kill(); err != nil { 603 return err 604 } 605 606 if d.pidFile != "" { 607 _ = os.Remove(d.pidFile) 608 } 609 return nil 610 } 611 612 // Pid returns the pid of the daemon 613 func (d *Daemon) Pid() int { 614 return d.cmd.Process.Pid 615 } 616 617 // Interrupt stops the daemon by sending it an Interrupt signal 618 func (d *Daemon) Interrupt() error { 619 return d.Signal(os.Interrupt) 620 } 621 622 // Signal sends the specified signal to the daemon if running 623 func (d *Daemon) Signal(signal os.Signal) error { 624 if d.cmd == nil || d.Wait == nil { 625 return errDaemonNotStarted 626 } 627 return d.cmd.Process.Signal(signal) 628 } 629 630 // DumpStackAndQuit sends SIGQUIT to the daemon, which triggers it to dump its 631 // stack to its log file and exit 632 // This is used primarily for gathering debug information on test timeout 633 func (d *Daemon) DumpStackAndQuit() { 634 if d.cmd == nil || d.cmd.Process == nil { 635 return 636 } 637 SignalDaemonDump(d.cmd.Process.Pid) 638 } 639 640 // Stop will send a SIGINT every second and wait for the daemon to stop. 641 // If it times out, a SIGKILL is sent. 642 // Stop will not delete the daemon directory. If a purged daemon is needed, 643 // instantiate a new one with NewDaemon. 644 // If an error occurs while starting the daemon, the test will fail. 645 func (d *Daemon) Stop(t testing.TB) { 646 t.Helper() 647 err := d.StopWithError() 648 if err != nil { 649 if err != errDaemonNotStarted { 650 t.Fatalf("[%s] error while stopping the daemon: %v", d.id, err) 651 } else { 652 t.Logf("[%s] daemon is not started", d.id) 653 } 654 } 655 } 656 657 // StopWithError will send a SIGINT every second and wait for the daemon to stop. 658 // If it timeouts, a SIGKILL is sent. 659 // Stop will not delete the daemon directory. If a purged daemon is needed, 660 // instantiate a new one with NewDaemon. 661 func (d *Daemon) StopWithError() (err error) { 662 if d.cmd == nil || d.Wait == nil { 663 return errDaemonNotStarted 664 } 665 defer func() { 666 if err != nil { 667 d.log.Logf("[%s] error while stopping daemon: %v", d.id, err) 668 } else { 669 d.log.Logf("[%s] daemon stopped", d.id) 670 if d.pidFile != "" { 671 _ = os.Remove(d.pidFile) 672 } 673 } 674 if err := d.logFile.Close(); err != nil { 675 d.log.Logf("[%s] failed to close daemon logfile: %v", d.id, err) 676 } 677 d.cmd = nil 678 }() 679 680 i := 1 681 ticker := time.NewTicker(time.Second) 682 defer ticker.Stop() 683 tick := ticker.C 684 685 d.log.Logf("[%s] stopping daemon", d.id) 686 687 if err := d.cmd.Process.Signal(os.Interrupt); err != nil { 688 if strings.Contains(err.Error(), "os: process already finished") { 689 return errDaemonNotStarted 690 } 691 return errors.Wrapf(err, "[%s] could not send signal", d.id) 692 } 693 694 out1: 695 for { 696 select { 697 case err := <-d.Wait: 698 return err 699 case <-time.After(20 * time.Second): 700 // time for stopping jobs and run onShutdown hooks 701 d.log.Logf("[%s] daemon stop timed out after 20 seconds", d.id) 702 break out1 703 } 704 } 705 706 out2: 707 for { 708 select { 709 case err := <-d.Wait: 710 return err 711 case <-tick: 712 i++ 713 if i > 5 { 714 d.log.Logf("[%s] tried to interrupt daemon for %d times, now try to kill it", d.id, i) 715 break out2 716 } 717 d.log.Logf("[%d] attempt #%d/5: daemon is still running with pid %d", i, d.cmd.Process.Pid) 718 if err := d.cmd.Process.Signal(os.Interrupt); err != nil { 719 return errors.Wrapf(err, "[%s] attempt #%d/5 could not send signal", d.id, i) 720 } 721 } 722 } 723 724 if err := d.cmd.Process.Kill(); err != nil { 725 d.log.Logf("[%s] failed to kill daemon: %v", d.id, err) 726 return err 727 } 728 729 return nil 730 } 731 732 // Restart will restart the daemon by first stopping it and the starting it. 733 // If an error occurs while starting the daemon, the test will fail. 734 func (d *Daemon) Restart(t testing.TB, args ...string) { 735 t.Helper() 736 d.Stop(t) 737 d.Start(t, args...) 738 } 739 740 // RestartWithError will restart the daemon by first stopping it and then starting it. 741 func (d *Daemon) RestartWithError(arg ...string) error { 742 if err := d.StopWithError(); err != nil { 743 return err 744 } 745 return d.StartWithError(arg...) 746 } 747 748 func (d *Daemon) handleUserns() { 749 // in the case of tests running a user namespace-enabled daemon, we have resolved 750 // d.Root to be the actual final path of the graph dir after the "uid.gid" of 751 // remapped root is added--we need to subtract it from the path before calling 752 // start or else we will continue making subdirectories rather than truly restarting 753 // with the same location/root: 754 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 755 d.Root = filepath.Dir(d.Root) 756 } 757 } 758 759 // ReloadConfig asks the daemon to reload its configuration 760 func (d *Daemon) ReloadConfig() error { 761 if d.cmd == nil || d.cmd.Process == nil { 762 return errors.New("daemon is not running") 763 } 764 765 errCh := make(chan error, 1) 766 started := make(chan struct{}) 767 go func() { 768 _, body, err := request.Get(context.TODO(), "/events", request.Host(d.Sock())) 769 close(started) 770 if err != nil { 771 errCh <- err 772 return 773 } 774 defer body.Close() 775 dec := json.NewDecoder(body) 776 for { 777 var e events.Message 778 if err := dec.Decode(&e); err != nil { 779 errCh <- err 780 return 781 } 782 if e.Type != events.DaemonEventType { 783 continue 784 } 785 if e.Action != events.ActionReload { 786 continue 787 } 788 close(errCh) // notify that we are done 789 return 790 } 791 }() 792 793 <-started 794 if err := signalDaemonReload(d.cmd.Process.Pid); err != nil { 795 return errors.Wrapf(err, "[%s] error signaling daemon reload", d.id) 796 } 797 select { 798 case err := <-errCh: 799 if err != nil { 800 return errors.Wrapf(err, "[%s] error waiting for daemon reload event", d.id) 801 } 802 case <-time.After(30 * time.Second): 803 return errors.Errorf("[%s] daemon reload event timed out after 30 seconds", d.id) 804 } 805 return nil 806 } 807 808 // LoadBusybox image into the daemon 809 func (d *Daemon) LoadBusybox(ctx context.Context, t testing.TB) { 810 t.Helper() 811 clientHost, err := client.NewClientWithOpts(client.FromEnv) 812 assert.NilError(t, err, "[%s] failed to create client", d.id) 813 defer clientHost.Close() 814 815 reader, err := clientHost.ImageSave(ctx, []string{"busybox:latest"}) 816 assert.NilError(t, err, "[%s] failed to download busybox", d.id) 817 defer reader.Close() 818 819 c := d.NewClientT(t) 820 defer c.Close() 821 822 resp, err := c.ImageLoad(ctx, reader, true) 823 assert.NilError(t, err, "[%s] failed to load busybox", d.id) 824 defer resp.Body.Close() 825 } 826 827 func (d *Daemon) getClientConfig() (*clientConfig, error) { 828 var ( 829 transport *http.Transport 830 scheme string 831 addr string 832 proto string 833 ) 834 if d.UseDefaultTLSHost { 835 option := &tlsconfig.Options{ 836 CAFile: "fixtures/https/ca.pem", 837 CertFile: "fixtures/https/client-cert.pem", 838 KeyFile: "fixtures/https/client-key.pem", 839 } 840 tlsConfig, err := tlsconfig.Client(*option) 841 if err != nil { 842 return nil, err 843 } 844 transport = &http.Transport{ 845 TLSClientConfig: tlsConfig, 846 } 847 addr = defaultTLSHost 848 scheme = "https" 849 proto = "tcp" 850 } else if d.UseDefaultHost { 851 addr = defaultUnixSocket 852 proto = "unix" 853 scheme = "http" 854 transport = &http.Transport{} 855 } else { 856 addr = d.sockPath() 857 proto = "unix" 858 scheme = "http" 859 transport = &http.Transport{} 860 } 861 862 if err := sockets.ConfigureTransport(transport, proto, addr); err != nil { 863 return nil, err 864 } 865 transport.DisableKeepAlives = true 866 if proto == "unix" { 867 addr = filepath.Base(addr) 868 } 869 return &clientConfig{ 870 transport: transport, 871 scheme: scheme, 872 addr: addr, 873 }, nil 874 } 875 876 func (d *Daemon) queryRootDir() (string, error) { 877 // update daemon root by asking /info endpoint (to support user 878 // namespaced daemon with root remapped uid.gid directory) 879 clientConfig, err := d.getClientConfig() 880 if err != nil { 881 return "", err 882 } 883 884 c := &http.Client{ 885 Transport: clientConfig.transport, 886 } 887 888 req, err := http.NewRequest(http.MethodGet, "/info", nil) 889 if err != nil { 890 return "", err 891 } 892 req.Header.Set("Content-Type", "application/json") 893 req.URL.Host = clientConfig.addr 894 req.URL.Scheme = clientConfig.scheme 895 896 resp, err := c.Do(req) 897 if err != nil { 898 return "", err 899 } 900 body := ioutils.NewReadCloserWrapper(resp.Body, func() error { 901 return resp.Body.Close() 902 }) 903 904 type Info struct { 905 DockerRootDir string 906 } 907 var b []byte 908 var i Info 909 b, err = request.ReadBody(body) 910 if err == nil && resp.StatusCode == http.StatusOK { 911 // read the docker root dir 912 if err = json.Unmarshal(b, &i); err == nil { 913 return i.DockerRootDir, nil 914 } 915 } 916 return "", err 917 } 918 919 // Info returns the info struct for this daemon 920 func (d *Daemon) Info(t testing.TB) system.Info { 921 t.Helper() 922 c := d.NewClientT(t) 923 info, err := c.Info(context.Background()) 924 assert.NilError(t, err) 925 assert.NilError(t, c.Close()) 926 return info 927 } 928 929 // TamperWithContainerConfig modifies the on-disk config of a container. 930 func (d *Daemon) TamperWithContainerConfig(t testing.TB, containerID string, tamper func(*container.Container)) { 931 t.Helper() 932 933 configPath := filepath.Join(d.Root, "containers", containerID, "config.v2.json") 934 configBytes, err := os.ReadFile(configPath) 935 assert.NilError(t, err) 936 937 var c container.Container 938 assert.NilError(t, json.Unmarshal(configBytes, &c)) 939 c.State = container.NewState() 940 tamper(&c) 941 configBytes, err = json.Marshal(&c) 942 assert.NilError(t, err) 943 assert.NilError(t, os.WriteFile(configPath, configBytes, 0o600)) 944 } 945 946 // cleanupRaftDir removes swarmkit wal files if present 947 func cleanupRaftDir(t testing.TB, d *Daemon) { 948 t.Helper() 949 for _, p := range []string{"wal", "wal-v3-encrypted", "snap-v3-encrypted"} { 950 dir := filepath.Join(d.Root, "swarm/raft", p) 951 if err := os.RemoveAll(dir); err != nil { 952 t.Logf("[%s] error removing %v: %v", d.id, dir, err) 953 } 954 } 955 } 956 957 // cleanupDaemonStorage removes the daemon's storage directory. 958 // 959 // Note that we don't delete the whole directory, as some files (e.g. daemon 960 // logs) are collected for inclusion in the "bundles" that are stored as Jenkins 961 // artifacts. 962 // 963 // We currently do not include container logs in the bundles, so this also 964 // removes the "containers" sub-directory. 965 func cleanupDaemonStorage(t testing.TB, d *Daemon) { 966 t.Helper() 967 dirs := []string{ 968 "builder", 969 "buildkit", 970 "containers", 971 "image", 972 "network", 973 "plugins", 974 "tmp", 975 "trust", 976 "volumes", 977 // note: this assumes storage-driver name matches the subdirectory, 978 // which is currently true, but not guaranteed. 979 d.storageDriver, 980 } 981 982 for _, p := range dirs { 983 dir := filepath.Join(d.Root, p) 984 if err := os.RemoveAll(dir); err != nil { 985 t.Logf("[%s] error removing %v: %v", d.id, dir, err) 986 } 987 } 988 }