github.com/lacework-dev/go-moby@v20.10.12+incompatible/testutil/daemon/daemon.go (about) 1 package daemon // import "github.com/docker/docker/testutil/daemon" 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "io/ioutil" 8 "net/http" 9 "os" 10 "os/exec" 11 "os/user" 12 "path/filepath" 13 "strconv" 14 "strings" 15 "testing" 16 "time" 17 18 "github.com/docker/docker/api/types" 19 "github.com/docker/docker/api/types/events" 20 "github.com/docker/docker/client" 21 "github.com/docker/docker/opts" 22 "github.com/docker/docker/pkg/ioutils" 23 "github.com/docker/docker/pkg/stringid" 24 "github.com/docker/docker/testutil/request" 25 "github.com/docker/go-connections/sockets" 26 "github.com/docker/go-connections/tlsconfig" 27 "github.com/pkg/errors" 28 "gotest.tools/v3/assert" 29 ) 30 31 // LogT is the subset of the testing.TB interface used by the daemon. 32 type LogT interface { 33 Logf(string, ...interface{}) 34 } 35 36 // nopLog is a no-op implementation of LogT that is used in daemons created by 37 // NewDaemon (where no testing.TB is available). 38 type nopLog struct{} 39 40 func (nopLog) Logf(string, ...interface{}) {} 41 42 const ( 43 defaultDockerdBinary = "dockerd" 44 defaultContainerdSocket = "/var/run/docker/containerd/containerd.sock" 45 defaultDockerdRootlessBinary = "dockerd-rootless.sh" 46 ) 47 48 var errDaemonNotStarted = errors.New("daemon not started") 49 50 // SockRoot holds the path of the default docker integration daemon socket 51 var SockRoot = filepath.Join(os.TempDir(), "docker-integration") 52 53 type clientConfig struct { 54 transport *http.Transport 55 scheme string 56 addr string 57 } 58 59 // Daemon represents a Docker daemon for the testing framework 60 type Daemon struct { 61 Root string 62 Folder string 63 Wait chan error 64 UseDefaultHost bool 65 UseDefaultTLSHost bool 66 67 id string 68 logFile *os.File 69 cmd *exec.Cmd 70 storageDriver string 71 userlandProxy bool 72 defaultCgroupNamespaceMode string 73 execRoot string 74 experimental bool 75 init bool 76 dockerdBinary string 77 log LogT 78 pidFile string 79 args []string 80 containerdSocket string 81 rootlessUser *user.User 82 rootlessXDGRuntimeDir string 83 84 // swarm related field 85 swarmListenAddr string 86 SwarmPort int // FIXME(vdemeester) should probably not be exported 87 DefaultAddrPool []string 88 SubnetSize uint32 89 DataPathPort uint32 90 OOMScoreAdjust int 91 // cached information 92 CachedInfo types.Info 93 } 94 95 // NewDaemon returns a Daemon instance to be used for testing. 96 // The daemon will not automatically start. 97 // The daemon will modify and create files under workingDir. 98 func NewDaemon(workingDir string, ops ...Option) (*Daemon, error) { 99 storageDriver := os.Getenv("DOCKER_GRAPHDRIVER") 100 101 if err := os.MkdirAll(SockRoot, 0700); err != nil { 102 return nil, errors.Wrapf(err, "failed to create daemon socket root %q", SockRoot) 103 } 104 105 id := fmt.Sprintf("d%s", stringid.TruncateID(stringid.GenerateRandomID())) 106 dir := filepath.Join(workingDir, id) 107 daemonFolder, err := filepath.Abs(dir) 108 if err != nil { 109 return nil, err 110 } 111 daemonRoot := filepath.Join(daemonFolder, "root") 112 if err := os.MkdirAll(daemonRoot, 0755); err != nil { 113 return nil, errors.Wrapf(err, "failed to create daemon root %q", daemonRoot) 114 } 115 116 userlandProxy := true 117 if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { 118 if val, err := strconv.ParseBool(env); err != nil { 119 userlandProxy = val 120 } 121 } 122 d := &Daemon{ 123 id: id, 124 Folder: daemonFolder, 125 Root: daemonRoot, 126 storageDriver: storageDriver, 127 userlandProxy: userlandProxy, 128 // dxr stands for docker-execroot (shortened for avoiding unix(7) path length limitation) 129 execRoot: filepath.Join(os.TempDir(), "dxr", id), 130 dockerdBinary: defaultDockerdBinary, 131 swarmListenAddr: defaultSwarmListenAddr, 132 SwarmPort: DefaultSwarmPort, 133 log: nopLog{}, 134 containerdSocket: defaultContainerdSocket, 135 } 136 137 for _, op := range ops { 138 op(d) 139 } 140 141 if d.rootlessUser != nil { 142 if err := os.Chmod(SockRoot, 0777); err != nil { 143 return nil, err 144 } 145 uid, err := strconv.Atoi(d.rootlessUser.Uid) 146 if err != nil { 147 return nil, err 148 } 149 gid, err := strconv.Atoi(d.rootlessUser.Gid) 150 if err != nil { 151 return nil, err 152 } 153 if err := os.Chown(d.Folder, uid, gid); err != nil { 154 return nil, err 155 } 156 if err := os.Chown(d.Root, uid, gid); err != nil { 157 return nil, err 158 } 159 if err := os.MkdirAll(filepath.Dir(d.execRoot), 0700); err != nil { 160 return nil, err 161 } 162 if err := os.Chown(filepath.Dir(d.execRoot), uid, gid); err != nil { 163 return nil, err 164 } 165 if err := os.MkdirAll(d.execRoot, 0700); err != nil { 166 return nil, err 167 } 168 if err := os.Chown(d.execRoot, uid, gid); err != nil { 169 return nil, err 170 } 171 d.rootlessXDGRuntimeDir = filepath.Join(d.Folder, "xdgrun") 172 if err := os.MkdirAll(d.rootlessXDGRuntimeDir, 0700); err != nil { 173 return nil, err 174 } 175 if err := os.Chown(d.rootlessXDGRuntimeDir, uid, gid); err != nil { 176 return nil, err 177 } 178 d.containerdSocket = "" 179 } 180 181 return d, nil 182 } 183 184 // New returns a Daemon instance to be used for testing. 185 // This will create a directory such as d123456789 in the folder specified by 186 // $DOCKER_INTEGRATION_DAEMON_DEST or $DEST. 187 // The daemon will not automatically start. 188 func New(t testing.TB, ops ...Option) *Daemon { 189 t.Helper() 190 dest := os.Getenv("DOCKER_INTEGRATION_DAEMON_DEST") 191 if dest == "" { 192 dest = os.Getenv("DEST") 193 } 194 dest = filepath.Join(dest, t.Name()) 195 196 assert.Check(t, dest != "", "Please set the DOCKER_INTEGRATION_DAEMON_DEST or the DEST environment variable") 197 198 if os.Getenv("DOCKER_ROOTLESS") != "" { 199 if os.Getenv("DOCKER_REMAP_ROOT") != "" { 200 t.Skip("DOCKER_ROOTLESS doesn't support DOCKER_REMAP_ROOT currently") 201 } 202 if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { 203 if val, err := strconv.ParseBool(env); err == nil && !val { 204 t.Skip("DOCKER_ROOTLESS doesn't support DOCKER_USERLANDPROXY=false") 205 } 206 } 207 ops = append(ops, WithRootlessUser("unprivilegeduser")) 208 } 209 ops = append(ops, WithOOMScoreAdjust(-500)) 210 211 d, err := NewDaemon(dest, ops...) 212 assert.NilError(t, err, "could not create daemon at %q", dest) 213 if d.rootlessUser != nil && d.dockerdBinary != defaultDockerdBinary { 214 t.Skipf("DOCKER_ROOTLESS doesn't support specifying non-default dockerd binary path %q", d.dockerdBinary) 215 } 216 217 return d 218 } 219 220 // ContainersNamespace returns the containerd namespace used for containers. 221 func (d *Daemon) ContainersNamespace() string { 222 return d.id 223 } 224 225 // RootDir returns the root directory of the daemon. 226 func (d *Daemon) RootDir() string { 227 return d.Root 228 } 229 230 // ID returns the generated id of the daemon 231 func (d *Daemon) ID() string { 232 return d.id 233 } 234 235 // StorageDriver returns the configured storage driver of the daemon 236 func (d *Daemon) StorageDriver() string { 237 return d.storageDriver 238 } 239 240 // Sock returns the socket path of the daemon 241 func (d *Daemon) Sock() string { 242 return fmt.Sprintf("unix://" + d.sockPath()) 243 } 244 245 func (d *Daemon) sockPath() string { 246 return filepath.Join(SockRoot, d.id+".sock") 247 } 248 249 // LogFileName returns the path the daemon's log file 250 func (d *Daemon) LogFileName() string { 251 return d.logFile.Name() 252 } 253 254 // ReadLogFile returns the content of the daemon log file 255 func (d *Daemon) ReadLogFile() ([]byte, error) { 256 return ioutil.ReadFile(d.logFile.Name()) 257 } 258 259 // NewClientT creates new client based on daemon's socket path 260 func (d *Daemon) NewClientT(t testing.TB, extraOpts ...client.Opt) *client.Client { 261 t.Helper() 262 263 c, err := d.NewClient(extraOpts...) 264 assert.NilError(t, err, "[%s] could not create daemon client", d.id) 265 return c 266 } 267 268 // NewClient creates new client based on daemon's socket path 269 func (d *Daemon) NewClient(extraOpts ...client.Opt) (*client.Client, error) { 270 clientOpts := []client.Opt{ 271 client.FromEnv, 272 client.WithHost(d.Sock()), 273 } 274 clientOpts = append(clientOpts, extraOpts...) 275 276 return client.NewClientWithOpts(clientOpts...) 277 } 278 279 // Cleanup cleans the daemon files : exec root (network namespaces, ...), swarmkit files 280 func (d *Daemon) Cleanup(t testing.TB) { 281 t.Helper() 282 cleanupMount(t, d) 283 cleanupRaftDir(t, d) 284 cleanupDaemonStorage(t, d) 285 cleanupNetworkNamespace(t, d) 286 } 287 288 // Start starts the daemon and return once it is ready to receive requests. 289 func (d *Daemon) Start(t testing.TB, args ...string) { 290 t.Helper() 291 if err := d.StartWithError(args...); err != nil { 292 d.DumpStackAndQuit() // in case the daemon is stuck 293 t.Fatalf("[%s] failed to start daemon with arguments %v : %v", d.id, d.args, err) 294 } 295 } 296 297 // StartWithError starts the daemon and return once it is ready to receive requests. 298 // It returns an error in case it couldn't start. 299 func (d *Daemon) StartWithError(args ...string) error { 300 logFile, err := os.OpenFile(filepath.Join(d.Folder, "docker.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) 301 if err != nil { 302 return errors.Wrapf(err, "[%s] failed to create logfile", d.id) 303 } 304 305 return d.StartWithLogFile(logFile, args...) 306 } 307 308 // StartWithLogFile will start the daemon and attach its streams to a given file. 309 func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error { 310 d.handleUserns() 311 dockerdBinary, err := exec.LookPath(d.dockerdBinary) 312 if err != nil { 313 return errors.Wrapf(err, "[%s] could not find docker binary in $PATH", d.id) 314 } 315 316 if d.pidFile == "" { 317 d.pidFile = filepath.Join(d.Folder, "docker.pid") 318 } 319 320 d.args = []string{} 321 if d.rootlessUser != nil { 322 if d.dockerdBinary != defaultDockerdBinary { 323 return errors.Errorf("[%s] DOCKER_ROOTLESS doesn't support non-default dockerd binary path %q", d.id, d.dockerdBinary) 324 } 325 dockerdBinary = "sudo" 326 d.args = append(d.args, 327 "-u", d.rootlessUser.Username, 328 "-E", "XDG_RUNTIME_DIR="+d.rootlessXDGRuntimeDir, 329 "-E", "HOME="+d.rootlessUser.HomeDir, 330 "-E", "PATH="+os.Getenv("PATH"), 331 "--", 332 defaultDockerdRootlessBinary, 333 ) 334 } 335 336 d.args = append(d.args, 337 "--data-root", d.Root, 338 "--exec-root", d.execRoot, 339 "--pidfile", d.pidFile, 340 fmt.Sprintf("--userland-proxy=%t", d.userlandProxy), 341 "--containerd-namespace", d.id, 342 "--containerd-plugins-namespace", d.id+"p", 343 ) 344 if d.containerdSocket != "" { 345 d.args = append(d.args, "--containerd", d.containerdSocket) 346 } 347 348 if d.defaultCgroupNamespaceMode != "" { 349 d.args = append(d.args, "--default-cgroupns-mode", d.defaultCgroupNamespaceMode) 350 } 351 if d.experimental { 352 d.args = append(d.args, "--experimental") 353 } 354 if d.init { 355 d.args = append(d.args, "--init") 356 } 357 if !(d.UseDefaultHost || d.UseDefaultTLSHost) { 358 d.args = append(d.args, "--host", d.Sock()) 359 } 360 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 361 d.args = append(d.args, "--userns-remap", root) 362 } 363 364 // If we don't explicitly set the log-level or debug flag(-D) then 365 // turn on debug mode 366 foundLog := false 367 foundSd := false 368 for _, a := range providedArgs { 369 if strings.Contains(a, "--log-level") || strings.Contains(a, "-D") || strings.Contains(a, "--debug") { 370 foundLog = true 371 } 372 if strings.Contains(a, "--storage-driver") { 373 foundSd = true 374 } 375 } 376 if !foundLog { 377 d.args = append(d.args, "--debug") 378 } 379 if d.storageDriver != "" && !foundSd { 380 d.args = append(d.args, "--storage-driver", d.storageDriver) 381 } 382 383 d.args = append(d.args, providedArgs...) 384 d.cmd = exec.Command(dockerdBinary, d.args...) 385 d.cmd.Env = append(os.Environ(), "DOCKER_SERVICE_PREFER_OFFLINE_IMAGE=1") 386 d.cmd.Stdout = out 387 d.cmd.Stderr = out 388 d.logFile = out 389 if d.rootlessUser != nil { 390 // sudo requires this for propagating signals 391 setsid(d.cmd) 392 } 393 394 if err := d.cmd.Start(); err != nil { 395 return errors.Wrapf(err, "[%s] could not start daemon container", d.id) 396 } 397 398 wait := make(chan error, 1) 399 400 go func() { 401 ret := d.cmd.Wait() 402 d.log.Logf("[%s] exiting daemon", d.id) 403 // If we send before logging, we might accidentally log _after_ the test is done. 404 // As of Go 1.12, this incurs a panic instead of silently being dropped. 405 wait <- ret 406 close(wait) 407 }() 408 409 d.Wait = wait 410 411 clientConfig, err := d.getClientConfig() 412 if err != nil { 413 return err 414 } 415 client := &http.Client{ 416 Transport: clientConfig.transport, 417 } 418 419 req, err := http.NewRequest(http.MethodGet, "/_ping", nil) 420 if err != nil { 421 return errors.Wrapf(err, "[%s] could not create new request", d.id) 422 } 423 req.URL.Host = clientConfig.addr 424 req.URL.Scheme = clientConfig.scheme 425 426 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) 427 defer cancel() 428 429 // make sure daemon is ready to receive requests 430 for i := 0; ; i++ { 431 d.log.Logf("[%s] waiting for daemon to start", d.id) 432 433 select { 434 case <-ctx.Done(): 435 return errors.Wrapf(ctx.Err(), "[%s] daemon exited and never started", d.id) 436 case err := <-d.Wait: 437 return errors.Wrapf(err, "[%s] daemon exited during startup", d.id) 438 default: 439 rctx, rcancel := context.WithTimeout(context.TODO(), 2*time.Second) 440 defer rcancel() 441 442 resp, err := client.Do(req.WithContext(rctx)) 443 if err != nil { 444 if i > 2 { // don't log the first couple, this ends up just being noise 445 d.log.Logf("[%s] error pinging daemon on start: %v", d.id, err) 446 } 447 448 select { 449 case <-ctx.Done(): 450 case <-time.After(500 * time.Millisecond): 451 } 452 continue 453 } 454 455 resp.Body.Close() 456 if resp.StatusCode != http.StatusOK { 457 d.log.Logf("[%s] received status != 200 OK: %s\n", d.id, resp.Status) 458 } 459 d.log.Logf("[%s] daemon started\n", d.id) 460 d.Root, err = d.queryRootDir() 461 if err != nil { 462 return errors.Wrapf(err, "[%s] error querying daemon for root directory", d.id) 463 } 464 return nil 465 } 466 } 467 } 468 469 // StartWithBusybox will first start the daemon with Daemon.Start() 470 // then save the busybox image from the main daemon and load it into this Daemon instance. 471 func (d *Daemon) StartWithBusybox(t testing.TB, arg ...string) { 472 t.Helper() 473 d.Start(t, arg...) 474 d.LoadBusybox(t) 475 } 476 477 // Kill will send a SIGKILL to the daemon 478 func (d *Daemon) Kill() error { 479 if d.cmd == nil || d.Wait == nil { 480 return errDaemonNotStarted 481 } 482 483 defer func() { 484 d.logFile.Close() 485 d.cmd = nil 486 }() 487 488 if err := d.cmd.Process.Kill(); err != nil { 489 return err 490 } 491 492 if d.pidFile != "" { 493 _ = os.Remove(d.pidFile) 494 } 495 return nil 496 } 497 498 // Pid returns the pid of the daemon 499 func (d *Daemon) Pid() int { 500 return d.cmd.Process.Pid 501 } 502 503 // Interrupt stops the daemon by sending it an Interrupt signal 504 func (d *Daemon) Interrupt() error { 505 return d.Signal(os.Interrupt) 506 } 507 508 // Signal sends the specified signal to the daemon if running 509 func (d *Daemon) Signal(signal os.Signal) error { 510 if d.cmd == nil || d.Wait == nil { 511 return errDaemonNotStarted 512 } 513 return d.cmd.Process.Signal(signal) 514 } 515 516 // DumpStackAndQuit sends SIGQUIT to the daemon, which triggers it to dump its 517 // stack to its log file and exit 518 // This is used primarily for gathering debug information on test timeout 519 func (d *Daemon) DumpStackAndQuit() { 520 if d.cmd == nil || d.cmd.Process == nil { 521 return 522 } 523 SignalDaemonDump(d.cmd.Process.Pid) 524 } 525 526 // Stop will send a SIGINT every second and wait for the daemon to stop. 527 // If it times out, a SIGKILL is sent. 528 // Stop will not delete the daemon directory. If a purged daemon is needed, 529 // instantiate a new one with NewDaemon. 530 // If an error occurs while starting the daemon, the test will fail. 531 func (d *Daemon) Stop(t testing.TB) { 532 t.Helper() 533 err := d.StopWithError() 534 if err != nil { 535 if err != errDaemonNotStarted { 536 t.Fatalf("[%s] error while stopping the daemon: %v", d.id, err) 537 } else { 538 t.Logf("[%s] daemon is not started", d.id) 539 } 540 } 541 } 542 543 // StopWithError will send a SIGINT every second and wait for the daemon to stop. 544 // If it timeouts, a SIGKILL is sent. 545 // Stop will not delete the daemon directory. If a purged daemon is needed, 546 // instantiate a new one with NewDaemon. 547 func (d *Daemon) StopWithError() (err error) { 548 if d.cmd == nil || d.Wait == nil { 549 return errDaemonNotStarted 550 } 551 defer func() { 552 if err != nil { 553 d.log.Logf("[%s] error while stopping daemon: %v", d.id, err) 554 } else { 555 d.log.Logf("[%s] daemon stopped", d.id) 556 if d.pidFile != "" { 557 _ = os.Remove(d.pidFile) 558 } 559 } 560 if err := d.logFile.Close(); err != nil { 561 d.log.Logf("[%s] failed to close daemon logfile: %v", d.id, err) 562 } 563 d.cmd = nil 564 }() 565 566 i := 1 567 ticker := time.NewTicker(time.Second) 568 defer ticker.Stop() 569 tick := ticker.C 570 571 d.log.Logf("[%s] stopping daemon", d.id) 572 573 if err := d.cmd.Process.Signal(os.Interrupt); err != nil { 574 if strings.Contains(err.Error(), "os: process already finished") { 575 return errDaemonNotStarted 576 } 577 return errors.Wrapf(err, "[%s] could not send signal", d.id) 578 } 579 580 out1: 581 for { 582 select { 583 case err := <-d.Wait: 584 return err 585 case <-time.After(20 * time.Second): 586 // time for stopping jobs and run onShutdown hooks 587 d.log.Logf("[%s] daemon stop timed out after 20 seconds", d.id) 588 break out1 589 } 590 } 591 592 out2: 593 for { 594 select { 595 case err := <-d.Wait: 596 return err 597 case <-tick: 598 i++ 599 if i > 5 { 600 d.log.Logf("[%s] tried to interrupt daemon for %d times, now try to kill it", d.id, i) 601 break out2 602 } 603 d.log.Logf("[%d] attempt #%d/5: daemon is still running with pid %d", i, d.cmd.Process.Pid) 604 if err := d.cmd.Process.Signal(os.Interrupt); err != nil { 605 return errors.Wrapf(err, "[%s] attempt #%d/5 could not send signal", d.id, i) 606 } 607 } 608 } 609 610 if err := d.cmd.Process.Kill(); err != nil { 611 d.log.Logf("[%s] failed to kill daemon: %v", d.id, err) 612 return err 613 } 614 615 return nil 616 } 617 618 // Restart will restart the daemon by first stopping it and the starting it. 619 // If an error occurs while starting the daemon, the test will fail. 620 func (d *Daemon) Restart(t testing.TB, args ...string) { 621 t.Helper() 622 d.Stop(t) 623 d.Start(t, args...) 624 } 625 626 // RestartWithError will restart the daemon by first stopping it and then starting it. 627 func (d *Daemon) RestartWithError(arg ...string) error { 628 if err := d.StopWithError(); err != nil { 629 return err 630 } 631 return d.StartWithError(arg...) 632 } 633 634 func (d *Daemon) handleUserns() { 635 // in the case of tests running a user namespace-enabled daemon, we have resolved 636 // d.Root to be the actual final path of the graph dir after the "uid.gid" of 637 // remapped root is added--we need to subtract it from the path before calling 638 // start or else we will continue making subdirectories rather than truly restarting 639 // with the same location/root: 640 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 641 d.Root = filepath.Dir(d.Root) 642 } 643 } 644 645 // ReloadConfig asks the daemon to reload its configuration 646 func (d *Daemon) ReloadConfig() error { 647 if d.cmd == nil || d.cmd.Process == nil { 648 return errors.New("daemon is not running") 649 } 650 651 errCh := make(chan error, 1) 652 started := make(chan struct{}) 653 go func() { 654 _, body, err := request.Get("/events", request.Host(d.Sock())) 655 close(started) 656 if err != nil { 657 errCh <- err 658 return 659 } 660 defer body.Close() 661 dec := json.NewDecoder(body) 662 for { 663 var e events.Message 664 if err := dec.Decode(&e); err != nil { 665 errCh <- err 666 return 667 } 668 if e.Type != events.DaemonEventType { 669 continue 670 } 671 if e.Action != "reload" { 672 continue 673 } 674 close(errCh) // notify that we are done 675 return 676 } 677 }() 678 679 <-started 680 if err := signalDaemonReload(d.cmd.Process.Pid); err != nil { 681 return errors.Wrapf(err, "[%s] error signaling daemon reload", d.id) 682 } 683 select { 684 case err := <-errCh: 685 if err != nil { 686 return errors.Wrapf(err, "[%s] error waiting for daemon reload event", d.id) 687 } 688 case <-time.After(30 * time.Second): 689 return errors.Errorf("[%s] daemon reload event timed out after 30 seconds", d.id) 690 } 691 return nil 692 } 693 694 // LoadBusybox image into the daemon 695 func (d *Daemon) LoadBusybox(t testing.TB) { 696 t.Helper() 697 clientHost, err := client.NewClientWithOpts(client.FromEnv) 698 assert.NilError(t, err, "[%s] failed to create client", d.id) 699 defer clientHost.Close() 700 701 ctx := context.Background() 702 reader, err := clientHost.ImageSave(ctx, []string{"busybox:latest"}) 703 assert.NilError(t, err, "[%s] failed to download busybox", d.id) 704 defer reader.Close() 705 706 c := d.NewClientT(t) 707 defer c.Close() 708 709 resp, err := c.ImageLoad(ctx, reader, true) 710 assert.NilError(t, err, "[%s] failed to load busybox", d.id) 711 defer resp.Body.Close() 712 } 713 714 func (d *Daemon) getClientConfig() (*clientConfig, error) { 715 var ( 716 transport *http.Transport 717 scheme string 718 addr string 719 proto string 720 ) 721 if d.UseDefaultTLSHost { 722 option := &tlsconfig.Options{ 723 CAFile: "fixtures/https/ca.pem", 724 CertFile: "fixtures/https/client-cert.pem", 725 KeyFile: "fixtures/https/client-key.pem", 726 } 727 tlsConfig, err := tlsconfig.Client(*option) 728 if err != nil { 729 return nil, err 730 } 731 transport = &http.Transport{ 732 TLSClientConfig: tlsConfig, 733 } 734 addr = fmt.Sprintf("%s:%d", opts.DefaultHTTPHost, opts.DefaultTLSHTTPPort) 735 scheme = "https" 736 proto = "tcp" 737 } else if d.UseDefaultHost { 738 addr = opts.DefaultUnixSocket 739 proto = "unix" 740 scheme = "http" 741 transport = &http.Transport{} 742 } else { 743 addr = d.sockPath() 744 proto = "unix" 745 scheme = "http" 746 transport = &http.Transport{} 747 } 748 749 if err := sockets.ConfigureTransport(transport, proto, addr); err != nil { 750 return nil, err 751 } 752 transport.DisableKeepAlives = true 753 if proto == "unix" { 754 addr = filepath.Base(addr) 755 } 756 return &clientConfig{ 757 transport: transport, 758 scheme: scheme, 759 addr: addr, 760 }, nil 761 } 762 763 func (d *Daemon) queryRootDir() (string, error) { 764 // update daemon root by asking /info endpoint (to support user 765 // namespaced daemon with root remapped uid.gid directory) 766 clientConfig, err := d.getClientConfig() 767 if err != nil { 768 return "", err 769 } 770 771 c := &http.Client{ 772 Transport: clientConfig.transport, 773 } 774 775 req, err := http.NewRequest(http.MethodGet, "/info", nil) 776 if err != nil { 777 return "", err 778 } 779 req.Header.Set("Content-Type", "application/json") 780 req.URL.Host = clientConfig.addr 781 req.URL.Scheme = clientConfig.scheme 782 783 resp, err := c.Do(req) 784 if err != nil { 785 return "", err 786 } 787 body := ioutils.NewReadCloserWrapper(resp.Body, func() error { 788 return resp.Body.Close() 789 }) 790 791 type Info struct { 792 DockerRootDir string 793 } 794 var b []byte 795 var i Info 796 b, err = request.ReadBody(body) 797 if err == nil && resp.StatusCode == http.StatusOK { 798 // read the docker root dir 799 if err = json.Unmarshal(b, &i); err == nil { 800 return i.DockerRootDir, nil 801 } 802 } 803 return "", err 804 } 805 806 // Info returns the info struct for this daemon 807 func (d *Daemon) Info(t testing.TB) types.Info { 808 t.Helper() 809 c := d.NewClientT(t) 810 info, err := c.Info(context.Background()) 811 assert.NilError(t, err) 812 assert.NilError(t, c.Close()) 813 return info 814 } 815 816 // cleanupRaftDir removes swarmkit wal files if present 817 func cleanupRaftDir(t testing.TB, d *Daemon) { 818 t.Helper() 819 for _, p := range []string{"wal", "wal-v3-encrypted", "snap-v3-encrypted"} { 820 dir := filepath.Join(d.Root, "swarm/raft", p) 821 if err := os.RemoveAll(dir); err != nil { 822 t.Logf("[%s] error removing %v: %v", d.id, dir, err) 823 } 824 } 825 } 826 827 // cleanupDaemonStorage removes the daemon's storage directory. 828 // 829 // Note that we don't delete the whole directory, as some files (e.g. daemon 830 // logs) are collected for inclusion in the "bundles" that are stored as Jenkins 831 // artifacts. 832 // 833 // We currently do not include container logs in the bundles, so this also 834 // removes the "containers" sub-directory. 835 func cleanupDaemonStorage(t testing.TB, d *Daemon) { 836 t.Helper() 837 dirs := []string{ 838 "builder", 839 "buildkit", 840 "containers", 841 "image", 842 "network", 843 "plugins", 844 "tmp", 845 "trust", 846 "volumes", 847 // note: this assumes storage-driver name matches the subdirectory, 848 // which is currently true, but not guaranteed. 849 d.storageDriver, 850 } 851 852 for _, p := range dirs { 853 dir := filepath.Join(d.Root, p) 854 if err := os.RemoveAll(dir); err != nil { 855 t.Logf("[%s] error removing %v: %v", d.id, dir, err) 856 } 857 } 858 }