github.com/jwhonce/docker@v0.6.7-0.20190327063223-da823cf3a5a3/internal/test/daemon/daemon.go (about) 1 package daemon // import "github.com/docker/docker/internal/test/daemon" 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "io/ioutil" 8 "net/http" 9 "os" 10 "os/exec" 11 "path/filepath" 12 "strconv" 13 "strings" 14 "time" 15 16 "github.com/docker/docker/api/types" 17 "github.com/docker/docker/api/types/events" 18 "github.com/docker/docker/client" 19 "github.com/docker/docker/internal/test" 20 "github.com/docker/docker/internal/test/request" 21 "github.com/docker/docker/opts" 22 "github.com/docker/docker/pkg/ioutils" 23 "github.com/docker/docker/pkg/stringid" 24 "github.com/docker/go-connections/sockets" 25 "github.com/docker/go-connections/tlsconfig" 26 "github.com/pkg/errors" 27 "gotest.tools/assert" 28 ) 29 30 type testingT interface { 31 assert.TestingT 32 logT 33 Fatalf(string, ...interface{}) 34 } 35 36 type logT interface { 37 Logf(string, ...interface{}) 38 } 39 40 const defaultDockerdBinary = "dockerd" 41 const containerdSocket = "/var/run/docker/containerd/containerd.sock" 42 43 var errDaemonNotStarted = errors.New("daemon not started") 44 45 // SockRoot holds the path of the default docker integration daemon socket 46 var SockRoot = filepath.Join(os.TempDir(), "docker-integration") 47 48 type clientConfig struct { 49 transport *http.Transport 50 scheme string 51 addr string 52 } 53 54 // Daemon represents a Docker daemon for the testing framework 55 type Daemon struct { 56 GlobalFlags []string 57 Root string 58 Folder string 59 Wait chan error 60 UseDefaultHost bool 61 UseDefaultTLSHost bool 62 63 id string 64 logFile *os.File 65 cmd *exec.Cmd 66 storageDriver string 67 userlandProxy bool 68 execRoot string 69 experimental bool 70 init bool 71 dockerdBinary string 72 log logT 73 74 // swarm related field 75 swarmListenAddr string 76 SwarmPort int // FIXME(vdemeester) should probably not be exported 77 DefaultAddrPool []string 78 SubnetSize uint32 79 DataPathPort uint32 80 // cached information 81 CachedInfo types.Info 82 } 83 84 // New returns a Daemon instance to be used for testing. 85 // This will create a directory such as d123456789 in the folder specified by $DOCKER_INTEGRATION_DAEMON_DEST or $DEST. 86 // The daemon will not automatically start. 87 func New(t testingT, ops ...func(*Daemon)) *Daemon { 88 if ht, ok := t.(test.HelperT); ok { 89 ht.Helper() 90 } 91 t.Log("Creating a new daemon") 92 dest := os.Getenv("DOCKER_INTEGRATION_DAEMON_DEST") 93 if dest == "" { 94 dest = os.Getenv("DEST") 95 } 96 assert.Check(t, dest != "", "Please set the DOCKER_INTEGRATION_DAEMON_DEST or the DEST environment variable") 97 98 storageDriver := os.Getenv("DOCKER_GRAPHDRIVER") 99 100 assert.NilError(t, os.MkdirAll(SockRoot, 0700), "could not create daemon socket root") 101 102 id := fmt.Sprintf("d%s", stringid.TruncateID(stringid.GenerateRandomID())) 103 dir := filepath.Join(dest, id) 104 daemonFolder, err := filepath.Abs(dir) 105 assert.NilError(t, err, "Could not make %q an absolute path", dir) 106 daemonRoot := filepath.Join(daemonFolder, "root") 107 108 assert.NilError(t, os.MkdirAll(daemonRoot, 0755), "Could not create daemon root %q", dir) 109 110 userlandProxy := true 111 if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { 112 if val, err := strconv.ParseBool(env); err != nil { 113 userlandProxy = val 114 } 115 } 116 d := &Daemon{ 117 id: id, 118 Folder: daemonFolder, 119 Root: daemonRoot, 120 storageDriver: storageDriver, 121 userlandProxy: userlandProxy, 122 // dxr stands for docker-execroot (shortened for avoiding unix(7) path length limitation) 123 execRoot: filepath.Join(os.TempDir(), "dxr", id), 124 dockerdBinary: defaultDockerdBinary, 125 swarmListenAddr: defaultSwarmListenAddr, 126 SwarmPort: DefaultSwarmPort, 127 log: t, 128 } 129 130 for _, op := range ops { 131 op(d) 132 } 133 134 return d 135 } 136 137 // RootDir returns the root directory of the daemon. 138 func (d *Daemon) RootDir() string { 139 return d.Root 140 } 141 142 // ID returns the generated id of the daemon 143 func (d *Daemon) ID() string { 144 return d.id 145 } 146 147 // StorageDriver returns the configured storage driver of the daemon 148 func (d *Daemon) StorageDriver() string { 149 return d.storageDriver 150 } 151 152 // Sock returns the socket path of the daemon 153 func (d *Daemon) Sock() string { 154 return fmt.Sprintf("unix://" + d.sockPath()) 155 } 156 157 func (d *Daemon) sockPath() string { 158 return filepath.Join(SockRoot, d.id+".sock") 159 } 160 161 // LogFileName returns the path the daemon's log file 162 func (d *Daemon) LogFileName() string { 163 return d.logFile.Name() 164 } 165 166 // ReadLogFile returns the content of the daemon log file 167 func (d *Daemon) ReadLogFile() ([]byte, error) { 168 return ioutil.ReadFile(d.logFile.Name()) 169 } 170 171 // NewClientT creates new client based on daemon's socket path 172 func (d *Daemon) NewClientT(t assert.TestingT) *client.Client { 173 if ht, ok := t.(test.HelperT); ok { 174 ht.Helper() 175 } 176 c, err := client.NewClientWithOpts( 177 client.FromEnv, 178 client.WithHost(d.Sock())) 179 assert.NilError(t, err, "cannot create daemon client") 180 return c 181 } 182 183 // Cleanup cleans the daemon files : exec root (network namespaces, ...), swarmkit files 184 func (d *Daemon) Cleanup(t testingT) { 185 if ht, ok := t.(test.HelperT); ok { 186 ht.Helper() 187 } 188 // Cleanup swarmkit wal files if present 189 cleanupRaftDir(t, d.Root) 190 cleanupNetworkNamespace(t, d.execRoot) 191 } 192 193 // Start starts the daemon and return once it is ready to receive requests. 194 func (d *Daemon) Start(t testingT, args ...string) { 195 if ht, ok := t.(test.HelperT); ok { 196 ht.Helper() 197 } 198 if err := d.StartWithError(args...); err != nil { 199 t.Fatalf("failed to start daemon with arguments %v : %v", args, err) 200 } 201 } 202 203 // StartWithError starts the daemon and return once it is ready to receive requests. 204 // It returns an error in case it couldn't start. 205 func (d *Daemon) StartWithError(args ...string) error { 206 logFile, err := os.OpenFile(filepath.Join(d.Folder, "docker.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) 207 if err != nil { 208 return errors.Wrapf(err, "[%s] Could not create %s/docker.log", d.id, d.Folder) 209 } 210 211 return d.StartWithLogFile(logFile, args...) 212 } 213 214 // StartWithLogFile will start the daemon and attach its streams to a given file. 215 func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error { 216 d.handleUserns() 217 dockerdBinary, err := exec.LookPath(d.dockerdBinary) 218 if err != nil { 219 return errors.Wrapf(err, "[%s] could not find docker binary in $PATH", d.id) 220 } 221 args := append(d.GlobalFlags, 222 "--containerd", containerdSocket, 223 "--data-root", d.Root, 224 "--exec-root", d.execRoot, 225 "--pidfile", fmt.Sprintf("%s/docker.pid", d.Folder), 226 fmt.Sprintf("--userland-proxy=%t", d.userlandProxy), 227 ) 228 if d.experimental { 229 args = append(args, "--experimental") 230 } 231 if d.init { 232 args = append(args, "--init") 233 } 234 if !(d.UseDefaultHost || d.UseDefaultTLSHost) { 235 args = append(args, []string{"--host", d.Sock()}...) 236 } 237 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 238 args = append(args, []string{"--userns-remap", root}...) 239 } 240 241 // If we don't explicitly set the log-level or debug flag(-D) then 242 // turn on debug mode 243 foundLog := false 244 foundSd := false 245 for _, a := range providedArgs { 246 if strings.Contains(a, "--log-level") || strings.Contains(a, "-D") || strings.Contains(a, "--debug") { 247 foundLog = true 248 } 249 if strings.Contains(a, "--storage-driver") { 250 foundSd = true 251 } 252 } 253 if !foundLog { 254 args = append(args, "--debug") 255 } 256 if d.storageDriver != "" && !foundSd { 257 args = append(args, "--storage-driver", d.storageDriver) 258 } 259 260 args = append(args, providedArgs...) 261 d.cmd = exec.Command(dockerdBinary, args...) 262 d.cmd.Env = append(os.Environ(), "DOCKER_SERVICE_PREFER_OFFLINE_IMAGE=1") 263 d.cmd.Stdout = out 264 d.cmd.Stderr = out 265 d.logFile = out 266 267 if err := d.cmd.Start(); err != nil { 268 return errors.Errorf("[%s] could not start daemon container: %v", d.id, err) 269 } 270 271 wait := make(chan error) 272 273 go func() { 274 wait <- d.cmd.Wait() 275 d.log.Logf("[%s] exiting daemon", d.id) 276 close(wait) 277 }() 278 279 d.Wait = wait 280 281 ticker := time.NewTicker(500 * time.Millisecond) 282 defer ticker.Stop() 283 tick := ticker.C 284 285 // make sure daemon is ready to receive requests 286 startTime := time.Now().Unix() 287 for { 288 d.log.Logf("[%s] waiting for daemon to start", d.id) 289 if time.Now().Unix()-startTime > 5 { 290 // After 5 seconds, give up 291 return errors.Errorf("[%s] Daemon exited and never started", d.id) 292 } 293 select { 294 case <-time.After(2 * time.Second): 295 return errors.Errorf("[%s] timeout: daemon does not respond", d.id) 296 case <-tick: 297 clientConfig, err := d.getClientConfig() 298 if err != nil { 299 return err 300 } 301 302 client := &http.Client{ 303 Transport: clientConfig.transport, 304 } 305 306 req, err := http.NewRequest("GET", "/_ping", nil) 307 if err != nil { 308 return errors.Wrapf(err, "[%s] could not create new request", d.id) 309 } 310 req.URL.Host = clientConfig.addr 311 req.URL.Scheme = clientConfig.scheme 312 resp, err := client.Do(req) 313 if err != nil { 314 continue 315 } 316 resp.Body.Close() 317 if resp.StatusCode != http.StatusOK { 318 d.log.Logf("[%s] received status != 200 OK: %s\n", d.id, resp.Status) 319 } 320 d.log.Logf("[%s] daemon started\n", d.id) 321 d.Root, err = d.queryRootDir() 322 if err != nil { 323 return errors.Errorf("[%s] error querying daemon for root directory: %v", d.id, err) 324 } 325 return nil 326 case err := <-d.Wait: 327 return errors.Errorf("[%s] Daemon exited during startup: %v", d.id, err) 328 } 329 } 330 } 331 332 // StartWithBusybox will first start the daemon with Daemon.Start() 333 // then save the busybox image from the main daemon and load it into this Daemon instance. 334 func (d *Daemon) StartWithBusybox(t testingT, arg ...string) { 335 if ht, ok := t.(test.HelperT); ok { 336 ht.Helper() 337 } 338 d.Start(t, arg...) 339 d.LoadBusybox(t) 340 } 341 342 // Kill will send a SIGKILL to the daemon 343 func (d *Daemon) Kill() error { 344 if d.cmd == nil || d.Wait == nil { 345 return errDaemonNotStarted 346 } 347 348 defer func() { 349 d.logFile.Close() 350 d.cmd = nil 351 }() 352 353 if err := d.cmd.Process.Kill(); err != nil { 354 return err 355 } 356 357 return os.Remove(fmt.Sprintf("%s/docker.pid", d.Folder)) 358 } 359 360 // Pid returns the pid of the daemon 361 func (d *Daemon) Pid() int { 362 return d.cmd.Process.Pid 363 } 364 365 // Interrupt stops the daemon by sending it an Interrupt signal 366 func (d *Daemon) Interrupt() error { 367 return d.Signal(os.Interrupt) 368 } 369 370 // Signal sends the specified signal to the daemon if running 371 func (d *Daemon) Signal(signal os.Signal) error { 372 if d.cmd == nil || d.Wait == nil { 373 return errDaemonNotStarted 374 } 375 return d.cmd.Process.Signal(signal) 376 } 377 378 // DumpStackAndQuit sends SIGQUIT to the daemon, which triggers it to dump its 379 // stack to its log file and exit 380 // This is used primarily for gathering debug information on test timeout 381 func (d *Daemon) DumpStackAndQuit() { 382 if d.cmd == nil || d.cmd.Process == nil { 383 return 384 } 385 SignalDaemonDump(d.cmd.Process.Pid) 386 } 387 388 // Stop will send a SIGINT every second and wait for the daemon to stop. 389 // If it times out, a SIGKILL is sent. 390 // Stop will not delete the daemon directory. If a purged daemon is needed, 391 // instantiate a new one with NewDaemon. 392 // If an error occurs while starting the daemon, the test will fail. 393 func (d *Daemon) Stop(t testingT) { 394 if ht, ok := t.(test.HelperT); ok { 395 ht.Helper() 396 } 397 err := d.StopWithError() 398 if err != nil { 399 if err != errDaemonNotStarted { 400 t.Fatalf("Error while stopping the daemon %s : %v", d.id, err) 401 } else { 402 t.Logf("Daemon %s is not started", d.id) 403 } 404 } 405 } 406 407 // StopWithError will send a SIGINT every second and wait for the daemon to stop. 408 // If it timeouts, a SIGKILL is sent. 409 // Stop will not delete the daemon directory. If a purged daemon is needed, 410 // instantiate a new one with NewDaemon. 411 func (d *Daemon) StopWithError() error { 412 if d.cmd == nil || d.Wait == nil { 413 return errDaemonNotStarted 414 } 415 416 defer func() { 417 d.logFile.Close() 418 d.cmd = nil 419 }() 420 421 i := 1 422 ticker := time.NewTicker(time.Second) 423 defer ticker.Stop() 424 tick := ticker.C 425 426 if err := d.cmd.Process.Signal(os.Interrupt); err != nil { 427 if strings.Contains(err.Error(), "os: process already finished") { 428 return errDaemonNotStarted 429 } 430 return errors.Errorf("could not send signal: %v", err) 431 } 432 out1: 433 for { 434 select { 435 case err := <-d.Wait: 436 return err 437 case <-time.After(20 * time.Second): 438 // time for stopping jobs and run onShutdown hooks 439 d.log.Logf("[%s] daemon stop timeout", d.id) 440 break out1 441 } 442 } 443 444 out2: 445 for { 446 select { 447 case err := <-d.Wait: 448 return err 449 case <-tick: 450 i++ 451 if i > 5 { 452 d.log.Logf("tried to interrupt daemon for %d times, now try to kill it", i) 453 break out2 454 } 455 d.log.Logf("Attempt #%d: daemon is still running with pid %d", i, d.cmd.Process.Pid) 456 if err := d.cmd.Process.Signal(os.Interrupt); err != nil { 457 return errors.Errorf("could not send signal: %v", err) 458 } 459 } 460 } 461 462 if err := d.cmd.Process.Kill(); err != nil { 463 d.log.Logf("Could not kill daemon: %v", err) 464 return err 465 } 466 467 d.cmd.Wait() 468 469 return os.Remove(fmt.Sprintf("%s/docker.pid", d.Folder)) 470 } 471 472 // Restart will restart the daemon by first stopping it and the starting it. 473 // If an error occurs while starting the daemon, the test will fail. 474 func (d *Daemon) Restart(t testingT, args ...string) { 475 if ht, ok := t.(test.HelperT); ok { 476 ht.Helper() 477 } 478 d.Stop(t) 479 d.Start(t, args...) 480 } 481 482 // RestartWithError will restart the daemon by first stopping it and then starting it. 483 func (d *Daemon) RestartWithError(arg ...string) error { 484 if err := d.StopWithError(); err != nil { 485 return err 486 } 487 return d.StartWithError(arg...) 488 } 489 490 func (d *Daemon) handleUserns() { 491 // in the case of tests running a user namespace-enabled daemon, we have resolved 492 // d.Root to be the actual final path of the graph dir after the "uid.gid" of 493 // remapped root is added--we need to subtract it from the path before calling 494 // start or else we will continue making subdirectories rather than truly restarting 495 // with the same location/root: 496 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 497 d.Root = filepath.Dir(d.Root) 498 } 499 } 500 501 // ReloadConfig asks the daemon to reload its configuration 502 func (d *Daemon) ReloadConfig() error { 503 if d.cmd == nil || d.cmd.Process == nil { 504 return errors.New("daemon is not running") 505 } 506 507 errCh := make(chan error) 508 started := make(chan struct{}) 509 go func() { 510 _, body, err := request.Get("/events", request.Host(d.Sock())) 511 close(started) 512 if err != nil { 513 errCh <- err 514 } 515 defer body.Close() 516 dec := json.NewDecoder(body) 517 for { 518 var e events.Message 519 if err := dec.Decode(&e); err != nil { 520 errCh <- err 521 return 522 } 523 if e.Type != events.DaemonEventType { 524 continue 525 } 526 if e.Action != "reload" { 527 continue 528 } 529 close(errCh) // notify that we are done 530 return 531 } 532 }() 533 534 <-started 535 if err := signalDaemonReload(d.cmd.Process.Pid); err != nil { 536 return errors.Errorf("error signaling daemon reload: %v", err) 537 } 538 select { 539 case err := <-errCh: 540 if err != nil { 541 return errors.Errorf("error waiting for daemon reload event: %v", err) 542 } 543 case <-time.After(30 * time.Second): 544 return errors.New("timeout waiting for daemon reload event") 545 } 546 return nil 547 } 548 549 // LoadBusybox image into the daemon 550 func (d *Daemon) LoadBusybox(t assert.TestingT) { 551 if ht, ok := t.(test.HelperT); ok { 552 ht.Helper() 553 } 554 clientHost, err := client.NewClientWithOpts(client.FromEnv) 555 assert.NilError(t, err, "failed to create client") 556 defer clientHost.Close() 557 558 ctx := context.Background() 559 reader, err := clientHost.ImageSave(ctx, []string{"busybox:latest"}) 560 assert.NilError(t, err, "failed to download busybox") 561 defer reader.Close() 562 563 c := d.NewClientT(t) 564 defer c.Close() 565 566 resp, err := c.ImageLoad(ctx, reader, true) 567 assert.NilError(t, err, "failed to load busybox") 568 defer resp.Body.Close() 569 } 570 571 func (d *Daemon) getClientConfig() (*clientConfig, error) { 572 var ( 573 transport *http.Transport 574 scheme string 575 addr string 576 proto string 577 ) 578 if d.UseDefaultTLSHost { 579 option := &tlsconfig.Options{ 580 CAFile: "fixtures/https/ca.pem", 581 CertFile: "fixtures/https/client-cert.pem", 582 KeyFile: "fixtures/https/client-key.pem", 583 } 584 tlsConfig, err := tlsconfig.Client(*option) 585 if err != nil { 586 return nil, err 587 } 588 transport = &http.Transport{ 589 TLSClientConfig: tlsConfig, 590 } 591 addr = fmt.Sprintf("%s:%d", opts.DefaultHTTPHost, opts.DefaultTLSHTTPPort) 592 scheme = "https" 593 proto = "tcp" 594 } else if d.UseDefaultHost { 595 addr = opts.DefaultUnixSocket 596 proto = "unix" 597 scheme = "http" 598 transport = &http.Transport{} 599 } else { 600 addr = d.sockPath() 601 proto = "unix" 602 scheme = "http" 603 transport = &http.Transport{} 604 } 605 606 if err := sockets.ConfigureTransport(transport, proto, addr); err != nil { 607 return nil, err 608 } 609 transport.DisableKeepAlives = true 610 611 return &clientConfig{ 612 transport: transport, 613 scheme: scheme, 614 addr: addr, 615 }, nil 616 } 617 618 func (d *Daemon) queryRootDir() (string, error) { 619 // update daemon root by asking /info endpoint (to support user 620 // namespaced daemon with root remapped uid.gid directory) 621 clientConfig, err := d.getClientConfig() 622 if err != nil { 623 return "", err 624 } 625 626 c := &http.Client{ 627 Transport: clientConfig.transport, 628 } 629 630 req, err := http.NewRequest("GET", "/info", nil) 631 if err != nil { 632 return "", err 633 } 634 req.Header.Set("Content-Type", "application/json") 635 req.URL.Host = clientConfig.addr 636 req.URL.Scheme = clientConfig.scheme 637 638 resp, err := c.Do(req) 639 if err != nil { 640 return "", err 641 } 642 body := ioutils.NewReadCloserWrapper(resp.Body, func() error { 643 return resp.Body.Close() 644 }) 645 646 type Info struct { 647 DockerRootDir string 648 } 649 var b []byte 650 var i Info 651 b, err = request.ReadBody(body) 652 if err == nil && resp.StatusCode == http.StatusOK { 653 // read the docker root dir 654 if err = json.Unmarshal(b, &i); err == nil { 655 return i.DockerRootDir, nil 656 } 657 } 658 return "", err 659 } 660 661 // Info returns the info struct for this daemon 662 func (d *Daemon) Info(t assert.TestingT) types.Info { 663 if ht, ok := t.(test.HelperT); ok { 664 ht.Helper() 665 } 666 c := d.NewClientT(t) 667 info, err := c.Info(context.Background()) 668 assert.NilError(t, err) 669 return info 670 } 671 672 func cleanupRaftDir(t testingT, rootPath string) { 673 if ht, ok := t.(test.HelperT); ok { 674 ht.Helper() 675 } 676 walDir := filepath.Join(rootPath, "swarm/raft/wal") 677 if err := os.RemoveAll(walDir); err != nil { 678 t.Logf("error removing %v: %v", walDir, err) 679 } 680 }