github.com/endophage/docker@v1.4.2-0.20161027011718-242853499895/integration-cli/daemon.go (about) 1 package main 2 3 import ( 4 "bytes" 5 "encoding/json" 6 "errors" 7 "fmt" 8 "io" 9 "net/http" 10 "os" 11 "os/exec" 12 "path/filepath" 13 "strconv" 14 "strings" 15 "time" 16 17 "github.com/docker/docker/api/types/events" 18 "github.com/docker/docker/opts" 19 "github.com/docker/docker/pkg/integration/checker" 20 "github.com/docker/docker/pkg/ioutils" 21 "github.com/docker/docker/pkg/stringid" 22 "github.com/docker/go-connections/sockets" 23 "github.com/docker/go-connections/tlsconfig" 24 "github.com/go-check/check" 25 ) 26 27 var daemonSockRoot = filepath.Join(os.TempDir(), "docker-integration") 28 29 // Daemon represents a Docker daemon for the testing framework. 30 type Daemon struct { 31 GlobalFlags []string 32 33 id string 34 c *check.C 35 logFile *os.File 36 folder string 37 root string 38 stdin io.WriteCloser 39 stdout, stderr io.ReadCloser 40 cmd *exec.Cmd 41 storageDriver string 42 wait chan error 43 userlandProxy bool 44 useDefaultHost bool 45 useDefaultTLSHost bool 46 execRoot string 47 } 48 49 type clientConfig struct { 50 transport *http.Transport 51 scheme string 52 addr string 53 } 54 55 // NewDaemon returns a Daemon instance to be used for testing. 56 // This will create a directory such as d123456789 in the folder specified by $DEST. 57 // The daemon will not automatically start. 58 func NewDaemon(c *check.C) *Daemon { 59 dest := os.Getenv("DEST") 60 c.Assert(dest, check.Not(check.Equals), "", check.Commentf("Please set the DEST environment variable")) 61 62 err := os.MkdirAll(daemonSockRoot, 0700) 63 c.Assert(err, checker.IsNil, check.Commentf("could not create daemon socket root")) 64 65 id := fmt.Sprintf("d%s", stringid.TruncateID(stringid.GenerateRandomID())) 66 dir := filepath.Join(dest, id) 67 daemonFolder, err := filepath.Abs(dir) 68 c.Assert(err, check.IsNil, check.Commentf("Could not make %q an absolute path", dir)) 69 daemonRoot := filepath.Join(daemonFolder, "root") 70 71 c.Assert(os.MkdirAll(daemonRoot, 0755), check.IsNil, check.Commentf("Could not create daemon root %q", dir)) 72 73 userlandProxy := true 74 if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { 75 if val, err := strconv.ParseBool(env); err != nil { 76 userlandProxy = val 77 } 78 } 79 80 return &Daemon{ 81 id: id, 82 c: c, 83 folder: daemonFolder, 84 root: daemonRoot, 85 storageDriver: os.Getenv("DOCKER_GRAPHDRIVER"), 86 userlandProxy: userlandProxy, 87 execRoot: filepath.Join(os.TempDir(), "docker-execroot", id), 88 } 89 } 90 91 func (d *Daemon) getClientConfig() (*clientConfig, error) { 92 var ( 93 transport *http.Transport 94 scheme string 95 addr string 96 proto string 97 ) 98 if d.useDefaultTLSHost { 99 option := &tlsconfig.Options{ 100 CAFile: "fixtures/https/ca.pem", 101 CertFile: "fixtures/https/client-cert.pem", 102 KeyFile: "fixtures/https/client-key.pem", 103 } 104 tlsConfig, err := tlsconfig.Client(*option) 105 if err != nil { 106 return nil, err 107 } 108 transport = &http.Transport{ 109 TLSClientConfig: tlsConfig, 110 } 111 addr = fmt.Sprintf("%s:%d", opts.DefaultHTTPHost, opts.DefaultTLSHTTPPort) 112 scheme = "https" 113 proto = "tcp" 114 } else if d.useDefaultHost { 115 addr = opts.DefaultUnixSocket 116 proto = "unix" 117 scheme = "http" 118 transport = &http.Transport{} 119 } else { 120 addr = d.sockPath() 121 proto = "unix" 122 scheme = "http" 123 transport = &http.Transport{} 124 } 125 126 d.c.Assert(sockets.ConfigureTransport(transport, proto, addr), check.IsNil) 127 128 return &clientConfig{ 129 transport: transport, 130 scheme: scheme, 131 addr: addr, 132 }, nil 133 } 134 135 // Start will start the daemon and return once it is ready to receive requests. 136 // You can specify additional daemon flags. 137 func (d *Daemon) Start(args ...string) error { 138 logFile, err := os.OpenFile(filepath.Join(d.folder, "docker.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) 139 d.c.Assert(err, check.IsNil, check.Commentf("[%s] Could not create %s/docker.log", d.id, d.folder)) 140 141 return d.StartWithLogFile(logFile, args...) 142 } 143 144 // StartWithLogFile will start the daemon and attach its streams to a given file. 145 func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error { 146 dockerdBinary, err := exec.LookPath(dockerdBinary) 147 d.c.Assert(err, check.IsNil, check.Commentf("[%s] could not find docker binary in $PATH", d.id)) 148 149 args := append(d.GlobalFlags, 150 "--containerd", "/var/run/docker/libcontainerd/docker-containerd.sock", 151 "--graph", d.root, 152 "--exec-root", d.execRoot, 153 "--pidfile", fmt.Sprintf("%s/docker.pid", d.folder), 154 fmt.Sprintf("--userland-proxy=%t", d.userlandProxy), 155 ) 156 if experimentalDaemon { 157 args = append(args, "--experimental") 158 } 159 if !(d.useDefaultHost || d.useDefaultTLSHost) { 160 args = append(args, []string{"--host", d.sock()}...) 161 } 162 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 163 args = append(args, []string{"--userns-remap", root}...) 164 } 165 166 // If we don't explicitly set the log-level or debug flag(-D) then 167 // turn on debug mode 168 foundLog := false 169 foundSd := false 170 for _, a := range providedArgs { 171 if strings.Contains(a, "--log-level") || strings.Contains(a, "-D") || strings.Contains(a, "--debug") { 172 foundLog = true 173 } 174 if strings.Contains(a, "--storage-driver") { 175 foundSd = true 176 } 177 } 178 if !foundLog { 179 args = append(args, "--debug") 180 } 181 if d.storageDriver != "" && !foundSd { 182 args = append(args, "--storage-driver", d.storageDriver) 183 } 184 185 args = append(args, providedArgs...) 186 d.cmd = exec.Command(dockerdBinary, args...) 187 d.cmd.Env = append(os.Environ(), "DOCKER_SERVICE_PREFER_OFFLINE_IMAGE=1") 188 d.cmd.Stdout = out 189 d.cmd.Stderr = out 190 d.logFile = out 191 192 if err := d.cmd.Start(); err != nil { 193 return fmt.Errorf("[%s] could not start daemon container: %v", d.id, err) 194 } 195 196 wait := make(chan error) 197 198 go func() { 199 wait <- d.cmd.Wait() 200 d.c.Logf("[%s] exiting daemon", d.id) 201 close(wait) 202 }() 203 204 d.wait = wait 205 206 tick := time.Tick(500 * time.Millisecond) 207 // make sure daemon is ready to receive requests 208 startTime := time.Now().Unix() 209 for { 210 d.c.Logf("[%s] waiting for daemon to start", d.id) 211 if time.Now().Unix()-startTime > 5 { 212 // After 5 seconds, give up 213 return fmt.Errorf("[%s] Daemon exited and never started", d.id) 214 } 215 select { 216 case <-time.After(2 * time.Second): 217 return fmt.Errorf("[%s] timeout: daemon does not respond", d.id) 218 case <-tick: 219 clientConfig, err := d.getClientConfig() 220 if err != nil { 221 return err 222 } 223 224 client := &http.Client{ 225 Transport: clientConfig.transport, 226 } 227 228 req, err := http.NewRequest("GET", "/_ping", nil) 229 d.c.Assert(err, check.IsNil, check.Commentf("[%s] could not create new request", d.id)) 230 req.URL.Host = clientConfig.addr 231 req.URL.Scheme = clientConfig.scheme 232 resp, err := client.Do(req) 233 if err != nil { 234 continue 235 } 236 if resp.StatusCode != http.StatusOK { 237 d.c.Logf("[%s] received status != 200 OK: %s", d.id, resp.Status) 238 } 239 d.c.Logf("[%s] daemon started", d.id) 240 d.root, err = d.queryRootDir() 241 if err != nil { 242 return fmt.Errorf("[%s] error querying daemon for root directory: %v", d.id, err) 243 } 244 return nil 245 case <-d.wait: 246 return fmt.Errorf("[%s] Daemon exited during startup", d.id) 247 } 248 } 249 } 250 251 // StartWithBusybox will first start the daemon with Daemon.Start() 252 // then save the busybox image from the main daemon and load it into this Daemon instance. 253 func (d *Daemon) StartWithBusybox(arg ...string) error { 254 if err := d.Start(arg...); err != nil { 255 return err 256 } 257 return d.LoadBusybox() 258 } 259 260 // Kill will send a SIGKILL to the daemon 261 func (d *Daemon) Kill() error { 262 if d.cmd == nil || d.wait == nil { 263 return errors.New("daemon not started") 264 } 265 266 defer func() { 267 d.logFile.Close() 268 d.cmd = nil 269 }() 270 271 if err := d.cmd.Process.Kill(); err != nil { 272 d.c.Logf("Could not kill daemon: %v", err) 273 return err 274 } 275 276 if err := os.Remove(fmt.Sprintf("%s/docker.pid", d.folder)); err != nil { 277 return err 278 } 279 280 return nil 281 } 282 283 // DumpStackAndQuit sends SIGQUIT to the daemon, which triggers it to dump its 284 // stack to its log file and exit 285 // This is used primarily for gathering debug information on test timeout 286 func (d *Daemon) DumpStackAndQuit() { 287 if d.cmd == nil || d.cmd.Process == nil { 288 return 289 } 290 signalDaemonDump(d.cmd.Process.Pid) 291 } 292 293 // Stop will send a SIGINT every second and wait for the daemon to stop. 294 // If it timeouts, a SIGKILL is sent. 295 // Stop will not delete the daemon directory. If a purged daemon is needed, 296 // instantiate a new one with NewDaemon. 297 func (d *Daemon) Stop() error { 298 if d.cmd == nil || d.wait == nil { 299 return errors.New("daemon not started") 300 } 301 302 defer func() { 303 d.logFile.Close() 304 d.cmd = nil 305 }() 306 307 i := 1 308 tick := time.Tick(time.Second) 309 310 if err := d.cmd.Process.Signal(os.Interrupt); err != nil { 311 return fmt.Errorf("could not send signal: %v", err) 312 } 313 out1: 314 for { 315 select { 316 case err := <-d.wait: 317 return err 318 case <-time.After(20 * time.Second): 319 // time for stopping jobs and run onShutdown hooks 320 d.c.Logf("timeout: %v", d.id) 321 break out1 322 } 323 } 324 325 out2: 326 for { 327 select { 328 case err := <-d.wait: 329 return err 330 case <-tick: 331 i++ 332 if i > 5 { 333 d.c.Logf("tried to interrupt daemon for %d times, now try to kill it", i) 334 break out2 335 } 336 d.c.Logf("Attempt #%d: daemon is still running with pid %d", i, d.cmd.Process.Pid) 337 if err := d.cmd.Process.Signal(os.Interrupt); err != nil { 338 return fmt.Errorf("could not send signal: %v", err) 339 } 340 } 341 } 342 343 if err := d.cmd.Process.Kill(); err != nil { 344 d.c.Logf("Could not kill daemon: %v", err) 345 return err 346 } 347 348 if err := os.Remove(fmt.Sprintf("%s/docker.pid", d.folder)); err != nil { 349 return err 350 } 351 352 return nil 353 } 354 355 // Restart will restart the daemon by first stopping it and then starting it. 356 func (d *Daemon) Restart(arg ...string) error { 357 d.Stop() 358 // in the case of tests running a user namespace-enabled daemon, we have resolved 359 // d.root to be the actual final path of the graph dir after the "uid.gid" of 360 // remapped root is added--we need to subtract it from the path before calling 361 // start or else we will continue making subdirectories rather than truly restarting 362 // with the same location/root: 363 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 364 d.root = filepath.Dir(d.root) 365 } 366 return d.Start(arg...) 367 } 368 369 // LoadBusybox will load the stored busybox into a newly started daemon 370 func (d *Daemon) LoadBusybox() error { 371 bb := filepath.Join(d.folder, "busybox.tar") 372 if _, err := os.Stat(bb); err != nil { 373 if !os.IsNotExist(err) { 374 return fmt.Errorf("unexpected error on busybox.tar stat: %v", err) 375 } 376 // saving busybox image from main daemon 377 if out, err := exec.Command(dockerBinary, "save", "--output", bb, "busybox:latest").CombinedOutput(); err != nil { 378 imagesOut, _ := exec.Command(dockerBinary, "images", "--format", "{{ .Repository }}:{{ .Tag }}").CombinedOutput() 379 return fmt.Errorf("could not save busybox image: %s\n%s", string(out), strings.TrimSpace(string(imagesOut))) 380 } 381 } 382 // loading busybox image to this daemon 383 if out, err := d.Cmd("load", "--input", bb); err != nil { 384 return fmt.Errorf("could not load busybox image: %s", out) 385 } 386 if err := os.Remove(bb); err != nil { 387 d.c.Logf("could not remove %s: %v", bb, err) 388 } 389 return nil 390 } 391 392 func (d *Daemon) queryRootDir() (string, error) { 393 // update daemon root by asking /info endpoint (to support user 394 // namespaced daemon with root remapped uid.gid directory) 395 clientConfig, err := d.getClientConfig() 396 if err != nil { 397 return "", err 398 } 399 400 client := &http.Client{ 401 Transport: clientConfig.transport, 402 } 403 404 req, err := http.NewRequest("GET", "/info", nil) 405 if err != nil { 406 return "", err 407 } 408 req.Header.Set("Content-Type", "application/json") 409 req.URL.Host = clientConfig.addr 410 req.URL.Scheme = clientConfig.scheme 411 412 resp, err := client.Do(req) 413 if err != nil { 414 return "", err 415 } 416 body := ioutils.NewReadCloserWrapper(resp.Body, func() error { 417 return resp.Body.Close() 418 }) 419 420 type Info struct { 421 DockerRootDir string 422 } 423 var b []byte 424 var i Info 425 b, err = readBody(body) 426 if err == nil && resp.StatusCode == http.StatusOK { 427 // read the docker root dir 428 if err = json.Unmarshal(b, &i); err == nil { 429 return i.DockerRootDir, nil 430 } 431 } 432 return "", err 433 } 434 435 func (d *Daemon) sock() string { 436 return fmt.Sprintf("unix://" + d.sockPath()) 437 } 438 439 func (d *Daemon) sockPath() string { 440 return filepath.Join(daemonSockRoot, d.id+".sock") 441 } 442 443 func (d *Daemon) waitRun(contID string) error { 444 args := []string{"--host", d.sock()} 445 return waitInspectWithArgs(contID, "{{.State.Running}}", "true", 10*time.Second, args...) 446 } 447 448 func (d *Daemon) getBaseDeviceSize(c *check.C) int64 { 449 infoCmdOutput, _, err := runCommandPipelineWithOutput( 450 exec.Command(dockerBinary, "-H", d.sock(), "info"), 451 exec.Command("grep", "Base Device Size"), 452 ) 453 c.Assert(err, checker.IsNil) 454 basesizeSlice := strings.Split(infoCmdOutput, ":") 455 basesize := strings.Trim(basesizeSlice[1], " ") 456 basesize = strings.Trim(basesize, "\n")[:len(basesize)-3] 457 basesizeFloat, err := strconv.ParseFloat(strings.Trim(basesize, " "), 64) 458 c.Assert(err, checker.IsNil) 459 basesizeBytes := int64(basesizeFloat) * (1024 * 1024 * 1024) 460 return basesizeBytes 461 } 462 463 // Cmd will execute a docker CLI command against this Daemon. 464 // Example: d.Cmd("version") will run docker -H unix://path/to/unix.sock version 465 func (d *Daemon) Cmd(args ...string) (string, error) { 466 c := exec.Command(dockerBinary, d.prependHostArg(args)...) 467 b, err := c.CombinedOutput() 468 return string(b), err 469 } 470 471 func (d *Daemon) prependHostArg(args []string) []string { 472 for _, arg := range args { 473 if arg == "--host" || arg == "-H" { 474 return args 475 } 476 } 477 return append([]string{"--host", d.sock()}, args...) 478 } 479 480 // SockRequest executes a socket request on a daemon and returns statuscode and output. 481 func (d *Daemon) SockRequest(method, endpoint string, data interface{}) (int, []byte, error) { 482 jsonData := bytes.NewBuffer(nil) 483 if err := json.NewEncoder(jsonData).Encode(data); err != nil { 484 return -1, nil, err 485 } 486 487 res, body, err := d.SockRequestRaw(method, endpoint, jsonData, "application/json") 488 if err != nil { 489 return -1, nil, err 490 } 491 b, err := readBody(body) 492 return res.StatusCode, b, err 493 } 494 495 // SockRequestRaw executes a socket request on a daemon and returns a http 496 // response and a reader for the output data. 497 func (d *Daemon) SockRequestRaw(method, endpoint string, data io.Reader, ct string) (*http.Response, io.ReadCloser, error) { 498 return sockRequestRawToDaemon(method, endpoint, data, ct, d.sock()) 499 } 500 501 // LogFileName returns the path the the daemon's log file 502 func (d *Daemon) LogFileName() string { 503 return d.logFile.Name() 504 } 505 506 func (d *Daemon) getIDByName(name string) (string, error) { 507 return d.inspectFieldWithError(name, "Id") 508 } 509 510 func (d *Daemon) activeContainers() (ids []string) { 511 out, _ := d.Cmd("ps", "-q") 512 for _, id := range strings.Split(out, "\n") { 513 if id = strings.TrimSpace(id); id != "" { 514 ids = append(ids, id) 515 } 516 } 517 return 518 } 519 520 func (d *Daemon) inspectFilter(name, filter string) (string, error) { 521 format := fmt.Sprintf("{{%s}}", filter) 522 out, err := d.Cmd("inspect", "-f", format, name) 523 if err != nil { 524 return "", fmt.Errorf("failed to inspect %s: %s", name, out) 525 } 526 return strings.TrimSpace(out), nil 527 } 528 529 func (d *Daemon) inspectFieldWithError(name, field string) (string, error) { 530 return d.inspectFilter(name, fmt.Sprintf(".%s", field)) 531 } 532 533 func (d *Daemon) findContainerIP(id string) string { 534 out, err := d.Cmd("inspect", fmt.Sprintf("--format='{{ .NetworkSettings.Networks.bridge.IPAddress }}'"), id) 535 if err != nil { 536 d.c.Log(err) 537 } 538 return strings.Trim(out, " \r\n'") 539 } 540 541 func (d *Daemon) buildImageWithOut(name, dockerfile string, useCache bool, buildFlags ...string) (string, int, error) { 542 buildCmd := buildImageCmdWithHost(name, dockerfile, d.sock(), useCache, buildFlags...) 543 return runCommandWithOutput(buildCmd) 544 } 545 546 func (d *Daemon) checkActiveContainerCount(c *check.C) (interface{}, check.CommentInterface) { 547 out, err := d.Cmd("ps", "-q") 548 c.Assert(err, checker.IsNil) 549 if len(strings.TrimSpace(out)) == 0 { 550 return 0, nil 551 } 552 return len(strings.Split(strings.TrimSpace(out), "\n")), check.Commentf("output: %q", string(out)) 553 } 554 555 func (d *Daemon) reloadConfig() error { 556 if d.cmd == nil || d.cmd.Process == nil { 557 return fmt.Errorf("daemon is not running") 558 } 559 560 errCh := make(chan error) 561 started := make(chan struct{}) 562 go func() { 563 _, body, err := sockRequestRawToDaemon("GET", "/events", nil, "", d.sock()) 564 close(started) 565 if err != nil { 566 errCh <- err 567 } 568 defer body.Close() 569 dec := json.NewDecoder(body) 570 for { 571 var e events.Message 572 if err := dec.Decode(&e); err != nil { 573 errCh <- err 574 return 575 } 576 if e.Type != events.DaemonEventType { 577 continue 578 } 579 if e.Action != "reload" { 580 continue 581 } 582 close(errCh) // notify that we are done 583 return 584 } 585 }() 586 587 <-started 588 if err := signalDaemonReload(d.cmd.Process.Pid); err != nil { 589 return fmt.Errorf("error signaling daemon reload: %v", err) 590 } 591 select { 592 case err := <-errCh: 593 if err != nil { 594 return fmt.Errorf("error waiting for daemon reload event: %v", err) 595 } 596 case <-time.After(30 * time.Second): 597 return fmt.Errorf("timeout waiting for daemon reload event") 598 } 599 return nil 600 }