github.com/sijibomii/docker@v0.0.0-20231230191044-5cf6ca554647/integration-cli/daemon.go (about) 1 package main 2 3 import ( 4 "encoding/json" 5 "errors" 6 "fmt" 7 "io" 8 "net/http" 9 "os" 10 "os/exec" 11 "path/filepath" 12 "strconv" 13 "strings" 14 "time" 15 16 "github.com/docker/docker/opts" 17 "github.com/docker/docker/pkg/integration/checker" 18 "github.com/docker/docker/pkg/ioutils" 19 "github.com/docker/docker/pkg/tlsconfig" 20 "github.com/docker/go-connections/sockets" 21 "github.com/go-check/check" 22 ) 23 24 // Daemon represents a Docker daemon for the testing framework. 25 type Daemon struct { 26 // Defaults to "daemon" 27 // Useful to set to --daemon or -d for checking backwards compatibility 28 Command string 29 GlobalFlags []string 30 31 id string 32 c *check.C 33 logFile *os.File 34 folder string 35 root string 36 stdin io.WriteCloser 37 stdout, stderr io.ReadCloser 38 cmd *exec.Cmd 39 storageDriver string 40 wait chan error 41 userlandProxy bool 42 useDefaultHost bool 43 useDefaultTLSHost bool 44 } 45 46 type clientConfig struct { 47 transport *http.Transport 48 scheme string 49 addr string 50 } 51 52 // NewDaemon returns a Daemon instance to be used for testing. 53 // This will create a directory such as d123456789 in the folder specified by $DEST. 54 // The daemon will not automatically start. 55 func NewDaemon(c *check.C) *Daemon { 56 dest := os.Getenv("DEST") 57 c.Assert(dest, check.Not(check.Equals), "", check.Commentf("Please set the DEST environment variable")) 58 59 id := fmt.Sprintf("d%d", time.Now().UnixNano()%100000000) 60 dir := filepath.Join(dest, id) 61 daemonFolder, err := filepath.Abs(dir) 62 c.Assert(err, check.IsNil, check.Commentf("Could not make %q an absolute path", dir)) 63 daemonRoot := filepath.Join(daemonFolder, "root") 64 65 c.Assert(os.MkdirAll(daemonRoot, 0755), check.IsNil, check.Commentf("Could not create daemon root %q", dir)) 66 67 userlandProxy := true 68 if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { 69 if val, err := strconv.ParseBool(env); err != nil { 70 userlandProxy = val 71 } 72 } 73 74 return &Daemon{ 75 Command: "daemon", 76 id: id, 77 c: c, 78 folder: daemonFolder, 79 root: daemonRoot, 80 storageDriver: os.Getenv("DOCKER_GRAPHDRIVER"), 81 userlandProxy: userlandProxy, 82 } 83 } 84 85 func (d *Daemon) getClientConfig() (*clientConfig, error) { 86 var ( 87 transport *http.Transport 88 scheme string 89 addr string 90 proto string 91 ) 92 if d.useDefaultTLSHost { 93 option := &tlsconfig.Options{ 94 CAFile: "fixtures/https/ca.pem", 95 CertFile: "fixtures/https/client-cert.pem", 96 KeyFile: "fixtures/https/client-key.pem", 97 } 98 tlsConfig, err := tlsconfig.Client(*option) 99 if err != nil { 100 return nil, err 101 } 102 transport = &http.Transport{ 103 TLSClientConfig: tlsConfig, 104 } 105 addr = fmt.Sprintf("%s:%d", opts.DefaultHTTPHost, opts.DefaultTLSHTTPPort) 106 scheme = "https" 107 proto = "tcp" 108 } else if d.useDefaultHost { 109 addr = opts.DefaultUnixSocket 110 proto = "unix" 111 scheme = "http" 112 transport = &http.Transport{} 113 } else { 114 addr = filepath.Join(d.folder, "docker.sock") 115 proto = "unix" 116 scheme = "http" 117 transport = &http.Transport{} 118 } 119 120 d.c.Assert(sockets.ConfigureTransport(transport, proto, addr), check.IsNil) 121 122 return &clientConfig{ 123 transport: transport, 124 scheme: scheme, 125 addr: addr, 126 }, nil 127 } 128 129 // Start will start the daemon and return once it is ready to receive requests. 130 // You can specify additional daemon flags. 131 func (d *Daemon) Start(args ...string) error { 132 logFile, err := os.OpenFile(filepath.Join(d.folder, "docker.log"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600) 133 d.c.Assert(err, check.IsNil, check.Commentf("[%s] Could not create %s/docker.log", d.id, d.folder)) 134 135 return d.StartWithLogFile(logFile, args...) 136 } 137 138 // StartWithLogFile will start the daemon and attach its streams to a given file. 139 func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error { 140 dockerBinary, err := exec.LookPath(dockerBinary) 141 d.c.Assert(err, check.IsNil, check.Commentf("[%s] could not find docker binary in $PATH", d.id)) 142 143 args := append(d.GlobalFlags, 144 d.Command, 145 "--containerd", "/var/run/docker/libcontainerd/docker-containerd.sock", 146 "--graph", d.root, 147 "--exec-root", filepath.Join(d.folder, "exec-root"), 148 "--pidfile", fmt.Sprintf("%s/docker.pid", d.folder), 149 fmt.Sprintf("--userland-proxy=%t", d.userlandProxy), 150 ) 151 if !(d.useDefaultHost || d.useDefaultTLSHost) { 152 args = append(args, []string{"--host", d.sock()}...) 153 } 154 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 155 args = append(args, []string{"--userns-remap", root}...) 156 } 157 158 // If we don't explicitly set the log-level or debug flag(-D) then 159 // turn on debug mode 160 foundLog := false 161 foundSd := false 162 for _, a := range providedArgs { 163 if strings.Contains(a, "--log-level") || strings.Contains(a, "-D") || strings.Contains(a, "--debug") { 164 foundLog = true 165 } 166 if strings.Contains(a, "--storage-driver") { 167 foundSd = true 168 } 169 } 170 if !foundLog { 171 args = append(args, "--debug") 172 } 173 if d.storageDriver != "" && !foundSd { 174 args = append(args, "--storage-driver", d.storageDriver) 175 } 176 177 args = append(args, providedArgs...) 178 d.cmd = exec.Command(dockerBinary, args...) 179 180 d.cmd.Stdout = out 181 d.cmd.Stderr = out 182 d.logFile = out 183 184 if err := d.cmd.Start(); err != nil { 185 return fmt.Errorf("[%s] could not start daemon container: %v", d.id, err) 186 } 187 188 wait := make(chan error) 189 190 go func() { 191 wait <- d.cmd.Wait() 192 d.c.Logf("[%s] exiting daemon", d.id) 193 close(wait) 194 }() 195 196 d.wait = wait 197 198 tick := time.Tick(500 * time.Millisecond) 199 // make sure daemon is ready to receive requests 200 startTime := time.Now().Unix() 201 for { 202 d.c.Logf("[%s] waiting for daemon to start", d.id) 203 if time.Now().Unix()-startTime > 5 { 204 // After 5 seconds, give up 205 return fmt.Errorf("[%s] Daemon exited and never started", d.id) 206 } 207 select { 208 case <-time.After(2 * time.Second): 209 return fmt.Errorf("[%s] timeout: daemon does not respond", d.id) 210 case <-tick: 211 clientConfig, err := d.getClientConfig() 212 if err != nil { 213 return err 214 } 215 216 client := &http.Client{ 217 Transport: clientConfig.transport, 218 } 219 220 req, err := http.NewRequest("GET", "/_ping", nil) 221 d.c.Assert(err, check.IsNil, check.Commentf("[%s] could not create new request", d.id)) 222 req.URL.Host = clientConfig.addr 223 req.URL.Scheme = clientConfig.scheme 224 resp, err := client.Do(req) 225 if err != nil { 226 continue 227 } 228 if resp.StatusCode != http.StatusOK { 229 d.c.Logf("[%s] received status != 200 OK: %s", d.id, resp.Status) 230 } 231 d.c.Logf("[%s] daemon started", d.id) 232 d.root, err = d.queryRootDir() 233 if err != nil { 234 return fmt.Errorf("[%s] error querying daemon for root directory: %v", d.id, err) 235 } 236 return nil 237 case <-d.wait: 238 return fmt.Errorf("[%s] Daemon exited during startup", d.id) 239 } 240 } 241 } 242 243 // StartWithBusybox will first start the daemon with Daemon.Start() 244 // then save the busybox image from the main daemon and load it into this Daemon instance. 245 func (d *Daemon) StartWithBusybox(arg ...string) error { 246 if err := d.Start(arg...); err != nil { 247 return err 248 } 249 return d.LoadBusybox() 250 } 251 252 // Kill will send a SIGKILL to the daemon 253 func (d *Daemon) Kill() error { 254 if d.cmd == nil || d.wait == nil { 255 return errors.New("daemon not started") 256 } 257 258 defer func() { 259 d.logFile.Close() 260 d.cmd = nil 261 }() 262 263 if err := d.cmd.Process.Kill(); err != nil { 264 d.c.Logf("Could not kill daemon: %v", err) 265 return err 266 } 267 268 if err := os.Remove(fmt.Sprintf("%s/docker.pid", d.folder)); err != nil { 269 return err 270 } 271 272 return nil 273 } 274 275 // Stop will send a SIGINT every second and wait for the daemon to stop. 276 // If it timeouts, a SIGKILL is sent. 277 // Stop will not delete the daemon directory. If a purged daemon is needed, 278 // instantiate a new one with NewDaemon. 279 func (d *Daemon) Stop() error { 280 if d.cmd == nil || d.wait == nil { 281 return errors.New("daemon not started") 282 } 283 284 defer func() { 285 d.logFile.Close() 286 d.cmd = nil 287 }() 288 289 i := 1 290 tick := time.Tick(time.Second) 291 292 if err := d.cmd.Process.Signal(os.Interrupt); err != nil { 293 return fmt.Errorf("could not send signal: %v", err) 294 } 295 out1: 296 for { 297 select { 298 case err := <-d.wait: 299 return err 300 case <-time.After(15 * time.Second): 301 // time for stopping jobs and run onShutdown hooks 302 d.c.Log("timeout") 303 break out1 304 } 305 } 306 307 out2: 308 for { 309 select { 310 case err := <-d.wait: 311 return err 312 case <-tick: 313 i++ 314 if i > 4 { 315 d.c.Logf("tried to interrupt daemon for %d times, now try to kill it", i) 316 break out2 317 } 318 d.c.Logf("Attempt #%d: daemon is still running with pid %d", i, d.cmd.Process.Pid) 319 if err := d.cmd.Process.Signal(os.Interrupt); err != nil { 320 return fmt.Errorf("could not send signal: %v", err) 321 } 322 } 323 } 324 325 if err := d.cmd.Process.Kill(); err != nil { 326 d.c.Logf("Could not kill daemon: %v", err) 327 return err 328 } 329 330 if err := os.Remove(fmt.Sprintf("%s/docker.pid", d.folder)); err != nil { 331 return err 332 } 333 334 return nil 335 } 336 337 // Restart will restart the daemon by first stopping it and then starting it. 338 func (d *Daemon) Restart(arg ...string) error { 339 d.Stop() 340 // in the case of tests running a user namespace-enabled daemon, we have resolved 341 // d.root to be the actual final path of the graph dir after the "uid.gid" of 342 // remapped root is added--we need to subtract it from the path before calling 343 // start or else we will continue making subdirectories rather than truly restarting 344 // with the same location/root: 345 if root := os.Getenv("DOCKER_REMAP_ROOT"); root != "" { 346 d.root = filepath.Dir(d.root) 347 } 348 return d.Start(arg...) 349 } 350 351 // LoadBusybox will load the stored busybox into a newly started daemon 352 func (d *Daemon) LoadBusybox() error { 353 bb := filepath.Join(d.folder, "busybox.tar") 354 if _, err := os.Stat(bb); err != nil { 355 if !os.IsNotExist(err) { 356 return fmt.Errorf("unexpected error on busybox.tar stat: %v", err) 357 } 358 // saving busybox image from main daemon 359 if err := exec.Command(dockerBinary, "save", "--output", bb, "busybox:latest").Run(); err != nil { 360 return fmt.Errorf("could not save busybox image: %v", err) 361 } 362 } 363 // loading busybox image to this daemon 364 if out, err := d.Cmd("load", "--input", bb); err != nil { 365 return fmt.Errorf("could not load busybox image: %s", out) 366 } 367 if err := os.Remove(bb); err != nil { 368 d.c.Logf("could not remove %s: %v", bb, err) 369 } 370 return nil 371 } 372 373 func (d *Daemon) queryRootDir() (string, error) { 374 // update daemon root by asking /info endpoint (to support user 375 // namespaced daemon with root remapped uid.gid directory) 376 clientConfig, err := d.getClientConfig() 377 if err != nil { 378 return "", err 379 } 380 381 client := &http.Client{ 382 Transport: clientConfig.transport, 383 } 384 385 req, err := http.NewRequest("GET", "/info", nil) 386 if err != nil { 387 return "", err 388 } 389 req.Header.Set("Content-Type", "application/json") 390 req.URL.Host = clientConfig.addr 391 req.URL.Scheme = clientConfig.scheme 392 393 resp, err := client.Do(req) 394 if err != nil { 395 return "", err 396 } 397 body := ioutils.NewReadCloserWrapper(resp.Body, func() error { 398 return resp.Body.Close() 399 }) 400 401 type Info struct { 402 DockerRootDir string 403 } 404 var b []byte 405 var i Info 406 b, err = readBody(body) 407 if err == nil && resp.StatusCode == 200 { 408 // read the docker root dir 409 if err = json.Unmarshal(b, &i); err == nil { 410 return i.DockerRootDir, nil 411 } 412 } 413 return "", err 414 } 415 416 func (d *Daemon) sock() string { 417 return fmt.Sprintf("unix://%s/docker.sock", d.folder) 418 } 419 420 func (d *Daemon) waitRun(contID string) error { 421 args := []string{"--host", d.sock()} 422 return waitInspectWithArgs(contID, "{{.State.Running}}", "true", 10*time.Second, args...) 423 } 424 425 func (d *Daemon) getBaseDeviceSize(c *check.C) int64 { 426 infoCmdOutput, _, err := runCommandPipelineWithOutput( 427 exec.Command(dockerBinary, "-H", d.sock(), "info"), 428 exec.Command("grep", "Base Device Size"), 429 ) 430 c.Assert(err, checker.IsNil) 431 basesizeSlice := strings.Split(infoCmdOutput, ":") 432 basesize := strings.Trim(basesizeSlice[1], " ") 433 basesize = strings.Trim(basesize, "\n")[:len(basesize)-3] 434 basesizeFloat, err := strconv.ParseFloat(strings.Trim(basesize, " "), 64) 435 c.Assert(err, checker.IsNil) 436 basesizeBytes := int64(basesizeFloat) * (1024 * 1024 * 1024) 437 return basesizeBytes 438 } 439 440 // Cmd will execute a docker CLI command against this Daemon. 441 // Example: d.Cmd("version") will run docker -H unix://path/to/unix.sock version 442 func (d *Daemon) Cmd(name string, arg ...string) (string, error) { 443 args := []string{"--host", d.sock(), name} 444 args = append(args, arg...) 445 c := exec.Command(dockerBinary, args...) 446 b, err := c.CombinedOutput() 447 return string(b), err 448 } 449 450 // CmdWithArgs will execute a docker CLI command against a daemon with the 451 // given additional arguments 452 func (d *Daemon) CmdWithArgs(daemonArgs []string, name string, arg ...string) (string, error) { 453 args := append(daemonArgs, name) 454 args = append(args, arg...) 455 c := exec.Command(dockerBinary, args...) 456 b, err := c.CombinedOutput() 457 return string(b), err 458 } 459 460 // LogFileName returns the path the the daemon's log file 461 func (d *Daemon) LogFileName() string { 462 return d.logFile.Name() 463 } 464 465 func (d *Daemon) getIDByName(name string) (string, error) { 466 return d.inspectFieldWithError(name, "Id") 467 } 468 469 func (d *Daemon) inspectFilter(name, filter string) (string, error) { 470 format := fmt.Sprintf("{{%s}}", filter) 471 out, err := d.Cmd("inspect", "-f", format, name) 472 if err != nil { 473 return "", fmt.Errorf("failed to inspect %s: %s", name, out) 474 } 475 return strings.TrimSpace(out), nil 476 } 477 478 func (d *Daemon) inspectFieldWithError(name, field string) (string, error) { 479 return d.inspectFilter(name, fmt.Sprintf(".%s", field)) 480 } 481 482 func (d *Daemon) findContainerIP(id string) string { 483 out, err := d.Cmd("inspect", fmt.Sprintf("--format='{{ .NetworkSettings.Networks.bridge.IPAddress }}'"), id) 484 if err != nil { 485 d.c.Log(err) 486 } 487 return strings.Trim(out, " \r\n'") 488 }