github.com/docker/containerd@v0.2.9-0.20170509230648-8ef7df579710/runtime/container.go (about) 1 package runtime 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "os" 9 "os/exec" 10 "path/filepath" 11 "strings" 12 "syscall" 13 "time" 14 15 "github.com/Sirupsen/logrus" 16 "github.com/containerd/containerd/specs" 17 ocs "github.com/opencontainers/runtime-spec/specs-go" 18 "golang.org/x/net/context" 19 "golang.org/x/sys/unix" 20 ) 21 22 // Container defines the operations allowed on a container 23 type Container interface { 24 // ID returns the container ID 25 ID() string 26 // Path returns the path to the bundle 27 Path() string 28 // Start starts the init process of the container 29 Start(ctx context.Context, checkpointPath string, s Stdio) (Process, error) 30 // Exec starts another process in an existing container 31 Exec(context.Context, string, specs.ProcessSpec, Stdio) (Process, error) 32 // Delete removes the container's state and any resources 33 Delete() error 34 // Processes returns all the containers processes that have been added 35 Processes() ([]Process, error) 36 // State returns the containers runtime state 37 State() State 38 // Resume resumes a paused container 39 Resume() error 40 // Pause pauses a running container 41 Pause() error 42 // RemoveProcess removes the specified process from the container 43 RemoveProcess(string) error 44 // Checkpoints returns all the checkpoints for a container 45 Checkpoints(checkpointDir string) ([]Checkpoint, error) 46 // Checkpoint creates a new checkpoint 47 Checkpoint(checkpoint Checkpoint, checkpointDir string) error 48 // DeleteCheckpoint deletes the checkpoint for the provided name 49 DeleteCheckpoint(name string, checkpointDir string) error 50 // Labels are user provided labels for the container 51 Labels() []string 52 // Pids returns all pids inside the container 53 Pids() ([]int, error) 54 // Stats returns realtime container stats and resource information 55 Stats() (*Stat, error) 56 // Name or path of the OCI compliant runtime used to execute the container 57 Runtime() string 58 // OOM signals the channel if the container received an OOM notification 59 OOM() (OOM, error) 60 // UpdateResource updates the containers resources to new values 61 UpdateResources(*Resource) error 62 63 // Status return the current status of the container. 64 Status() (State, error) 65 } 66 67 // OOM wraps a container OOM. 68 type OOM interface { 69 io.Closer 70 FD() int 71 ContainerID() string 72 Flush() 73 Removed() bool 74 } 75 76 // Stdio holds the path to the 3 pipes used for the standard ios. 77 type Stdio struct { 78 Stdin string 79 Stdout string 80 Stderr string 81 } 82 83 // NewStdio wraps the given standard io path into an Stdio struct. 84 // If a given parameter is the empty string, it is replaced by "/dev/null" 85 func NewStdio(stdin, stdout, stderr string) Stdio { 86 for _, s := range []*string{ 87 &stdin, &stdout, &stderr, 88 } { 89 if *s == "" { 90 *s = "/dev/null" 91 } 92 } 93 return Stdio{ 94 Stdin: stdin, 95 Stdout: stdout, 96 Stderr: stderr, 97 } 98 } 99 100 // ContainerOpts keeps the options passed at container creation 101 type ContainerOpts struct { 102 Root string 103 ID string 104 Bundle string 105 Runtime string 106 RuntimeArgs []string 107 Shim string 108 Labels []string 109 NoPivotRoot bool 110 Timeout time.Duration 111 } 112 113 // New returns a new container 114 func New(opts ContainerOpts) (Container, error) { 115 c := &container{ 116 root: opts.Root, 117 id: opts.ID, 118 bundle: opts.Bundle, 119 labels: opts.Labels, 120 processes: make(map[string]*process), 121 runtime: opts.Runtime, 122 runtimeArgs: opts.RuntimeArgs, 123 shim: opts.Shim, 124 noPivotRoot: opts.NoPivotRoot, 125 timeout: opts.Timeout, 126 } 127 if err := os.Mkdir(filepath.Join(c.root, c.id), 0755); err != nil { 128 return nil, err 129 } 130 f, err := os.Create(filepath.Join(c.root, c.id, StateFile)) 131 if err != nil { 132 return nil, err 133 } 134 defer f.Close() 135 if err := json.NewEncoder(f).Encode(state{ 136 Bundle: c.bundle, 137 Labels: c.labels, 138 Runtime: c.runtime, 139 RuntimeArgs: c.runtimeArgs, 140 Shim: c.shim, 141 NoPivotRoot: opts.NoPivotRoot, 142 }); err != nil { 143 return nil, err 144 } 145 return c, nil 146 } 147 148 // Load return a new container from the matchin state file on disk. 149 func Load(root, id, shimName string, timeout time.Duration) (Container, error) { 150 var s state 151 f, err := os.Open(filepath.Join(root, id, StateFile)) 152 if err != nil { 153 return nil, err 154 } 155 defer f.Close() 156 if err := json.NewDecoder(f).Decode(&s); err != nil { 157 return nil, err 158 } 159 c := &container{ 160 root: root, 161 id: id, 162 bundle: s.Bundle, 163 labels: s.Labels, 164 runtime: s.Runtime, 165 runtimeArgs: s.RuntimeArgs, 166 shim: s.Shim, 167 noPivotRoot: s.NoPivotRoot, 168 processes: make(map[string]*process), 169 timeout: timeout, 170 } 171 172 if c.shim == "" { 173 c.shim = shimName 174 } 175 176 dirs, err := ioutil.ReadDir(filepath.Join(root, id)) 177 if err != nil { 178 return nil, err 179 } 180 for _, d := range dirs { 181 if !d.IsDir() { 182 continue 183 } 184 pid := d.Name() 185 s, err := readProcessState(filepath.Join(root, id, pid)) 186 if err != nil { 187 return nil, err 188 } 189 p, err := loadProcess(filepath.Join(root, id, pid), pid, c, s) 190 if err != nil { 191 logrus.WithField("id", id).WithField("pid", pid).Debugf("containerd: error loading process %s", err) 192 continue 193 } 194 c.processes[pid] = p 195 } 196 197 _, err = os.Stat(c.bundle) 198 if err != nil && !os.IsExist(err) { 199 for key, p := range c.processes { 200 if key == InitProcessID { 201 p.Delete() 202 break 203 } 204 } 205 return nil, fmt.Errorf("bundle dir %s don't exist", c.bundle) 206 } 207 return c, nil 208 } 209 210 func readProcessState(dir string) (*ProcessState, error) { 211 f, err := os.Open(filepath.Join(dir, "process.json")) 212 if err != nil { 213 return nil, err 214 } 215 defer f.Close() 216 var s ProcessState 217 if err := json.NewDecoder(f).Decode(&s); err != nil { 218 return nil, err 219 } 220 return &s, nil 221 } 222 223 type container struct { 224 // path to store runtime state information 225 root string 226 id string 227 bundle string 228 runtime string 229 runtimeArgs []string 230 shim string 231 processes map[string]*process 232 labels []string 233 oomFds []int 234 noPivotRoot bool 235 timeout time.Duration 236 } 237 238 func (c *container) ID() string { 239 return c.id 240 } 241 242 func (c *container) Path() string { 243 return c.bundle 244 } 245 246 func (c *container) Labels() []string { 247 return c.labels 248 } 249 250 func (c *container) readSpec() (*specs.Spec, error) { 251 var spec specs.Spec 252 f, err := os.Open(filepath.Join(c.bundle, "config.json")) 253 if err != nil { 254 return nil, err 255 } 256 defer f.Close() 257 if err := json.NewDecoder(f).Decode(&spec); err != nil { 258 return nil, err 259 } 260 return &spec, nil 261 } 262 263 func (c *container) Delete() error { 264 var err error 265 args := append(c.runtimeArgs, "delete", c.id) 266 if b, derr := exec.Command(c.runtime, args...).CombinedOutput(); derr != nil && !strings.Contains(string(b), "does not exist") { 267 err = fmt.Errorf("%s: %q", derr, string(b)) 268 } 269 if rerr := os.RemoveAll(filepath.Join(c.root, c.id)); rerr != nil { 270 if err != nil { 271 err = fmt.Errorf("%s; failed to remove %s: %s", err, filepath.Join(c.root, c.id), rerr) 272 } else { 273 err = rerr 274 } 275 } 276 return err 277 } 278 279 func (c *container) Processes() ([]Process, error) { 280 out := []Process{} 281 for _, p := range c.processes { 282 out = append(out, p) 283 } 284 return out, nil 285 } 286 287 func (c *container) RemoveProcess(pid string) error { 288 delete(c.processes, pid) 289 return os.RemoveAll(filepath.Join(c.root, c.id, pid)) 290 } 291 292 func (c *container) State() State { 293 proc := c.processes[InitProcessID] 294 if proc == nil { 295 return Stopped 296 } 297 return proc.State() 298 } 299 300 func (c *container) Runtime() string { 301 return c.runtime 302 } 303 304 func (c *container) Pause() error { 305 args := c.runtimeArgs 306 args = append(args, "pause", c.id) 307 b, err := exec.Command(c.runtime, args...).CombinedOutput() 308 if err != nil { 309 return fmt.Errorf("%s: %q", err.Error(), string(b)) 310 } 311 return nil 312 } 313 314 func (c *container) Resume() error { 315 args := c.runtimeArgs 316 args = append(args, "resume", c.id) 317 b, err := exec.Command(c.runtime, args...).CombinedOutput() 318 if err != nil { 319 return fmt.Errorf("%s: %q", err.Error(), string(b)) 320 } 321 return nil 322 } 323 324 func (c *container) Checkpoints(checkpointDir string) ([]Checkpoint, error) { 325 if checkpointDir == "" { 326 checkpointDir = filepath.Join(c.bundle, "checkpoints") 327 } 328 329 dirs, err := ioutil.ReadDir(checkpointDir) 330 if err != nil { 331 return nil, err 332 } 333 var out []Checkpoint 334 for _, d := range dirs { 335 if !d.IsDir() { 336 continue 337 } 338 path := filepath.Join(checkpointDir, d.Name(), "config.json") 339 data, err := ioutil.ReadFile(path) 340 if err != nil { 341 return nil, err 342 } 343 var cpt Checkpoint 344 if err := json.Unmarshal(data, &cpt); err != nil { 345 return nil, err 346 } 347 out = append(out, cpt) 348 } 349 return out, nil 350 } 351 352 func (c *container) Checkpoint(cpt Checkpoint, checkpointDir string) error { 353 if checkpointDir == "" { 354 checkpointDir = filepath.Join(c.bundle, "checkpoints") 355 } 356 357 if err := os.MkdirAll(checkpointDir, 0755); err != nil { 358 return err 359 } 360 361 path := filepath.Join(checkpointDir, cpt.Name) 362 if err := os.Mkdir(path, 0755); err != nil { 363 return err 364 } 365 f, err := os.Create(filepath.Join(path, "config.json")) 366 if err != nil { 367 return err 368 } 369 cpt.Created = time.Now() 370 err = json.NewEncoder(f).Encode(cpt) 371 f.Close() 372 if err != nil { 373 return err 374 } 375 args := []string{ 376 "checkpoint", 377 "--image-path", path, 378 "--work-path", filepath.Join(path, "criu.work"), 379 } 380 add := func(flags ...string) { 381 args = append(args, flags...) 382 } 383 add(c.runtimeArgs...) 384 if !cpt.Exit { 385 add("--leave-running") 386 } 387 if cpt.Shell { 388 add("--shell-job") 389 } 390 if cpt.TCP { 391 add("--tcp-established") 392 } 393 if cpt.UnixSockets { 394 add("--ext-unix-sk") 395 } 396 for _, ns := range cpt.EmptyNS { 397 add("--empty-ns", ns) 398 } 399 add(c.id) 400 out, err := exec.Command(c.runtime, args...).CombinedOutput() 401 if err != nil { 402 return fmt.Errorf("%s: %q", err.Error(), string(out)) 403 } 404 return err 405 } 406 407 func (c *container) DeleteCheckpoint(name string, checkpointDir string) error { 408 if checkpointDir == "" { 409 checkpointDir = filepath.Join(c.bundle, "checkpoints") 410 } 411 return os.RemoveAll(filepath.Join(checkpointDir, name)) 412 } 413 414 func (c *container) Start(ctx context.Context, checkpointPath string, s Stdio) (Process, error) { 415 processRoot := filepath.Join(c.root, c.id, InitProcessID) 416 if err := os.Mkdir(processRoot, 0755); err != nil { 417 return nil, err 418 } 419 cmd := exec.Command(c.shim, 420 c.id, c.bundle, c.runtime, 421 ) 422 cmd.Dir = processRoot 423 cmd.SysProcAttr = &syscall.SysProcAttr{ 424 Setpgid: true, 425 } 426 spec, err := c.readSpec() 427 if err != nil { 428 return nil, err 429 } 430 config := &processConfig{ 431 checkpoint: checkpointPath, 432 root: processRoot, 433 id: InitProcessID, 434 c: c, 435 stdio: s, 436 spec: spec, 437 processSpec: specs.ProcessSpec(spec.Process), 438 } 439 p, err := newProcess(config) 440 if err != nil { 441 return nil, err 442 } 443 if err := c.createCmd(ctx, InitProcessID, cmd, p); err != nil { 444 return nil, err 445 } 446 return p, nil 447 } 448 449 func (c *container) Exec(ctx context.Context, pid string, pspec specs.ProcessSpec, s Stdio) (pp Process, err error) { 450 processRoot := filepath.Join(c.root, c.id, pid) 451 if err := os.Mkdir(processRoot, 0755); err != nil { 452 return nil, err 453 } 454 defer func() { 455 if err != nil { 456 c.RemoveProcess(pid) 457 } 458 }() 459 cmd := exec.Command(c.shim, 460 c.id, c.bundle, c.runtime, 461 ) 462 cmd.Dir = processRoot 463 cmd.SysProcAttr = &syscall.SysProcAttr{ 464 Setpgid: true, 465 } 466 spec, err := c.readSpec() 467 if err != nil { 468 return nil, err 469 } 470 config := &processConfig{ 471 exec: true, 472 id: pid, 473 root: processRoot, 474 c: c, 475 processSpec: pspec, 476 spec: spec, 477 stdio: s, 478 } 479 p, err := newProcess(config) 480 if err != nil { 481 return nil, err 482 } 483 if err := c.createCmd(ctx, pid, cmd, p); err != nil { 484 return nil, err 485 } 486 return p, nil 487 } 488 489 func (c *container) createCmd(ctx context.Context, pid string, cmd *exec.Cmd, p *process) error { 490 p.cmd = cmd 491 if err := cmd.Start(); err != nil { 492 close(p.cmdDoneCh) 493 if exErr, ok := err.(*exec.Error); ok { 494 if exErr.Err == exec.ErrNotFound || exErr.Err == os.ErrNotExist { 495 return fmt.Errorf("%s not installed on system", c.shim) 496 } 497 } 498 return err 499 } 500 // We need the pid file to have been written to run 501 defer func() { 502 go func() { 503 err := p.cmd.Wait() 504 if err == nil { 505 p.cmdSuccess = true 506 } 507 508 if same, err := p.isSameProcess(); same && p.pid > 0 { 509 // The process changed its PR_SET_PDEATHSIG, so force 510 // kill it 511 logrus.Infof("containerd: %s:%s (pid %v) has become an orphan, killing it", p.container.id, p.id, p.pid) 512 err = unix.Kill(p.pid, syscall.SIGKILL) 513 if err != nil && err != syscall.ESRCH { 514 logrus.Errorf("containerd: unable to SIGKILL %s:%s (pid %v): %v", p.container.id, p.id, p.pid, err) 515 } else { 516 for { 517 err = unix.Kill(p.pid, 0) 518 if err != nil { 519 break 520 } 521 time.Sleep(5 * time.Millisecond) 522 } 523 } 524 } 525 close(p.cmdDoneCh) 526 }() 527 }() 528 529 ch := make(chan error) 530 go func() { 531 if err := c.waitForCreate(p, cmd); err != nil { 532 ch <- err 533 return 534 } 535 c.processes[pid] = p 536 ch <- nil 537 }() 538 select { 539 case <-ctx.Done(): 540 cmd.Process.Kill() 541 cmd.Wait() 542 <-ch 543 return ctx.Err() 544 case err := <-ch: 545 return err 546 } 547 return nil 548 } 549 550 func hostIDFromMap(id uint32, mp []ocs.LinuxIDMapping) int { 551 for _, m := range mp { 552 if (id >= m.ContainerID) && (id <= (m.ContainerID + m.Size - 1)) { 553 return int(m.HostID + (id - m.ContainerID)) 554 } 555 } 556 return 0 557 } 558 559 func (c *container) Stats() (*Stat, error) { 560 now := time.Now() 561 args := c.runtimeArgs 562 args = append(args, "events", "--stats", c.id) 563 out, err := exec.Command(c.runtime, args...).CombinedOutput() 564 if err != nil { 565 return nil, fmt.Errorf("%s: %q", err.Error(), out) 566 } 567 s := struct { 568 Data *Stat `json:"data"` 569 }{} 570 if err := json.Unmarshal(out, &s); err != nil { 571 return nil, err 572 } 573 s.Data.Timestamp = now 574 return s.Data, nil 575 } 576 577 // Status implements the runtime Container interface. 578 func (c *container) Status() (State, error) { 579 args := c.runtimeArgs 580 args = append(args, "state", c.id) 581 582 out, err := exec.Command(c.runtime, args...).CombinedOutput() 583 if err != nil { 584 return "", fmt.Errorf("%s: %q", err.Error(), out) 585 } 586 587 // We only require the runtime json output to have a top level Status field. 588 var s struct { 589 Status State `json:"status"` 590 } 591 if err := json.Unmarshal(out, &s); err != nil { 592 return "", err 593 } 594 return s.Status, nil 595 } 596 597 func (c *container) writeEventFD(root string, cfd, efd int) error { 598 f, err := os.OpenFile(filepath.Join(root, "cgroup.event_control"), os.O_WRONLY, 0) 599 if err != nil { 600 return err 601 } 602 defer f.Close() 603 _, err = f.WriteString(fmt.Sprintf("%d %d", efd, cfd)) 604 return err 605 } 606 607 type waitArgs struct { 608 pid int 609 err error 610 } 611 612 func (c *container) waitForCreate(p *process, cmd *exec.Cmd) error { 613 wc := make(chan error, 1) 614 go func() { 615 for { 616 if _, err := p.getPidFromFile(); err != nil { 617 if os.IsNotExist(err) || err == errInvalidPidInt || err == errContainerNotFound { 618 alive, err := isAlive(cmd) 619 if err != nil { 620 wc <- err 621 return 622 } 623 if !alive { 624 // runc could have failed to run the container so lets get the error 625 // out of the logs or the shim could have encountered an error 626 messages, err := readLogMessages(filepath.Join(p.root, "shim-log.json")) 627 if err != nil { 628 wc <- err 629 return 630 } 631 for _, m := range messages { 632 if m.Level == "error" { 633 wc <- fmt.Errorf("shim error: %v", m.Msg) 634 return 635 } 636 } 637 // no errors reported back from shim, check for runc/runtime errors 638 messages, err = readLogMessages(filepath.Join(p.root, "log.json")) 639 if err != nil { 640 if os.IsNotExist(err) { 641 err = ErrContainerNotStarted 642 } 643 wc <- err 644 return 645 } 646 for _, m := range messages { 647 if m.Level == "error" { 648 wc <- fmt.Errorf("oci runtime error: %v", m.Msg) 649 return 650 } 651 } 652 wc <- ErrContainerNotStarted 653 return 654 } 655 time.Sleep(15 * time.Millisecond) 656 continue 657 } 658 wc <- err 659 return 660 } 661 // the pid file was read successfully 662 wc <- nil 663 return 664 } 665 }() 666 select { 667 case err := <-wc: 668 if err != nil { 669 return err 670 } 671 err = p.saveStartTime() 672 if err != nil && !os.IsNotExist(err) { 673 logrus.Warnf("containerd: unable to save %s:%s starttime: %v", p.container.id, p.id, err) 674 } 675 return nil 676 case <-time.After(c.timeout): 677 cmd.Process.Kill() 678 cmd.Wait() 679 return ErrContainerStartTimeout 680 } 681 } 682 683 // isAlive checks if the shim that launched the container is still alive 684 func isAlive(cmd *exec.Cmd) (bool, error) { 685 if _, err := syscall.Wait4(cmd.Process.Pid, nil, syscall.WNOHANG, nil); err == nil { 686 return true, nil 687 } 688 if err := syscall.Kill(cmd.Process.Pid, 0); err != nil { 689 if err == syscall.ESRCH { 690 return false, nil 691 } 692 return false, err 693 } 694 return true, nil 695 } 696 697 type oom struct { 698 id string 699 root string 700 eventfd int 701 } 702 703 func (o *oom) ContainerID() string { 704 return o.id 705 } 706 707 func (o *oom) FD() int { 708 return o.eventfd 709 } 710 711 func (o *oom) Flush() { 712 buf := make([]byte, 8) 713 syscall.Read(o.eventfd, buf) 714 } 715 716 func (o *oom) Removed() bool { 717 _, err := os.Lstat(filepath.Join(o.root, "cgroup.event_control")) 718 return os.IsNotExist(err) 719 } 720 721 func (o *oom) Close() error { 722 return syscall.Close(o.eventfd) 723 } 724 725 type message struct { 726 Level string `json:"level"` 727 Msg string `json:"msg"` 728 } 729 730 func readLogMessages(path string) ([]message, error) { 731 var out []message 732 f, err := os.Open(path) 733 if err != nil { 734 return nil, err 735 } 736 defer f.Close() 737 dec := json.NewDecoder(f) 738 for { 739 var m message 740 if err := dec.Decode(&m); err != nil { 741 if err == io.EOF { 742 break 743 } 744 return nil, err 745 } 746 out = append(out, m) 747 } 748 return out, nil 749 }