github.com/windmilleng/containerd@v0.2.7/runtime/container.go (about) 1 package runtime 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "os" 9 "os/exec" 10 "path/filepath" 11 "strings" 12 "syscall" 13 "time" 14 15 "github.com/Sirupsen/logrus" 16 "github.com/docker/containerd/specs" 17 ocs "github.com/opencontainers/runtime-spec/specs-go" 18 "golang.org/x/net/context" 19 "golang.org/x/sys/unix" 20 ) 21 22 // Container defines the operations allowed on a container 23 type Container interface { 24 // ID returns the container ID 25 ID() string 26 // Path returns the path to the bundle 27 Path() string 28 // Start starts the init process of the container 29 Start(ctx context.Context, checkpointPath string, s Stdio) (Process, error) 30 // Exec starts another process in an existing container 31 Exec(context.Context, string, specs.ProcessSpec, Stdio) (Process, error) 32 // Delete removes the container's state and any resources 33 Delete() error 34 // Processes returns all the containers processes that have been added 35 Processes() ([]Process, error) 36 // State returns the containers runtime state 37 State() State 38 // Resume resumes a paused container 39 Resume() error 40 // Pause pauses a running container 41 Pause() error 42 // RemoveProcess removes the specified process from the container 43 RemoveProcess(string) error 44 // Checkpoints returns all the checkpoints for a container 45 Checkpoints(checkpointDir string) ([]Checkpoint, error) 46 // Checkpoint creates a new checkpoint 47 Checkpoint(checkpoint Checkpoint, checkpointDir string) error 48 // DeleteCheckpoint deletes the checkpoint for the provided name 49 DeleteCheckpoint(name string, checkpointDir string) error 50 // Labels are user provided labels for the container 51 Labels() []string 52 // Pids returns all pids inside the container 53 Pids() ([]int, error) 54 // Stats returns realtime container stats and resource information 55 Stats() (*Stat, error) 56 // Name or path of the OCI compliant runtime used to execute the container 57 Runtime() string 58 // OOM signals the channel if the container received an OOM notification 59 OOM() (OOM, error) 60 // UpdateResource updates the containers resources to new values 61 UpdateResources(*Resource) error 62 63 // Status return the current status of the container. 64 Status() (State, error) 65 } 66 67 // OOM wraps a container OOM. 68 type OOM interface { 69 io.Closer 70 FD() int 71 ContainerID() string 72 Flush() 73 Removed() bool 74 } 75 76 // Stdio holds the path to the 3 pipes used for the standard ios. 77 type Stdio struct { 78 Stdin string 79 Stdout string 80 Stderr string 81 } 82 83 // NewStdio wraps the given standard io path into an Stdio struct. 84 // If a given parameter is the empty string, it is replaced by "/dev/null" 85 func NewStdio(stdin, stdout, stderr string) Stdio { 86 for _, s := range []*string{ 87 &stdin, &stdout, &stderr, 88 } { 89 if *s == "" { 90 *s = "/dev/null" 91 } 92 } 93 return Stdio{ 94 Stdin: stdin, 95 Stdout: stdout, 96 Stderr: stderr, 97 } 98 } 99 100 // ContainerOpts keeps the options passed at container creation 101 type ContainerOpts struct { 102 Root string 103 ID string 104 Bundle string 105 Runtime string 106 RuntimeArgs []string 107 Shim string 108 Labels []string 109 NoPivotRoot bool 110 Timeout time.Duration 111 } 112 113 // New returns a new container 114 func New(opts ContainerOpts) (Container, error) { 115 c := &container{ 116 root: opts.Root, 117 id: opts.ID, 118 bundle: opts.Bundle, 119 labels: opts.Labels, 120 processes: make(map[string]*process), 121 runtime: opts.Runtime, 122 runtimeArgs: opts.RuntimeArgs, 123 shim: opts.Shim, 124 noPivotRoot: opts.NoPivotRoot, 125 timeout: opts.Timeout, 126 } 127 if err := os.Mkdir(filepath.Join(c.root, c.id), 0755); err != nil { 128 return nil, err 129 } 130 f, err := os.Create(filepath.Join(c.root, c.id, StateFile)) 131 if err != nil { 132 return nil, err 133 } 134 defer f.Close() 135 if err := json.NewEncoder(f).Encode(state{ 136 Bundle: c.bundle, 137 Labels: c.labels, 138 Runtime: c.runtime, 139 RuntimeArgs: c.runtimeArgs, 140 Shim: c.shim, 141 NoPivotRoot: opts.NoPivotRoot, 142 }); err != nil { 143 return nil, err 144 } 145 return c, nil 146 } 147 148 // Load return a new container from the matchin state file on disk. 149 func Load(root, id, shimName string, timeout time.Duration) (Container, error) { 150 var s state 151 f, err := os.Open(filepath.Join(root, id, StateFile)) 152 if err != nil { 153 return nil, err 154 } 155 defer f.Close() 156 if err := json.NewDecoder(f).Decode(&s); err != nil { 157 return nil, err 158 } 159 c := &container{ 160 root: root, 161 id: id, 162 bundle: s.Bundle, 163 labels: s.Labels, 164 runtime: s.Runtime, 165 runtimeArgs: s.RuntimeArgs, 166 shim: s.Shim, 167 noPivotRoot: s.NoPivotRoot, 168 processes: make(map[string]*process), 169 timeout: timeout, 170 } 171 172 if c.shim == "" { 173 c.shim = shimName 174 } 175 176 dirs, err := ioutil.ReadDir(filepath.Join(root, id)) 177 if err != nil { 178 return nil, err 179 } 180 for _, d := range dirs { 181 if !d.IsDir() { 182 continue 183 } 184 pid := d.Name() 185 s, err := readProcessState(filepath.Join(root, id, pid)) 186 if err != nil { 187 return nil, err 188 } 189 p, err := loadProcess(filepath.Join(root, id, pid), pid, c, s) 190 if err != nil { 191 logrus.WithField("id", id).WithField("pid", pid).Debugf("containerd: error loading process %s", err) 192 continue 193 } 194 c.processes[pid] = p 195 } 196 return c, nil 197 } 198 199 func readProcessState(dir string) (*ProcessState, error) { 200 f, err := os.Open(filepath.Join(dir, "process.json")) 201 if err != nil { 202 return nil, err 203 } 204 defer f.Close() 205 var s ProcessState 206 if err := json.NewDecoder(f).Decode(&s); err != nil { 207 return nil, err 208 } 209 return &s, nil 210 } 211 212 type container struct { 213 // path to store runtime state information 214 root string 215 id string 216 bundle string 217 runtime string 218 runtimeArgs []string 219 shim string 220 processes map[string]*process 221 labels []string 222 oomFds []int 223 noPivotRoot bool 224 timeout time.Duration 225 } 226 227 func (c *container) ID() string { 228 return c.id 229 } 230 231 func (c *container) Path() string { 232 return c.bundle 233 } 234 235 func (c *container) Labels() []string { 236 return c.labels 237 } 238 239 func (c *container) readSpec() (*specs.Spec, error) { 240 var spec specs.Spec 241 f, err := os.Open(filepath.Join(c.bundle, "config.json")) 242 if err != nil { 243 return nil, err 244 } 245 defer f.Close() 246 if err := json.NewDecoder(f).Decode(&spec); err != nil { 247 return nil, err 248 } 249 return &spec, nil 250 } 251 252 func (c *container) Delete() error { 253 var err error 254 args := append(c.runtimeArgs, "delete", c.id) 255 if b, derr := exec.Command(c.runtime, args...).CombinedOutput(); derr != nil && !strings.Contains(string(b), "does not exist") { 256 err = fmt.Errorf("%s: %q", derr, string(b)) 257 } 258 if rerr := os.RemoveAll(filepath.Join(c.root, c.id)); rerr != nil { 259 if err != nil { 260 err = fmt.Errorf("%s; failed to remove %s: %s", err, filepath.Join(c.root, c.id), rerr) 261 } else { 262 err = rerr 263 } 264 } 265 return err 266 } 267 268 func (c *container) Processes() ([]Process, error) { 269 out := []Process{} 270 for _, p := range c.processes { 271 out = append(out, p) 272 } 273 return out, nil 274 } 275 276 func (c *container) RemoveProcess(pid string) error { 277 delete(c.processes, pid) 278 return os.RemoveAll(filepath.Join(c.root, c.id, pid)) 279 } 280 281 func (c *container) State() State { 282 proc := c.processes[InitProcessID] 283 if proc == nil { 284 return Stopped 285 } 286 return proc.State() 287 } 288 289 func (c *container) Runtime() string { 290 return c.runtime 291 } 292 293 func (c *container) Pause() error { 294 args := c.runtimeArgs 295 args = append(args, "pause", c.id) 296 b, err := exec.Command(c.runtime, args...).CombinedOutput() 297 if err != nil { 298 return fmt.Errorf("%s: %q", err.Error(), string(b)) 299 } 300 return nil 301 } 302 303 func (c *container) Resume() error { 304 args := c.runtimeArgs 305 args = append(args, "resume", c.id) 306 b, err := exec.Command(c.runtime, args...).CombinedOutput() 307 if err != nil { 308 return fmt.Errorf("%s: %q", err.Error(), string(b)) 309 } 310 return nil 311 } 312 313 func (c *container) Checkpoints(checkpointDir string) ([]Checkpoint, error) { 314 if checkpointDir == "" { 315 checkpointDir = filepath.Join(c.bundle, "checkpoints") 316 } 317 318 dirs, err := ioutil.ReadDir(checkpointDir) 319 if err != nil { 320 return nil, err 321 } 322 var out []Checkpoint 323 for _, d := range dirs { 324 if !d.IsDir() { 325 continue 326 } 327 path := filepath.Join(checkpointDir, d.Name(), "config.json") 328 data, err := ioutil.ReadFile(path) 329 if err != nil { 330 return nil, err 331 } 332 var cpt Checkpoint 333 if err := json.Unmarshal(data, &cpt); err != nil { 334 return nil, err 335 } 336 out = append(out, cpt) 337 } 338 return out, nil 339 } 340 341 func (c *container) Checkpoint(cpt Checkpoint, checkpointDir string) error { 342 if checkpointDir == "" { 343 checkpointDir = filepath.Join(c.bundle, "checkpoints") 344 } 345 346 if err := os.MkdirAll(checkpointDir, 0755); err != nil { 347 return err 348 } 349 350 path := filepath.Join(checkpointDir, cpt.Name) 351 if err := os.Mkdir(path, 0755); err != nil { 352 return err 353 } 354 f, err := os.Create(filepath.Join(path, "config.json")) 355 if err != nil { 356 return err 357 } 358 cpt.Created = time.Now() 359 err = json.NewEncoder(f).Encode(cpt) 360 f.Close() 361 if err != nil { 362 return err 363 } 364 args := []string{ 365 "checkpoint", 366 "--image-path", path, 367 "--work-path", filepath.Join(path, "criu.work"), 368 } 369 add := func(flags ...string) { 370 args = append(args, flags...) 371 } 372 add(c.runtimeArgs...) 373 if !cpt.Exit { 374 add("--leave-running") 375 } 376 if cpt.Shell { 377 add("--shell-job") 378 } 379 if cpt.TCP { 380 add("--tcp-established") 381 } 382 if cpt.UnixSockets { 383 add("--ext-unix-sk") 384 } 385 for _, ns := range cpt.EmptyNS { 386 add("--empty-ns", ns) 387 } 388 add(c.id) 389 out, err := exec.Command(c.runtime, args...).CombinedOutput() 390 if err != nil { 391 return fmt.Errorf("%s: %q", err.Error(), string(out)) 392 } 393 return err 394 } 395 396 func (c *container) DeleteCheckpoint(name string, checkpointDir string) error { 397 if checkpointDir == "" { 398 checkpointDir = filepath.Join(c.bundle, "checkpoints") 399 } 400 return os.RemoveAll(filepath.Join(checkpointDir, name)) 401 } 402 403 func (c *container) Start(ctx context.Context, checkpointPath string, s Stdio) (Process, error) { 404 processRoot := filepath.Join(c.root, c.id, InitProcessID) 405 if err := os.Mkdir(processRoot, 0755); err != nil { 406 return nil, err 407 } 408 cmd := exec.Command(c.shim, 409 c.id, c.bundle, c.runtime, 410 ) 411 cmd.Dir = processRoot 412 cmd.SysProcAttr = &syscall.SysProcAttr{ 413 Setpgid: true, 414 } 415 spec, err := c.readSpec() 416 if err != nil { 417 return nil, err 418 } 419 config := &processConfig{ 420 checkpoint: checkpointPath, 421 root: processRoot, 422 id: InitProcessID, 423 c: c, 424 stdio: s, 425 spec: spec, 426 processSpec: specs.ProcessSpec(spec.Process), 427 } 428 p, err := newProcess(config) 429 if err != nil { 430 return nil, err 431 } 432 if err := c.createCmd(ctx, InitProcessID, cmd, p); err != nil { 433 return nil, err 434 } 435 return p, nil 436 } 437 438 func (c *container) Exec(ctx context.Context, pid string, pspec specs.ProcessSpec, s Stdio) (pp Process, err error) { 439 processRoot := filepath.Join(c.root, c.id, pid) 440 if err := os.Mkdir(processRoot, 0755); err != nil { 441 return nil, err 442 } 443 defer func() { 444 if err != nil { 445 c.RemoveProcess(pid) 446 } 447 }() 448 cmd := exec.Command(c.shim, 449 c.id, c.bundle, c.runtime, 450 ) 451 cmd.Dir = processRoot 452 cmd.SysProcAttr = &syscall.SysProcAttr{ 453 Setpgid: true, 454 } 455 spec, err := c.readSpec() 456 if err != nil { 457 return nil, err 458 } 459 config := &processConfig{ 460 exec: true, 461 id: pid, 462 root: processRoot, 463 c: c, 464 processSpec: pspec, 465 spec: spec, 466 stdio: s, 467 } 468 p, err := newProcess(config) 469 if err != nil { 470 return nil, err 471 } 472 if err := c.createCmd(ctx, pid, cmd, p); err != nil { 473 return nil, err 474 } 475 return p, nil 476 } 477 478 func (c *container) createCmd(ctx context.Context, pid string, cmd *exec.Cmd, p *process) error { 479 p.cmd = cmd 480 if err := cmd.Start(); err != nil { 481 close(p.cmdDoneCh) 482 if exErr, ok := err.(*exec.Error); ok { 483 if exErr.Err == exec.ErrNotFound || exErr.Err == os.ErrNotExist { 484 return fmt.Errorf("%s not installed on system", c.shim) 485 } 486 } 487 return err 488 } 489 // We need the pid file to have been written to run 490 defer func() { 491 go func() { 492 err := p.cmd.Wait() 493 if err == nil { 494 p.cmdSuccess = true 495 } 496 497 if same, err := p.isSameProcess(); same && p.pid > 0 { 498 // The process changed its PR_SET_PDEATHSIG, so force 499 // kill it 500 logrus.Infof("containerd: %s:%s (pid %v) has become an orphan, killing it", p.container.id, p.id, p.pid) 501 err = unix.Kill(p.pid, syscall.SIGKILL) 502 if err != nil && err != syscall.ESRCH { 503 logrus.Errorf("containerd: unable to SIGKILL %s:%s (pid %v): %v", p.container.id, p.id, p.pid, err) 504 } else { 505 for { 506 err = unix.Kill(p.pid, 0) 507 if err != nil { 508 break 509 } 510 time.Sleep(5 * time.Millisecond) 511 } 512 } 513 } 514 close(p.cmdDoneCh) 515 }() 516 }() 517 518 ch := make(chan error) 519 go func() { 520 if err := c.waitForCreate(p, cmd); err != nil { 521 ch <- err 522 return 523 } 524 c.processes[pid] = p 525 ch <- nil 526 }() 527 select { 528 case <-ctx.Done(): 529 cmd.Process.Kill() 530 cmd.Wait() 531 <-ch 532 return ctx.Err() 533 case err := <-ch: 534 return err 535 } 536 return nil 537 } 538 539 func hostIDFromMap(id uint32, mp []ocs.IDMapping) int { 540 for _, m := range mp { 541 if (id >= m.ContainerID) && (id <= (m.ContainerID + m.Size - 1)) { 542 return int(m.HostID + (id - m.ContainerID)) 543 } 544 } 545 return 0 546 } 547 548 func (c *container) Stats() (*Stat, error) { 549 now := time.Now() 550 args := c.runtimeArgs 551 args = append(args, "events", "--stats", c.id) 552 out, err := exec.Command(c.runtime, args...).CombinedOutput() 553 if err != nil { 554 return nil, fmt.Errorf("%s: %q", err.Error(), out) 555 } 556 s := struct { 557 Data *Stat `json:"data"` 558 }{} 559 if err := json.Unmarshal(out, &s); err != nil { 560 return nil, err 561 } 562 s.Data.Timestamp = now 563 return s.Data, nil 564 } 565 566 // Status implements the runtime Container interface. 567 func (c *container) Status() (State, error) { 568 args := c.runtimeArgs 569 args = append(args, "state", c.id) 570 571 out, err := exec.Command(c.runtime, args...).CombinedOutput() 572 if err != nil { 573 return "", fmt.Errorf("%s: %q", err.Error(), out) 574 } 575 576 // We only require the runtime json output to have a top level Status field. 577 var s struct { 578 Status State `json:"status"` 579 } 580 if err := json.Unmarshal(out, &s); err != nil { 581 return "", err 582 } 583 return s.Status, nil 584 } 585 586 func (c *container) writeEventFD(root string, cfd, efd int) error { 587 f, err := os.OpenFile(filepath.Join(root, "cgroup.event_control"), os.O_WRONLY, 0) 588 if err != nil { 589 return err 590 } 591 defer f.Close() 592 _, err = f.WriteString(fmt.Sprintf("%d %d", efd, cfd)) 593 return err 594 } 595 596 type waitArgs struct { 597 pid int 598 err error 599 } 600 601 func (c *container) waitForCreate(p *process, cmd *exec.Cmd) error { 602 wc := make(chan error, 1) 603 go func() { 604 for { 605 if _, err := p.getPidFromFile(); err != nil { 606 if os.IsNotExist(err) || err == errInvalidPidInt || err == errContainerNotFound { 607 alive, err := isAlive(cmd) 608 if err != nil { 609 wc <- err 610 return 611 } 612 if !alive { 613 // runc could have failed to run the container so lets get the error 614 // out of the logs or the shim could have encountered an error 615 messages, err := readLogMessages(filepath.Join(p.root, "shim-log.json")) 616 if err != nil { 617 wc <- err 618 return 619 } 620 for _, m := range messages { 621 if m.Level == "error" { 622 wc <- fmt.Errorf("shim error: %v", m.Msg) 623 return 624 } 625 } 626 // no errors reported back from shim, check for runc/runtime errors 627 messages, err = readLogMessages(filepath.Join(p.root, "log.json")) 628 if err != nil { 629 if os.IsNotExist(err) { 630 err = ErrContainerNotStarted 631 } 632 wc <- err 633 return 634 } 635 for _, m := range messages { 636 if m.Level == "error" { 637 wc <- fmt.Errorf("oci runtime error: %v", m.Msg) 638 return 639 } 640 } 641 wc <- ErrContainerNotStarted 642 return 643 } 644 time.Sleep(15 * time.Millisecond) 645 continue 646 } 647 wc <- err 648 return 649 } 650 // the pid file was read successfully 651 wc <- nil 652 return 653 } 654 }() 655 select { 656 case err := <-wc: 657 if err != nil { 658 return err 659 } 660 err = p.saveStartTime() 661 if err != nil && !os.IsNotExist(err) { 662 logrus.Warnf("containerd: unable to save %s:%s starttime: %v", p.container.id, p.id, err) 663 } 664 return nil 665 case <-time.After(c.timeout): 666 cmd.Process.Kill() 667 cmd.Wait() 668 return ErrContainerStartTimeout 669 } 670 } 671 672 // isAlive checks if the shim that launched the container is still alive 673 func isAlive(cmd *exec.Cmd) (bool, error) { 674 if _, err := syscall.Wait4(cmd.Process.Pid, nil, syscall.WNOHANG, nil); err == nil { 675 return true, nil 676 } 677 if err := syscall.Kill(cmd.Process.Pid, 0); err != nil { 678 if err == syscall.ESRCH { 679 return false, nil 680 } 681 return false, err 682 } 683 return true, nil 684 } 685 686 type oom struct { 687 id string 688 root string 689 eventfd int 690 } 691 692 func (o *oom) ContainerID() string { 693 return o.id 694 } 695 696 func (o *oom) FD() int { 697 return o.eventfd 698 } 699 700 func (o *oom) Flush() { 701 buf := make([]byte, 8) 702 syscall.Read(o.eventfd, buf) 703 } 704 705 func (o *oom) Removed() bool { 706 _, err := os.Lstat(filepath.Join(o.root, "cgroup.event_control")) 707 return os.IsNotExist(err) 708 } 709 710 func (o *oom) Close() error { 711 return syscall.Close(o.eventfd) 712 } 713 714 type message struct { 715 Level string `json:"level"` 716 Msg string `json:"msg"` 717 } 718 719 func readLogMessages(path string) ([]message, error) { 720 var out []message 721 f, err := os.Open(path) 722 if err != nil { 723 return nil, err 724 } 725 defer f.Close() 726 dec := json.NewDecoder(f) 727 for { 728 var m message 729 if err := dec.Decode(&m); err != nil { 730 if err == io.EOF { 731 break 732 } 733 return nil, err 734 } 735 out = append(out, m) 736 } 737 return out, nil 738 }