github.com/darciopacifico/docker@v1.9.0-rc1/daemon/monitor.go (about) 1 package daemon 2 3 import ( 4 "io" 5 "os/exec" 6 "sync" 7 "time" 8 9 "github.com/Sirupsen/logrus" 10 "github.com/docker/docker/daemon/execdriver" 11 "github.com/docker/docker/pkg/stringid" 12 "github.com/docker/docker/runconfig" 13 ) 14 15 const ( 16 defaultTimeIncrement = 100 17 loggerCloseTimeout = 10 * time.Second 18 ) 19 20 // containerMonitor monitors the execution of a container's main process. 21 // If a restart policy is specified for the container the monitor will ensure that the 22 // process is restarted based on the rules of the policy. When the container is finally stopped 23 // the monitor will reset and cleanup any of the container resources such as networking allocations 24 // and the rootfs 25 type containerMonitor struct { 26 mux sync.Mutex 27 28 // container is the container being monitored 29 container *Container 30 31 // restartPolicy is the current policy being applied to the container monitor 32 restartPolicy runconfig.RestartPolicy 33 34 // failureCount is the number of times the container has failed to 35 // start in a row 36 failureCount int 37 38 // shouldStop signals the monitor that the next time the container exits it is 39 // either because docker or the user asked for the container to be stopped 40 shouldStop bool 41 42 // startSignal is a channel that is closes after the container initially starts 43 startSignal chan struct{} 44 45 // stopChan is used to signal to the monitor whenever there is a wait for the 46 // next restart so that the timeIncrement is not honored and the user is not 47 // left waiting for nothing to happen during this time 48 stopChan chan struct{} 49 50 // timeIncrement is the amount of time to wait between restarts 51 // this is in milliseconds 52 timeIncrement int 53 54 // lastStartTime is the time which the monitor last exec'd the container's process 55 lastStartTime time.Time 56 } 57 58 // newContainerMonitor returns an initialized containerMonitor for the provided container 59 // honoring the provided restart policy 60 func newContainerMonitor(container *Container, policy runconfig.RestartPolicy) *containerMonitor { 61 return &containerMonitor{ 62 container: container, 63 restartPolicy: policy, 64 timeIncrement: defaultTimeIncrement, 65 stopChan: make(chan struct{}), 66 startSignal: make(chan struct{}), 67 } 68 } 69 70 // Stop signals to the container monitor that it should stop monitoring the container 71 // for exits the next time the process dies 72 func (m *containerMonitor) ExitOnNext() { 73 m.mux.Lock() 74 75 // we need to protect having a double close of the channel when stop is called 76 // twice or else we will get a panic 77 if !m.shouldStop { 78 m.shouldStop = true 79 close(m.stopChan) 80 } 81 82 m.mux.Unlock() 83 } 84 85 // Close closes the container's resources such as networking allocations and 86 // unmounts the contatiner's root filesystem 87 func (m *containerMonitor) Close() error { 88 // Cleanup networking and mounts 89 m.container.cleanup() 90 91 // FIXME: here is race condition between two RUN instructions in Dockerfile 92 // because they share same runconfig and change image. Must be fixed 93 // in builder/builder.go 94 if err := m.container.toDisk(); err != nil { 95 logrus.Errorf("Error dumping container %s state to disk: %s", m.container.ID, err) 96 97 return err 98 } 99 100 return nil 101 } 102 103 // Start starts the containers process and monitors it according to the restart policy 104 func (m *containerMonitor) Start() error { 105 var ( 106 err error 107 exitStatus execdriver.ExitStatus 108 // this variable indicates where we in execution flow: 109 // before Run or after 110 afterRun bool 111 ) 112 113 // ensure that when the monitor finally exits we release the networking and unmount the rootfs 114 defer func() { 115 if afterRun { 116 m.container.Lock() 117 m.container.setStopped(&exitStatus) 118 defer m.container.Unlock() 119 } 120 m.Close() 121 }() 122 // reset stopped flag 123 if m.container.HasBeenManuallyStopped { 124 m.container.HasBeenManuallyStopped = false 125 } 126 127 // reset the restart count 128 m.container.RestartCount = -1 129 130 for { 131 m.container.RestartCount++ 132 133 if err := m.container.startLogging(); err != nil { 134 m.resetContainer(false) 135 136 return err 137 } 138 139 pipes := execdriver.NewPipes(m.container.stdin, m.container.stdout, m.container.stderr, m.container.Config.OpenStdin) 140 141 m.container.logEvent("start") 142 143 m.lastStartTime = time.Now() 144 145 if exitStatus, err = m.container.daemon.run(m.container, pipes, m.callback); err != nil { 146 // if we receive an internal error from the initial start of a container then lets 147 // return it instead of entering the restart loop 148 if m.container.RestartCount == 0 { 149 m.container.ExitCode = -1 150 m.resetContainer(false) 151 152 return err 153 } 154 155 logrus.Errorf("Error running container: %s", err) 156 } 157 158 // here container.Lock is already lost 159 afterRun = true 160 161 m.resetMonitor(err == nil && exitStatus.ExitCode == 0) 162 163 if m.shouldRestart(exitStatus.ExitCode) { 164 m.container.setRestarting(&exitStatus) 165 m.container.logEvent("die") 166 m.resetContainer(true) 167 168 // sleep with a small time increment between each restart to help avoid issues cased by quickly 169 // restarting the container because of some types of errors ( networking cut out, etc... ) 170 m.waitForNextRestart() 171 172 // we need to check this before reentering the loop because the waitForNextRestart could have 173 // been terminated by a request from a user 174 if m.shouldStop { 175 return err 176 } 177 continue 178 } 179 180 m.container.logEvent("die") 181 m.resetContainer(true) 182 return err 183 } 184 } 185 186 // resetMonitor resets the stateful fields on the containerMonitor based on the 187 // previous runs success or failure. Regardless of success, if the container had 188 // an execution time of more than 10s then reset the timer back to the default 189 func (m *containerMonitor) resetMonitor(successful bool) { 190 executionTime := time.Now().Sub(m.lastStartTime).Seconds() 191 192 if executionTime > 10 { 193 m.timeIncrement = defaultTimeIncrement 194 } else { 195 // otherwise we need to increment the amount of time we wait before restarting 196 // the process. We will build up by multiplying the increment by 2 197 m.timeIncrement *= 2 198 } 199 200 // the container exited successfully so we need to reset the failure counter 201 if successful { 202 m.failureCount = 0 203 } else { 204 m.failureCount++ 205 } 206 } 207 208 // waitForNextRestart waits with the default time increment to restart the container unless 209 // a user or docker asks for the container to be stopped 210 func (m *containerMonitor) waitForNextRestart() { 211 select { 212 case <-time.After(time.Duration(m.timeIncrement) * time.Millisecond): 213 case <-m.stopChan: 214 } 215 } 216 217 // shouldRestart checks the restart policy and applies the rules to determine if 218 // the container's process should be restarted 219 func (m *containerMonitor) shouldRestart(exitCode int) bool { 220 m.mux.Lock() 221 defer m.mux.Unlock() 222 223 // do not restart if the user or docker has requested that this container be stopped 224 if m.shouldStop { 225 m.container.HasBeenManuallyStopped = !m.container.daemon.shutdown 226 return false 227 } 228 229 switch { 230 case m.restartPolicy.IsAlways(), m.restartPolicy.IsUnlessStopped(): 231 return true 232 case m.restartPolicy.IsOnFailure(): 233 // the default value of 0 for MaximumRetryCount means that we will not enforce a maximum count 234 if max := m.restartPolicy.MaximumRetryCount; max != 0 && m.failureCount > max { 235 logrus.Debugf("stopping restart of container %s because maximum failure could of %d has been reached", 236 stringid.TruncateID(m.container.ID), max) 237 return false 238 } 239 240 return exitCode != 0 241 } 242 243 return false 244 } 245 246 // callback ensures that the container's state is properly updated after we 247 // received ack from the execution drivers 248 func (m *containerMonitor) callback(processConfig *execdriver.ProcessConfig, pid int, chOOM <-chan struct{}) error { 249 go func() { 250 _, ok := <-chOOM 251 if ok { 252 m.container.logEvent("oom") 253 } 254 }() 255 256 if processConfig.Tty { 257 // The callback is called after the process Start() 258 // so we are in the parent process. In TTY mode, stdin/out/err is the PtySlave 259 // which we close here. 260 if c, ok := processConfig.Stdout.(io.Closer); ok { 261 c.Close() 262 } 263 } 264 265 m.container.setRunning(pid) 266 267 // signal that the process has started 268 // close channel only if not closed 269 select { 270 case <-m.startSignal: 271 default: 272 close(m.startSignal) 273 } 274 275 if err := m.container.toDiskLocking(); err != nil { 276 logrus.Errorf("Error saving container to disk: %v", err) 277 } 278 return nil 279 } 280 281 // resetContainer resets the container's IO and ensures that the command is able to be executed again 282 // by copying the data into a new struct 283 // if lock is true, then container locked during reset 284 func (m *containerMonitor) resetContainer(lock bool) { 285 container := m.container 286 if lock { 287 container.Lock() 288 defer container.Unlock() 289 } 290 291 if container.Config.OpenStdin { 292 if err := container.stdin.Close(); err != nil { 293 logrus.Errorf("%s: Error close stdin: %s", container.ID, err) 294 } 295 } 296 297 if err := container.stdout.Clean(); err != nil { 298 logrus.Errorf("%s: Error close stdout: %s", container.ID, err) 299 } 300 301 if err := container.stderr.Clean(); err != nil { 302 logrus.Errorf("%s: Error close stderr: %s", container.ID, err) 303 } 304 305 if container.command != nil && container.command.ProcessConfig.Terminal != nil { 306 if err := container.command.ProcessConfig.Terminal.Close(); err != nil { 307 logrus.Errorf("%s: Error closing terminal: %s", container.ID, err) 308 } 309 } 310 311 // Re-create a brand new stdin pipe once the container exited 312 if container.Config.OpenStdin { 313 container.stdin, container.stdinPipe = io.Pipe() 314 } 315 316 if container.logDriver != nil { 317 if container.logCopier != nil { 318 exit := make(chan struct{}) 319 go func() { 320 container.logCopier.Wait() 321 close(exit) 322 }() 323 select { 324 case <-time.After(loggerCloseTimeout): 325 logrus.Warnf("Logger didn't exit in time: logs may be truncated") 326 case <-exit: 327 } 328 } 329 container.logDriver.Close() 330 container.logCopier = nil 331 container.logDriver = nil 332 } 333 334 c := container.command.ProcessConfig.Cmd 335 336 container.command.ProcessConfig.Cmd = exec.Cmd{ 337 Stdin: c.Stdin, 338 Stdout: c.Stdout, 339 Stderr: c.Stderr, 340 Path: c.Path, 341 Env: c.Env, 342 ExtraFiles: c.ExtraFiles, 343 Args: c.Args, 344 Dir: c.Dir, 345 SysProcAttr: c.SysProcAttr, 346 } 347 }