github.com/technosophos/deis@v1.7.1-0.20150915173815-f9005256004b/mesos/pkg/confd/confd.go (about)

     1  package confd
     2  
     3  import (
     4  	"bufio"
     5  	"fmt"
     6  	"io"
     7  	"os"
     8  	"os/exec"
     9  	"regexp"
    10  	"strings"
    11  	"syscall"
    12  	"time"
    13  
    14  	logger "github.com/deis/deis/mesos/pkg/log"
    15  	oswrapper "github.com/deis/deis/mesos/pkg/os"
    16  )
    17  
    18  const (
    19  	confdInterval       = 5                // seconds
    20  	errorTickInterval   = 60 * time.Second // 1 minute
    21  	maxErrorsInInterval = 5                // up to 5 errors per time interval
    22  )
    23  
    24  var (
    25  	log                = logger.New()
    26  	templateErrorRegex = "(\\d{4})-(\\d{2})-(\\d{2})T(\\d{2}):(\\d{2}):\\d{2}Z.*ERROR template:"
    27  )
    28  
    29  // WaitForInitialConf wait until the compilation of the templates is correct
    30  func WaitForInitialConf(etcd []string, timeout time.Duration) {
    31  	log.Info("waiting for confd to write initial templates...")
    32  	for {
    33  		cmdAsString := fmt.Sprintf("confd -onetime -node %v -confdir /app", strings.Join(etcd, ","))
    34  		log.Debugf("running %s", cmdAsString)
    35  		cmd, args := oswrapper.BuildCommandFromString(cmdAsString)
    36  		err := oswrapper.RunCommand(cmd, args)
    37  		if err == nil {
    38  			break
    39  		}
    40  
    41  		time.Sleep(timeout)
    42  	}
    43  }
    44  
    45  // Launch launch confd as a daemon process.
    46  func Launch(signalChan chan os.Signal, etcd []string) {
    47  	confdLogLevel := "error"
    48  	if log.Level.String() == "debug" {
    49  		confdLogLevel = "debug"
    50  	}
    51  	cmdAsString := fmt.Sprintf("confd -node %v -confdir /app --interval %v --log-level %v", confdInterval, strings.Join(etcd, ","), confdLogLevel)
    52  	cmd, args := oswrapper.BuildCommandFromString(cmdAsString)
    53  	go runConfdDaemon(signalChan, cmd, args)
    54  }
    55  
    56  func runConfdDaemon(signalChan chan os.Signal, command string, args []string) {
    57  	cmd := exec.Command(command, args...)
    58  
    59  	stdout, err := cmd.StdoutPipe()
    60  	checkError(signalChan, err)
    61  	// stderr, err := cmd.StderrPipe()
    62  	// checkError(signalChan, err)
    63  
    64  	go io.Copy(os.Stdout, stdout)
    65  	// go io.Copy(os.Stderr, stderr)
    66  
    67  	go checkNumberOfErrors(stdout, maxErrorsInInterval, errorTickInterval, signalChan)
    68  
    69  	err = cmd.Start()
    70  	if err != nil {
    71  		log.Errorf("an error ocurred executing confd: [%s params %v], %v", command, args, err)
    72  		signalChan <- syscall.SIGKILL
    73  	}
    74  
    75  	err = cmd.Wait()
    76  	log.Errorf("confd command finished with error: %v", err)
    77  	signalChan <- syscall.SIGKILL
    78  }
    79  
    80  func checkError(signalChan chan os.Signal, err error) {
    81  	if err != nil {
    82  		log.Errorf("%v", err)
    83  		signalChan <- syscall.SIGKILL
    84  	}
    85  }
    86  
    87  func checkNumberOfErrors(std io.ReadCloser, count uint64, tick time.Duration, signalChan chan os.Signal) {
    88  	testRegex := regexp.MustCompile(templateErrorRegex)
    89  	var tickErrors uint64
    90  	lines := make(chan string)
    91  	go func() {
    92  		scanner := bufio.NewScanner(std)
    93  		scanner.Split(bufio.ScanLines)
    94  		for scanner.Scan() {
    95  			lines <- scanner.Text()
    96  		}
    97  	}()
    98  
    99  	timer := time.NewTicker(tick)
   100  
   101  	for {
   102  		select {
   103  		case <-timer.C:
   104  			if tickErrors > count {
   105  				log.Debugf("number of errors %v", tickErrors)
   106  				log.Error("too many confd errors in the last minute. restarting component")
   107  				signalChan <- syscall.SIGKILL
   108  				return
   109  			}
   110  
   111  			tickErrors = 0
   112  		case line := <-lines:
   113  			match := testRegex.FindStringSubmatch(line)
   114  			if match != nil {
   115  				tickErrors++
   116  			}
   117  		}
   118  	}
   119  }