github.com/cloud-green/juju@v0.0.0-20151002100041-a00291338d3d/worker/rsyslog/worker.go (about)

     1  // Copyright 2014 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package rsyslog
     5  
     6  import (
     7  	"crypto/tls"
     8  	"crypto/x509"
     9  	"fmt"
    10  	"io"
    11  	"net"
    12  	"os"
    13  	"os/user"
    14  	"path/filepath"
    15  	"runtime"
    16  	"strconv"
    17  	"time"
    18  
    19  	"github.com/juju/errors"
    20  	"github.com/juju/loggo"
    21  	"github.com/juju/names"
    22  	rsyslog "github.com/juju/syslog"
    23  	"github.com/juju/utils"
    24  
    25  	"github.com/juju/juju/agent"
    26  	apirsyslog "github.com/juju/juju/api/rsyslog"
    27  	"github.com/juju/juju/api/watcher"
    28  	"github.com/juju/juju/cert"
    29  	"github.com/juju/juju/utils/syslog"
    30  	"github.com/juju/juju/worker"
    31  	jujuos "github.com/juju/utils/os"
    32  )
    33  
    34  var logger = loggo.GetLogger("juju.worker.rsyslog")
    35  
    36  var (
    37  	rsyslogConfDir = "/etc/rsyslog.d"
    38  	logDir         = agent.DefaultPaths.LogDir
    39  	syslogTargets  = []*rsyslog.Writer{}
    40  )
    41  
    42  // RsyslogMode describes how to configure rsyslog.
    43  type RsyslogMode int
    44  
    45  const (
    46  	RsyslogModeInvalid RsyslogMode = iota
    47  	// RsyslogModeForwarding is the mode in which
    48  	// rsyslog will be configured to forward logging
    49  	// to state servers.
    50  	RsyslogModeForwarding
    51  	// RsyslogModeAccumulate is the mode in which
    52  	// rsyslog will be configured to accumulate logging
    53  	// from other machines into an "all-machines.log".
    54  	RsyslogModeAccumulate
    55  )
    56  
    57  // RsyslogConfigHandler implements worker.NotifyWatchHandler, watching
    58  // environment configuration changes and generating new rsyslog
    59  // configuration.
    60  type RsyslogConfigHandler struct {
    61  	st              *apirsyslog.State
    62  	mode            RsyslogMode
    63  	syslogConfig    *syslog.SyslogConfig
    64  	rsyslogConfPath string
    65  	tag             names.Tag
    66  	// We store the syslog-port and rsyslog-ca-cert
    67  	// values after writing the rsyslog configuration,
    68  	// so we can decide whether a change has occurred.
    69  	syslogPort    int
    70  	rsyslogCACert string
    71  	rsyslogCAKey  string
    72  }
    73  
    74  // certPair holds the path and contents for a certificate.
    75  type certPair struct {
    76  	path string
    77  	data string
    78  }
    79  
    80  var _ worker.NotifyWatchHandler = (*RsyslogConfigHandler)(nil)
    81  
    82  func syslogUser() string {
    83  	switch jujuos.HostOS() {
    84  	case jujuos.CentOS:
    85  		return "root"
    86  	default:
    87  		return "syslog"
    88  	}
    89  }
    90  
    91  var NewRsyslogConfigWorker = newRsyslogConfigWorker
    92  
    93  // newRsyslogConfigWorker returns a worker.Worker that uses
    94  // WatchForRsyslogChanges and updates rsyslog configuration based
    95  // on changes. The worker will remove the configuration file
    96  // on teardown.
    97  func newRsyslogConfigWorker(st *apirsyslog.State, mode RsyslogMode, tag names.Tag, namespace string, stateServerAddrs []string, jujuConfigDir string) (worker.Worker, error) {
    98  	if jujuos.HostOS() == jujuos.Windows && mode == RsyslogModeAccumulate {
    99  		return worker.NewNoOpWorker(), nil
   100  	}
   101  	handler, err := newRsyslogConfigHandler(st, mode, tag, namespace, stateServerAddrs, jujuConfigDir)
   102  	if err != nil {
   103  		return nil, err
   104  	}
   105  	logger.Debugf("starting rsyslog worker mode %v for %q %q", mode, tag, namespace)
   106  	return worker.NewNotifyWorker(handler), nil
   107  }
   108  
   109  func newRsyslogConfigHandler(st *apirsyslog.State, mode RsyslogMode, tag names.Tag, namespace string, stateServerAddrs []string, jujuConfigDir string) (*RsyslogConfigHandler, error) {
   110  	if namespace != "" {
   111  		jujuConfigDir += "-" + namespace
   112  	}
   113  	jujuConfigDir = filepath.Join(jujuConfigDir, "rsyslog")
   114  	if err := os.MkdirAll(jujuConfigDir, 0755); err != nil {
   115  		return nil, errors.Trace(err)
   116  	}
   117  
   118  	syslogConfig := &syslog.SyslogConfig{
   119  		LogFileName:          tag.String(),
   120  		LogDir:               logDir,
   121  		JujuConfigDir:        jujuConfigDir,
   122  		Port:                 0,
   123  		Namespace:            namespace,
   124  		StateServerAddresses: stateServerAddrs,
   125  	}
   126  	if mode == RsyslogModeAccumulate {
   127  		syslog.NewAccumulateConfig(syslogConfig)
   128  	} else {
   129  		syslog.NewForwardConfig(syslogConfig)
   130  	}
   131  
   132  	// Historically only machine-0 includes the namespace in the log
   133  	// dir/file; for backwards compatibility we continue the tradition.
   134  	if tag != names.NewMachineTag("0") {
   135  		namespace = ""
   136  	}
   137  	switch tag := tag.(type) {
   138  	case names.MachineTag:
   139  		if namespace == "" {
   140  			syslogConfig.ConfigFileName = "25-juju.conf"
   141  		} else {
   142  			syslogConfig.ConfigFileName = fmt.Sprintf("25-juju-%s.conf", namespace)
   143  		}
   144  	default:
   145  		syslogConfig.ConfigFileName = fmt.Sprintf("26-juju-%s.conf", tag)
   146  	}
   147  
   148  	syslogConfig.ConfigDir = rsyslogConfDir
   149  	syslogConfig.LogDir = logDir
   150  	if namespace != "" {
   151  		syslogConfig.LogDir += "-" + namespace
   152  	}
   153  
   154  	return &RsyslogConfigHandler{
   155  		st:           st,
   156  		mode:         mode,
   157  		syslogConfig: syslogConfig,
   158  		tag:          tag,
   159  	}, nil
   160  }
   161  
   162  func (h *RsyslogConfigHandler) SetUp() (watcher.NotifyWatcher, error) {
   163  	if h.mode == RsyslogModeAccumulate {
   164  		if err := h.ensureCA(); err != nil {
   165  			return nil, errors.Annotate(err, "failed to write rsyslog certificates")
   166  		}
   167  
   168  		if err := h.ensureLogrotate(); err != nil {
   169  			return nil, errors.Annotate(err, "failed to write rsyslog logrotate scripts")
   170  		}
   171  
   172  	}
   173  	return h.st.WatchForRsyslogChanges(h.tag.String())
   174  }
   175  
   176  var restartRsyslog = syslog.Restart
   177  var dialSyslog = rsyslog.Dial
   178  
   179  func (h *RsyslogConfigHandler) TearDown() error {
   180  	if err := os.Remove(h.syslogConfig.ConfigFilePath()); err == nil {
   181  		restartRsyslog()
   182  	}
   183  	return nil
   184  }
   185  
   186  // composeTLS generates a new client certificate for connecting to the rsyslog server.
   187  // We explicitly set the ServerName field, this ensures that even if we are connecting
   188  // via an IP address and are using an old certificate (pre 1.20.9), we can still
   189  // successfully connect.
   190  func (h *RsyslogConfigHandler) composeTLS(caCert string) (*tls.Config, error) {
   191  	cert := x509.NewCertPool()
   192  	ok := cert.AppendCertsFromPEM([]byte(caCert))
   193  	if !ok {
   194  		return nil, errors.Errorf("Failed to parse rsyslog root certificate")
   195  	}
   196  	return &tls.Config{
   197  		RootCAs:    cert,
   198  		ServerName: "juju-rsyslog",
   199  	}, nil
   200  }
   201  
   202  func (h *RsyslogConfigHandler) replaceRemoteLogger(caCert string) error {
   203  	tlsConf, err := h.composeTLS(caCert)
   204  	if err != nil {
   205  		return err
   206  	}
   207  
   208  	var newLoggers []*rsyslog.Writer
   209  	var wrapLoggers []io.Writer
   210  	for _, j := range h.syslogConfig.StateServerAddresses {
   211  		host, _, err := net.SplitHostPort(j)
   212  		if err != nil {
   213  			// No port was found
   214  			host = j
   215  		}
   216  		target := fmt.Sprintf("%s:%d", host, h.syslogConfig.Port)
   217  		namespace := h.syslogConfig.Namespace
   218  		if namespace != "" {
   219  			namespace = "-" + namespace
   220  		}
   221  		logTag := "juju" + namespace + "-" + h.tag.String()
   222  		logger.Debugf("making syslog connection for %q to %s", logTag, target)
   223  		writer, err := dialSyslog("tcp", target, rsyslog.LOG_DEBUG, logTag, tlsConf)
   224  		if err != nil {
   225  			return err
   226  		}
   227  		wrapLoggers = append(wrapLoggers, writer)
   228  		newLoggers = append(newLoggers, writer)
   229  	}
   230  	wapper := io.MultiWriter(wrapLoggers...)
   231  	writer := loggo.NewSimpleWriter(wapper, &loggo.DefaultFormatter{})
   232  
   233  	loggo.RemoveWriter("syslog")
   234  	err = loggo.RegisterWriter("syslog", writer, loggo.TRACE)
   235  	if err != nil {
   236  		return err
   237  	}
   238  
   239  	// Close old targets
   240  	for _, j := range syslogTargets {
   241  		if err := j.Close(); err != nil {
   242  			logger.Warningf("Failed to close syslog writer: %s", err)
   243  		}
   244  	}
   245  	// record new targets
   246  	syslogTargets = newLoggers
   247  	return nil
   248  }
   249  
   250  func (h *RsyslogConfigHandler) Handle(_ <-chan struct{}) error {
   251  	cfg, err := h.st.GetRsyslogConfig(h.tag.String())
   252  	if err != nil {
   253  		return errors.Annotate(err, "cannot get environ config")
   254  	}
   255  
   256  	rsyslogCACert := cfg.CACert
   257  	if rsyslogCACert == "" {
   258  		return nil
   259  	}
   260  
   261  	rsyslogCAKey := cfg.CAKey
   262  	if rsyslogCAKey == "" {
   263  		return nil
   264  	}
   265  
   266  	h.syslogConfig.Port = cfg.Port
   267  	if h.mode == RsyslogModeForwarding {
   268  		if err := writeFileAtomic(h.syslogConfig.CACertPath(), []byte(rsyslogCACert), 0644, 0, 0); err != nil {
   269  			return errors.Annotate(err, "cannot write CA certificate")
   270  		}
   271  		if err := h.replaceRemoteLogger(rsyslogCACert); err != nil {
   272  			return err
   273  		}
   274  	} else {
   275  		rsyslogCertPEM, rsyslogKeyPEM, err := h.rsyslogServerCerts(rsyslogCACert, rsyslogCAKey)
   276  		if err != nil {
   277  			return errors.Trace(err)
   278  		}
   279  
   280  		if err := writeCertificates([]certPair{
   281  			{h.syslogConfig.ServerCertPath(), rsyslogCertPEM},
   282  			{h.syslogConfig.ServerKeyPath(), rsyslogKeyPEM},
   283  			{h.syslogConfig.CACertPath(), rsyslogCACert},
   284  		}); err != nil {
   285  			return errors.Trace(err)
   286  		}
   287  
   288  		data, err := h.syslogConfig.Render()
   289  		if err != nil {
   290  			return errors.Annotate(err, "failed to render rsyslog configuration file")
   291  		}
   292  		if err := writeFileAtomic(h.syslogConfig.ConfigFilePath(), []byte(data), 0644, 0, 0); err != nil {
   293  			return errors.Annotate(err, "failed to write rsyslog configuration file")
   294  		}
   295  		logger.Debugf("Reloading rsyslog configuration")
   296  		if err := restartRsyslog(); err != nil {
   297  			logger.Errorf("failed to reload rsyslog configuration")
   298  			return errors.Annotate(err, "cannot restart rsyslog")
   299  		}
   300  	}
   301  	// Record config values so we don't try again.
   302  	// Do this last so we recover from intermittent
   303  	// failures.
   304  	h.syslogPort = cfg.Port
   305  	h.rsyslogCACert = rsyslogCACert
   306  	h.rsyslogCAKey = rsyslogCAKey
   307  	return nil
   308  }
   309  
   310  var lookupUser = func(username string) (uid, gid int, err error) {
   311  	u, err := user.Lookup(username)
   312  	if err != nil {
   313  		return -1, -1, err
   314  	}
   315  	uid, err = strconv.Atoi(u.Uid)
   316  	if err != nil {
   317  		return -1, -1, err
   318  	}
   319  	gid, err = strconv.Atoi(u.Gid)
   320  	if err != nil {
   321  		return -1, -1, err
   322  	}
   323  	return uid, gid, nil
   324  }
   325  
   326  func localIPS() ([]string, error) {
   327  	var ips []string
   328  	addrs, err := net.InterfaceAddrs()
   329  	if err != nil {
   330  		return nil, err
   331  	}
   332  	for _, j := range addrs {
   333  		ip, _, err := net.ParseCIDR(j.String())
   334  		if err != nil {
   335  			return nil, err
   336  		}
   337  		if ip.IsLoopback() {
   338  			continue
   339  		}
   340  		ips = append(ips, ip.String())
   341  	}
   342  	return ips, nil
   343  }
   344  
   345  func (h *RsyslogConfigHandler) rsyslogHosts() ([]string, error) {
   346  	var hosts []string
   347  	cfg, err := h.st.GetRsyslogConfig(h.tag.String())
   348  	if err != nil {
   349  		return nil, err
   350  	}
   351  	for _, j := range cfg.HostPorts {
   352  		if j.Value != "" {
   353  			hosts = append(hosts, j.Address.Value)
   354  		}
   355  	}
   356  
   357  	// Explicitly add the '*' wildcard host. This will ensure that rsyslog
   358  	// clients will always be able to connect even if their hostnames and/or IPAddresses
   359  	// are changed. This also ensures we can continue to use SSL for our rsyslog connections
   360  	// and we can avoid having to use the skipVerify flag.
   361  	hosts = append(hosts, "*")
   362  
   363  	return hosts, nil
   364  }
   365  
   366  // ensureCA ensures that a CA certificate and key exist in state,
   367  // to be picked up by all rsyslog workers in the environment.
   368  func (h *RsyslogConfigHandler) ensureCA() error {
   369  	// We never write the CA key to local disk, so
   370  	// we must check state to know whether or not
   371  	// we need to generate new certs and keys.
   372  	cfg, err := h.st.GetRsyslogConfig(h.tag.String())
   373  	if err != nil {
   374  		return errors.Annotate(err, "cannot get environ config")
   375  	}
   376  	if cfg.CACert != "" && cfg.CAKey != "" {
   377  		return nil
   378  	}
   379  
   380  	// Generate a new CA and server cert/key pairs, and
   381  	// publish to state. Rsyslog workers will observe
   382  	// this and generate certificates and keys for
   383  	// rsyslog in response.
   384  	expiry := time.Now().UTC().AddDate(10, 0, 0)
   385  	caCertPEM, caKeyPEM, err := cert.NewCA("rsyslog", expiry)
   386  	if err != nil {
   387  		return err
   388  	}
   389  	return h.st.SetRsyslogCert(caCertPEM, caKeyPEM)
   390  }
   391  
   392  // writeCertificates persists any certPair to disk. If any
   393  // of the write attempts fail it will return an error immediately.
   394  // It is up to the caller to ensure the order of pairs represents
   395  // a suitable order in the case of failue.
   396  func writeCertificates(pairs []certPair) error {
   397  	// Files must be chowned to syslog:adm.
   398  	syslogUid, syslogGid, err := lookupUser(syslogUser())
   399  	if err != nil {
   400  		return err
   401  	}
   402  
   403  	for _, pair := range pairs {
   404  		if err := writeFileAtomic(pair.path, []byte(pair.data), 0600, syslogUid, syslogGid); err != nil {
   405  			return err
   406  		}
   407  	}
   408  	return nil
   409  }
   410  
   411  // rsyslogServerCerts generates new certificates for RsyslogConfigHandler
   412  // using the provider caCert and caKey. This is used during the setup of the
   413  // rsyslog worker as well as when handling any changes to the rsyslog configuration,
   414  // usually adding and removing of state machines through ensure-availability.
   415  func (h *RsyslogConfigHandler) rsyslogServerCerts(caCert, caKey string) (string, string, error) {
   416  	if caCert == "" {
   417  		return "", "", errors.New("CACert is not set")
   418  	}
   419  	if caKey == "" {
   420  		return "", "", errors.New("CAKey is not set")
   421  	}
   422  
   423  	expiry := time.Now().UTC().AddDate(10, 0, 0)
   424  	// Add rsyslog servers in the subjectAltName so we can
   425  	// successfully validate when connectiong via SSL
   426  	hosts, err := h.rsyslogHosts()
   427  	if err != nil {
   428  		return "", "", err
   429  	}
   430  	// Add local IPs to SAN. When connecting via IP address,
   431  	// the client will validate the server against any IP in
   432  	// the subjectAltName. We add all local ips to make sure
   433  	// this does not cause an error
   434  	ips, err := localIPS()
   435  	if err != nil {
   436  		return "", "", err
   437  	}
   438  	hosts = append(hosts, ips...)
   439  	return cert.NewServer(caCert, caKey, expiry, hosts)
   440  }
   441  
   442  // ensureLogrotate ensures that the logrotate
   443  // configuration file and logrotate helper script
   444  // exist in the log directory and creates them if they do not.
   445  func (h *RsyslogConfigHandler) ensureLogrotate() error {
   446  	// Files must be chowned to syslog
   447  	syslogUid, syslogGid, err := lookupUser(syslogUser())
   448  	if err != nil {
   449  		return err
   450  	}
   451  
   452  	logrotateConfPath := h.syslogConfig.LogrotateConfPath()
   453  	// check for the logrotate conf
   454  	if _, err := os.Stat(logrotateConfPath); os.IsNotExist(err) {
   455  		logrotateConfFile, err := h.syslogConfig.LogrotateConfFile()
   456  		if err != nil {
   457  			return err
   458  		}
   459  		// create the logrotate conf
   460  		if err := writeFileAtomic(logrotateConfPath, logrotateConfFile, 0600, syslogUid, syslogGid); err != nil {
   461  			return err
   462  		}
   463  	} else {
   464  		return err
   465  	}
   466  
   467  	logrotateHelperPath := h.syslogConfig.LogrotateHelperPath()
   468  	// check for the logrotate helper
   469  	if _, err := os.Stat(logrotateHelperPath); os.IsNotExist(err) {
   470  		logrotateHelperFile, err := h.syslogConfig.LogrotateHelperFile()
   471  		if err != nil {
   472  			return err
   473  		}
   474  		// create the logrotate helper
   475  		if err := writeFileAtomic(logrotateHelperPath, logrotateHelperFile, 0700, syslogUid, syslogGid); err != nil {
   476  			return err
   477  		}
   478  	} else {
   479  		return err
   480  	}
   481  
   482  	return nil
   483  }
   484  
   485  func writeFileAtomic(path string, data []byte, mode os.FileMode, uid, gid int) error {
   486  	chmodAndChown := func(f *os.File) error {
   487  		// f.Chmod() and f.Chown() are not implemented on Windows
   488  		// There is currently no good way of doing file permission
   489  		// management for Windows, directly from Go. The behavior of os.Chmod()
   490  		// is different from its linux implementation.
   491  		if runtime.GOOS == "windows" {
   492  			return nil
   493  		}
   494  		if err := f.Chmod(mode); err != nil {
   495  			return err
   496  		}
   497  		if uid != 0 {
   498  			if err := f.Chown(uid, gid); err != nil {
   499  				return err
   500  			}
   501  		}
   502  		return nil
   503  	}
   504  	return utils.AtomicWriteFileAndChange(path, data, chmodAndChown)
   505  }