github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/daemon/logger/fluentd/fluentd.go (about)

     1  // Package fluentd provides the log driver for forwarding server logs
     2  // to fluentd endpoints.
     3  package fluentd // import "github.com/demonoid81/moby/daemon/logger/fluentd"
     4  
     5  import (
     6  	"math"
     7  	"net"
     8  	"net/url"
     9  	"strconv"
    10  	"strings"
    11  	"time"
    12  
    13  	"github.com/demonoid81/moby/daemon/logger"
    14  	"github.com/demonoid81/moby/daemon/logger/loggerutils"
    15  	"github.com/demonoid81/moby/errdefs"
    16  	"github.com/demonoid81/moby/pkg/urlutil"
    17  	units "github.com/docker/go-units"
    18  	"github.com/fluent/fluent-logger-golang/fluent"
    19  	"github.com/pkg/errors"
    20  	"github.com/sirupsen/logrus"
    21  )
    22  
    23  type fluentd struct {
    24  	tag           string
    25  	containerID   string
    26  	containerName string
    27  	writer        *fluent.Fluent
    28  	extra         map[string]string
    29  }
    30  
    31  type location struct {
    32  	protocol string
    33  	host     string
    34  	port     int
    35  	path     string
    36  }
    37  
    38  const (
    39  	name = "fluentd"
    40  
    41  	defaultBufferLimit = 1024 * 1024
    42  	defaultHost        = "127.0.0.1"
    43  	defaultPort        = 24224
    44  	defaultProtocol    = "tcp"
    45  
    46  	// logger tries to reconnect 2**32 - 1 times
    47  	// failed (and panic) after 204 years [ 1.5 ** (2**32 - 1) - 1 seconds]
    48  	defaultMaxRetries = math.MaxInt32
    49  	defaultRetryWait  = 1000
    50  
    51  	addressKey            = "fluentd-address"
    52  	asyncKey              = "fluentd-async"
    53  	asyncConnectKey       = "fluentd-async-connect" // deprecated option (use fluent-async instead)
    54  	bufferLimitKey        = "fluentd-buffer-limit"
    55  	maxRetriesKey         = "fluentd-max-retries"
    56  	requestAckKey         = "fluentd-request-ack"
    57  	retryWaitKey          = "fluentd-retry-wait"
    58  	subSecondPrecisionKey = "fluentd-sub-second-precision"
    59  )
    60  
    61  func init() {
    62  	if err := logger.RegisterLogDriver(name, New); err != nil {
    63  		logrus.Fatal(err)
    64  	}
    65  	if err := logger.RegisterLogOptValidator(name, ValidateLogOpt); err != nil {
    66  		logrus.Fatal(err)
    67  	}
    68  }
    69  
    70  // New creates a fluentd logger using the configuration passed in on
    71  // the context. The supported context configuration variable is
    72  // fluentd-address.
    73  func New(info logger.Info) (logger.Logger, error) {
    74  	fluentConfig, err := parseConfig(info.Config)
    75  	if err != nil {
    76  		return nil, errdefs.InvalidParameter(err)
    77  	}
    78  
    79  	tag, err := loggerutils.ParseLogTag(info, loggerutils.DefaultTemplate)
    80  	if err != nil {
    81  		return nil, errdefs.InvalidParameter(err)
    82  	}
    83  
    84  	extra, err := info.ExtraAttributes(nil)
    85  	if err != nil {
    86  		return nil, errdefs.InvalidParameter(err)
    87  	}
    88  
    89  	logrus.WithField("container", info.ContainerID).WithField("config", fluentConfig).
    90  		Debug("logging driver fluentd configured")
    91  
    92  	log, err := fluent.New(fluentConfig)
    93  	if err != nil {
    94  		return nil, err
    95  	}
    96  	return &fluentd{
    97  		tag:           tag,
    98  		containerID:   info.ContainerID,
    99  		containerName: info.ContainerName,
   100  		writer:        log,
   101  		extra:         extra,
   102  	}, nil
   103  }
   104  
   105  func (f *fluentd) Log(msg *logger.Message) error {
   106  	data := map[string]string{
   107  		"container_id":   f.containerID,
   108  		"container_name": f.containerName,
   109  		"source":         msg.Source,
   110  		"log":            string(msg.Line),
   111  	}
   112  	for k, v := range f.extra {
   113  		data[k] = v
   114  	}
   115  	if msg.PLogMetaData != nil {
   116  		data["partial_message"] = "true"
   117  		data["partial_id"] = msg.PLogMetaData.ID
   118  		data["partial_ordinal"] = strconv.Itoa(msg.PLogMetaData.Ordinal)
   119  		data["partial_last"] = strconv.FormatBool(msg.PLogMetaData.Last)
   120  	}
   121  
   122  	ts := msg.Timestamp
   123  	logger.PutMessage(msg)
   124  	// fluent-logger-golang buffers logs from failures and disconnections,
   125  	// and these are transferred again automatically.
   126  	return f.writer.PostWithTime(f.tag, ts, data)
   127  }
   128  
   129  func (f *fluentd) Close() error {
   130  	return f.writer.Close()
   131  }
   132  
   133  func (f *fluentd) Name() string {
   134  	return name
   135  }
   136  
   137  // ValidateLogOpt looks for fluentd specific log option fluentd-address.
   138  func ValidateLogOpt(cfg map[string]string) error {
   139  	for key := range cfg {
   140  		switch key {
   141  		case "env":
   142  		case "env-regex":
   143  		case "labels":
   144  		case "labels-regex":
   145  		case "tag":
   146  
   147  		case addressKey:
   148  		case asyncKey:
   149  		case asyncConnectKey:
   150  		case bufferLimitKey:
   151  		case maxRetriesKey:
   152  		case requestAckKey:
   153  		case retryWaitKey:
   154  		case subSecondPrecisionKey:
   155  			// Accepted
   156  		default:
   157  			return errors.Errorf("unknown log opt '%s' for fluentd log driver", key)
   158  		}
   159  	}
   160  
   161  	_, err := parseConfig(cfg)
   162  	return err
   163  }
   164  
   165  func parseConfig(cfg map[string]string) (fluent.Config, error) {
   166  	var config fluent.Config
   167  
   168  	loc, err := parseAddress(cfg[addressKey])
   169  	if err != nil {
   170  		return config, err
   171  	}
   172  
   173  	bufferLimit := defaultBufferLimit
   174  	if cfg[bufferLimitKey] != "" {
   175  		bl64, err := units.RAMInBytes(cfg[bufferLimitKey])
   176  		if err != nil {
   177  			return config, err
   178  		}
   179  		bufferLimit = int(bl64)
   180  	}
   181  
   182  	retryWait := defaultRetryWait
   183  	if cfg[retryWaitKey] != "" {
   184  		rwd, err := time.ParseDuration(cfg[retryWaitKey])
   185  		if err != nil {
   186  			return config, err
   187  		}
   188  		retryWait = int(rwd.Seconds() * 1000)
   189  	}
   190  
   191  	maxRetries := defaultMaxRetries
   192  	if cfg[maxRetriesKey] != "" {
   193  		mr64, err := strconv.ParseUint(cfg[maxRetriesKey], 10, strconv.IntSize)
   194  		if err != nil {
   195  			return config, err
   196  		}
   197  		maxRetries = int(mr64)
   198  	}
   199  
   200  	if cfg[asyncKey] != "" && cfg[asyncConnectKey] != "" {
   201  		return config, errors.Errorf("conflicting options: cannot specify both '%s' and '%s", asyncKey, asyncConnectKey)
   202  	}
   203  
   204  	async := false
   205  	if cfg[asyncKey] != "" {
   206  		if async, err = strconv.ParseBool(cfg[asyncKey]); err != nil {
   207  			return config, err
   208  		}
   209  	}
   210  
   211  	// TODO fluentd-async-connect is deprecated in driver v1.4.0. Remove after two stable releases
   212  	asyncConnect := false
   213  	if cfg[asyncConnectKey] != "" {
   214  		if asyncConnect, err = strconv.ParseBool(cfg[asyncConnectKey]); err != nil {
   215  			return config, err
   216  		}
   217  	}
   218  
   219  	subSecondPrecision := false
   220  	if cfg[subSecondPrecisionKey] != "" {
   221  		if subSecondPrecision, err = strconv.ParseBool(cfg[subSecondPrecisionKey]); err != nil {
   222  			return config, err
   223  		}
   224  	}
   225  
   226  	requestAck := false
   227  	if cfg[requestAckKey] != "" {
   228  		if requestAck, err = strconv.ParseBool(cfg[requestAckKey]); err != nil {
   229  			return config, err
   230  		}
   231  	}
   232  
   233  	config = fluent.Config{
   234  		FluentPort:         loc.port,
   235  		FluentHost:         loc.host,
   236  		FluentNetwork:      loc.protocol,
   237  		FluentSocketPath:   loc.path,
   238  		BufferLimit:        bufferLimit,
   239  		RetryWait:          retryWait,
   240  		MaxRetry:           maxRetries,
   241  		Async:              async,
   242  		AsyncConnect:       asyncConnect,
   243  		SubSecondPrecision: subSecondPrecision,
   244  		RequestAck:         requestAck,
   245  	}
   246  
   247  	return config, nil
   248  }
   249  
   250  func parseAddress(address string) (*location, error) {
   251  	if address == "" {
   252  		return &location{
   253  			protocol: defaultProtocol,
   254  			host:     defaultHost,
   255  			port:     defaultPort,
   256  			path:     "",
   257  		}, nil
   258  	}
   259  
   260  	protocol := defaultProtocol
   261  	givenAddress := address
   262  	if urlutil.IsTransportURL(address) {
   263  		url, err := url.Parse(address)
   264  		if err != nil {
   265  			return nil, errors.Wrapf(err, "invalid fluentd-address %s", givenAddress)
   266  		}
   267  		// unix and unixgram socket
   268  		if url.Scheme == "unix" || url.Scheme == "unixgram" {
   269  			return &location{
   270  				protocol: url.Scheme,
   271  				host:     "",
   272  				port:     0,
   273  				path:     url.Path,
   274  			}, nil
   275  		}
   276  		// tcp|udp
   277  		protocol = url.Scheme
   278  		address = url.Host
   279  	}
   280  
   281  	host, port, err := net.SplitHostPort(address)
   282  	if err != nil {
   283  		if !strings.Contains(err.Error(), "missing port in address") {
   284  			return nil, errors.Wrapf(err, "invalid fluentd-address %s", givenAddress)
   285  		}
   286  		return &location{
   287  			protocol: protocol,
   288  			host:     host,
   289  			port:     defaultPort,
   290  			path:     "",
   291  		}, nil
   292  	}
   293  
   294  	portnum, err := strconv.Atoi(port)
   295  	if err != nil {
   296  		return nil, errors.Wrapf(err, "invalid fluentd-address %s", givenAddress)
   297  	}
   298  	return &location{
   299  		protocol: protocol,
   300  		host:     host,
   301  		port:     portnum,
   302  		path:     "",
   303  	}, nil
   304  }