github.com/honeycombio/honeytail@v1.9.0/parsers/htjson/htjson.go (about)

     1  // Package htjson (honeytail-json, renamed to not conflict with the json module)
     2  // parses logs that are one json blob per line.
     3  package htjson
     4  
     5  import (
     6  	"encoding/json"
     7  	"strings"
     8  	"sync"
     9  
    10  	"github.com/sirupsen/logrus"
    11  
    12  	"github.com/honeycombio/honeytail/event"
    13  	"github.com/honeycombio/honeytail/httime"
    14  	"github.com/honeycombio/honeytail/parsers"
    15  )
    16  
    17  type Options struct {
    18  	TimeFieldName   string `long:"timefield" description:"Name of the field that contains a timestamp" yaml:"timefield,omitempty"`
    19  	TimeFieldFormat string `long:"format" description:"Format of the timestamp found in timefield (supports strftime and Golang time formats)" yaml:"format,omitempty"`
    20  
    21  	NumParsers int `hidden:"true" description:"number of htjson parsers to spin up" yaml:"-"`
    22  }
    23  
    24  type Parser struct {
    25  	conf       Options
    26  	lineParser parsers.LineParser
    27  
    28  	warnedAboutTime bool
    29  }
    30  
    31  func (p *Parser) Init(options interface{}) error {
    32  	p.conf = *options.(*Options)
    33  
    34  	p.lineParser = &JSONLineParser{}
    35  	return nil
    36  }
    37  
    38  type JSONLineParser struct {
    39  }
    40  
    41  // ParseLine will unmarshal the thing it read in to detect errors in the JSON
    42  // (by failing to parse) and give us an object that can be mutated by the
    43  // various filters honeytail might apply.
    44  func (j *JSONLineParser) ParseLine(line string) (map[string]interface{}, error) {
    45  	parsed := make(map[string]interface{})
    46  	err := json.Unmarshal([]byte(line), &parsed)
    47  	return parsed, err
    48  }
    49  
    50  func (p *Parser) ProcessLines(lines <-chan string, send chan<- event.Event, prefixRegex *parsers.ExtRegexp) {
    51  	wg := sync.WaitGroup{}
    52  	numParsers := 1
    53  	if p.conf.NumParsers > 0 {
    54  		numParsers = p.conf.NumParsers
    55  	}
    56  	for i := 0; i < numParsers; i++ {
    57  		wg.Add(1)
    58  		go func() {
    59  			for line := range lines {
    60  				line = strings.TrimSpace(line)
    61  				logrus.WithFields(logrus.Fields{
    62  					"line": line,
    63  				}).Debug("Attempting to process json log line")
    64  
    65  				// take care of any headers on the line
    66  				var prefixFields map[string]string
    67  				if prefixRegex != nil {
    68  					var prefix string
    69  					prefix, prefixFields = prefixRegex.FindStringSubmatchMap(line)
    70  					line = strings.TrimPrefix(line, prefix)
    71  				}
    72  
    73  				parsedLine, err := p.lineParser.ParseLine(line)
    74  
    75  				if err != nil {
    76  					// skip lines that won't parse
    77  					logrus.WithFields(logrus.Fields{
    78  						"line":  line,
    79  						"error": err,
    80  					}).Debug("skipping line; failed to parse.")
    81  					continue
    82  				}
    83  				timestamp := httime.GetTimestamp(parsedLine, p.conf.TimeFieldName, p.conf.TimeFieldFormat)
    84  
    85  				// merge the prefix fields and the parsed line contents
    86  				for k, v := range prefixFields {
    87  					parsedLine[k] = v
    88  				}
    89  
    90  				// send an event to Transmission
    91  				e := event.Event{
    92  					Timestamp: timestamp,
    93  					Data:      parsedLine,
    94  				}
    95  				send <- e
    96  			}
    97  
    98  			wg.Done()
    99  		}()
   100  	}
   101  	wg.Wait()
   102  	logrus.Debug("lines channel is closed, ending json processor")
   103  }