github.com/honeycombio/honeytail@v1.9.0/parsers/htjson/htjson.go (about) 1 // Package htjson (honeytail-json, renamed to not conflict with the json module) 2 // parses logs that are one json blob per line. 3 package htjson 4 5 import ( 6 "encoding/json" 7 "strings" 8 "sync" 9 10 "github.com/sirupsen/logrus" 11 12 "github.com/honeycombio/honeytail/event" 13 "github.com/honeycombio/honeytail/httime" 14 "github.com/honeycombio/honeytail/parsers" 15 ) 16 17 type Options struct { 18 TimeFieldName string `long:"timefield" description:"Name of the field that contains a timestamp" yaml:"timefield,omitempty"` 19 TimeFieldFormat string `long:"format" description:"Format of the timestamp found in timefield (supports strftime and Golang time formats)" yaml:"format,omitempty"` 20 21 NumParsers int `hidden:"true" description:"number of htjson parsers to spin up" yaml:"-"` 22 } 23 24 type Parser struct { 25 conf Options 26 lineParser parsers.LineParser 27 28 warnedAboutTime bool 29 } 30 31 func (p *Parser) Init(options interface{}) error { 32 p.conf = *options.(*Options) 33 34 p.lineParser = &JSONLineParser{} 35 return nil 36 } 37 38 type JSONLineParser struct { 39 } 40 41 // ParseLine will unmarshal the thing it read in to detect errors in the JSON 42 // (by failing to parse) and give us an object that can be mutated by the 43 // various filters honeytail might apply. 44 func (j *JSONLineParser) ParseLine(line string) (map[string]interface{}, error) { 45 parsed := make(map[string]interface{}) 46 err := json.Unmarshal([]byte(line), &parsed) 47 return parsed, err 48 } 49 50 func (p *Parser) ProcessLines(lines <-chan string, send chan<- event.Event, prefixRegex *parsers.ExtRegexp) { 51 wg := sync.WaitGroup{} 52 numParsers := 1 53 if p.conf.NumParsers > 0 { 54 numParsers = p.conf.NumParsers 55 } 56 for i := 0; i < numParsers; i++ { 57 wg.Add(1) 58 go func() { 59 for line := range lines { 60 line = strings.TrimSpace(line) 61 logrus.WithFields(logrus.Fields{ 62 "line": line, 63 }).Debug("Attempting to process json log line") 64 65 // take care of any headers on the line 66 var prefixFields map[string]string 67 if prefixRegex != nil { 68 var prefix string 69 prefix, prefixFields = prefixRegex.FindStringSubmatchMap(line) 70 line = strings.TrimPrefix(line, prefix) 71 } 72 73 parsedLine, err := p.lineParser.ParseLine(line) 74 75 if err != nil { 76 // skip lines that won't parse 77 logrus.WithFields(logrus.Fields{ 78 "line": line, 79 "error": err, 80 }).Debug("skipping line; failed to parse.") 81 continue 82 } 83 timestamp := httime.GetTimestamp(parsedLine, p.conf.TimeFieldName, p.conf.TimeFieldFormat) 84 85 // merge the prefix fields and the parsed line contents 86 for k, v := range prefixFields { 87 parsedLine[k] = v 88 } 89 90 // send an event to Transmission 91 e := event.Event{ 92 Timestamp: timestamp, 93 Data: parsedLine, 94 } 95 send <- e 96 } 97 98 wg.Done() 99 }() 100 } 101 wg.Wait() 102 logrus.Debug("lines channel is closed, ending json processor") 103 }