github.com/webonyx/up@v0.7.4-0.20180808230834-91b94e551323/platform/aws/logs/logs.go (about)

     1  // Package logs provides log management for AWS platforms.
     2  package logs
     3  
     4  import (
     5  	"encoding/json"
     6  	"io"
     7  	"os"
     8  	"strings"
     9  	"time"
    10  
    11  	"github.com/apex/log"
    12  	jsonlog "github.com/apex/log/handlers/json"
    13  	"github.com/apex/up"
    14  
    15  	"github.com/aws/aws-sdk-go/aws"
    16  	"github.com/aws/aws-sdk-go/aws/session"
    17  	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
    18  	"github.com/tj/aws/logs"
    19  
    20  	"github.com/apex/up/internal/logs/parser"
    21  	"github.com/apex/up/internal/logs/text"
    22  	"github.com/apex/up/internal/util"
    23  )
    24  
    25  // Logs implementation.
    26  type Logs struct {
    27  	up.LogsConfig
    28  	group string
    29  	query string
    30  	w     io.WriteCloser
    31  	io.Reader
    32  }
    33  
    34  // New log tailer.
    35  func New(group string, c up.LogsConfig) up.Logs {
    36  	r, w := io.Pipe()
    37  
    38  	query, err := parseQuery(c.Query)
    39  	if err != nil {
    40  		w.CloseWithError(err)
    41  	}
    42  	log.Debugf("query %q", query)
    43  
    44  	l := &Logs{
    45  		LogsConfig: c,
    46  		query:      query,
    47  		group:      group,
    48  		Reader:     r,
    49  		w:          w,
    50  	}
    51  
    52  	go l.start()
    53  
    54  	return l
    55  }
    56  
    57  // start fetching logs.
    58  func (l *Logs) start() {
    59  	tailer := logs.New(logs.Config{
    60  		Service:       cloudwatchlogs.New(session.New(aws.NewConfig().WithRegion(l.Region))),
    61  		StartTime:     l.Since,
    62  		PollInterval:  2 * time.Second,
    63  		Follow:        l.Follow,
    64  		FilterPattern: l.query,
    65  		GroupNames:    []string{l.group},
    66  	})
    67  
    68  	var handler log.Handler
    69  
    70  	if l.OutputJSON {
    71  		handler = jsonlog.New(os.Stdout)
    72  	} else {
    73  		handler = text.New(os.Stdout).WithExpandedFields(l.Expand)
    74  	}
    75  
    76  	// TODO: transform to reader of nl-delimited json, move to apex/log?
    77  	// TODO: marshal/unmarshal as JSON so that numeric values are always float64... remove util.ToFloat()
    78  	for l := range tailer.Start() {
    79  		line := strings.TrimSpace(l.Message)
    80  
    81  		// json log
    82  		if util.IsJSONLog(line) {
    83  			var e log.Entry
    84  			err := json.Unmarshal([]byte(line), &e)
    85  			if err != nil {
    86  				log.Fatalf("error parsing json: %s", err)
    87  			}
    88  
    89  			handler.HandleLog(&e)
    90  			continue
    91  		}
    92  
    93  		// skip START / END logs since they are redundant
    94  		if skippable(l.Message) {
    95  			continue
    96  		}
    97  
    98  		// lambda textual logs
    99  		handler.HandleLog(&log.Entry{
   100  			Timestamp: l.Timestamp,
   101  			Level:     log.InfoLevel,
   102  			Message:   strings.TrimRight(l.Message, " \n"),
   103  		})
   104  	}
   105  
   106  	// TODO: refactor interface to delegate
   107  	if err := tailer.Err(); err != nil {
   108  		panic(err)
   109  	}
   110  
   111  	l.w.Close()
   112  }
   113  
   114  // parseQuery parses and converts the query to a CW friendly syntax.
   115  func parseQuery(s string) (string, error) {
   116  	if s == "" {
   117  		return s, nil
   118  	}
   119  
   120  	n, err := parser.Parse(s)
   121  	if err != nil {
   122  		return "", err
   123  	}
   124  
   125  	return n.String(), nil
   126  }
   127  
   128  // skippable returns true if the message is skippable.
   129  func skippable(s string) bool {
   130  	return strings.Contains(s, "END RequestId") ||
   131  		strings.Contains(s, "START RequestId")
   132  }