github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/clients/pkg/promtail/targets/gcplog/formatter.go (about)

     1  package gcplog
     2  
     3  import (
     4  	"fmt"
     5  	"strings"
     6  	"time"
     7  
     8  	"cloud.google.com/go/pubsub"
     9  	json "github.com/json-iterator/go"
    10  	"github.com/prometheus/common/model"
    11  	"github.com/prometheus/prometheus/model/labels"
    12  	"github.com/prometheus/prometheus/model/relabel"
    13  
    14  	"github.com/grafana/loki/clients/pkg/promtail/api"
    15  
    16  	"github.com/grafana/loki/pkg/logproto"
    17  	"github.com/grafana/loki/pkg/util"
    18  )
    19  
    20  // LogEntry that will be written to the pubsub topic.
    21  // According to the following spec.
    22  // https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry
    23  // nolint:revive
    24  type GCPLogEntry struct {
    25  	LogName  string `json:"logName"`
    26  	Resource struct {
    27  		Type   string            `json:"type"`
    28  		Labels map[string]string `json:"labels"`
    29  	} `json:"resource"`
    30  	Timestamp string `json:"timestamp"`
    31  
    32  	// The time the log entry was received by Logging.
    33  	// Its important that `Timestamp` is optional in GCE log entry.
    34  	ReceiveTimestamp string `json:"receiveTimestamp"`
    35  
    36  	TextPayload string `json:"textPayload"`
    37  
    38  	// NOTE(kavi): There are other fields on GCPLogEntry. but we need only need above fields for now
    39  	// anyway we will be sending the entire entry to Loki.
    40  }
    41  
    42  func format(
    43  	m *pubsub.Message,
    44  	other model.LabelSet,
    45  	useIncomingTimestamp bool,
    46  	relabelConfig []*relabel.Config,
    47  ) (api.Entry, error) {
    48  	var ge GCPLogEntry
    49  
    50  	if err := json.Unmarshal(m.Data, &ge); err != nil {
    51  		return api.Entry{}, err
    52  	}
    53  
    54  	// mandatory label for gcplog
    55  	lbs := labels.NewBuilder(nil)
    56  	lbs.Set("__gcp_logname", ge.LogName)
    57  	lbs.Set("__gcp_resource_type", ge.Resource.Type)
    58  
    59  	// labels from gcp log entry. Add it as internal labels
    60  	for k, v := range ge.Resource.Labels {
    61  		lbs.Set("__gcp_resource_labels_"+util.SnakeCase(k), v)
    62  	}
    63  
    64  	var processed labels.Labels
    65  
    66  	// apply relabeling
    67  	if len(relabelConfig) > 0 {
    68  		processed = relabel.Process(lbs.Labels(), relabelConfig...)
    69  	} else {
    70  		processed = lbs.Labels()
    71  	}
    72  
    73  	// final labelset that will be sent to loki
    74  	labels := make(model.LabelSet)
    75  	for _, lbl := range processed {
    76  		// ignore internal labels
    77  		if strings.HasPrefix(lbl.Name, "__") {
    78  			continue
    79  		}
    80  		// ignore invalid labels
    81  		if !model.LabelName(lbl.Name).IsValid() || !model.LabelValue(lbl.Value).IsValid() {
    82  			continue
    83  		}
    84  		labels[model.LabelName(lbl.Name)] = model.LabelValue(lbl.Value)
    85  	}
    86  
    87  	// add labels coming from scrapeconfig
    88  	labels = labels.Merge(other)
    89  
    90  	ts := time.Now()
    91  	line := string(m.Data)
    92  
    93  	if useIncomingTimestamp {
    94  		tt := ge.Timestamp
    95  		if tt == "" {
    96  			tt = ge.ReceiveTimestamp
    97  		}
    98  		var err error
    99  		ts, err = time.Parse(time.RFC3339, tt)
   100  		if err != nil {
   101  			return api.Entry{}, fmt.Errorf("invalid timestamp format: %w", err)
   102  		}
   103  
   104  		if ts.IsZero() {
   105  			return api.Entry{}, fmt.Errorf("no timestamp found in the log entry")
   106  		}
   107  	}
   108  
   109  	// Send only `ge.textPaylload` as log line if its present.
   110  	if strings.TrimSpace(ge.TextPayload) != "" {
   111  		line = ge.TextPayload
   112  	}
   113  
   114  	return api.Entry{
   115  		Labels: labels,
   116  		Entry: logproto.Entry{
   117  			Timestamp: ts,
   118  			Line:      line,
   119  		},
   120  	}, nil
   121  }