github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/clients/pkg/promtail/targets/gcplog/push_translation.go (about)

     1  package gcplog
     2  
     3  import (
     4  	"encoding/base64"
     5  	"fmt"
     6  	"regexp"
     7  	"strings"
     8  	"time"
     9  
    10  	"github.com/prometheus/common/model"
    11  	"github.com/prometheus/prometheus/model/labels"
    12  	"github.com/prometheus/prometheus/model/relabel"
    13  
    14  	"github.com/grafana/loki/clients/pkg/promtail/api"
    15  	lokiClient "github.com/grafana/loki/clients/pkg/promtail/client"
    16  
    17  	"github.com/grafana/loki/pkg/logproto"
    18  	"github.com/grafana/loki/pkg/util"
    19  )
    20  
    21  // Configured as a global in this file to avoid recompiling this regex everywhere.
    22  var labelToLokiCompatible *regexp.Regexp
    23  
    24  func init() {
    25  	// TODO: Maybe use a regexp negative filter and grab everything non-alphanumeric non-underscore?
    26  	labelToLokiCompatible = regexp.MustCompile("[.-/]")
    27  }
    28  
    29  // PushMessage is the POST body format sent by GCP PubSub push subscriptions.
    30  type PushMessage struct {
    31  	Message struct {
    32  		Attributes       map[string]string `json:"attributes"`
    33  		Data             string            `json:"data"`
    34  		ID               string            `json:"message_id"`
    35  		PublishTimestamp string            `json:"publish_time"`
    36  	} `json:"message"`
    37  	Subscription string `json:"subscription"`
    38  }
    39  
    40  // translate converts a GCP PushMessage into a loki api.Entry. It is responsible for decoding the log line (contained in the Message.Data)
    41  // attribute, using the incoming timestamp if necessary, and formatting and passing down the incoming Message.Attributes
    42  // if relabel is configured.
    43  func translate(m PushMessage, other model.LabelSet, useIncomingTimestamp bool, relabelConfigs []*relabel.Config, xScopeOrgID string) (api.Entry, error) {
    44  	// mandatory label for gcplog
    45  	lbs := labels.NewBuilder(nil)
    46  	lbs.Set("__gcp_message_id", m.Message.ID)
    47  
    48  	// labels from gcp log entry. Add it as internal labels
    49  	for k, v := range m.Message.Attributes {
    50  		lbs.Set(fmt.Sprintf("__gcp_attributes_%s", convertToLokiCompatibleLabel(k)), v)
    51  	}
    52  
    53  	var processed labels.Labels
    54  
    55  	// apply relabeling
    56  	if len(relabelConfigs) > 0 {
    57  		processed = relabel.Process(lbs.Labels(), relabelConfigs...)
    58  	} else {
    59  		processed = lbs.Labels()
    60  	}
    61  
    62  	// final labelset that will be sent to loki
    63  	labels := make(model.LabelSet)
    64  	for _, lbl := range processed {
    65  		// ignore internal labels
    66  		if strings.HasPrefix(lbl.Name, "__") {
    67  			continue
    68  		}
    69  		// ignore invalid labels
    70  		if !model.LabelName(lbl.Name).IsValid() || !model.LabelValue(lbl.Value).IsValid() {
    71  			continue
    72  		}
    73  		labels[model.LabelName(lbl.Name)] = model.LabelValue(lbl.Value)
    74  	}
    75  
    76  	// add labels coming from scrapeconfig
    77  	labels = labels.Merge(other)
    78  
    79  	ts := time.Now()
    80  
    81  	decodedData, err := base64.StdEncoding.DecodeString(m.Message.Data)
    82  	if err != nil {
    83  		return api.Entry{}, fmt.Errorf("failed to decode data: %w", err)
    84  	}
    85  	line := string(decodedData)
    86  
    87  	if useIncomingTimestamp {
    88  		var err error
    89  		ts, err = time.Parse(time.RFC3339, m.Message.PublishTimestamp)
    90  		if err != nil {
    91  			return api.Entry{}, fmt.Errorf("invalid publish timestamp format: %w", err)
    92  		}
    93  	}
    94  
    95  	// If the incoming request carries the tenant id, inject it as the reserved label, so it's used by the
    96  	// remote write client.
    97  	if xScopeOrgID != "" {
    98  		labels[lokiClient.ReservedLabelTenantID] = model.LabelValue(xScopeOrgID)
    99  	}
   100  
   101  	return api.Entry{
   102  		Labels: labels,
   103  		Entry: logproto.Entry{
   104  			Timestamp: ts,
   105  			Line:      line,
   106  		},
   107  	}, nil
   108  }
   109  
   110  // convertToLokiCompatibleLabel converts an incoming GCP Push message label to a loki compatible format. There are lables
   111  // such as `logging.googleapis.com/timestamp`, which contain non-loki-compatible characters, which is just alphanumeric
   112  // and _. The approach taken is to translate every non-alphanumeric separator character to an underscore.
   113  func convertToLokiCompatibleLabel(label string) string {
   114  	// TODO: Since this is running for every incoming message, maybe it's more performant to do something
   115  	// like a loop over characters, and checking if it's not loki compatible, instead of a regexp.
   116  	return util.SnakeCase(labelToLokiCompatible.ReplaceAllString(label, "_"))
   117  }