github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/clients/pkg/promtail/targets/gcplog/pull_target.go (about)

     1  package gcplog
     2  
     3  import (
     4  	"context"
     5  	"sync"
     6  
     7  	"cloud.google.com/go/pubsub"
     8  	"github.com/go-kit/log"
     9  	"github.com/go-kit/log/level"
    10  	"github.com/prometheus/common/model"
    11  	"github.com/prometheus/prometheus/model/relabel"
    12  
    13  	"github.com/grafana/loki/clients/pkg/promtail/api"
    14  	"github.com/grafana/loki/clients/pkg/promtail/scrapeconfig"
    15  	"github.com/grafana/loki/clients/pkg/promtail/targets/target"
    16  )
    17  
    18  // pullTarget represents the target specific to GCP project, with a pull subscription type.
    19  // It collects logs from GCP and push it to Loki.
    20  // nolint:revive
    21  type pullTarget struct {
    22  	metrics       *Metrics
    23  	logger        log.Logger
    24  	handler       api.EntryHandler
    25  	config        *scrapeconfig.GcplogTargetConfig
    26  	relabelConfig []*relabel.Config
    27  	jobName       string
    28  
    29  	// lifecycle management
    30  	ctx    context.Context
    31  	cancel context.CancelFunc
    32  	wg     sync.WaitGroup
    33  
    34  	// pubsub
    35  	ps   *pubsub.Client
    36  	msgs chan *pubsub.Message
    37  }
    38  
    39  // newPullTarget returns the new instance of pullTarget for
    40  // the given `project-id`. It scraps logs from the GCP project
    41  // and push it Loki via given `api.EntryHandler.`
    42  // It starts the `run` loop to consume log entries that can be
    43  // stopped via `target.Stop()`
    44  // nolint:revive,govet
    45  func newPullTarget(
    46  	metrics *Metrics,
    47  	logger log.Logger,
    48  	handler api.EntryHandler,
    49  	relabel []*relabel.Config,
    50  	jobName string,
    51  	config *scrapeconfig.GcplogTargetConfig,
    52  ) (*pullTarget, error) {
    53  	ctx, cancel := context.WithCancel(context.Background())
    54  
    55  	ps, err := pubsub.NewClient(ctx, config.ProjectID)
    56  	if err != nil {
    57  		return nil, err
    58  	}
    59  
    60  	target := &pullTarget{
    61  		metrics:       metrics,
    62  		logger:        logger,
    63  		handler:       handler,
    64  		relabelConfig: relabel,
    65  		config:        config,
    66  		jobName:       jobName,
    67  		ctx:           ctx,
    68  		cancel:        cancel,
    69  		ps:            ps,
    70  		msgs:          make(chan *pubsub.Message),
    71  	}
    72  
    73  	go func() {
    74  		_ = target.run()
    75  	}()
    76  
    77  	return target, nil
    78  }
    79  
    80  func (t *pullTarget) run() error {
    81  	t.wg.Add(1)
    82  	defer t.wg.Done()
    83  
    84  	send := t.handler.Chan()
    85  
    86  	sub := t.ps.SubscriptionInProject(t.config.Subscription, t.config.ProjectID)
    87  	go func() {
    88  		// NOTE(kavi): `cancel` the context as exiting from this goroutine should stop main `run` loop
    89  		// It makesense as no more messages will be received.
    90  		defer t.cancel()
    91  
    92  		err := sub.Receive(t.ctx, func(ctx context.Context, m *pubsub.Message) {
    93  			t.msgs <- m
    94  		})
    95  		if err != nil {
    96  			level.Error(t.logger).Log("msg", "failed to receive pubsub messages", "error", err)
    97  			t.metrics.gcplogErrors.WithLabelValues(t.config.ProjectID).Inc()
    98  			t.metrics.gcplogTargetLastSuccessScrape.WithLabelValues(t.config.ProjectID, t.config.Subscription).SetToCurrentTime()
    99  		}
   100  	}()
   101  
   102  	for {
   103  		select {
   104  		case <-t.ctx.Done():
   105  			return t.ctx.Err()
   106  		case m := <-t.msgs:
   107  			entry, err := format(m, t.config.Labels, t.config.UseIncomingTimestamp, t.relabelConfig)
   108  			if err != nil {
   109  				level.Error(t.logger).Log("event", "error formating log entry", "cause", err)
   110  				m.Ack()
   111  				break
   112  			}
   113  			send <- entry
   114  			m.Ack() // Ack only after log is sent.
   115  			t.metrics.gcplogEntries.WithLabelValues(t.config.ProjectID).Inc()
   116  		}
   117  	}
   118  }
   119  
   120  func (t *pullTarget) Type() target.TargetType {
   121  	return target.GcplogTargetType
   122  }
   123  
   124  func (t *pullTarget) Ready() bool {
   125  	// Return true just like all other targets.
   126  	// Rationale is gcplog scraping shouldn't stop because of some transient timeout errors.
   127  	// This transient failure can cause promtail readyness probe to fail which may prevent pod from starting.
   128  	// We have metrics now to track if scraping failed (`gcplog_target_last_success_scrape`).
   129  	return true
   130  }
   131  
   132  func (t *pullTarget) DiscoveredLabels() model.LabelSet {
   133  	return nil
   134  }
   135  
   136  func (t *pullTarget) Labels() model.LabelSet {
   137  	return t.config.Labels
   138  }
   139  
   140  func (t *pullTarget) Details() interface{} {
   141  	return nil
   142  }
   143  
   144  func (t *pullTarget) Stop() error {
   145  	t.cancel()
   146  	t.wg.Wait()
   147  	t.handler.Stop()
   148  	return nil
   149  }