github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/querier/worker/frontend_processor.go (about)

     1  package worker
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"net/http"
     7  	"time"
     8  
     9  	"github.com/go-kit/log"
    10  	"github.com/go-kit/log/level"
    11  	"github.com/grafana/dskit/backoff"
    12  	"github.com/weaveworks/common/httpgrpc"
    13  	"google.golang.org/grpc"
    14  
    15  	"github.com/grafana/loki/pkg/lokifrontend/frontend/v1/frontendv1pb"
    16  	querier_stats "github.com/grafana/loki/pkg/querier/stats"
    17  )
    18  
    19  var (
    20  	processorBackoffConfig = backoff.Config{
    21  		MinBackoff: 500 * time.Millisecond,
    22  		MaxBackoff: 5 * time.Second,
    23  	}
    24  )
    25  
    26  func newFrontendProcessor(cfg Config, handler RequestHandler, log log.Logger) processor {
    27  	return &frontendProcessor{
    28  		log:            log,
    29  		handler:        handler,
    30  		maxMessageSize: cfg.GRPCClientConfig.MaxSendMsgSize,
    31  		querierID:      cfg.QuerierID,
    32  	}
    33  }
    34  
    35  // Handles incoming queries from frontend.
    36  type frontendProcessor struct {
    37  	handler        RequestHandler
    38  	maxMessageSize int
    39  	querierID      string
    40  
    41  	log log.Logger
    42  }
    43  
    44  // notifyShutdown implements processor.
    45  func (fp *frontendProcessor) notifyShutdown(ctx context.Context, conn *grpc.ClientConn, address string) {
    46  	client := frontendv1pb.NewFrontendClient(conn)
    47  
    48  	req := &frontendv1pb.NotifyClientShutdownRequest{ClientID: fp.querierID}
    49  	if _, err := client.NotifyClientShutdown(ctx, req); err != nil {
    50  		// Since we're shutting down there's nothing we can do except logging it.
    51  		level.Warn(fp.log).Log("msg", "failed to notify querier shutdown to query-frontend", "address", address, "err", err)
    52  	}
    53  }
    54  
    55  // runOne loops, trying to establish a stream to the frontend to begin request processing.
    56  func (fp *frontendProcessor) processQueriesOnSingleStream(ctx context.Context, conn *grpc.ClientConn, address string) {
    57  	client := frontendv1pb.NewFrontendClient(conn)
    58  
    59  	backoff := backoff.New(ctx, processorBackoffConfig)
    60  	for backoff.Ongoing() {
    61  		c, err := client.Process(ctx)
    62  		if err != nil {
    63  			level.Error(fp.log).Log("msg", "error contacting frontend", "address", address, "err", err)
    64  			backoff.Wait()
    65  			continue
    66  		}
    67  
    68  		if err := fp.process(c); err != nil {
    69  			level.Error(fp.log).Log("msg", "error processing requests", "address", address, "err", err)
    70  			backoff.Wait()
    71  			continue
    72  		}
    73  
    74  		backoff.Reset()
    75  	}
    76  }
    77  
    78  // process loops processing requests on an established stream.
    79  func (fp *frontendProcessor) process(c frontendv1pb.Frontend_ProcessClient) error {
    80  	// Build a child context so we can cancel a query when the stream is closed.
    81  	ctx, cancel := context.WithCancel(c.Context())
    82  	defer cancel()
    83  
    84  	for {
    85  		request, err := c.Recv()
    86  		if err != nil {
    87  			return err
    88  		}
    89  
    90  		switch request.Type {
    91  		case frontendv1pb.HTTP_REQUEST:
    92  			// Handle the request on a "background" goroutine, so we go back to
    93  			// blocking on c.Recv().  This allows us to detect the stream closing
    94  			// and cancel the query.  We don't actually handle queries in parallel
    95  			// here, as we're running in lock step with the server - each Recv is
    96  			// paired with a Send.
    97  			go fp.runRequest(ctx, request.HttpRequest, request.StatsEnabled, func(response *httpgrpc.HTTPResponse, stats *querier_stats.Stats) error {
    98  				return c.Send(&frontendv1pb.ClientToFrontend{
    99  					HttpResponse: response,
   100  					Stats:        stats,
   101  				})
   102  			})
   103  
   104  		case frontendv1pb.GET_ID:
   105  			err := c.Send(&frontendv1pb.ClientToFrontend{ClientID: fp.querierID})
   106  			if err != nil {
   107  				return err
   108  			}
   109  
   110  		default:
   111  			return fmt.Errorf("unknown request type: %v", request.Type)
   112  		}
   113  	}
   114  }
   115  
   116  func (fp *frontendProcessor) runRequest(ctx context.Context, request *httpgrpc.HTTPRequest, statsEnabled bool, sendHTTPResponse func(response *httpgrpc.HTTPResponse, stats *querier_stats.Stats) error) {
   117  	var stats *querier_stats.Stats
   118  	if statsEnabled {
   119  		stats, ctx = querier_stats.ContextWithEmptyStats(ctx)
   120  	}
   121  
   122  	response, err := fp.handler.Handle(ctx, request)
   123  	if err != nil {
   124  		var ok bool
   125  		response, ok = httpgrpc.HTTPResponseFromError(err)
   126  		if !ok {
   127  			response = &httpgrpc.HTTPResponse{
   128  				Code: http.StatusInternalServerError,
   129  				Body: []byte(err.Error()),
   130  			}
   131  		}
   132  	}
   133  
   134  	// Ensure responses that are too big are not retried.
   135  	if len(response.Body) >= fp.maxMessageSize {
   136  		errMsg := fmt.Sprintf("response larger than the max (%d vs %d)", len(response.Body), fp.maxMessageSize)
   137  		response = &httpgrpc.HTTPResponse{
   138  			Code: http.StatusRequestEntityTooLarge,
   139  			Body: []byte(errMsg),
   140  		}
   141  		level.Error(fp.log).Log("msg", "error processing query", "err", errMsg)
   142  	}
   143  
   144  	if err := sendHTTPResponse(response, stats); err != nil {
   145  		level.Error(fp.log).Log("msg", "error processing requests", "err", err)
   146  	}
   147  }