github.com/go-graphite/carbonapi@v0.17.0/zipper/helper/requests.go (about)

     1  package helper
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"io"
     7  	"net/http"
     8  	"net/url"
     9  	"strings"
    10  	"sync/atomic"
    11  	"unicode/utf8"
    12  
    13  	"github.com/ansel1/merry"
    14  	"go.uber.org/zap"
    15  
    16  	"github.com/go-graphite/carbonapi/limiter"
    17  	util "github.com/go-graphite/carbonapi/util/ctx"
    18  	"github.com/go-graphite/carbonapi/zipper/types"
    19  )
    20  
    21  func min(a, b int) int {
    22  	if a < b {
    23  		return a
    24  	}
    25  	return b
    26  }
    27  
    28  const (
    29  	htmlTagStart = 60 // Unicode `<`
    30  	htmlTagEnd   = 62 // Unicode `>`
    31  )
    32  
    33  // Aggressively strips HTML tags from a string.
    34  // It will only keep anything between `>` and `<`.
    35  func stripHtmlTags(s string, maxLen int) string {
    36  	var n int
    37  	if !strings.Contains(s, "<html>") {
    38  		if maxLen == 0 || maxLen > len(s) {
    39  			return s
    40  		}
    41  		return s[:maxLen]
    42  	}
    43  	// Setup a string builder and allocate enough memory for the new string.
    44  	var builder strings.Builder
    45  	if maxLen == 0 {
    46  		n = len(s) + utf8.UTFMax
    47  	} else {
    48  		n = min(len(s), maxLen)
    49  	}
    50  
    51  	builder.Grow(n)
    52  
    53  	in := false // True if we are inside an HTML tag.
    54  	start := 0  // The index of the previous start tag character `<`
    55  	end := 0    // The index of the previous end tag character `>`
    56  
    57  	for i, c := range s {
    58  		// If this is the last character and we are not in an HTML tag, save it.
    59  		if (i+1) == len(s) && end >= start {
    60  			builder.WriteString(s[end:])
    61  		}
    62  
    63  		if c == htmlTagStart {
    64  			// Only update the start if we are not in a tag.
    65  			// This make sure we strip out `<<br>` not just `<br>`
    66  			if !in {
    67  				start = i
    68  			}
    69  			in = true
    70  
    71  			// Write the valid string between the close and start of the two tags.
    72  			builder.WriteString(s[end:start])
    73  			end = i + 1
    74  		} else if c == htmlTagEnd {
    75  			in = false
    76  			end = i + 1
    77  		}
    78  	}
    79  	s = strings.Trim(builder.String(), "\r\n")
    80  	return s
    81  }
    82  
    83  type ServerResponse struct {
    84  	Server   string
    85  	Response []byte
    86  }
    87  
    88  type HttpQuery struct {
    89  	groupName string
    90  	servers   []string
    91  	maxTries  int
    92  	limiter   limiter.ServerLimiter
    93  	client    *http.Client
    94  	encoding  string
    95  
    96  	counter uint64
    97  }
    98  
    99  func NewHttpQuery(groupName string, servers []string, maxTries int, limiter limiter.ServerLimiter, client *http.Client, encoding string) *HttpQuery {
   100  	return &HttpQuery{
   101  		groupName: groupName,
   102  		servers:   servers,
   103  		maxTries:  maxTries,
   104  		limiter:   limiter,
   105  		client:    client,
   106  		encoding:  encoding,
   107  	}
   108  }
   109  
   110  func (c *HttpQuery) pickServer(logger *zap.Logger) string {
   111  	if len(c.servers) == 1 {
   112  		// No need to do heavy operations here
   113  		return c.servers[0]
   114  	}
   115  	logger = logger.With(zap.String("function", "picker"))
   116  	counter := atomic.AddUint64(&(c.counter), 1)
   117  	idx := counter % uint64(len(c.servers))
   118  	srv := c.servers[int(idx)]
   119  	logger.Debug("picked",
   120  		zap.Uint64("counter", counter),
   121  		zap.Uint64("idx", idx),
   122  		zap.String("server", srv),
   123  	)
   124  
   125  	return srv
   126  }
   127  
   128  func (c *HttpQuery) doRequest(ctx context.Context, logger *zap.Logger, server, uri string, r types.Request) (*ServerResponse, merry.Error) {
   129  	logger = logger.With(
   130  		zap.String("function", "HttpQuery.doRequest"),
   131  	)
   132  
   133  	u, err := url.Parse(server + uri)
   134  	if err != nil {
   135  		return nil, merry.Here(err).WithValue("server", server)
   136  	}
   137  
   138  	var reader io.Reader
   139  	var body []byte
   140  	if r != nil {
   141  		body, err = r.Marshal()
   142  		if err != nil {
   143  			return nil, merry.Here(err).WithValue("server", server)
   144  		}
   145  		if body != nil {
   146  			reader = bytes.NewReader(body)
   147  		}
   148  	}
   149  	logger = logger.With(
   150  		zap.String("server", server),
   151  		zap.String("name", c.groupName),
   152  		zap.String("uri", u.String()),
   153  	)
   154  
   155  	// TODO: change to NewRequestWithContext
   156  	req, err := http.NewRequest("GET", u.String(), reader)
   157  	if err != nil {
   158  		return nil, merry.Here(err).WithValue("server", server)
   159  	}
   160  
   161  	req.Header.Set("Accept", c.encoding)
   162  	req = util.MarshalPassHeaders(ctx, util.MarshalCtx(ctx, util.MarshalCtx(ctx, req, util.HeaderUUIDZipper), util.HeaderUUIDAPI))
   163  
   164  	logger.Debug("trying to get slot",
   165  		zap.String("name", server),
   166  	)
   167  	err = c.limiter.Enter(ctx, server)
   168  	if err != nil {
   169  		logger.Debug("timeout waiting for a slot")
   170  		return nil, merry.Here(err).WithValue("server", server)
   171  	}
   172  
   173  	defer c.limiter.Leave(ctx, server)
   174  
   175  	logger.Debug("got slot for server",
   176  		zap.String("name", server),
   177  	)
   178  
   179  	if r != nil {
   180  		logger = logger.With(zap.Any("payloadData", r.LogInfo()))
   181  	}
   182  	resp, err := c.client.Do(req.WithContext(ctx))
   183  	if err != nil {
   184  		logger.Debug("error fetching result",
   185  			zap.Error(err),
   186  		)
   187  
   188  		return nil, requestError(err, server)
   189  
   190  	}
   191  	defer func() {
   192  		_ = resp.Body.Close()
   193  	}()
   194  
   195  	// we don't need to process any further if the response is empty.
   196  	if resp.StatusCode == http.StatusNotFound {
   197  		return &ServerResponse{Server: server}, nil
   198  	}
   199  
   200  	body, err = io.ReadAll(resp.Body)
   201  	if err != nil {
   202  		logger.Debug("error reading body",
   203  			zap.Error(err),
   204  		)
   205  		return nil, merry.Here(err).WithValue("server", server)
   206  	}
   207  
   208  	if resp.StatusCode != http.StatusOK {
   209  		return nil, types.ErrFailedToFetch.WithValue("server", server).WithMessage(string(body)).WithHTTPCode(resp.StatusCode)
   210  	}
   211  
   212  	return &ServerResponse{Server: server, Response: body}, nil
   213  }
   214  
   215  func (c *HttpQuery) DoQuery(ctx context.Context, logger *zap.Logger, uri string, r types.Request) (resp *ServerResponse, err merry.Error) {
   216  	maxTries := c.maxTries
   217  	if len(c.servers) > maxTries {
   218  		maxTries = len(c.servers)
   219  	}
   220  
   221  	e := types.ErrFailedToFetch.WithValue("uri", uri)
   222  	code := http.StatusInternalServerError
   223  	for try := 0; try < maxTries; try++ {
   224  		server := c.pickServer(logger)
   225  		res, err := c.doRequest(ctx, logger, server, uri, r)
   226  		if err != nil {
   227  			logger.Debug("have errors",
   228  				zap.String("error", err.Error()),
   229  				zap.String("server", server),
   230  			)
   231  
   232  			e = e.WithCause(err).WithHTTPCode(merry.HTTPCode(err))
   233  			code = merry.HTTPCode(err)
   234  			// TODO (msaf1980): may be metric for server failures ?
   235  			// TODO (msaf1980): may be retry policy for avoid retry bad queries ?
   236  			continue
   237  		}
   238  
   239  		return res, nil
   240  	}
   241  
   242  	return nil, types.ErrMaxTriesExceeded.WithCause(e).WithHTTPCode(code)
   243  }
   244  
   245  func (c *HttpQuery) DoQueryToAll(ctx context.Context, logger *zap.Logger, uri string, r types.Request) (resp []*ServerResponse, err merry.Error) {
   246  	maxTries := c.maxTries
   247  	if len(c.servers) > maxTries {
   248  		maxTries = len(c.servers)
   249  	}
   250  
   251  	res := make([]*ServerResponse, len(c.servers))
   252  	e := types.ErrFailedToFetch.WithValue("uri", uri)
   253  	responseCount := 0
   254  	code := http.StatusInternalServerError
   255  	for i := range c.servers {
   256  		for try := 0; try < maxTries; try++ {
   257  			response, err := c.doRequest(ctx, logger, c.servers[i], uri, r)
   258  			if err != nil {
   259  				logger.Debug("have errors",
   260  					zap.Error(err),
   261  				)
   262  
   263  				e = e.WithCause(err)
   264  				code = merry.HTTPCode(err)
   265  				continue
   266  			}
   267  
   268  			res[i] = response
   269  			responseCount++
   270  			break
   271  		}
   272  	}
   273  
   274  	if responseCount == len(c.servers) {
   275  		return res, nil
   276  	}
   277  
   278  	return res, types.ErrMaxTriesExceeded.WithCause(e).WithHTTPCode(code)
   279  }