github.com/GuanceCloud/cliutils@v1.1.21/pipeline/offload/receiver_datakit_http.go (about)

     1  // Unless explicitly stated otherwise all files in this repository are licensed
     2  // under the MIT License.
     3  // This product includes software developed at Guance Cloud (https://www.guance.com/).
     4  // Copyright 2021-present Guance, Inc.
     5  
     6  // Package offload offload data to other data sinks
     7  package offload
     8  
     9  import (
    10  	"bytes"
    11  	"fmt"
    12  	"net"
    13  	"net/http"
    14  	"net/url"
    15  	"sync"
    16  	"time"
    17  
    18  	"github.com/GuanceCloud/cliutils/point"
    19  	"github.com/avast/retry-go"
    20  )
    21  
    22  const DKRcv = "datakit-http"
    23  
    24  func sendReq(req *http.Request, cli *http.Client) (resp *http.Response, err error) {
    25  	if err := retry.Do(
    26  		func() error {
    27  			resp, err = cli.Do(req)
    28  			if err != nil {
    29  				return err
    30  			}
    31  			if resp.StatusCode/100 == 5 { // server-side error
    32  				return fmt.Errorf("doSendReq: %s", resp.Status)
    33  			}
    34  			return nil
    35  		},
    36  
    37  		retry.Attempts(4),
    38  		retry.Delay(time.Second*1),
    39  		retry.OnRetry(func(n uint, err error) {
    40  			l.Warnf("on %dth retry, error: %s", n, err)
    41  		}),
    42  	); err != nil {
    43  		return resp, err
    44  	}
    45  
    46  	return resp, err
    47  }
    48  
    49  type DKRecver struct {
    50  	httpCli *http.Client
    51  
    52  	Addresses []string
    53  	AddrMap   []map[point.Category]string
    54  
    55  	sync.Mutex
    56  }
    57  
    58  func NewDKRecver(addresses []string) (*DKRecver, error) {
    59  	receiver := &DKRecver{
    60  		httpCli: &http.Client{
    61  			Transport: &http.Transport{
    62  				DialContext: (&net.Dialer{
    63  					Timeout:   time.Second * 30,
    64  					KeepAlive: time.Second * 90,
    65  				}).DialContext,
    66  				MaxIdleConns:          100,
    67  				MaxConnsPerHost:       64,
    68  				IdleConnTimeout:       time.Second * 90,
    69  				TLSHandshakeTimeout:   time.Second * 10,
    70  				ExpectContinueTimeout: time.Second,
    71  			},
    72  		},
    73  	}
    74  
    75  	receiver.Addresses = append([]string{}, addresses...)
    76  	receiver.AddrMap = make([]map[point.Category]string, len(addresses))
    77  
    78  	allCat := point.AllCategories()
    79  
    80  	for i, addr := range addresses {
    81  		u, err := url.Parse(addr)
    82  		if err != nil {
    83  			return nil, fmt.Errorf("parse url '%s' failed: %w", addr, err)
    84  		}
    85  		receiver.AddrMap[i] = map[point.Category]string{}
    86  		for _, cat := range allCat {
    87  			receiver.AddrMap[i][cat] = fmt.Sprintf("%s://%s%s",
    88  				u.Scheme, u.Host, cat.URL())
    89  		}
    90  	}
    91  
    92  	return receiver, nil
    93  }
    94  
    95  const batchSize = 128
    96  
    97  func (recevier *DKRecver) Send(s uint64, cat point.Category, data []*point.Point) error {
    98  	if len(data) == 0 {
    99  		return nil
   100  	}
   101  
   102  	i := s % (uint64)(len(recevier.AddrMap))
   103  	addr := recevier.AddrMap[i][cat]
   104  
   105  	if len(recevier.AddrMap) == 0 {
   106  		return fmt.Errorf("no server address")
   107  	}
   108  
   109  	enc := point.GetEncoder(point.WithEncEncoding(point.LineProtocol),
   110  		point.WithEncBatchSize(batchSize))
   111  	defer point.PutEncoder(enc)
   112  
   113  	dataList, err := enc.Encode(data)
   114  	if err != nil {
   115  		return err
   116  	}
   117  
   118  	if len(dataList) == 0 {
   119  		return nil
   120  	}
   121  
   122  	for _, d := range dataList {
   123  		buffer := bytes.NewReader(d)
   124  		req, err := http.NewRequest("POST", addr, buffer)
   125  		if err != nil {
   126  			return err
   127  		}
   128  
   129  		resp, err := sendReq(req, recevier.httpCli)
   130  		if err != nil {
   131  			return err
   132  		}
   133  		defer resp.Body.Close() //nolint:errcheck
   134  
   135  		if resp.StatusCode != http.StatusOK {
   136  			r := make([]byte, 256)
   137  			_, _ = resp.Body.Read(r)
   138  			return fmt.Errorf("lastErrPostURL, http status code: %d, body: %s", resp.StatusCode, string(r))
   139  		}
   140  	}
   141  
   142  	return nil
   143  }