github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/sink/pulsar/config.go (about)

     1  // Copyright 2023 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package pulsar
    15  
    16  import (
    17  	"fmt"
    18  	"net/url"
    19  
    20  	"github.com/pingcap/log"
    21  	"github.com/pingcap/tiflow/pkg/config"
    22  	"go.uber.org/zap"
    23  )
    24  
    25  // sink config default Value
    26  const (
    27  	defaultConnectionTimeout = 5 // 5s
    28  
    29  	defaultOperationTimeout = 30 // 30s
    30  
    31  	defaultBatchingMaxSize = uint(1000)
    32  
    33  	defaultBatchingMaxPublishDelay = 10 // 10ms
    34  
    35  	// defaultSendTimeout 30s
    36  	defaultSendTimeout = 30 // 30s
    37  
    38  )
    39  
    40  func checkSinkURI(sinkURI *url.URL) error {
    41  	if sinkURI.Scheme == "" {
    42  		return fmt.Errorf("scheme is empty")
    43  	}
    44  	if sinkURI.Host == "" {
    45  		return fmt.Errorf("host is empty")
    46  	}
    47  	if sinkURI.Path == "" {
    48  		return fmt.Errorf("path is empty")
    49  	}
    50  	return nil
    51  }
    52  
    53  // NewPulsarConfig new pulsar config
    54  // TODO(dongmen): make this method more concise.
    55  func NewPulsarConfig(sinkURI *url.URL, pulsarConfig *config.PulsarConfig) (*config.PulsarConfig, error) {
    56  	c := &config.PulsarConfig{
    57  		ConnectionTimeout:       toSec(defaultConnectionTimeout),
    58  		OperationTimeout:        toSec(defaultOperationTimeout),
    59  		BatchingMaxMessages:     toUint(defaultBatchingMaxSize),
    60  		BatchingMaxPublishDelay: toMill(defaultBatchingMaxPublishDelay),
    61  		SendTimeout:             toSec(defaultSendTimeout),
    62  	}
    63  	err := checkSinkURI(sinkURI)
    64  	if err != nil {
    65  		return nil, err
    66  	}
    67  
    68  	c.SinkURI = sinkURI
    69  	c.BrokerURL = sinkURI.Scheme + "://" + sinkURI.Host
    70  
    71  	if pulsarConfig == nil {
    72  		log.L().Debug("new pulsar config", zap.Any("config", c))
    73  		return c, nil
    74  	}
    75  
    76  	pulsarConfig.SinkURI = c.SinkURI
    77  
    78  	if len(sinkURI.Scheme) == 0 || len(sinkURI.Host) == 0 {
    79  		return nil, fmt.Errorf("BrokerURL is empty")
    80  	}
    81  	pulsarConfig.BrokerURL = c.BrokerURL
    82  
    83  	// merge default config
    84  	if pulsarConfig.ConnectionTimeout == nil {
    85  		pulsarConfig.ConnectionTimeout = c.ConnectionTimeout
    86  	}
    87  	if pulsarConfig.OperationTimeout == nil {
    88  		pulsarConfig.OperationTimeout = c.OperationTimeout
    89  	}
    90  	if pulsarConfig.BatchingMaxMessages == nil {
    91  		pulsarConfig.BatchingMaxMessages = c.BatchingMaxMessages
    92  	}
    93  	if pulsarConfig.BatchingMaxPublishDelay == nil {
    94  		pulsarConfig.BatchingMaxPublishDelay = c.BatchingMaxPublishDelay
    95  	}
    96  	if pulsarConfig.SendTimeout == nil {
    97  		pulsarConfig.SendTimeout = c.SendTimeout
    98  	}
    99  
   100  	log.L().Debug("new pulsar config success", zap.Any("config", pulsarConfig))
   101  
   102  	return pulsarConfig, nil
   103  }
   104  
   105  func toSec(x int) *config.TimeSec {
   106  	t := config.TimeSec(x)
   107  	return &t
   108  }
   109  
   110  func toMill(x int) *config.TimeMill {
   111  	t := config.TimeMill(x)
   112  	return &t
   113  }
   114  
   115  func toUint(x uint) *uint {
   116  	return &x
   117  }