github.com/matrixorigin/matrixone@v0.7.0/pkg/cnservice/cnclient/client.go (about)

     1  // Copyright 2021 - 2022 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package cnclient
    16  
    17  import (
    18  	"context"
    19  	"sync"
    20  
    21  	"time"
    22  
    23  	"github.com/fagongzi/goetty/v2"
    24  	"github.com/matrixorigin/matrixone/pkg/common/morpc"
    25  	"github.com/matrixorigin/matrixone/pkg/logutil"
    26  	"github.com/matrixorigin/matrixone/pkg/pb/pipeline"
    27  	"github.com/matrixorigin/matrixone/pkg/txn/rpc"
    28  )
    29  
    30  // client each node will hold only one client.
    31  // It is responsible for sending messages to other nodes. and messages were received
    32  // and handled by cn-server.
    33  var client *CNClient
    34  
    35  func CloseCNClient() error {
    36  	return client.Close()
    37  }
    38  
    39  func GetStreamSender(backend string) (morpc.Stream, error) {
    40  	return client.NewStream(backend)
    41  }
    42  
    43  func AcquireMessage() *pipeline.Message {
    44  	return client.acquireMessage().(*pipeline.Message)
    45  }
    46  
    47  func IsCNClientReady() bool {
    48  	return client != nil && client.ready
    49  }
    50  
    51  type CNClient struct {
    52  	ready  bool
    53  	config *ClientConfig
    54  	client morpc.RPCClient
    55  
    56  	// pool for send message
    57  	requestPool *sync.Pool
    58  }
    59  
    60  func (c *CNClient) Send(ctx context.Context, backend string, request morpc.Message) (*morpc.Future, error) {
    61  	return c.client.Send(ctx, backend, request)
    62  }
    63  
    64  func (c *CNClient) NewStream(backend string) (morpc.Stream, error) {
    65  	return c.client.NewStream(backend, true)
    66  }
    67  
    68  func (c *CNClient) Close() error {
    69  	lock.Lock()
    70  	defer lock.Unlock()
    71  	if client == nil || !c.ready {
    72  		return nil
    73  	}
    74  
    75  	c.ready = false
    76  	return c.client.Close()
    77  }
    78  
    79  const (
    80  	dfMaxSenderNumber       = 100000
    81  	dfConnectTimeout        = 5 * time.Second
    82  	dfClientReadBufferSize  = 1 << 10
    83  	dfClientWriteBufferSize = 1 << 10
    84  )
    85  
    86  // ClientConfig a config to init a CNClient
    87  type ClientConfig struct {
    88  	// MaxSenderNumber is the max number of backends per host for compute node service.
    89  	MaxSenderNumber int
    90  	// TimeOutForEachConnect is the out time for each tcp connect.
    91  	TimeOutForEachConnect time.Duration
    92  	// related buffer size.
    93  	ReadBufferSize  int
    94  	WriteBufferSize int
    95  	// RPC rpc config
    96  	RPC rpc.Config
    97  }
    98  
    99  var (
   100  	lock sync.Mutex
   101  )
   102  
   103  // TODO: Here it needs to be refactored together with Runtime
   104  func NewCNClient(cfg *ClientConfig) error {
   105  	lock.Lock()
   106  	defer lock.Unlock()
   107  	if client != nil {
   108  		return nil
   109  	}
   110  
   111  	var err error
   112  	cfg.Fill()
   113  	client = &CNClient{config: cfg}
   114  	client.requestPool = &sync.Pool{New: func() any { return &pipeline.Message{} }}
   115  
   116  	codec := morpc.NewMessageCodec(client.acquireMessage,
   117  		morpc.WithCodecMaxBodySize(int(cfg.RPC.MaxMessageSize)))
   118  	factory := morpc.NewGoettyBasedBackendFactory(codec,
   119  		morpc.WithBackendGoettyOptions(
   120  			goetty.WithSessionRWBUfferSize(cfg.ReadBufferSize, cfg.WriteBufferSize),
   121  			goetty.WithSessionReleaseMsgFunc(func(v any) {
   122  				m := v.(morpc.RPCMessage)
   123  				if !m.InternalMessage() {
   124  					client.releaseMessage(m.Message.(*pipeline.Message))
   125  				}
   126  			}),
   127  		),
   128  		morpc.WithBackendConnectTimeout(cfg.TimeOutForEachConnect),
   129  		morpc.WithBackendLogger(logutil.GetGlobalLogger().Named("cn-backend")),
   130  	)
   131  
   132  	client.client, err = morpc.NewClient(factory,
   133  		morpc.WithClientMaxBackendPerHost(cfg.MaxSenderNumber),
   134  		morpc.WithClientTag("cn-client"),
   135  	)
   136  	client.ready = true
   137  	return err
   138  }
   139  
   140  func (c *CNClient) acquireMessage() morpc.Message {
   141  	// TODO: pipeline.Message has many []byte fields, maybe can use PayloadMessage to avoid mem copy.
   142  	return c.requestPool.Get().(*pipeline.Message)
   143  }
   144  
   145  func (c *CNClient) releaseMessage(m *pipeline.Message) {
   146  	if c.requestPool != nil {
   147  		m.Sid = 0
   148  		m.Err = nil
   149  		m.Data = nil
   150  		m.ProcInfoData = nil
   151  		m.Analyse = nil
   152  		c.requestPool.Put(m)
   153  	}
   154  }
   155  
   156  // Fill set some default value for client config.
   157  func (cfg *ClientConfig) Fill() {
   158  	if cfg.MaxSenderNumber <= 0 {
   159  		cfg.MaxSenderNumber = dfMaxSenderNumber
   160  	}
   161  	if cfg.ReadBufferSize < 0 {
   162  		cfg.ReadBufferSize = dfClientReadBufferSize
   163  	}
   164  	if cfg.WriteBufferSize < 0 {
   165  		cfg.WriteBufferSize = dfClientWriteBufferSize
   166  	}
   167  	if cfg.TimeOutForEachConnect <= 0 {
   168  		cfg.TimeOutForEachConnect = dfConnectTimeout
   169  	}
   170  }