github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/sink/dmlsink/mq/dmlproducer/pulsar_dml_mock_producer.go (about)

     1  // Copyright 2023 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package dmlproducer
    15  
    16  import (
    17  	"context"
    18  	"fmt"
    19  
    20  	"github.com/apache/pulsar-client-go/pulsar"
    21  	lru "github.com/hashicorp/golang-lru"
    22  	"github.com/pingcap/tiflow/cdc/model"
    23  	"github.com/pingcap/tiflow/pkg/config"
    24  	cerror "github.com/pingcap/tiflow/pkg/errors"
    25  	"github.com/pingcap/tiflow/pkg/sink/codec/common"
    26  )
    27  
    28  var _ DMLProducer = (*pulsarDMLProducerMock)(nil)
    29  
    30  // pulsarDMLProducer is used to send messages to kafka.
    31  type pulsarDMLProducerMock struct {
    32  	// id indicates which processor (changefeed) this sink belongs to.
    33  	id model.ChangeFeedID
    34  	// We hold the client to make close operation faster.
    35  	// Please see the comment of Close().
    36  	client pulsar.Client
    37  	// producers is used to send messages to kafka asynchronously.
    38  	// support multiple topics
    39  	producers *lru.Cache
    40  
    41  	// failpointCh is used to inject failpoints to the run loop.
    42  	// Only used in test.
    43  	failpointCh chan error
    44  	// closeCh is send error
    45  	errChan chan error
    46  
    47  	pConfig *config.PulsarConfig
    48  }
    49  
    50  // NewPulsarDMLProducerMock creates a new pulsar producer.
    51  func NewPulsarDMLProducerMock(
    52  	ctx context.Context,
    53  	changefeedID model.ChangeFeedID,
    54  	client pulsar.Client,
    55  	sinkConfig *config.SinkConfig,
    56  	errCh chan error,
    57  	failpointCh chan error,
    58  ) (DMLProducer, error) {
    59  	pulsarConfig := sinkConfig.PulsarConfig
    60  	defaultTopicName := pulsarConfig.GetDefaultTopicName()
    61  	defaultProducer, err := newProducerMock(pulsarConfig, client, defaultTopicName)
    62  	if err != nil {
    63  		// Close the client to prevent the goroutine leak.
    64  		// Because it may be a long time to close the client,
    65  		// so close it asynchronously.
    66  		// follow kafka
    67  		go func() {
    68  			client.Close()
    69  		}()
    70  		return nil, cerror.WrapError(cerror.ErrPulsarNewProducer, err)
    71  	}
    72  	producerCacheSize := config.DefaultPulsarProducerCacheSize
    73  	if pulsarConfig != nil && pulsarConfig.PulsarProducerCacheSize != nil {
    74  		producerCacheSize = int(*pulsarConfig.PulsarProducerCacheSize)
    75  	}
    76  
    77  	producers, err := lru.NewWithEvict(producerCacheSize, func(key interface{}, value interface{}) {
    78  		// remove producer
    79  		pulsarProducer, ok := value.(pulsar.Producer)
    80  		if ok && pulsarProducer != nil {
    81  			pulsarProducer.Close()
    82  		}
    83  	})
    84  	if err != nil {
    85  		go client.Close()
    86  		return nil, cerror.WrapError(cerror.ErrPulsarNewProducer, err)
    87  	}
    88  
    89  	producers.Add(defaultTopicName, defaultProducer)
    90  
    91  	p := &pulsarDMLProducerMock{
    92  		id:          changefeedID,
    93  		client:      client,
    94  		producers:   producers,
    95  		pConfig:     pulsarConfig,
    96  		failpointCh: failpointCh,
    97  		errChan:     errCh,
    98  	}
    99  
   100  	return p, nil
   101  }
   102  
   103  // AsyncSendMessage  Async send one message
   104  func (p *pulsarDMLProducerMock) AsyncSendMessage(
   105  	ctx context.Context, topic string,
   106  	partition int32, message *common.Message,
   107  ) error {
   108  	data := &pulsar.ProducerMessage{
   109  		Payload: message.Value,
   110  		Key:     message.GetPartitionKey(),
   111  	}
   112  
   113  	fmt.Printf("pulsar send message %+v\n", data)
   114  
   115  	return nil
   116  }
   117  
   118  func (p *pulsarDMLProducerMock) Close() {
   119  }
   120  
   121  // newProducerMock creates a pulsar producer mock
   122  func newProducerMock(
   123  	pConfig *config.PulsarConfig,
   124  	client pulsar.Client,
   125  	topicName string,
   126  ) (p pulsar.Producer, err error) {
   127  	return p, nil
   128  }