github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/sink/ddlsink/mq/pulsar_ddl_sink.go (about)

     1  // Copyright 2023 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package mq
    15  
    16  import (
    17  	"context"
    18  	"net/url"
    19  	"time"
    20  
    21  	"github.com/pingcap/errors"
    22  	"github.com/pingcap/log"
    23  	"github.com/pingcap/tiflow/cdc/model"
    24  	"github.com/pingcap/tiflow/cdc/sink/ddlsink/mq/ddlproducer"
    25  	"github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dispatcher"
    26  	"github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/manager"
    27  	"github.com/pingcap/tiflow/cdc/sink/util"
    28  	"github.com/pingcap/tiflow/pkg/config"
    29  	cerror "github.com/pingcap/tiflow/pkg/errors"
    30  	"github.com/pingcap/tiflow/pkg/sink/codec/builder"
    31  	pulsarConfig "github.com/pingcap/tiflow/pkg/sink/pulsar"
    32  	tiflowutil "github.com/pingcap/tiflow/pkg/util"
    33  	"go.uber.org/zap"
    34  )
    35  
    36  // NewPulsarDDLSink will verify the config and create a Pulsar DDL Sink.
    37  func NewPulsarDDLSink(
    38  	ctx context.Context,
    39  	changefeedID model.ChangeFeedID,
    40  	sinkURI *url.URL,
    41  	replicaConfig *config.ReplicaConfig,
    42  	pulsarTopicManagerCreator manager.PulsarTopicManager,
    43  	clientCreator pulsarConfig.FactoryCreator,
    44  	producerCreator ddlproducer.PulsarFactory,
    45  ) (_ *DDLSink, err error) {
    46  	log.Info("Starting pulsar DDL producer ...",
    47  		zap.String("namespace", changefeedID.Namespace),
    48  		zap.String("changefeed", changefeedID.ID))
    49  
    50  	defaultTopic, err := util.GetTopic(sinkURI)
    51  	if err != nil {
    52  		return nil, errors.Trace(err)
    53  	}
    54  
    55  	protocol, err := util.GetProtocol(tiflowutil.GetOrZero(replicaConfig.Sink.Protocol))
    56  	if err != nil {
    57  		return nil, errors.Trace(err)
    58  	}
    59  
    60  	pConfig, err := pulsarConfig.NewPulsarConfig(sinkURI, replicaConfig.Sink.PulsarConfig)
    61  	if err != nil {
    62  		return nil, errors.Trace(err)
    63  	}
    64  
    65  	log.Info("Try to create a DDL sink producer",
    66  		zap.String("changefeed", changefeedID.String()))
    67  
    68  	// NewEventRouter
    69  	eventRouter, err := dispatcher.NewEventRouter(replicaConfig, protocol, defaultTopic, sinkURI.Scheme)
    70  	if err != nil {
    71  		return nil, errors.Trace(err)
    72  	}
    73  
    74  	encoderConfig, err := util.GetEncoderConfig(changefeedID,
    75  		sinkURI, protocol, replicaConfig, config.DefaultMaxMessageBytes)
    76  	if err != nil {
    77  		return nil, errors.Trace(err)
    78  	}
    79  
    80  	encoderBuilder, err := builder.NewRowEventEncoderBuilder(ctx, encoderConfig)
    81  	if err != nil {
    82  		return nil, cerror.WrapError(cerror.ErrKafkaInvalidConfig, err)
    83  	}
    84  
    85  	start := time.Now()
    86  	client, err := clientCreator(pConfig, changefeedID, replicaConfig.Sink)
    87  	if err != nil {
    88  		log.Error("DDL sink producer client create fail", zap.Error(err))
    89  		return nil, cerror.WrapError(cerror.ErrPulsarNewClient, err)
    90  	}
    91  
    92  	p, err := producerCreator(ctx, changefeedID, pConfig, client, replicaConfig.Sink)
    93  	log.Info("DDL sink producer client created", zap.Duration("duration", time.Since(start)))
    94  	if err != nil {
    95  		return nil, cerror.WrapError(cerror.ErrPulsarNewProducer, err)
    96  	}
    97  
    98  	topicManager, err := pulsarTopicManagerCreator(pConfig, client)
    99  	if err != nil {
   100  		return nil, errors.Trace(err)
   101  	}
   102  
   103  	s := newDDLSink(ctx, changefeedID, p, nil, topicManager, eventRouter, encoderBuilder, protocol)
   104  
   105  	return s, nil
   106  }