github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/sink/ddlsink/mq/kafka_ddl_sink.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package mq
    15  
    16  import (
    17  	"context"
    18  	"net/url"
    19  	"time"
    20  
    21  	"github.com/pingcap/errors"
    22  	"github.com/pingcap/log"
    23  	"github.com/pingcap/tiflow/cdc/model"
    24  	"github.com/pingcap/tiflow/cdc/sink/ddlsink/mq/ddlproducer"
    25  	"github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dispatcher"
    26  	"github.com/pingcap/tiflow/cdc/sink/util"
    27  	"github.com/pingcap/tiflow/pkg/config"
    28  	cerror "github.com/pingcap/tiflow/pkg/errors"
    29  	"github.com/pingcap/tiflow/pkg/sink/codec/builder"
    30  	"github.com/pingcap/tiflow/pkg/sink/kafka"
    31  	tiflowutil "github.com/pingcap/tiflow/pkg/util"
    32  	"go.uber.org/zap"
    33  )
    34  
    35  // NewKafkaDDLSink will verify the config and create a Kafka DDL Sink.
    36  func NewKafkaDDLSink(
    37  	ctx context.Context,
    38  	changefeedID model.ChangeFeedID,
    39  	sinkURI *url.URL,
    40  	replicaConfig *config.ReplicaConfig,
    41  	factoryCreator kafka.FactoryCreator,
    42  	producerCreator ddlproducer.Factory,
    43  ) (_ *DDLSink, err error) {
    44  	topic, err := util.GetTopic(sinkURI)
    45  	if err != nil {
    46  		return nil, errors.Trace(err)
    47  	}
    48  
    49  	options := kafka.NewOptions()
    50  	if err := options.Apply(changefeedID, sinkURI, replicaConfig); err != nil {
    51  		return nil, cerror.WrapError(cerror.ErrKafkaInvalidConfig, err)
    52  	}
    53  
    54  	factory, err := factoryCreator(options, changefeedID)
    55  	if err != nil {
    56  		return nil, cerror.WrapError(cerror.ErrKafkaNewProducer, err)
    57  	}
    58  
    59  	adminClient, err := factory.AdminClient(ctx)
    60  	if err != nil {
    61  		return nil, cerror.WrapError(cerror.ErrKafkaNewProducer, err)
    62  	}
    63  	// We must close adminClient when this func return cause by an error
    64  	// otherwise the adminClient will never be closed and lead to a goroutine leak.
    65  	defer func() {
    66  		if err != nil && adminClient != nil {
    67  			adminClient.Close()
    68  		}
    69  	}()
    70  
    71  	if err := kafka.AdjustOptions(ctx, adminClient, options, topic); err != nil {
    72  		return nil, cerror.WrapError(cerror.ErrKafkaNewProducer, err)
    73  	}
    74  
    75  	protocol, err := util.GetProtocol(tiflowutil.GetOrZero(replicaConfig.Sink.Protocol))
    76  	if err != nil {
    77  		return nil, errors.Trace(err)
    78  	}
    79  
    80  	topicManager, err := util.GetTopicManagerAndTryCreateTopic(
    81  		ctx,
    82  		changefeedID,
    83  		topic,
    84  		options.DeriveTopicConfig(),
    85  		adminClient,
    86  	)
    87  	if err != nil {
    88  		return nil, errors.Trace(err)
    89  	}
    90  
    91  	eventRouter, err := dispatcher.NewEventRouter(replicaConfig, protocol, topic, sinkURI.Scheme)
    92  	if err != nil {
    93  		return nil, errors.Trace(err)
    94  	}
    95  
    96  	encoderConfig, err := util.GetEncoderConfig(changefeedID, sinkURI, protocol, replicaConfig, options.MaxMessageBytes)
    97  	if err != nil {
    98  		return nil, errors.Trace(err)
    99  	}
   100  
   101  	encoderBuilder, err := builder.NewRowEventEncoderBuilder(ctx, encoderConfig)
   102  	if err != nil {
   103  		return nil, cerror.WrapError(cerror.ErrKafkaInvalidConfig, err)
   104  	}
   105  
   106  	start := time.Now()
   107  	log.Info("Try to create a DDL sink producer",
   108  		zap.String("changefeed", changefeedID.String()))
   109  	syncProducer, err := factory.SyncProducer(ctx)
   110  	if err != nil {
   111  		return nil, errors.Trace(err)
   112  	}
   113  
   114  	ddlProducer := producerCreator(ctx, changefeedID, syncProducer)
   115  	s := newDDLSink(ctx, changefeedID, ddlProducer, adminClient, topicManager, eventRouter, encoderBuilder, protocol)
   116  	log.Info("DDL sink producer client created", zap.Duration("duration", time.Since(start)))
   117  	return s, nil
   118  }