github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/sink/ddlsink/mq/kafka_ddl_sink.go (about) 1 // Copyright 2022 PingCAP, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package mq 15 16 import ( 17 "context" 18 "net/url" 19 "time" 20 21 "github.com/pingcap/errors" 22 "github.com/pingcap/log" 23 "github.com/pingcap/tiflow/cdc/model" 24 "github.com/pingcap/tiflow/cdc/sink/ddlsink/mq/ddlproducer" 25 "github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dispatcher" 26 "github.com/pingcap/tiflow/cdc/sink/util" 27 "github.com/pingcap/tiflow/pkg/config" 28 cerror "github.com/pingcap/tiflow/pkg/errors" 29 "github.com/pingcap/tiflow/pkg/sink/codec/builder" 30 "github.com/pingcap/tiflow/pkg/sink/kafka" 31 tiflowutil "github.com/pingcap/tiflow/pkg/util" 32 "go.uber.org/zap" 33 ) 34 35 // NewKafkaDDLSink will verify the config and create a Kafka DDL Sink. 36 func NewKafkaDDLSink( 37 ctx context.Context, 38 changefeedID model.ChangeFeedID, 39 sinkURI *url.URL, 40 replicaConfig *config.ReplicaConfig, 41 factoryCreator kafka.FactoryCreator, 42 producerCreator ddlproducer.Factory, 43 ) (_ *DDLSink, err error) { 44 topic, err := util.GetTopic(sinkURI) 45 if err != nil { 46 return nil, errors.Trace(err) 47 } 48 49 options := kafka.NewOptions() 50 if err := options.Apply(changefeedID, sinkURI, replicaConfig); err != nil { 51 return nil, cerror.WrapError(cerror.ErrKafkaInvalidConfig, err) 52 } 53 54 factory, err := factoryCreator(options, changefeedID) 55 if err != nil { 56 return nil, cerror.WrapError(cerror.ErrKafkaNewProducer, err) 57 } 58 59 adminClient, err := factory.AdminClient(ctx) 60 if err != nil { 61 return nil, cerror.WrapError(cerror.ErrKafkaNewProducer, err) 62 } 63 // We must close adminClient when this func return cause by an error 64 // otherwise the adminClient will never be closed and lead to a goroutine leak. 65 defer func() { 66 if err != nil && adminClient != nil { 67 adminClient.Close() 68 } 69 }() 70 71 if err := kafka.AdjustOptions(ctx, adminClient, options, topic); err != nil { 72 return nil, cerror.WrapError(cerror.ErrKafkaNewProducer, err) 73 } 74 75 protocol, err := util.GetProtocol(tiflowutil.GetOrZero(replicaConfig.Sink.Protocol)) 76 if err != nil { 77 return nil, errors.Trace(err) 78 } 79 80 topicManager, err := util.GetTopicManagerAndTryCreateTopic( 81 ctx, 82 changefeedID, 83 topic, 84 options.DeriveTopicConfig(), 85 adminClient, 86 ) 87 if err != nil { 88 return nil, errors.Trace(err) 89 } 90 91 eventRouter, err := dispatcher.NewEventRouter(replicaConfig, protocol, topic, sinkURI.Scheme) 92 if err != nil { 93 return nil, errors.Trace(err) 94 } 95 96 encoderConfig, err := util.GetEncoderConfig(changefeedID, sinkURI, protocol, replicaConfig, options.MaxMessageBytes) 97 if err != nil { 98 return nil, errors.Trace(err) 99 } 100 101 encoderBuilder, err := builder.NewRowEventEncoderBuilder(ctx, encoderConfig) 102 if err != nil { 103 return nil, cerror.WrapError(cerror.ErrKafkaInvalidConfig, err) 104 } 105 106 start := time.Now() 107 log.Info("Try to create a DDL sink producer", 108 zap.String("changefeed", changefeedID.String())) 109 syncProducer, err := factory.SyncProducer(ctx) 110 if err != nil { 111 return nil, errors.Trace(err) 112 } 113 114 ddlProducer := producerCreator(ctx, changefeedID, syncProducer) 115 s := newDDLSink(ctx, changefeedID, ddlProducer, adminClient, topicManager, eventRouter, encoderBuilder, protocol) 116 log.Info("DDL sink producer client created", zap.Duration("duration", time.Since(start))) 117 return s, nil 118 }