github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/sink/ddlsink/mq/pulsar_ddl_sink_test.go (about)

     1  // Copyright 2023 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package mq
    15  
    16  import (
    17  	"context"
    18  	"net/url"
    19  	"testing"
    20  
    21  	"github.com/aws/aws-sdk-go/aws"
    22  	mm "github.com/pingcap/tidb/pkg/parser/model"
    23  	"github.com/pingcap/tiflow/cdc/model"
    24  	"github.com/pingcap/tiflow/cdc/sink/ddlsink/mq/ddlproducer"
    25  	"github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/manager"
    26  	"github.com/pingcap/tiflow/pkg/config"
    27  	pulsarConfig "github.com/pingcap/tiflow/pkg/sink/pulsar"
    28  	"github.com/stretchr/testify/require"
    29  )
    30  
    31  const (
    32  	// MockPulsarTopic is the mock topic for pulsar
    33  	MockPulsarTopic = "pulsar_test"
    34  )
    35  
    36  // newPulsarConfig set config
    37  func newPulsarConfig(t *testing.T) (*config.PulsarConfig, *url.URL) {
    38  	sinkURL := "pulsar://127.0.0.1:6650/persistent://public/default/test?" +
    39  		"protocol=canal-json&pulsar-version=v2.10.0&enable-tidb-extension=true&" +
    40  		"authentication-token=eyJhbcGcixxxxxxxxxxxxxx"
    41  
    42  	sinkURI, err := url.Parse(sinkURL)
    43  	require.NoError(t, err)
    44  	replicaConfig := config.GetDefaultReplicaConfig()
    45  	require.NoError(t, replicaConfig.ValidateAndAdjust(sinkURI))
    46  	require.NoError(t, err)
    47  	c, err := pulsarConfig.NewPulsarConfig(sinkURI, replicaConfig.Sink.PulsarConfig)
    48  	require.NoError(t, err)
    49  	return c, sinkURI
    50  }
    51  
    52  // TestNewPulsarDDLSink tests the NewPulsarDDLSink
    53  func TestNewPulsarDDLSink(t *testing.T) {
    54  	t.Parallel()
    55  
    56  	_, sinkURI := newPulsarConfig(t)
    57  	changefeedID := model.DefaultChangeFeedID("test")
    58  	replicaConfig := config.GetDefaultReplicaConfig()
    59  	replicaConfig.Sink = &config.SinkConfig{
    60  		Protocol: aws.String("canal-json"),
    61  	}
    62  
    63  	ctx, cancel := context.WithCancel(context.Background())
    64  	defer cancel()
    65  
    66  	ctx = context.WithValue(ctx, "testing.T", t)
    67  	ddlSink, err := NewPulsarDDLSink(ctx, changefeedID, sinkURI, replicaConfig,
    68  		manager.NewMockPulsarTopicManager, pulsarConfig.NewMockCreatorFactory, ddlproducer.NewMockPulsarProducerDDL)
    69  
    70  	require.NoError(t, err)
    71  	require.NotNil(t, ddlSink)
    72  
    73  	checkpointTs := uint64(417318403368288260)
    74  	tables := []*model.TableInfo{
    75  		{
    76  			TableName: model.TableName{
    77  				Schema: "cdc",
    78  				Table:  "person",
    79  			},
    80  		},
    81  		{
    82  			TableName: model.TableName{
    83  				Schema: "cdc",
    84  				Table:  "person1",
    85  			},
    86  		},
    87  		{
    88  			TableName: model.TableName{
    89  				Schema: "cdc",
    90  				Table:  "person2",
    91  			},
    92  		},
    93  	}
    94  
    95  	err = ddlSink.WriteCheckpointTs(ctx, checkpointTs, tables)
    96  	require.NoError(t, err)
    97  
    98  	events := ddlSink.producer.(*ddlproducer.PulsarMockProducers).GetAllEvents()
    99  	require.Len(t, events, 1, "All topics and partitions should be broadcast")
   100  }
   101  
   102  // TestPulsarDDLSinkNewSuccess tests the NewPulsarDDLSink write a event to pulsar
   103  func TestPulsarDDLSinkNewSuccess(t *testing.T) {
   104  	t.Parallel()
   105  
   106  	_, sinkURI := newPulsarConfig(t)
   107  	changefeedID := model.DefaultChangeFeedID("test")
   108  	replicaConfig := config.GetDefaultReplicaConfig()
   109  	replicaConfig.Sink = &config.SinkConfig{
   110  		Protocol: aws.String("canal-json"),
   111  	}
   112  
   113  	ctx, cancel := context.WithCancel(context.Background())
   114  	defer cancel()
   115  
   116  	ctx = context.WithValue(ctx, "testing.T", t)
   117  	s, err := NewPulsarDDLSink(ctx, changefeedID, sinkURI, replicaConfig, manager.NewMockPulsarTopicManager,
   118  		pulsarConfig.NewMockCreatorFactory, ddlproducer.NewMockPulsarProducerDDL)
   119  	require.NoError(t, err)
   120  	require.NotNil(t, s)
   121  }
   122  
   123  func TestPulsarWriteDDLEventToZeroPartition(t *testing.T) {
   124  	t.Parallel()
   125  
   126  	_, sinkURI := newPulsarConfig(t)
   127  	changefeedID := model.DefaultChangeFeedID("test")
   128  	replicaConfig := config.GetDefaultReplicaConfig()
   129  	replicaConfig.Sink = &config.SinkConfig{
   130  		Protocol: aws.String("canal-json"),
   131  	}
   132  
   133  	ctx, cancel := context.WithCancel(context.Background())
   134  	defer cancel()
   135  
   136  	ctx = context.WithValue(ctx, "testing.T", t)
   137  	ddlSink, err := NewPulsarDDLSink(ctx, changefeedID, sinkURI, replicaConfig,
   138  		manager.NewMockPulsarTopicManager, pulsarConfig.NewMockCreatorFactory, ddlproducer.NewMockPulsarProducerDDL)
   139  
   140  	require.NoError(t, err)
   141  	require.NotNil(t, ddlSink)
   142  
   143  	ddl := &model.DDLEvent{
   144  		CommitTs: 417318403368288260,
   145  		TableInfo: &model.TableInfo{
   146  			TableName: model.TableName{
   147  				Schema: "cdc", Table: "person",
   148  			},
   149  		},
   150  		Query: "create table person(id int, name varchar(32), primary key(id))",
   151  		Type:  mm.ActionCreateTable,
   152  	}
   153  	err = ddlSink.WriteDDLEvent(ctx, ddl)
   154  	require.NoError(t, err)
   155  
   156  	err = ddlSink.WriteDDLEvent(ctx, ddl)
   157  	require.NoError(t, err)
   158  
   159  	require.Len(t, ddlSink.producer.(*ddlproducer.PulsarMockProducers).GetAllEvents(),
   160  		2, "Write DDL 2 Events")
   161  }