github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/sink/codec/csv/csv_decoder_test.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //	http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  package csv
    14  
    15  import (
    16  	"context"
    17  	"testing"
    18  
    19  	timodel "github.com/pingcap/tidb/pkg/parser/model"
    20  	"github.com/pingcap/tidb/pkg/parser/mysql"
    21  	"github.com/pingcap/tidb/pkg/parser/types"
    22  	"github.com/pingcap/tiflow/cdc/model"
    23  	"github.com/pingcap/tiflow/pkg/sink/codec/common"
    24  	"github.com/stretchr/testify/require"
    25  )
    26  
    27  func TestCSVBatchDecoder(t *testing.T) {
    28  	csvData := `"I","employee","hr",433305438660591626,101,"Smith","Bob","2014-06-04","New York"
    29  "U","employee","hr",433305438660591627,101,"Smith","Bob","2015-10-08","Los Angeles"
    30  "D","employee","hr",433305438660591629,101,"Smith","Bob","2017-03-13","Dallas"
    31  "I","employee","hr",433305438660591630,102,"Alex","Alice","2017-03-14","Shanghai"
    32  "U","employee","hr",433305438660591630,102,"Alex","Alice","2018-06-15","Beijing"
    33  `
    34  	ctx := context.Background()
    35  	tableInfo := &model.TableInfo{
    36  		TableName: model.TableName{
    37  			Schema: "hr",
    38  			Table:  "employee",
    39  		},
    40  		TableInfo: &timodel.TableInfo{
    41  			Name: timodel.NewCIStr("employee"),
    42  			Columns: []*timodel.ColumnInfo{
    43  				{
    44  					Name:      timodel.NewCIStr("Id"),
    45  					FieldType: *types.NewFieldType(mysql.TypeInt24),
    46  				},
    47  				{
    48  					Name:      timodel.NewCIStr("LastName"),
    49  					FieldType: *types.NewFieldType(mysql.TypeVarchar),
    50  				},
    51  				{
    52  					Name:      timodel.NewCIStr("FirstName"),
    53  					FieldType: *types.NewFieldType(mysql.TypeVarchar),
    54  				},
    55  				{
    56  					Name:      timodel.NewCIStr("HireDate"),
    57  					FieldType: *types.NewFieldType(mysql.TypeDate),
    58  				},
    59  				{
    60  					Name:      timodel.NewCIStr("OfficeLocation"),
    61  					FieldType: *types.NewFieldType(mysql.TypeVarchar),
    62  				},
    63  			},
    64  		},
    65  	}
    66  	decoder, err := NewBatchDecoder(ctx, &common.Config{
    67  		Delimiter:       ",",
    68  		Quote:           "\"",
    69  		Terminator:      "\n",
    70  		NullString:      "\\N",
    71  		IncludeCommitTs: true,
    72  	}, tableInfo, []byte(csvData))
    73  	require.Nil(t, err)
    74  
    75  	for i := 0; i < 5; i++ {
    76  		tp, hasNext, err := decoder.HasNext()
    77  		require.Nil(t, err)
    78  		require.True(t, hasNext)
    79  		require.Equal(t, model.MessageTypeRow, tp)
    80  		event, err := decoder.NextRowChangedEvent()
    81  		require.NoError(t, err)
    82  		require.NotNil(t, event)
    83  	}
    84  
    85  	_, hasNext, _ := decoder.HasNext()
    86  	require.False(t, hasNext)
    87  }