github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/integration/framework/avro/kafka_single_table.go (about)

     1  // Copyright 2020 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package avro
    15  
    16  import (
    17  	"bytes"
    18  	"database/sql"
    19  	"fmt"
    20  	"io/ioutil"
    21  	"net/http"
    22  	"time"
    23  
    24  	"github.com/pingcap/errors"
    25  	"github.com/pingcap/log"
    26  	"github.com/pingcap/ticdc/integration/framework"
    27  	"go.uber.org/zap"
    28  )
    29  
    30  // SingleTableTask provides a basic implementation for an Avro test case
    31  type SingleTableTask struct {
    32  	TableName string
    33  }
    34  
    35  // Name implements Task
    36  func (a *SingleTableTask) Name() string {
    37  	log.Warn("SingleTableTask should be embedded in another Task")
    38  	return "SingleTableTask-" + a.TableName
    39  }
    40  
    41  // GetCDCProfile implements Task
    42  func (a *SingleTableTask) GetCDCProfile() *framework.CDCProfile {
    43  	return &framework.CDCProfile{
    44  		PDUri:   "http://upstream-pd:2379",
    45  		SinkURI: "kafka://kafka:9092/testdb_" + a.TableName + "?kafka-version=2.6.0&protocol=avro",
    46  		Opts:    map[string]string{"registry": "http://schema-registry:8081"},
    47  	}
    48  }
    49  
    50  // Prepare implements Task
    51  func (a *SingleTableTask) Prepare(taskContext *framework.TaskContext) error {
    52  	err := taskContext.CreateDB("testdb")
    53  	if err != nil {
    54  		return err
    55  	}
    56  
    57  	_ = taskContext.Upstream.Close()
    58  	taskContext.Upstream, err = sql.Open("mysql", framework.UpstreamDSN+"testdb")
    59  	if err != nil {
    60  		return err
    61  	}
    62  
    63  	_ = taskContext.Downstream.Close()
    64  	taskContext.Downstream, err = sql.Open("mysql", framework.DownstreamDSN+"testdb")
    65  	if err != nil {
    66  		return err
    67  	}
    68  	taskContext.Downstream.SetConnMaxLifetime(5 * time.Second)
    69  	if taskContext.WaitForReady != nil {
    70  		log.Info("Waiting for env to be ready")
    71  		return taskContext.WaitForReady()
    72  	}
    73  
    74  	return nil
    75  }
    76  
    77  // Run implements Task
    78  func (a *SingleTableTask) Run(taskContext *framework.TaskContext) error {
    79  	log.Warn("SingleTableTask has been run")
    80  	return nil
    81  }
    82  
    83  func createConnector() error {
    84  	// TODO better way to generate JSON
    85  	connectorConfigFmt := `{
    86  		"name": "jdbc-sink-connector",
    87  		"config": {
    88  		  "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
    89  		  "tasks.max": "1",
    90  		  "topics": "testdb_%s",
    91  		  "connection.url": "jdbc:mysql://root@downstream-tidb:4000/testdb",
    92  		  "connection.ds.pool.size": 5,
    93  		  "table.name.format": "%s",
    94  		  "insert.mode": "upsert",
    95  		  "delete.enabled": true,
    96  		  "pk.mode": "record_key",
    97  		  "auto.create": true,
    98  		  "auto.evolve": true
    99  		}
   100  	  }`
   101  	connectorConfig := fmt.Sprintf(connectorConfigFmt, "test", "test")
   102  	log.Debug("Creating Kafka sink connector", zap.String("config", connectorConfig))
   103  
   104  	resp, err := http.Post(
   105  		"http://127.0.0.1:8083/connectors",
   106  		"application/json",
   107  		bytes.NewReader([]byte(connectorConfig)))
   108  	if err != nil {
   109  		return err
   110  	}
   111  
   112  	if resp.Body == nil {
   113  		return errors.New("Kafka Connect Rest API returned empty body")
   114  	}
   115  	defer resp.Body.Close()
   116  
   117  	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
   118  		str, err := ioutil.ReadAll(resp.Body)
   119  		if err != nil {
   120  			return err
   121  		}
   122  		log.Warn(
   123  			"Kafka Connect Rest API returned",
   124  			zap.Int("status", resp.StatusCode),
   125  			zap.ByteString("body", str))
   126  		return errors.Errorf("Kafka Connect Rest API returned status code %d", resp.StatusCode)
   127  	}
   128  	return nil
   129  }