github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/integration/framework/sql_batch_op.go (about)

     1  // Copyright 2020 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package framework
    15  
    16  import (
    17  	"context"
    18  	"fmt"
    19  	"reflect"
    20  	"strings"
    21  	"time"
    22  
    23  	"github.com/jmoiron/sqlx"
    24  	"github.com/pingcap/errors"
    25  	"github.com/pingcap/log"
    26  	"go.uber.org/zap"
    27  )
    28  
    29  const (
    30  	selectQueryMaxBatchSize = 256
    31  )
    32  
    33  type sqlAllAwaiter struct {
    34  	helper          *SQLHelper
    35  	data            map[interface{}]sqlRowContainer
    36  	retrievedValues []map[string]interface{}
    37  	table           *Table
    38  }
    39  
    40  // All joins a slice of Awaitable sql requests. The request must be to the same table.
    41  // TODO does not support composite primary key for now!
    42  func All(helper *SQLHelper, awaitables []Awaitable) Awaitable {
    43  	if _, ok := awaitables[0].(sqlRowContainer); !ok {
    44  		return awaitables[0]
    45  	}
    46  
    47  	ret := &sqlAllAwaiter{
    48  		helper:          helper,
    49  		data:            make(map[interface{}]sqlRowContainer, len(awaitables)),
    50  		retrievedValues: make([]map[string]interface{}, 0),
    51  		table:           awaitables[0].(sqlRowContainer).getTable(),
    52  	}
    53  
    54  	for _, row := range awaitables {
    55  		rowContainer, ok := row.(sqlRowContainer)
    56  		if !ok {
    57  			return row
    58  		}
    59  		key := rowContainer.getData()[rowContainer.getTable().uniqueIndex[0]]
    60  		ret.data[normalizeKeys(key)] = rowContainer
    61  	}
    62  
    63  	return &basicAwaitable{
    64  		pollableAndCheckable: ret,
    65  		timeout:              120 * time.Second,
    66  	}
    67  }
    68  
    69  func (s *sqlAllAwaiter) poll(ctx context.Context) (bool, error) {
    70  	db := sqlx.NewDb(s.helper.downstream, "mysql")
    71  
    72  	batchSize := 0
    73  	counter := 0
    74  	indexValues := make([]interface{}, 0)
    75  	s.retrievedValues = make([]map[string]interface{}, 0)
    76  	for k, v := range s.data {
    77  		indexValues = append(indexValues, k)
    78  		batchSize++
    79  		counter++
    80  		if batchSize >= selectQueryMaxBatchSize || counter == len(s.data) {
    81  			log.Debug("Selecting", zap.String("table", s.table.tableName), zap.Any("keys", indexValues))
    82  			query, args, err := sqlx.In("select distinct * from "+s.table.tableName+" where "+v.getTable().uniqueIndex[0]+" in (?)", indexValues)
    83  			if err != nil {
    84  				return false, errors.AddStack(err)
    85  			}
    86  			query = db.Rebind(query)
    87  			rows, err := db.QueryContext(ctx, query, args...)
    88  			if err != nil {
    89  				if strings.Contains(err.Error(), "Error 1146") {
    90  					log.Info("table does not exist, will try again", zap.Error(err), zap.String("query", query))
    91  					return false, nil
    92  				}
    93  				return false, errors.AddStack(err)
    94  			}
    95  
    96  			for rows.Next() {
    97  				m, err := rowsToMap(rows)
    98  				if err != nil {
    99  					return false, errors.AddStack(err)
   100  				}
   101  				s.retrievedValues = append(s.retrievedValues, m)
   102  			}
   103  			batchSize = 0
   104  			indexValues = make([]interface{}, 0)
   105  		}
   106  	}
   107  
   108  	log.Debug("poll finished", zap.Int("total-retrieved", len(s.retrievedValues)))
   109  
   110  	if len(s.data) == len(s.retrievedValues) {
   111  		return true, nil
   112  	}
   113  
   114  	return false, nil
   115  }
   116  
   117  func (s *sqlAllAwaiter) Check() error {
   118  	for _, row := range s.retrievedValues {
   119  		key := row[s.table.uniqueIndex[0]]
   120  		expected := s.data[normalizeKeys(key)]
   121  		if !compareMaps(row, expected.getData()) {
   122  			log.Warn(
   123  				"Check failed",
   124  				zap.String("expected", fmt.Sprintf("%v", expected)),
   125  				zap.String("actual", fmt.Sprintf("%v", row)),
   126  			)
   127  			return errors.New("Check failed")
   128  		}
   129  	}
   130  	return nil
   131  }
   132  
   133  func normalizeKeys(key interface{}) interface{} {
   134  	switch key.(type) {
   135  	case int, int8, int16, int32, int64:
   136  		return reflect.ValueOf(key).Int()
   137  	case uint, uint8, uint16, uint32, uint64:
   138  		return reflect.ValueOf(key).Uint()
   139  	default:
   140  		return key
   141  	}
   142  }