github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/testing_utils/sorter_stress_test/sorter_stress.go (about)

     1  // Copyright 2020 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package main
    15  
    16  import (
    17  	"context"
    18  	"flag"
    19  	"math/rand"
    20  	"net/http"
    21  	_ "net/http/pprof"
    22  	"os"
    23  	"strings"
    24  
    25  	"github.com/pingcap/failpoint"
    26  
    27  	"github.com/pingcap/log"
    28  	"github.com/pingcap/ticdc/cdc/model"
    29  	"github.com/pingcap/ticdc/cdc/puller"
    30  	pullerSorter "github.com/pingcap/ticdc/cdc/puller/sorter"
    31  	"github.com/pingcap/ticdc/pkg/config"
    32  	"go.uber.org/zap"
    33  	"golang.org/x/sync/errgroup"
    34  )
    35  
    36  var (
    37  	sorterDir    = flag.String("dir", "./sorter", "temporary directory used for sorting")
    38  	numBatches   = flag.Int("num-batches", 256, "number of batches of ordered events")
    39  	msgsPerBatch = flag.Int("num-messages-per-batch", 1024, "number of events in a batch")
    40  	bytesPerMsg  = flag.Int("bytes-per-message", 1024, "number of bytes in an event")
    41  )
    42  
    43  func main() {
    44  	flag.Parse()
    45  	log.SetLevel(zap.DebugLevel)
    46  	err := failpoint.Enable("github.com/pingcap/ticdc/cdc/puller/sorter/sorterDebug", "return(true)")
    47  	if err != nil {
    48  		log.Fatal("Could not enable failpoint", zap.Error(err))
    49  	}
    50  
    51  	conf := config.GetDefaultServerConfig()
    52  	conf.Sorter = &config.SorterConfig{
    53  		NumConcurrentWorker:  8,
    54  		ChunkSizeLimit:       1 * 1024 * 1024 * 1024,
    55  		MaxMemoryPressure:    60,
    56  		MaxMemoryConsumption: 16 * 1024 * 1024 * 1024,
    57  	}
    58  	config.StoreGlobalServerConfig(conf)
    59  
    60  	go func() {
    61  		_ = http.ListenAndServe("localhost:6060", nil)
    62  	}()
    63  
    64  	err = os.MkdirAll(*sorterDir, 0o755)
    65  	if err != nil {
    66  		log.Error("sorter_stress_test:", zap.Error(err))
    67  	}
    68  
    69  	sorter, err := pullerSorter.NewUnifiedSorter(*sorterDir, "test-cf", "test", 0, "0.0.0.0:0")
    70  	if err != nil {
    71  		log.Panic("sorter_stress_test:", zap.Error(err))
    72  	}
    73  
    74  	ctx1, cancel := context.WithCancel(context.Background())
    75  
    76  	eg, ctx := errgroup.WithContext(ctx1)
    77  
    78  	eg.Go(func() error {
    79  		return pullerSorter.RunWorkerPool(ctx)
    80  	})
    81  
    82  	eg.Go(func() error {
    83  		return sorter.Run(ctx)
    84  	})
    85  
    86  	// launch the consumer
    87  	eg.Go(func() error {
    88  		counter := 0
    89  		lastTs := uint64(0)
    90  		for {
    91  			select {
    92  			case <-ctx.Done():
    93  				return ctx.Err()
    94  			case event := <-sorter.Output():
    95  				if event.RawKV.OpType != model.OpTypeResolved {
    96  					if event.CRTs < lastTs {
    97  						panic("regressed")
    98  					}
    99  					lastTs = event.CRTs
   100  					counter += 1
   101  					if counter%10000 == 0 {
   102  						log.Debug("Messages received", zap.Int("counter", counter))
   103  					}
   104  					if counter >= *numBatches**msgsPerBatch {
   105  						log.Debug("Unified Sorter test successful")
   106  						cancel()
   107  						return nil
   108  					}
   109  				}
   110  			}
   111  		}
   112  	})
   113  
   114  	eg1 := errgroup.Group{}
   115  	for i := 0; i < *numBatches; i++ {
   116  		eg1.Go(func() error {
   117  			generateGroup(ctx, sorter)
   118  			return nil
   119  		})
   120  	}
   121  
   122  	err = eg1.Wait()
   123  	if err != nil {
   124  		log.Error("sorter_stress_test:", zap.Error(err))
   125  	}
   126  
   127  	sorter.AddEntry(ctx, model.NewResolvedPolymorphicEvent(0, uint64((*msgsPerBatch<<5)+256)))
   128  
   129  	err = eg.Wait()
   130  	if err != nil {
   131  		if strings.Contains(err.Error(), "context canceled") {
   132  			return
   133  		}
   134  		log.Error("sorter_stress_test:", zap.Error(err))
   135  	}
   136  }
   137  
   138  func generateGroup(ctx context.Context, sorter puller.EventSorter) {
   139  	for i := 0; i < *msgsPerBatch; i++ {
   140  		ts := (i << 5) + rand.Intn(256)
   141  		event := model.NewPolymorphicEvent(newMockRawKV(uint64(ts)))
   142  		sorter.AddEntry(ctx, event)
   143  	}
   144  }
   145  
   146  var (
   147  	key   = []byte(randSeq(10))
   148  	value = []byte(randSeq(*bytesPerMsg))
   149  )
   150  
   151  func newMockRawKV(ts uint64) *model.RawKVEntry {
   152  	return &model.RawKVEntry{
   153  		OpType:   model.OpTypePut,
   154  		Key:      key,
   155  		Value:    value,
   156  		OldValue: nil,
   157  		StartTs:  ts - 5,
   158  		CRTs:     ts,
   159  		RegionID: 0,
   160  	}
   161  }
   162  
   163  var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
   164  
   165  func randSeq(n int) string {
   166  	b := make([]rune, n)
   167  	for i := range b {
   168  		b[i] = letters[rand.Intn(len(letters))]
   169  	}
   170  	return string(b)
   171  }