github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/testing_utils/sorter_stress_test/sorter_stress.go (about) 1 // Copyright 2020 PingCAP, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package main 15 16 import ( 17 "context" 18 "flag" 19 "math/rand" 20 "net/http" 21 _ "net/http/pprof" 22 "os" 23 "strings" 24 25 "github.com/pingcap/failpoint" 26 27 "github.com/pingcap/log" 28 "github.com/pingcap/ticdc/cdc/model" 29 "github.com/pingcap/ticdc/cdc/puller" 30 pullerSorter "github.com/pingcap/ticdc/cdc/puller/sorter" 31 "github.com/pingcap/ticdc/pkg/config" 32 "go.uber.org/zap" 33 "golang.org/x/sync/errgroup" 34 ) 35 36 var ( 37 sorterDir = flag.String("dir", "./sorter", "temporary directory used for sorting") 38 numBatches = flag.Int("num-batches", 256, "number of batches of ordered events") 39 msgsPerBatch = flag.Int("num-messages-per-batch", 1024, "number of events in a batch") 40 bytesPerMsg = flag.Int("bytes-per-message", 1024, "number of bytes in an event") 41 ) 42 43 func main() { 44 flag.Parse() 45 log.SetLevel(zap.DebugLevel) 46 err := failpoint.Enable("github.com/pingcap/ticdc/cdc/puller/sorter/sorterDebug", "return(true)") 47 if err != nil { 48 log.Fatal("Could not enable failpoint", zap.Error(err)) 49 } 50 51 conf := config.GetDefaultServerConfig() 52 conf.Sorter = &config.SorterConfig{ 53 NumConcurrentWorker: 8, 54 ChunkSizeLimit: 1 * 1024 * 1024 * 1024, 55 MaxMemoryPressure: 60, 56 MaxMemoryConsumption: 16 * 1024 * 1024 * 1024, 57 } 58 config.StoreGlobalServerConfig(conf) 59 60 go func() { 61 _ = http.ListenAndServe("localhost:6060", nil) 62 }() 63 64 err = os.MkdirAll(*sorterDir, 0o755) 65 if err != nil { 66 log.Error("sorter_stress_test:", zap.Error(err)) 67 } 68 69 sorter, err := pullerSorter.NewUnifiedSorter(*sorterDir, "test-cf", "test", 0, "0.0.0.0:0") 70 if err != nil { 71 log.Panic("sorter_stress_test:", zap.Error(err)) 72 } 73 74 ctx1, cancel := context.WithCancel(context.Background()) 75 76 eg, ctx := errgroup.WithContext(ctx1) 77 78 eg.Go(func() error { 79 return pullerSorter.RunWorkerPool(ctx) 80 }) 81 82 eg.Go(func() error { 83 return sorter.Run(ctx) 84 }) 85 86 // launch the consumer 87 eg.Go(func() error { 88 counter := 0 89 lastTs := uint64(0) 90 for { 91 select { 92 case <-ctx.Done(): 93 return ctx.Err() 94 case event := <-sorter.Output(): 95 if event.RawKV.OpType != model.OpTypeResolved { 96 if event.CRTs < lastTs { 97 panic("regressed") 98 } 99 lastTs = event.CRTs 100 counter += 1 101 if counter%10000 == 0 { 102 log.Debug("Messages received", zap.Int("counter", counter)) 103 } 104 if counter >= *numBatches**msgsPerBatch { 105 log.Debug("Unified Sorter test successful") 106 cancel() 107 return nil 108 } 109 } 110 } 111 } 112 }) 113 114 eg1 := errgroup.Group{} 115 for i := 0; i < *numBatches; i++ { 116 eg1.Go(func() error { 117 generateGroup(ctx, sorter) 118 return nil 119 }) 120 } 121 122 err = eg1.Wait() 123 if err != nil { 124 log.Error("sorter_stress_test:", zap.Error(err)) 125 } 126 127 sorter.AddEntry(ctx, model.NewResolvedPolymorphicEvent(0, uint64((*msgsPerBatch<<5)+256))) 128 129 err = eg.Wait() 130 if err != nil { 131 if strings.Contains(err.Error(), "context canceled") { 132 return 133 } 134 log.Error("sorter_stress_test:", zap.Error(err)) 135 } 136 } 137 138 func generateGroup(ctx context.Context, sorter puller.EventSorter) { 139 for i := 0; i < *msgsPerBatch; i++ { 140 ts := (i << 5) + rand.Intn(256) 141 event := model.NewPolymorphicEvent(newMockRawKV(uint64(ts))) 142 sorter.AddEntry(ctx, event) 143 } 144 } 145 146 var ( 147 key = []byte(randSeq(10)) 148 value = []byte(randSeq(*bytesPerMsg)) 149 ) 150 151 func newMockRawKV(ts uint64) *model.RawKVEntry { 152 return &model.RawKVEntry{ 153 OpType: model.OpTypePut, 154 Key: key, 155 Value: value, 156 OldValue: nil, 157 StartTs: ts - 5, 158 CRTs: ts, 159 RegionID: 0, 160 } 161 } 162 163 var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") 164 165 func randSeq(n int) string { 166 b := make([]rune, n) 167 for i := range b { 168 b[i] = letters[rand.Intn(len(letters))] 169 } 170 return string(b) 171 }