github.com/ari-anchor/sei-tendermint@v0.0.0-20230519144642-dc826b7b56bb/internal/statesync/block_queue_test.go (about) 1 package statesync 2 3 import ( 4 "context" 5 "math/rand" 6 "sync" 7 "testing" 8 "time" 9 10 "github.com/stretchr/testify/assert" 11 "github.com/stretchr/testify/require" 12 13 "github.com/ari-anchor/sei-tendermint/internal/test/factory" 14 "github.com/ari-anchor/sei-tendermint/types" 15 ) 16 17 var ( 18 startHeight int64 = 200 19 stopHeight int64 = 100 20 stopTime = time.Date(2019, 1, 1, 1, 0, 0, 0, time.UTC) 21 endTime = stopTime.Add(-1 * time.Second) 22 numWorkers = 1 23 ) 24 25 func TestBlockQueueBasic(t *testing.T) { 26 ctx, cancel := context.WithCancel(context.Background()) 27 defer cancel() 28 29 peerID, err := types.NewNodeID("0011223344556677889900112233445566778899") 30 require.NoError(t, err) 31 32 queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 1) 33 wg := &sync.WaitGroup{} 34 35 // asynchronously fetch blocks and add it to the queue 36 for i := 0; i <= numWorkers; i++ { 37 wg.Add(1) 38 go func() { 39 for { 40 select { 41 case height := <-queue.nextHeight(): 42 queue.add(mockLBResp(ctx, t, peerID, height, endTime)) 43 case <-queue.done(): 44 wg.Done() 45 return 46 } 47 } 48 }() 49 } 50 51 trackingHeight := startHeight 52 wg.Add(1) 53 54 loop: 55 for { 56 select { 57 case <-queue.done(): 58 wg.Done() 59 break loop 60 61 case resp := <-queue.verifyNext(): 62 // assert that the queue serializes the blocks 63 require.Equal(t, resp.block.Height, trackingHeight) 64 trackingHeight-- 65 queue.success() 66 } 67 68 } 69 70 wg.Wait() 71 assert.Less(t, trackingHeight, stopHeight) 72 } 73 74 // Test with spurious failures and retries 75 func TestBlockQueueWithFailures(t *testing.T) { 76 ctx, cancel := context.WithCancel(context.Background()) 77 defer cancel() 78 79 peerID, err := types.NewNodeID("0011223344556677889900112233445566778899") 80 require.NoError(t, err) 81 82 queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 200) 83 wg := &sync.WaitGroup{} 84 85 failureRate := 4 86 for i := 0; i <= numWorkers; i++ { 87 wg.Add(1) 88 go func() { 89 for { 90 select { 91 case height := <-queue.nextHeight(): 92 if rand.Intn(failureRate) == 0 { 93 queue.retry(height) 94 } else { 95 queue.add(mockLBResp(ctx, t, peerID, height, endTime)) 96 } 97 case <-queue.done(): 98 wg.Done() 99 return 100 } 101 } 102 }() 103 } 104 105 trackingHeight := startHeight 106 for { 107 select { 108 case resp := <-queue.verifyNext(): 109 // assert that the queue serializes the blocks 110 assert.Equal(t, resp.block.Height, trackingHeight) 111 if rand.Intn(failureRate) == 0 { 112 queue.retry(resp.block.Height) 113 } else { 114 trackingHeight-- 115 queue.success() 116 } 117 118 case <-queue.done(): 119 wg.Wait() 120 assert.Less(t, trackingHeight, stopHeight) 121 return 122 } 123 } 124 } 125 126 // Test that when all the blocks are retrieved that the queue still holds on to 127 // it's workers and in the event of failure can still fetch the failed block 128 func TestBlockQueueBlocks(t *testing.T) { 129 peerID, err := types.NewNodeID("0011223344556677889900112233445566778899") 130 require.NoError(t, err) 131 queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 2) 132 expectedHeight := startHeight 133 retryHeight := stopHeight + 2 134 135 ctx, cancel := context.WithCancel(context.Background()) 136 defer cancel() 137 138 loop: 139 for { 140 select { 141 case height := <-queue.nextHeight(): 142 require.Equal(t, height, expectedHeight) 143 require.GreaterOrEqual(t, height, stopHeight) 144 expectedHeight-- 145 queue.add(mockLBResp(ctx, t, peerID, height, endTime)) 146 case <-time.After(1 * time.Second): 147 if expectedHeight >= stopHeight { 148 t.Fatalf("expected next height %d", expectedHeight) 149 } 150 break loop 151 } 152 } 153 154 // close any waiter channels that the previous worker left hanging 155 for _, ch := range queue.waiters { 156 close(ch) 157 } 158 queue.waiters = make([]chan int64, 0) 159 160 wg := &sync.WaitGroup{} 161 wg.Add(1) 162 // so far so good. The worker is waiting. Now we fail a previous 163 // block and check that the worker fetches them 164 go func(t *testing.T) { 165 defer wg.Done() 166 select { 167 case height := <-queue.nextHeight(): 168 require.Equal(t, retryHeight, height) 169 case <-time.After(1 * time.Second): 170 require.Fail(t, "queue didn't ask worker to fetch failed height") 171 } 172 }(t) 173 queue.retry(retryHeight) 174 wg.Wait() 175 176 } 177 178 func TestBlockQueueAcceptsNoMoreBlocks(t *testing.T) { 179 peerID, err := types.NewNodeID("0011223344556677889900112233445566778899") 180 require.NoError(t, err) 181 queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 1) 182 defer queue.close() 183 184 ctx, cancel := context.WithCancel(context.Background()) 185 defer cancel() 186 187 loop: 188 for { 189 select { 190 case height := <-queue.nextHeight(): 191 require.GreaterOrEqual(t, height, stopHeight) 192 queue.add(mockLBResp(ctx, t, peerID, height, endTime)) 193 case <-time.After(1 * time.Second): 194 break loop 195 } 196 } 197 198 require.Len(t, queue.pending, int(startHeight-stopHeight)+1) 199 200 queue.add(mockLBResp(ctx, t, peerID, stopHeight-1, endTime)) 201 require.Len(t, queue.pending, int(startHeight-stopHeight)+1) 202 } 203 204 // Test a scenario where more blocks are needed then just the stopheight because 205 // we haven't found a block with a small enough time. 206 func TestBlockQueueStopTime(t *testing.T) { 207 peerID, err := types.NewNodeID("0011223344556677889900112233445566778899") 208 require.NoError(t, err) 209 210 queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 1) 211 wg := &sync.WaitGroup{} 212 213 ctx, cancel := context.WithCancel(context.Background()) 214 defer cancel() 215 216 baseTime := stopTime.Add(-50 * time.Second) 217 218 // asynchronously fetch blocks and add it to the queue 219 for i := 0; i <= numWorkers; i++ { 220 wg.Add(1) 221 go func() { 222 for { 223 select { 224 case height := <-queue.nextHeight(): 225 blockTime := baseTime.Add(time.Duration(height) * time.Second) 226 queue.add(mockLBResp(ctx, t, peerID, height, blockTime)) 227 case <-queue.done(): 228 wg.Done() 229 return 230 } 231 } 232 }() 233 } 234 235 trackingHeight := startHeight 236 for { 237 select { 238 case resp := <-queue.verifyNext(): 239 // assert that the queue serializes the blocks 240 assert.Equal(t, resp.block.Height, trackingHeight) 241 trackingHeight-- 242 queue.success() 243 244 case <-queue.done(): 245 wg.Wait() 246 assert.Less(t, trackingHeight, stopHeight-50) 247 return 248 } 249 } 250 } 251 252 func TestBlockQueueInitialHeight(t *testing.T) { 253 peerID, err := types.NewNodeID("0011223344556677889900112233445566778899") 254 require.NoError(t, err) 255 const initialHeight int64 = 120 256 257 queue := newBlockQueue(startHeight, stopHeight, initialHeight, stopTime, 1) 258 wg := &sync.WaitGroup{} 259 260 ctx, cancel := context.WithCancel(context.Background()) 261 defer cancel() 262 263 // asynchronously fetch blocks and add it to the queue 264 for i := 0; i <= numWorkers; i++ { 265 wg.Add(1) 266 go func() { 267 for { 268 select { 269 case height := <-queue.nextHeight(): 270 require.GreaterOrEqual(t, height, initialHeight) 271 queue.add(mockLBResp(ctx, t, peerID, height, endTime)) 272 case <-queue.done(): 273 wg.Done() 274 return 275 } 276 } 277 }() 278 } 279 280 loop: 281 for { 282 select { 283 case <-queue.done(): 284 wg.Wait() 285 require.NoError(t, queue.error()) 286 break loop 287 288 case resp := <-queue.verifyNext(): 289 require.GreaterOrEqual(t, resp.block.Height, initialHeight) 290 queue.success() 291 } 292 } 293 } 294 295 func mockLBResp(ctx context.Context, t *testing.T, peer types.NodeID, height int64, time time.Time) lightBlockResponse { 296 t.Helper() 297 vals, pv := factory.ValidatorSet(ctx, t, 3, 10) 298 _, _, lb := mockLB(ctx, t, height, time, factory.MakeBlockID(), vals, pv) 299 return lightBlockResponse{ 300 block: lb, 301 peer: peer, 302 } 303 }