github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/libraries/doltcore/remotestorage/concurrency_test.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package remotestorage
    16  
    17  import (
    18  	"errors"
    19  	"math/rand"
    20  	"sync"
    21  	"testing"
    22  	"time"
    23  )
    24  
    25  func TestBatchItr(t *testing.T) {
    26  	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
    27  
    28  	const maxSize = 5000
    29  	const numTests = 64
    30  
    31  	ints := make([]int, maxSize)
    32  	for i := 0; i < maxSize; i++ {
    33  		ints[i] = i
    34  	}
    35  
    36  	for i := 0; i < numTests; i++ {
    37  		batchSize := rng.Int()%200 + 1
    38  		size := rng.Int()%maxSize + 1
    39  		sl := ints[:size]
    40  
    41  		k := 0
    42  		batchItr(size, batchSize, func(start, end int) (stop bool) {
    43  			currSl := sl[start:end]
    44  
    45  			for j := 0; j < len(currSl); j++ {
    46  				if currSl[j] != k {
    47  					t.Fatal("failure. batchSize:", batchSize, "size:", size, "start", start, "end", end, "j", j, "k", k, "currSl[j]", currSl[j], "k", k)
    48  				}
    49  
    50  				k++
    51  			}
    52  
    53  			return false
    54  		})
    55  	}
    56  }
    57  
    58  func TestConcurrentExec(t *testing.T) {
    59  	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
    60  
    61  	const maxConcurrency = 256
    62  	const numWorkItems = 5000
    63  	const numUnbufferedTests = 16
    64  	const numBufferedTests = 16
    65  	const numErrTests = 16
    66  
    67  	t.Run("Unbuffered", func(t *testing.T) {
    68  		for i := 0; i < numUnbufferedTests; i++ {
    69  			concurrency := (rng.Int() % (maxConcurrency - 1)) + 1
    70  			concurrentTest(t, concurrency, numWorkItems, -1, make(chan int))
    71  		}
    72  	})
    73  
    74  	t.Run("Buffered", func(t *testing.T) {
    75  		for i := 0; i < numBufferedTests; i++ {
    76  			concurrency := (rng.Int() % (maxConcurrency - 1)) + 1
    77  			chanBuffSize := rng.Int() % numWorkItems
    78  			concurrentTest(t, concurrency, numWorkItems, -1, make(chan int, chanBuffSize))
    79  		}
    80  	})
    81  
    82  	t.Run("Error", func(t *testing.T) {
    83  		for i := 0; i < numErrTests; i++ {
    84  			concurrency := (rng.Int() % (maxConcurrency - 1)) + 1
    85  			chanBuffSize := rng.Int() % numWorkItems
    86  			firstErrIdx := rng.Int() % numWorkItems
    87  			concurrentTest(t, concurrency, numWorkItems, firstErrIdx, make(chan int, chanBuffSize))
    88  		}
    89  	})
    90  
    91  	t.Run("more concurrency than work", func(t *testing.T) {
    92  		concurrentTest(t, maxConcurrency*2, numWorkItems, -1, make(chan int))
    93  	})
    94  
    95  	t.Run("no work", func(t *testing.T) {
    96  		concurrentTest(t, maxConcurrency, 0, -1, make(chan int))
    97  	})
    98  
    99  }
   100  
   101  func concurrentTest(t *testing.T, concurrency, numWorkItems, firstErrIdx int, resultChan chan int) {
   102  	work := make([]func() error, numWorkItems)
   103  	shouldError := firstErrIdx > 0
   104  
   105  	var wg sync.WaitGroup
   106  	wg.Add(1)
   107  	go func() {
   108  		defer wg.Done()
   109  		count := 0
   110  		for z := range resultChan {
   111  			count++
   112  			if z != numWorkItems*2 {
   113  				t.Error("bad result value")
   114  			}
   115  		}
   116  
   117  		if count != numWorkItems {
   118  			t.Error("Didn't get all the results")
   119  		}
   120  	}()
   121  
   122  	for i := 0; i < numWorkItems; i++ {
   123  		x := i
   124  		y := numWorkItems*2 - i
   125  		work[i] = func() error {
   126  			if shouldError && i >= firstErrIdx {
   127  				return errors.New("an error")
   128  			}
   129  
   130  			resultChan <- x + y
   131  			return nil
   132  		}
   133  	}
   134  
   135  	err := concurrentExec(work, concurrency)
   136  
   137  	if err != nil != shouldError {
   138  		t.Error("unexpected error value")
   139  	}
   140  }