github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/lib/pool/pool_test.go (about)

     1  package pool
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"math/rand"
     7  	"testing"
     8  	"time"
     9  
    10  	"github.com/rclone/rclone/fstest/testy"
    11  	"github.com/stretchr/testify/assert"
    12  )
    13  
    14  // makes the allocations be unreliable
    15  func makeUnreliable(bp *Pool) {
    16  	bp.alloc = func(size int) ([]byte, error) {
    17  		if rand.Intn(3) != 0 {
    18  			return nil, errors.New("failed to allocate memory")
    19  		}
    20  		return make([]byte, size), nil
    21  	}
    22  	bp.free = func(b []byte) error {
    23  		if rand.Intn(3) != 0 {
    24  			return errors.New("failed to free memory")
    25  		}
    26  		return nil
    27  	}
    28  }
    29  
    30  func testGetPut(t *testing.T, useMmap bool, unreliable bool) {
    31  	bp := New(60*time.Second, 4096, 2, useMmap)
    32  	if unreliable {
    33  		makeUnreliable(bp)
    34  	}
    35  
    36  	assert.Equal(t, 0, bp.InUse())
    37  
    38  	b1 := bp.Get()
    39  	assert.Equal(t, 1, bp.InUse())
    40  	assert.Equal(t, 0, bp.InPool())
    41  	assert.Equal(t, 1, bp.Alloced())
    42  
    43  	b2 := bp.Get()
    44  	assert.Equal(t, 2, bp.InUse())
    45  	assert.Equal(t, 0, bp.InPool())
    46  	assert.Equal(t, 2, bp.Alloced())
    47  
    48  	b3 := bp.Get()
    49  	assert.Equal(t, 3, bp.InUse())
    50  	assert.Equal(t, 0, bp.InPool())
    51  	assert.Equal(t, 3, bp.Alloced())
    52  
    53  	bp.Put(b1)
    54  	assert.Equal(t, 2, bp.InUse())
    55  	assert.Equal(t, 1, bp.InPool())
    56  	assert.Equal(t, 3, bp.Alloced())
    57  
    58  	bp.Put(b2)
    59  	assert.Equal(t, 1, bp.InUse())
    60  	assert.Equal(t, 2, bp.InPool())
    61  	assert.Equal(t, 3, bp.Alloced())
    62  
    63  	bp.Put(b3)
    64  	assert.Equal(t, 0, bp.InUse())
    65  	assert.Equal(t, 2, bp.InPool())
    66  	assert.Equal(t, 2, bp.Alloced())
    67  
    68  	addr := func(b []byte) string {
    69  		return fmt.Sprintf("%p", &b[0])
    70  	}
    71  	b1a := bp.Get()
    72  	assert.Equal(t, addr(b2), addr(b1a))
    73  	assert.Equal(t, 1, bp.InUse())
    74  	assert.Equal(t, 1, bp.InPool())
    75  	assert.Equal(t, 2, bp.Alloced())
    76  
    77  	b2a := bp.Get()
    78  	assert.Equal(t, addr(b1), addr(b2a))
    79  	assert.Equal(t, 2, bp.InUse())
    80  	assert.Equal(t, 0, bp.InPool())
    81  	assert.Equal(t, 2, bp.Alloced())
    82  
    83  	bp.Put(b1a)
    84  	bp.Put(b2a)
    85  	assert.Equal(t, 0, bp.InUse())
    86  	assert.Equal(t, 2, bp.InPool())
    87  	assert.Equal(t, 2, bp.Alloced())
    88  
    89  	assert.Panics(t, func() {
    90  		bp.Put(make([]byte, 1))
    91  	})
    92  
    93  	bp.Flush()
    94  	assert.Equal(t, 0, bp.InUse())
    95  	assert.Equal(t, 0, bp.InPool())
    96  	assert.Equal(t, 0, bp.Alloced())
    97  }
    98  
    99  func testFlusher(t *testing.T, useMmap bool, unreliable bool) {
   100  	bp := New(50*time.Millisecond, 4096, 2, useMmap)
   101  	if unreliable {
   102  		makeUnreliable(bp)
   103  	}
   104  
   105  	b1 := bp.Get()
   106  	b2 := bp.Get()
   107  	b3 := bp.Get()
   108  	bp.Put(b1)
   109  	bp.Put(b2)
   110  	bp.Put(b3)
   111  	assert.Equal(t, 0, bp.InUse())
   112  	assert.Equal(t, 2, bp.InPool())
   113  	assert.Equal(t, 2, bp.Alloced())
   114  	bp.mu.Lock()
   115  	assert.Equal(t, 0, bp.minFill)
   116  	assert.Equal(t, true, bp.flushPending)
   117  	bp.mu.Unlock()
   118  
   119  	checkFlushHasHappened := func(desired int) {
   120  		var n int
   121  		for i := 0; i < 10; i++ {
   122  			time.Sleep(100 * time.Millisecond)
   123  			n = bp.InPool()
   124  			if n <= desired {
   125  				break
   126  			}
   127  		}
   128  		assert.Equal(t, desired, n)
   129  	}
   130  
   131  	checkFlushHasHappened(0)
   132  	assert.Equal(t, 0, bp.InUse())
   133  	assert.Equal(t, 0, bp.InPool())
   134  	assert.Equal(t, 0, bp.Alloced())
   135  	bp.mu.Lock()
   136  	assert.Equal(t, 0, bp.minFill)
   137  	assert.Equal(t, false, bp.flushPending)
   138  	bp.mu.Unlock()
   139  
   140  	// Now do manual aging to check it is working properly
   141  	bp = New(100*time.Second, 4096, 2, useMmap)
   142  
   143  	// Check the new one doesn't get flushed
   144  	b1 = bp.Get()
   145  	b2 = bp.Get()
   146  	bp.Put(b1)
   147  	bp.Put(b2)
   148  
   149  	bp.mu.Lock()
   150  	assert.Equal(t, 0, bp.minFill)
   151  	assert.Equal(t, true, bp.flushPending)
   152  	bp.mu.Unlock()
   153  
   154  	bp.flushAged()
   155  
   156  	assert.Equal(t, 0, bp.InUse())
   157  	assert.Equal(t, 2, bp.InPool())
   158  	assert.Equal(t, 2, bp.Alloced())
   159  	bp.mu.Lock()
   160  	assert.Equal(t, 2, bp.minFill)
   161  	assert.Equal(t, true, bp.flushPending)
   162  	bp.mu.Unlock()
   163  
   164  	bp.Put(bp.Get())
   165  
   166  	assert.Equal(t, 0, bp.InUse())
   167  	assert.Equal(t, 2, bp.InPool())
   168  	assert.Equal(t, 2, bp.Alloced())
   169  	bp.mu.Lock()
   170  	assert.Equal(t, 1, bp.minFill)
   171  	assert.Equal(t, true, bp.flushPending)
   172  	bp.mu.Unlock()
   173  
   174  	bp.flushAged()
   175  
   176  	assert.Equal(t, 0, bp.InUse())
   177  	assert.Equal(t, 1, bp.InPool())
   178  	assert.Equal(t, 1, bp.Alloced())
   179  	bp.mu.Lock()
   180  	assert.Equal(t, 1, bp.minFill)
   181  	assert.Equal(t, true, bp.flushPending)
   182  	bp.mu.Unlock()
   183  
   184  	bp.flushAged()
   185  
   186  	assert.Equal(t, 0, bp.InUse())
   187  	assert.Equal(t, 0, bp.InPool())
   188  	assert.Equal(t, 0, bp.Alloced())
   189  	bp.mu.Lock()
   190  	assert.Equal(t, 0, bp.minFill)
   191  	assert.Equal(t, false, bp.flushPending)
   192  	bp.mu.Unlock()
   193  }
   194  
   195  func TestPool(t *testing.T) {
   196  	for _, test := range []struct {
   197  		name       string
   198  		useMmap    bool
   199  		unreliable bool
   200  	}{
   201  		{
   202  			name:       "make",
   203  			useMmap:    false,
   204  			unreliable: false,
   205  		},
   206  		{
   207  			name:       "mmap",
   208  			useMmap:    true,
   209  			unreliable: false,
   210  		},
   211  		{
   212  			name:       "canFail",
   213  			useMmap:    false,
   214  			unreliable: true,
   215  		},
   216  	} {
   217  		t.Run(test.name, func(t *testing.T) {
   218  			t.Run("GetPut", func(t *testing.T) { testGetPut(t, test.useMmap, test.unreliable) })
   219  			t.Run("Flusher", func(t *testing.T) {
   220  				if test.name == "canFail" {
   221  					testy.SkipUnreliable(t) // fails regularly on macOS
   222  				}
   223  				testFlusher(t, test.useMmap, test.unreliable)
   224  			})
   225  		})
   226  	}
   227  }