code.gitea.io/gitea@v1.19.3/modules/queue/queue_disk_channel_test.go (about)

     1  // Copyright 2019 The Gitea Authors. All rights reserved.
     2  // SPDX-License-Identifier: MIT
     3  
     4  package queue
     5  
     6  import (
     7  	"sync"
     8  	"testing"
     9  	"time"
    10  
    11  	"code.gitea.io/gitea/modules/log"
    12  
    13  	"github.com/stretchr/testify/assert"
    14  )
    15  
    16  func TestPersistableChannelQueue(t *testing.T) {
    17  	handleChan := make(chan *testData)
    18  	handle := func(data ...Data) []Data {
    19  		for _, datum := range data {
    20  			if datum == nil {
    21  				continue
    22  			}
    23  			testDatum := datum.(*testData)
    24  			handleChan <- testDatum
    25  		}
    26  		return nil
    27  	}
    28  
    29  	lock := sync.Mutex{}
    30  	queueShutdown := []func(){}
    31  	queueTerminate := []func(){}
    32  
    33  	tmpDir := t.TempDir()
    34  
    35  	queue, err := NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{
    36  		DataDir:      tmpDir,
    37  		BatchLength:  2,
    38  		QueueLength:  20,
    39  		Workers:      1,
    40  		BoostWorkers: 0,
    41  		MaxWorkers:   10,
    42  		Name:         "test-queue",
    43  	}, &testData{})
    44  	assert.NoError(t, err)
    45  
    46  	readyForShutdown := make(chan struct{})
    47  	readyForTerminate := make(chan struct{})
    48  
    49  	go queue.Run(func(shutdown func()) {
    50  		lock.Lock()
    51  		defer lock.Unlock()
    52  		select {
    53  		case <-readyForShutdown:
    54  		default:
    55  			close(readyForShutdown)
    56  		}
    57  		queueShutdown = append(queueShutdown, shutdown)
    58  	}, func(terminate func()) {
    59  		lock.Lock()
    60  		defer lock.Unlock()
    61  		select {
    62  		case <-readyForTerminate:
    63  		default:
    64  			close(readyForTerminate)
    65  		}
    66  		queueTerminate = append(queueTerminate, terminate)
    67  	})
    68  
    69  	test1 := testData{"A", 1}
    70  	test2 := testData{"B", 2}
    71  
    72  	err = queue.Push(&test1)
    73  	assert.NoError(t, err)
    74  	go func() {
    75  		err := queue.Push(&test2)
    76  		assert.NoError(t, err)
    77  	}()
    78  
    79  	result1 := <-handleChan
    80  	assert.Equal(t, test1.TestString, result1.TestString)
    81  	assert.Equal(t, test1.TestInt, result1.TestInt)
    82  
    83  	result2 := <-handleChan
    84  	assert.Equal(t, test2.TestString, result2.TestString)
    85  	assert.Equal(t, test2.TestInt, result2.TestInt)
    86  
    87  	// test1 is a testData not a *testData so will be rejected
    88  	err = queue.Push(test1)
    89  	assert.Error(t, err)
    90  
    91  	<-readyForShutdown
    92  	// Now shutdown the queue
    93  	lock.Lock()
    94  	callbacks := make([]func(), len(queueShutdown))
    95  	copy(callbacks, queueShutdown)
    96  	lock.Unlock()
    97  	for _, callback := range callbacks {
    98  		callback()
    99  	}
   100  
   101  	// Wait til it is closed
   102  	<-queue.(*PersistableChannelQueue).closed
   103  
   104  	err = queue.Push(&test1)
   105  	assert.NoError(t, err)
   106  	err = queue.Push(&test2)
   107  	assert.NoError(t, err)
   108  	select {
   109  	case <-handleChan:
   110  		assert.Fail(t, "Handler processing should have stopped")
   111  	default:
   112  	}
   113  
   114  	// terminate the queue
   115  	<-readyForTerminate
   116  	lock.Lock()
   117  	callbacks = make([]func(), len(queueTerminate))
   118  	copy(callbacks, queueTerminate)
   119  	lock.Unlock()
   120  	for _, callback := range callbacks {
   121  		callback()
   122  	}
   123  
   124  	select {
   125  	case <-handleChan:
   126  		assert.Fail(t, "Handler processing should have stopped")
   127  	default:
   128  	}
   129  
   130  	// Reopen queue
   131  	queue, err = NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{
   132  		DataDir:      tmpDir,
   133  		BatchLength:  2,
   134  		QueueLength:  20,
   135  		Workers:      1,
   136  		BoostWorkers: 0,
   137  		MaxWorkers:   10,
   138  		Name:         "test-queue",
   139  	}, &testData{})
   140  	assert.NoError(t, err)
   141  
   142  	readyForShutdown = make(chan struct{})
   143  	readyForTerminate = make(chan struct{})
   144  
   145  	go queue.Run(func(shutdown func()) {
   146  		lock.Lock()
   147  		defer lock.Unlock()
   148  		select {
   149  		case <-readyForShutdown:
   150  		default:
   151  			close(readyForShutdown)
   152  		}
   153  		queueShutdown = append(queueShutdown, shutdown)
   154  	}, func(terminate func()) {
   155  		lock.Lock()
   156  		defer lock.Unlock()
   157  		select {
   158  		case <-readyForTerminate:
   159  		default:
   160  			close(readyForTerminate)
   161  		}
   162  		queueTerminate = append(queueTerminate, terminate)
   163  	})
   164  
   165  	result3 := <-handleChan
   166  	assert.Equal(t, test1.TestString, result3.TestString)
   167  	assert.Equal(t, test1.TestInt, result3.TestInt)
   168  
   169  	result4 := <-handleChan
   170  	assert.Equal(t, test2.TestString, result4.TestString)
   171  	assert.Equal(t, test2.TestInt, result4.TestInt)
   172  
   173  	<-readyForShutdown
   174  	lock.Lock()
   175  	callbacks = make([]func(), len(queueShutdown))
   176  	copy(callbacks, queueShutdown)
   177  	lock.Unlock()
   178  	for _, callback := range callbacks {
   179  		callback()
   180  	}
   181  	<-readyForTerminate
   182  	lock.Lock()
   183  	callbacks = make([]func(), len(queueTerminate))
   184  	copy(callbacks, queueTerminate)
   185  	lock.Unlock()
   186  	for _, callback := range callbacks {
   187  		callback()
   188  	}
   189  }
   190  
   191  func TestPersistableChannelQueue_Pause(t *testing.T) {
   192  	lock := sync.Mutex{}
   193  	var queue Queue
   194  	var err error
   195  	pushBack := false
   196  
   197  	handleChan := make(chan *testData)
   198  	handle := func(data ...Data) []Data {
   199  		lock.Lock()
   200  		if pushBack {
   201  			if pausable, ok := queue.(Pausable); ok {
   202  				log.Info("pausing")
   203  				pausable.Pause()
   204  			}
   205  			lock.Unlock()
   206  			return data
   207  		}
   208  		lock.Unlock()
   209  
   210  		for _, datum := range data {
   211  			testDatum := datum.(*testData)
   212  			handleChan <- testDatum
   213  		}
   214  		return nil
   215  	}
   216  
   217  	queueShutdown := []func(){}
   218  	queueTerminate := []func(){}
   219  	terminated := make(chan struct{})
   220  
   221  	tmpDir := t.TempDir()
   222  
   223  	queue, err = NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{
   224  		DataDir:      tmpDir,
   225  		BatchLength:  2,
   226  		QueueLength:  20,
   227  		Workers:      1,
   228  		BoostWorkers: 0,
   229  		MaxWorkers:   10,
   230  		Name:         "test-queue",
   231  	}, &testData{})
   232  	assert.NoError(t, err)
   233  
   234  	go func() {
   235  		queue.Run(func(shutdown func()) {
   236  			lock.Lock()
   237  			defer lock.Unlock()
   238  			queueShutdown = append(queueShutdown, shutdown)
   239  		}, func(terminate func()) {
   240  			lock.Lock()
   241  			defer lock.Unlock()
   242  			queueTerminate = append(queueTerminate, terminate)
   243  		})
   244  		close(terminated)
   245  	}()
   246  
   247  	// Shutdown and Terminate in defer
   248  	defer func() {
   249  		lock.Lock()
   250  		callbacks := make([]func(), len(queueShutdown))
   251  		copy(callbacks, queueShutdown)
   252  		lock.Unlock()
   253  		for _, callback := range callbacks {
   254  			callback()
   255  		}
   256  		lock.Lock()
   257  		log.Info("Finally terminating")
   258  		callbacks = make([]func(), len(queueTerminate))
   259  		copy(callbacks, queueTerminate)
   260  		lock.Unlock()
   261  		for _, callback := range callbacks {
   262  			callback()
   263  		}
   264  	}()
   265  
   266  	test1 := testData{"A", 1}
   267  	test2 := testData{"B", 2}
   268  
   269  	err = queue.Push(&test1)
   270  	assert.NoError(t, err)
   271  
   272  	pausable, ok := queue.(Pausable)
   273  	if !assert.True(t, ok) {
   274  		return
   275  	}
   276  	result1 := <-handleChan
   277  	assert.Equal(t, test1.TestString, result1.TestString)
   278  	assert.Equal(t, test1.TestInt, result1.TestInt)
   279  
   280  	pausable.Pause()
   281  	paused, _ := pausable.IsPausedIsResumed()
   282  
   283  	select {
   284  	case <-paused:
   285  	case <-time.After(100 * time.Millisecond):
   286  		assert.Fail(t, "Queue is not paused")
   287  		return
   288  	}
   289  
   290  	queue.Push(&test2)
   291  
   292  	var result2 *testData
   293  	select {
   294  	case result2 = <-handleChan:
   295  		assert.Fail(t, "handler chan should be empty")
   296  	case <-time.After(100 * time.Millisecond):
   297  	}
   298  
   299  	assert.Nil(t, result2)
   300  
   301  	pausable.Resume()
   302  	_, resumed := pausable.IsPausedIsResumed()
   303  
   304  	select {
   305  	case <-resumed:
   306  	case <-time.After(100 * time.Millisecond):
   307  		assert.Fail(t, "Queue should be resumed")
   308  		return
   309  	}
   310  
   311  	select {
   312  	case result2 = <-handleChan:
   313  	case <-time.After(500 * time.Millisecond):
   314  		assert.Fail(t, "handler chan should contain test2")
   315  	}
   316  
   317  	assert.Equal(t, test2.TestString, result2.TestString)
   318  	assert.Equal(t, test2.TestInt, result2.TestInt)
   319  
   320  	// Set pushBack to so that the next handle will result in a Pause
   321  	lock.Lock()
   322  	pushBack = true
   323  	lock.Unlock()
   324  
   325  	// Ensure that we're still resumed
   326  	_, resumed = pausable.IsPausedIsResumed()
   327  
   328  	select {
   329  	case <-resumed:
   330  	case <-time.After(100 * time.Millisecond):
   331  		assert.Fail(t, "Queue is not resumed")
   332  		return
   333  	}
   334  
   335  	// push test1
   336  	queue.Push(&test1)
   337  
   338  	// Now as this is handled it should pause
   339  	paused, _ = pausable.IsPausedIsResumed()
   340  
   341  	select {
   342  	case <-paused:
   343  	case <-handleChan:
   344  		assert.Fail(t, "handler chan should not contain test1")
   345  		return
   346  	case <-time.After(500 * time.Millisecond):
   347  		assert.Fail(t, "queue should be paused")
   348  		return
   349  	}
   350  
   351  	lock.Lock()
   352  	pushBack = false
   353  	lock.Unlock()
   354  
   355  	pausable.Resume()
   356  
   357  	_, resumed = pausable.IsPausedIsResumed()
   358  	select {
   359  	case <-resumed:
   360  	case <-time.After(500 * time.Millisecond):
   361  		assert.Fail(t, "Queue should be resumed")
   362  		return
   363  	}
   364  
   365  	select {
   366  	case result1 = <-handleChan:
   367  	case <-time.After(500 * time.Millisecond):
   368  		assert.Fail(t, "handler chan should contain test1")
   369  		return
   370  	}
   371  	assert.Equal(t, test1.TestString, result1.TestString)
   372  	assert.Equal(t, test1.TestInt, result1.TestInt)
   373  
   374  	lock.Lock()
   375  	callbacks := make([]func(), len(queueShutdown))
   376  	copy(callbacks, queueShutdown)
   377  	queueShutdown = queueShutdown[:0]
   378  	lock.Unlock()
   379  	// Now shutdown the queue
   380  	for _, callback := range callbacks {
   381  		callback()
   382  	}
   383  
   384  	// Wait til it is closed
   385  	select {
   386  	case <-queue.(*PersistableChannelQueue).closed:
   387  	case <-time.After(5 * time.Second):
   388  		assert.Fail(t, "queue should close")
   389  		return
   390  	}
   391  
   392  	err = queue.Push(&test1)
   393  	assert.NoError(t, err)
   394  	err = queue.Push(&test2)
   395  	assert.NoError(t, err)
   396  	select {
   397  	case <-handleChan:
   398  		assert.Fail(t, "Handler processing should have stopped")
   399  		return
   400  	default:
   401  	}
   402  
   403  	// terminate the queue
   404  	lock.Lock()
   405  	callbacks = make([]func(), len(queueTerminate))
   406  	copy(callbacks, queueTerminate)
   407  	queueShutdown = queueTerminate[:0]
   408  	lock.Unlock()
   409  	for _, callback := range callbacks {
   410  		callback()
   411  	}
   412  
   413  	select {
   414  	case <-handleChan:
   415  		assert.Fail(t, "Handler processing should have stopped")
   416  		return
   417  	case <-terminated:
   418  	case <-time.After(10 * time.Second):
   419  		assert.Fail(t, "Queue should have terminated")
   420  		return
   421  	}
   422  
   423  	lock.Lock()
   424  	pushBack = true
   425  	lock.Unlock()
   426  
   427  	// Reopen queue
   428  	terminated = make(chan struct{})
   429  	queue, err = NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{
   430  		DataDir:      tmpDir,
   431  		BatchLength:  1,
   432  		QueueLength:  20,
   433  		Workers:      1,
   434  		BoostWorkers: 0,
   435  		MaxWorkers:   10,
   436  		Name:         "test-queue",
   437  	}, &testData{})
   438  	assert.NoError(t, err)
   439  	pausable, ok = queue.(Pausable)
   440  	if !assert.True(t, ok) {
   441  		return
   442  	}
   443  
   444  	paused, _ = pausable.IsPausedIsResumed()
   445  
   446  	go func() {
   447  		queue.Run(func(shutdown func()) {
   448  			lock.Lock()
   449  			defer lock.Unlock()
   450  			queueShutdown = append(queueShutdown, shutdown)
   451  		}, func(terminate func()) {
   452  			lock.Lock()
   453  			defer lock.Unlock()
   454  			queueTerminate = append(queueTerminate, terminate)
   455  		})
   456  		close(terminated)
   457  	}()
   458  
   459  	select {
   460  	case <-handleChan:
   461  		assert.Fail(t, "Handler processing should have stopped")
   462  		return
   463  	case <-paused:
   464  	}
   465  
   466  	paused, _ = pausable.IsPausedIsResumed()
   467  
   468  	select {
   469  	case <-paused:
   470  	case <-time.After(500 * time.Millisecond):
   471  		assert.Fail(t, "Queue is not paused")
   472  		return
   473  	}
   474  
   475  	select {
   476  	case <-handleChan:
   477  		assert.Fail(t, "Handler processing should have stopped")
   478  		return
   479  	default:
   480  	}
   481  
   482  	lock.Lock()
   483  	pushBack = false
   484  	lock.Unlock()
   485  
   486  	pausable.Resume()
   487  	_, resumed = pausable.IsPausedIsResumed()
   488  	select {
   489  	case <-resumed:
   490  	case <-time.After(500 * time.Millisecond):
   491  		assert.Fail(t, "Queue should be resumed")
   492  		return
   493  	}
   494  
   495  	var result3, result4 *testData
   496  
   497  	select {
   498  	case result3 = <-handleChan:
   499  	case <-time.After(1 * time.Second):
   500  		assert.Fail(t, "Handler processing should have resumed")
   501  		return
   502  	}
   503  	select {
   504  	case result4 = <-handleChan:
   505  	case <-time.After(1 * time.Second):
   506  		assert.Fail(t, "Handler processing should have resumed")
   507  		return
   508  	}
   509  	if result4.TestString == test1.TestString {
   510  		result3, result4 = result4, result3
   511  	}
   512  	assert.Equal(t, test1.TestString, result3.TestString)
   513  	assert.Equal(t, test1.TestInt, result3.TestInt)
   514  
   515  	assert.Equal(t, test2.TestString, result4.TestString)
   516  	assert.Equal(t, test2.TestInt, result4.TestInt)
   517  
   518  	lock.Lock()
   519  	callbacks = make([]func(), len(queueShutdown))
   520  	copy(callbacks, queueShutdown)
   521  	queueShutdown = queueShutdown[:0]
   522  	lock.Unlock()
   523  	// Now shutdown the queue
   524  	for _, callback := range callbacks {
   525  		callback()
   526  	}
   527  
   528  	// terminate the queue
   529  	lock.Lock()
   530  	callbacks = make([]func(), len(queueTerminate))
   531  	copy(callbacks, queueTerminate)
   532  	queueShutdown = queueTerminate[:0]
   533  	lock.Unlock()
   534  	for _, callback := range callbacks {
   535  		callback()
   536  	}
   537  
   538  	select {
   539  	case <-time.After(10 * time.Second):
   540  		assert.Fail(t, "Queue should have terminated")
   541  		return
   542  	case <-terminated:
   543  	}
   544  }