github.com/matrixorigin/matrixone@v1.2.0/pkg/vm/engine/tae/blockio/pipeline_test.go (about)

     1  // Copyright 2023 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package blockio
    16  
    17  import (
    18  	"context"
    19  	"os"
    20  	"sync"
    21  	"testing"
    22  	"time"
    23  
    24  	"github.com/matrixorigin/matrixone/pkg/container/types"
    25  	"github.com/matrixorigin/matrixone/pkg/fileservice"
    26  	"github.com/matrixorigin/matrixone/pkg/objectio"
    27  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/logstore/sm"
    28  	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/tasks"
    29  	"github.com/stretchr/testify/assert"
    30  )
    31  
    32  func makeIOPipelineOptions(depth int) Option {
    33  	return func(p *IoPipeline) {
    34  		p.options.queueDepth = depth
    35  	}
    36  }
    37  
    38  func makeFileService(t *testing.T) fileservice.FileService {
    39  	dir := os.TempDir()
    40  	fs, err := fileservice.NewLocalFS(
    41  		context.TODO(), "local", dir, fileservice.DisabledCacheConfig, nil)
    42  	assert.Nil(t, err)
    43  	return fs
    44  }
    45  
    46  func makeLocation() objectio.Location {
    47  	uuid, _ := types.BuildUuid()
    48  	name := objectio.BuildObjectName(&uuid, 1)
    49  	extent := objectio.NewExtent(1, 1, 1, 1)
    50  	return objectio.BuildLocation(name, extent, 1, 1)
    51  }
    52  
    53  func makeTaskJob() *tasks.Job {
    54  	job := _jobPool.Get().(*tasks.Job)
    55  	job.Init(context.TODO(), "0", tasks.JTInvalid, func(context.Context) *tasks.JobResult {
    56  		return nil
    57  	})
    58  	return job
    59  }
    60  
    61  func TestNewIOPipeline(t *testing.T) {
    62  	p := NewIOPipeline(makeIOPipelineOptions(0))
    63  	p.Start()
    64  	assert.Equal(t, p.active.Load(), true)
    65  	// waiting pipeline's queue initial done
    66  	time.Sleep(time.Millisecond * 100)
    67  
    68  	service := makeFileService(t)
    69  	location := makeLocation()
    70  
    71  	// step 1: all queue can accept item
    72  	para := buildPrefetchParams(service, location)
    73  	item, err := p.prefetch.queue.Enqueue(para)
    74  	assert.Nil(t, err)
    75  	assert.NotNil(t, item)
    76  
    77  	item, err = p.fetch.queue.Enqueue(makeTaskJob())
    78  	assert.Nil(t, err)
    79  	assert.NotNil(t, item)
    80  
    81  	// step 2: shut down all queue
    82  	p.prefetch.queue.Stop()
    83  	item, err = p.prefetch.queue.Enqueue(para)
    84  	assert.Equal(t, err, sm.ErrClose)
    85  	assert.NotNil(t, item)
    86  
    87  	p.fetch.queue.Stop()
    88  	item, err = p.fetch.queue.Enqueue(makeTaskJob())
    89  	assert.Equal(t, err, sm.ErrClose)
    90  	assert.NotNil(t, item)
    91  
    92  	// step 3: recreate queue to make sure pipeline.close() will not try to
    93  	// close a closed channel
    94  	p.fetch.queue = sm.NewSafeQueue(0, 0, nil)
    95  	p.prefetch.queue = sm.NewSafeQueue(0, 0, nil)
    96  
    97  	// step 4: close pipeline
    98  	p.Stop()
    99  
   100  }
   101  
   102  func TestIoPipeline_Prefetch(t *testing.T) {
   103  	wait := sync.WaitGroup{}
   104  	wait.Add(1)
   105  
   106  	queueSize := 10
   107  	batchSize := 0
   108  	p := new(IoPipeline)
   109  
   110  	p.prefetch.queue = sm.NewNonBlockingQueue(queueSize, batchSize, func(items ...any) {
   111  		wait.Wait()
   112  	})
   113  
   114  	p.stats.prefetchDropStats.Reset()
   115  	p.prefetch.queue.Start()
   116  
   117  	for i := 0; i < queueSize+1; i++ {
   118  		err := p.doPrefetch(buildPrefetchParams(nil, nil))
   119  		assert.Nil(t, err)
   120  		assert.Equal(t, int64(0), p.stats.prefetchDropStats.Load())
   121  		time.Sleep(time.Millisecond * 10)
   122  	}
   123  
   124  	err := p.doPrefetch(buildPrefetchParams(nil, nil))
   125  	assert.Nil(t, err)
   126  	assert.Equal(t, int64(1), p.stats.prefetchDropStats.Load())
   127  
   128  	wait.Done()
   129  	time.Sleep(time.Millisecond * 100)
   130  
   131  }