github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/fsutil/file_allocator_test.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //	http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  package fsutil
    14  
    15  import (
    16  	"math"
    17  	"os"
    18  	"path/filepath"
    19  	"testing"
    20  
    21  	"github.com/stretchr/testify/require"
    22  )
    23  
    24  func TestFileAllocateSuccess(t *testing.T) {
    25  	fl := NewFileAllocator(t.TempDir(), "test", 64*1024*1024)
    26  	defer fl.Close()
    27  
    28  	f, err := fl.Open()
    29  	require.Nil(t, err)
    30  	f.Close()
    31  }
    32  
    33  func TestFileAllocateFailed(t *testing.T) {
    34  	// 1. the requested allocation space will cause disk full
    35  	fl := NewFileAllocator(t.TempDir(), "test", math.MaxInt64)
    36  
    37  	f, err := fl.Open()
    38  	require.Nil(t, f)
    39  	require.NotNil(t, err)
    40  	f.Close()
    41  	fl.Close()
    42  
    43  	// 2. the directory does not exist
    44  	fl = NewFileAllocator("not-exist-dir", "test", 1024)
    45  	f, err = fl.Open()
    46  	require.NotNil(t, err)
    47  	require.Nil(t, f)
    48  	fl.Close()
    49  }
    50  
    51  func benchmarkWriteData(b *testing.B, size int, useFileAlloctor bool) {
    52  	var f *os.File
    53  	var err error
    54  
    55  	if useFileAlloctor {
    56  		fl := NewFileAllocator(b.TempDir(), "bench", 64*1024*1024)
    57  		defer fl.Close()
    58  
    59  		f, err := fl.Open()
    60  		require.NotNil(b, f)
    61  		require.Nil(b, err)
    62  	} else {
    63  		f, err = os.OpenFile(filepath.Join(b.TempDir(), "bench"),
    64  			os.O_CREATE|os.O_WRONLY, 0o420)
    65  		require.Nil(b, err)
    66  	}
    67  
    68  	data := make([]byte, size)
    69  	for i := 0; i < size; i++ {
    70  		data[i] = byte(i)
    71  	}
    72  
    73  	b.ResetTimer()
    74  	b.SetBytes(int64(8 * size))
    75  	for i := 0; i < b.N; i++ {
    76  		_, err := f.Write(data)
    77  		f.Sync()
    78  		require.Nil(b, err)
    79  	}
    80  }
    81  
    82  // go test -bench ^BenchmarkWrite100EntryWithAllocator$ \
    83  // -benchtime=30000x github.com/pingcap/tiflow/pkg/fsutil
    84  // goos: linux
    85  // goarch: amd64
    86  // pkg: github.com/pingcap/tiflow/pkg/fsutil
    87  // cpu: Intel(R) Xeon(R) Gold 6240 CPU @ 2.60GHz
    88  // BenchmarkWrite100EntryWithAllocator-9   	   30000	    204681 ns/op	   3.91 MB/s
    89  func BenchmarkWrite100EntryWithAllocator(b *testing.B) {
    90  	benchmarkWriteData(b, 100, true)
    91  }
    92  
    93  // go test -bench ^BenchmarkWrite100EntryWithoutAllocator$ \
    94  // -benchtime=30000x github.com/pingcap/tiflow/pkg/fsutil
    95  // goos: linux
    96  // goarch: amd64
    97  // pkg: github.com/pingcap/tiflow/pkg/fsutil
    98  // cpu: Intel(R) Xeon(R) Gold 6240 CPU @ 2.60GHz
    99  // BenchmarkWrite100EntryWithoutAllocator-9   	   30000	    303590 ns/op	   2.64 MB/s
   100  func BenchmarkWrite100EntryWithoutAllocator(b *testing.B) {
   101  	benchmarkWriteData(b, 100, false)
   102  }
   103  
   104  // go test -bench ^BenchmarkWrite1000EntryWithAllocator$ \
   105  // -benchtime=30000x github.com/pingcap/tiflow/pkg/fsutil
   106  // goos: linux
   107  // goarch: amd64
   108  // pkg: github.com/pingcap/tiflow/pkg/fsutil
   109  // cpu: Intel(R) Xeon(R) Gold 6240 CPU @ 2.60GHz
   110  // BenchmarkWrite1000EntryWithAllocator-9   	   30000	    216718 ns/op	  36.91 MB/s
   111  func BenchmarkWrite1000EntryWithAllocator(b *testing.B) {
   112  	benchmarkWriteData(b, 1000, true)
   113  }
   114  
   115  // go test -bench ^BenchmarkWrite1000EntryWithoutAllocator$ \
   116  // -benchtime=30000x github.com/pingcap/tiflow/pkg/fsutil
   117  // goos: linux
   118  // goarch: amd64
   119  // pkg: github.com/pingcap/tiflow/pkg/fsutil
   120  // cpu: Intel(R) Xeon(R) Gold 6240 CPU @ 2.60GHz
   121  // BenchmarkWrite1000EntryWithoutAllocator-9   	   30000	    293505 ns/op	  27.26 MB/s
   122  func BenchmarkWrite1000EntryWithoutAllocator(b *testing.B) {
   123  	benchmarkWriteData(b, 1000, false)
   124  }
   125  
   126  // go test -bench ^BenchmarkWrite10000EntryWithAllocator$ \
   127  // -benchtime=30000x github.com/pingcap/tiflow/pkg/fsutil
   128  // goos: linux
   129  // goarch: amd64
   130  // pkg: github.com/pingcap/tiflow/pkg/fsutil
   131  // cpu: Intel(R) Xeon(R) Gold 6240 CPU @ 2.60GHz
   132  // BenchmarkWrite10000EntryWithAllocator-9   	   30000	    333734 ns/op	 239.71 MB/s
   133  func BenchmarkWrite10000EntryWithAllocator(b *testing.B) {
   134  	benchmarkWriteData(b, 10000, true)
   135  }
   136  
   137  // go test -bench ^BenchmarkWrite10000EntryWithoutAllocator$ \
   138  // -benchtime=30000x github.com/pingcap/tiflow/pkg/fsutil
   139  // goos: linux
   140  // goarch: amd64
   141  // pkg: github.com/pingcap/tiflow/pkg/fsutil
   142  // cpu: Intel(R) Xeon(R) Gold 6240 CPU @ 2.60GHz
   143  // BenchmarkWrite10000EntryWithoutAllocator-9   	   30000	    399723 ns/op	 200.14 MB/s
   144  func BenchmarkWrite10000EntryWithoutAllocator(b *testing.B) {
   145  	benchmarkWriteData(b, 10000, false)
   146  }