github.com/matrixorigin/matrixone@v0.7.0/pkg/fileservice/file_with_checksum_test.go (about)

     1  // Copyright 2022 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package fileservice
    16  
    17  import (
    18  	"crypto/rand"
    19  	"io"
    20  	"os"
    21  	"testing"
    22  	"testing/iotest"
    23  
    24  	"github.com/stretchr/testify/assert"
    25  )
    26  
    27  func TestFileWithChecksumOffsets(t *testing.T) {
    28  	f := NewFileWithChecksum[*os.File](nil, 64)
    29  
    30  	blockOffset, offsetInBlock := f.contentOffsetToBlockOffset(0)
    31  	assert.Equal(t, int64(0), blockOffset)
    32  	assert.Equal(t, int64(0), offsetInBlock)
    33  
    34  	blockOffset, offsetInBlock = f.contentOffsetToBlockOffset(1)
    35  	assert.Equal(t, int64(0), blockOffset)
    36  	assert.Equal(t, int64(1), offsetInBlock)
    37  
    38  	blockOffset, offsetInBlock = f.contentOffsetToBlockOffset(int64(f.blockContentSize))
    39  	assert.Equal(t, int64(f.blockSize), blockOffset)
    40  	assert.Equal(t, int64(0), offsetInBlock)
    41  
    42  	blockOffset, offsetInBlock = f.contentOffsetToBlockOffset(int64(f.blockContentSize) + 1)
    43  	assert.Equal(t, int64(f.blockSize), blockOffset)
    44  	assert.Equal(t, int64(1), offsetInBlock)
    45  
    46  	blockOffset, offsetInBlock = f.contentOffsetToBlockOffset(int64(f.blockContentSize)*2 + 1)
    47  	assert.Equal(t, int64(f.blockSize*2), blockOffset)
    48  	assert.Equal(t, int64(1), offsetInBlock)
    49  
    50  	blockOffset, offsetInBlock = f.contentOffsetToBlockOffset(int64(f.blockContentSize)*3 + 1)
    51  	assert.Equal(t, int64(f.blockSize*3), blockOffset)
    52  	assert.Equal(t, int64(1), offsetInBlock)
    53  }
    54  
    55  func TestFileWithChecksum(t *testing.T) {
    56  	blockContentSize := 8
    57  	tempDir := t.TempDir()
    58  
    59  	testFileWithChecksum(
    60  		t,
    61  		blockContentSize,
    62  		func() FileLike {
    63  			f, err := os.CreateTemp(tempDir, "*")
    64  			assert.Nil(t, err)
    65  			t.Cleanup(func() {
    66  				f.Close()
    67  			})
    68  			return f
    69  		},
    70  	)
    71  }
    72  
    73  func testFileWithChecksum(
    74  	t *testing.T,
    75  	blockContentSize int,
    76  	newUnderlying func() FileLike,
    77  ) {
    78  
    79  	for i := 0; i < blockContentSize*4; i++ {
    80  
    81  		underlying := newUnderlying()
    82  		fileWithChecksum := NewFileWithChecksum(underlying, blockContentSize)
    83  
    84  		check := func(data []byte) {
    85  			// check content
    86  			pos, err := fileWithChecksum.Seek(0, io.SeekStart)
    87  			assert.Nil(t, err)
    88  			assert.Equal(t, int64(0), pos)
    89  			content, err := io.ReadAll(fileWithChecksum)
    90  			assert.Nil(t, err)
    91  			assert.Equal(t, data, content)
    92  
    93  			// seek
    94  			n, err := fileWithChecksum.Seek(0, io.SeekEnd)
    95  			assert.Nil(t, err)
    96  			assert.Equal(t, int64(len(data)), n)
    97  
    98  			// iotest
    99  			pos, err = fileWithChecksum.Seek(0, io.SeekStart)
   100  			assert.Nil(t, err)
   101  			assert.Equal(t, int64(0), pos)
   102  			err = iotest.TestReader(fileWithChecksum, data)
   103  			if err != nil {
   104  				t.Logf("%s", err)
   105  			}
   106  			assert.Nil(t, err)
   107  		}
   108  
   109  		// random bytes
   110  		data := make([]byte, i)
   111  		_, err := rand.Read(data)
   112  		assert.Nil(t, err)
   113  
   114  		// write
   115  		n, err := fileWithChecksum.Write(data)
   116  		assert.Nil(t, err)
   117  		assert.Equal(t, i, n)
   118  
   119  		// underlying size
   120  		underlyingSize, err := underlying.Seek(0, io.SeekEnd)
   121  		assert.Nil(t, err)
   122  		expectedSize := len(data) / blockContentSize * (blockContentSize + _ChecksumSize)
   123  		mod := len(data) % blockContentSize
   124  		if mod != 0 {
   125  			expectedSize += _ChecksumSize + mod
   126  		}
   127  		assert.Equal(t, expectedSize, int(underlyingSize))
   128  
   129  		check(data)
   130  
   131  		for j := 0; j < len(data); j++ {
   132  
   133  			// seek and write random bytes
   134  			_, err = rand.Read(data[j:])
   135  			assert.Nil(t, err)
   136  			pos, err := fileWithChecksum.Seek(int64(j), io.SeekStart)
   137  			assert.Nil(t, err)
   138  			assert.Equal(t, int64(j), pos)
   139  			n, err = fileWithChecksum.Write(data[j:])
   140  			assert.Nil(t, err)
   141  			assert.Equal(t, len(data[j:]), n)
   142  
   143  			// seek and read
   144  			pos, err = fileWithChecksum.Seek(int64(j), io.SeekStart)
   145  			assert.Nil(t, err)
   146  			assert.Equal(t, int64(j), pos)
   147  			content, err := io.ReadAll(fileWithChecksum)
   148  			assert.Nil(t, err)
   149  			assert.Equal(t, data[j:], content)
   150  
   151  			check(data)
   152  
   153  		}
   154  
   155  	}
   156  }
   157  
   158  func TestMultiLayerFileWithChecksum(t *testing.T) {
   159  	blockContentSize := 8
   160  	tempDir := t.TempDir()
   161  
   162  	testFileWithChecksum(
   163  		t,
   164  		blockContentSize,
   165  		func() FileLike {
   166  			f, err := os.CreateTemp(tempDir, "*")
   167  			assert.Nil(t, err)
   168  			t.Cleanup(func() {
   169  				f.Close()
   170  			})
   171  			f2 := NewFileWithChecksum(f, blockContentSize)
   172  			f3 := NewFileWithChecksum(f2, blockContentSize)
   173  			f4 := NewFileWithChecksum(f3, blockContentSize)
   174  			return f4
   175  		},
   176  	)
   177  }