github.com/matrixorigin/matrixone@v1.2.0/pkg/logservice/snapshot_test.go (about)

     1  // Copyright 2021 - 2022 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package logservice
    16  
    17  import (
    18  	"fmt"
    19  	"testing"
    20  
    21  	"github.com/lni/vfs"
    22  	"github.com/stretchr/testify/assert"
    23  )
    24  
    25  func TestSnapshotItemValid(t *testing.T) {
    26  	si := &snapshotItem{
    27  		fs:    vfs.NewStrictMem(),
    28  		index: 100,
    29  		dir:   "/tmp/exported",
    30  	}
    31  	err := si.fs.MkdirAll(si.dir, defaultExportedDirMode)
    32  	assert.NoError(t, err)
    33  
    34  	_, err = si.fs.Create(si.dir + "/snapshot.metadata")
    35  	assert.NoError(t, err)
    36  	_, err = si.fs.Create(si.dir + "/snapshot-0000000000000064.gbsnap")
    37  	assert.NoError(t, err)
    38  
    39  	v, err := si.Valid()
    40  	assert.Equal(t, true, v)
    41  	assert.NoError(t, err)
    42  }
    43  
    44  func TestSnapshotItemInValid(t *testing.T) {
    45  	si := &snapshotItem{
    46  		fs:    vfs.NewStrictMem(),
    47  		index: 100,
    48  		dir:   "/tmp/exported",
    49  	}
    50  	err := si.fs.MkdirAll(si.dir, defaultExportedDirMode)
    51  	assert.NoError(t, err)
    52  
    53  	_, err = si.fs.Create(si.dir + "/snapshot.m")
    54  	assert.NoError(t, err)
    55  	_, err = si.fs.Create(si.dir + "/snapshot-0000000000000064.gbsnap")
    56  	assert.NoError(t, err)
    57  
    58  	v, err := si.Valid()
    59  	assert.Equal(t, false, v)
    60  	assert.Error(t, err)
    61  }
    62  
    63  func testPrepareSnapshot(
    64  	t *testing.T, mgr *snapshotManager, nid nodeID, index snapshotIndex,
    65  ) {
    66  	metadataFile := "/snapshot.metadata"
    67  	err := mgr.cfg.FS.MkdirAll(mgr.snapshotPath(nid, index), defaultExportedDirMode)
    68  	assert.NoError(t, err)
    69  	_, err = mgr.cfg.FS.Create(mgr.snapshotPath(nid, index) + metadataFile)
    70  	assert.NoError(t, err)
    71  	_, err = mgr.cfg.FS.Create(
    72  		fmt.Sprintf("%s/snapshot-%016X.gbsnap", mgr.snapshotPath(nid, index), index))
    73  	assert.NoError(t, err)
    74  }
    75  
    76  func TestInitSnapshotMgr(t *testing.T) {
    77  	var (
    78  		shardID   uint64 = 1
    79  		replicaID uint64 = 1
    80  	)
    81  	cfg := &Config{
    82  		FS:                vfs.NewStrictMem(),
    83  		SnapshotExportDir: "/tmp/exported",
    84  	}
    85  	mgr := newSnapshotManager(cfg)
    86  	err := mgr.cfg.FS.MkdirAll(mgr.exportPath(shardID, replicaID), defaultExportedDirMode)
    87  	assert.NoError(t, err)
    88  
    89  	nid := nodeID{shardID: shardID, replicaID: replicaID}
    90  	indexes := []snapshotIndex{
    91  		snapshotIndex(310),
    92  		snapshotIndex(1324),
    93  		snapshotIndex(1211),
    94  		snapshotIndex(1203),
    95  	}
    96  	for _, index := range indexes {
    97  		testPrepareSnapshot(t, mgr, nid, index)
    98  	}
    99  	// add a bad dir
   100  	err = mgr.cfg.FS.MkdirAll(mgr.exportPath(shardID, replicaID)+"/bad-dir", defaultExportedDirMode)
   101  	assert.NoError(t, err)
   102  
   103  	err = mgr.Init(shardID, replicaID)
   104  	assert.NoError(t, err)
   105  	assert.Equal(t, 4, mgr.Count(shardID, replicaID))
   106  	assert.Equal(t, mgr.snapshots[nid].items[0].index, snapshotIndex(310))
   107  	assert.Equal(t, mgr.snapshots[nid].items[1].index, snapshotIndex(1203))
   108  	assert.Equal(t, mgr.snapshots[nid].items[2].index, snapshotIndex(1211))
   109  	assert.Equal(t, mgr.snapshots[nid].items[3].index, snapshotIndex(1324))
   110  }
   111  
   112  func TestAddSnapshot(t *testing.T) {
   113  	var (
   114  		shardID   uint64 = 1
   115  		replicaID uint64 = 1
   116  	)
   117  	cfg := &Config{
   118  		FS:                vfs.NewStrictMem(),
   119  		SnapshotExportDir: "",
   120  	}
   121  	mgr := newSnapshotManager(cfg)
   122  	err := mgr.cfg.FS.MkdirAll(mgr.exportPath(shardID, replicaID), defaultExportedDirMode)
   123  	assert.NoError(t, err)
   124  
   125  	err = mgr.Init(shardID, replicaID)
   126  	assert.NoError(t, err)
   127  	assert.Equal(t, 0, mgr.Count(shardID, replicaID))
   128  
   129  	// add one
   130  	nid := nodeID{shardID: shardID, replicaID: replicaID}
   131  	index := snapshotIndex(621)
   132  	testPrepareSnapshot(t, mgr, nid, index)
   133  	err = mgr.Add(shardID, replicaID, uint64(index))
   134  	assert.NoError(t, err)
   135  	assert.Equal(t, 1, mgr.Count(shardID, replicaID))
   136  
   137  	// add another one, but smaller index, should get an error
   138  	index = snapshotIndex(96)
   139  	testPrepareSnapshot(t, mgr, nid, index)
   140  	err = mgr.Add(shardID, replicaID, uint64(index))
   141  	assert.Error(t, err)
   142  	assert.Equal(t, 1, mgr.Count(shardID, replicaID))
   143  
   144  	// add another one
   145  	index = snapshotIndex(794)
   146  	testPrepareSnapshot(t, mgr, nid, index)
   147  	err = mgr.Add(shardID, replicaID, uint64(index))
   148  	assert.NoError(t, err)
   149  	assert.Equal(t, 2, mgr.Count(shardID, replicaID))
   150  
   151  	// add an invalid one
   152  	index = snapshotIndex(795)
   153  	metadataFile := "/snapshot.bad"
   154  	err = mgr.cfg.FS.MkdirAll(mgr.snapshotPath(nid, index), defaultExportedDirMode)
   155  	assert.NoError(t, err)
   156  	_, err = mgr.cfg.FS.Create(mgr.snapshotPath(nid, index) + metadataFile)
   157  	assert.NoError(t, err)
   158  	_, err = mgr.cfg.FS.Create(
   159  		fmt.Sprintf("%s/snapshot-%016X.gbsnap", mgr.snapshotPath(nid, index), index))
   160  	assert.NoError(t, err)
   161  	err = mgr.Add(shardID, replicaID, uint64(index))
   162  	assert.Error(t, err)
   163  	assert.Equal(t, 2, mgr.Count(shardID, replicaID))
   164  
   165  	// add an invalid one
   166  	index = snapshotIndex(797)
   167  	metadataFile = "/snapshot.metadata"
   168  	err = mgr.cfg.FS.MkdirAll(mgr.snapshotPath(nid, index), defaultExportedDirMode)
   169  	assert.NoError(t, err)
   170  	_, err = mgr.cfg.FS.Create(mgr.snapshotPath(nid, index) + metadataFile)
   171  	assert.NoError(t, err)
   172  	_, err = mgr.cfg.FS.Create(
   173  		fmt.Sprintf("%s/snapshot.gbsnap", mgr.snapshotPath(nid, index)))
   174  	assert.NoError(t, err)
   175  	err = mgr.Add(shardID, replicaID, uint64(index))
   176  	assert.Error(t, err)
   177  	assert.Equal(t, 2, mgr.Count(shardID, replicaID))
   178  }
   179  
   180  func getInitializedMgr(t *testing.T, shardID uint64, replicaID uint64) *snapshotManager {
   181  	cfg := &Config{
   182  		FS:                vfs.NewStrictMem(),
   183  		SnapshotExportDir: "",
   184  	}
   185  	mgr := newSnapshotManager(cfg)
   186  	err := mgr.Init(shardID, replicaID)
   187  	assert.NoError(t, err)
   188  	assert.Equal(t, 0, mgr.Count(shardID, replicaID))
   189  
   190  	nid := nodeID{shardID: shardID, replicaID: replicaID}
   191  	index := snapshotIndex(100)
   192  	testPrepareSnapshot(t, mgr, nid, index)
   193  	err = mgr.Add(shardID, replicaID, uint64(index))
   194  	assert.NoError(t, err)
   195  	assert.Equal(t, 1, mgr.Count(shardID, replicaID))
   196  	assert.Equal(t, mgr.snapshots[nid].items[0].index, index)
   197  
   198  	index = snapshotIndex(200)
   199  	testPrepareSnapshot(t, mgr, nid, index)
   200  	err = mgr.Add(shardID, replicaID, uint64(index))
   201  	assert.NoError(t, err)
   202  	assert.Equal(t, 2, mgr.Count(shardID, replicaID))
   203  	assert.Equal(t, mgr.snapshots[nid].items[1].index, index)
   204  
   205  	index = snapshotIndex(300)
   206  	testPrepareSnapshot(t, mgr, nid, index)
   207  	err = mgr.Add(shardID, replicaID, uint64(index))
   208  	assert.NoError(t, err)
   209  	assert.Equal(t, 3, mgr.Count(shardID, replicaID))
   210  	assert.Equal(t, mgr.snapshots[nid].items[2].index, index)
   211  
   212  	return mgr
   213  }
   214  
   215  func TestRemoveSnapshot(t *testing.T) {
   216  	var (
   217  		shardID   uint64 = 1
   218  		replicaID uint64 = 1
   219  	)
   220  	mgr := getInitializedMgr(t, shardID, replicaID)
   221  	err := mgr.Remove(shardID, replicaID, 200)
   222  	nid := nodeID{shardID: shardID, replicaID: replicaID}
   223  	assert.NoError(t, err)
   224  	assert.Equal(t, 1, mgr.Count(shardID, replicaID))
   225  	assert.Equal(t, mgr.snapshots[nid].items[0].index, snapshotIndex(300))
   226  }
   227  
   228  func TestEvalImportSnapshot(t *testing.T) {
   229  	var (
   230  		shardID   uint64 = 1
   231  		replicaID uint64 = 1
   232  	)
   233  	mgr := getInitializedMgr(t, shardID, replicaID)
   234  
   235  	dir, index := mgr.EvalImportSnapshot(shardID, replicaID, 50)
   236  	assert.Equal(t, "", dir)
   237  	assert.Equal(t, uint64(0), index)
   238  
   239  	dir, index = mgr.EvalImportSnapshot(shardID, replicaID, 110)
   240  	assert.Equal(t, "shard-1/replica-1/snapshot-0000000000000064", dir)
   241  	assert.Equal(t, uint64(100), index)
   242  
   243  	dir, index = mgr.EvalImportSnapshot(shardID, replicaID, 210)
   244  	assert.Equal(t, "shard-1/replica-1/snapshot-00000000000000C8", dir)
   245  	assert.Equal(t, uint64(200), index)
   246  
   247  	err := mgr.Remove(shardID, replicaID, 150)
   248  	assert.NoError(t, err)
   249  
   250  	dir, index = mgr.EvalImportSnapshot(shardID, replicaID, 110)
   251  	assert.Equal(t, "", dir)
   252  	assert.Equal(t, uint64(0), index)
   253  
   254  	dir, index = mgr.EvalImportSnapshot(shardID, replicaID, 210)
   255  	assert.Equal(t, "shard-1/replica-1/snapshot-00000000000000C8", dir)
   256  	assert.Equal(t, uint64(200), index)
   257  
   258  	dir, index = mgr.EvalImportSnapshot(shardID, replicaID, 10000)
   259  	assert.Equal(t, "shard-1/replica-1/snapshot-000000000000012C", dir)
   260  	assert.Equal(t, uint64(300), index)
   261  }