github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/fs/health/fshc_internal_test.go (about)

     1  // Package health provides a basic mountpath health monitor.
     2  /*
     3   * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
     4   */
     5  package health
     6  
     7  import (
     8  	"fmt"
     9  	"os"
    10  	"testing"
    11  
    12  	"github.com/NVIDIA/aistore/cmn"
    13  	"github.com/NVIDIA/aistore/cmn/cos"
    14  	"github.com/NVIDIA/aistore/fs"
    15  	"github.com/NVIDIA/aistore/tools/tassert"
    16  )
    17  
    18  const (
    19  	fsCheckerTmpDir = "/tmp/fshc"
    20  )
    21  
    22  func initMountpaths(t *testing.T) {
    23  	t.Cleanup(func() {
    24  		os.RemoveAll(fsCheckerTmpDir)
    25  	})
    26  
    27  	cos.CreateDir(fsCheckerTmpDir)
    28  
    29  	config := cmn.GCO.BeginUpdate()
    30  	config.TestFSP.Count = 1
    31  	cmn.GCO.CommitUpdate(config)
    32  
    33  	fs.TestNew(nil)
    34  	for i := 1; i <= 4; i++ {
    35  		mpath := fmt.Sprintf("%s/%d", fsCheckerTmpDir, i)
    36  
    37  		err := cos.CreateDir(mpath)
    38  		tassert.CheckFatal(t, err)
    39  
    40  		_, err = fs.Add(mpath, "id")
    41  		tassert.CheckFatal(t, err)
    42  	}
    43  
    44  	os.RemoveAll(fsCheckerTmpDir + "/3") // One directory is deleted.
    45  	fs.Disable(fsCheckerTmpDir + "/4")
    46  }
    47  
    48  func updateTestConfig() {
    49  	config := cmn.GCO.BeginUpdate()
    50  	config.FSHC.Enabled = true
    51  	config.FSHC.ErrorLimit = 2
    52  	config.FSHC.TestFileCount = 4
    53  	config.Log.Level = "3"
    54  	cmn.GCO.CommitUpdate(config)
    55  }
    56  
    57  type MockFSDispatcher struct {
    58  	faultyPaths   []string
    59  	faultDetected bool
    60  }
    61  
    62  func newMockFSDispatcher(mpathsToFail ...string) *MockFSDispatcher {
    63  	return &MockFSDispatcher{
    64  		faultyPaths: mpathsToFail,
    65  	}
    66  }
    67  
    68  func (d *MockFSDispatcher) DisableMpath(mpath, reason string) (err error) {
    69  	d.faultDetected = cos.StringInSlice(mpath, d.faultyPaths)
    70  	if d.faultDetected {
    71  		err = fmt.Errorf("fault detected: %s", reason)
    72  	}
    73  	return
    74  }
    75  
    76  func setupTests(t *testing.T) {
    77  	updateTestConfig()
    78  	initMountpaths(t)
    79  }
    80  
    81  func TestFSCheckerInaccessibleMountpath(t *testing.T) {
    82  	setupTests(t)
    83  
    84  	var (
    85  		failedMpath = fsCheckerTmpDir + "/3"
    86  		filePath    = failedMpath + "/testfile"
    87  	)
    88  	_, _, exists := testMountpath(cmn.GCO.Get(), filePath, failedMpath, cos.KiB)
    89  	tassert.Errorf(t, !exists, "testing non-existing mountpath must fail")
    90  }
    91  
    92  func TestFSCheckerFailedMountpath(t *testing.T) {
    93  	setupTests(t)
    94  
    95  	var (
    96  		failedMpath = fsCheckerTmpDir + "/3"
    97  
    98  		dispatcher = newMockFSDispatcher(failedMpath)
    99  		fshc       = NewFSHC(dispatcher)
   100  	)
   101  	// Failed mountpath must be disabled.
   102  	fshc.runMpathTest(failedMpath, failedMpath+"/dir/testfile")
   103  	tassert.Errorf(t, dispatcher.faultDetected, "faulty mountpath %s was not detected", failedMpath)
   104  }
   105  
   106  func TestFSCheckerDecisionFn(t *testing.T) {
   107  	updateTestConfig()
   108  
   109  	// Decision making function.
   110  	type tstInfo struct {
   111  		title               string
   112  		readErrs, writeErrs int
   113  		avail, result       bool
   114  	}
   115  	testList := []tstInfo{
   116  		{"Inaccessible mountpath", 0, 0, false, false},
   117  		{"Healthy mountpath", 0, 0, true, true},
   118  		{"Unstable but OK mountpath", 1, 1, true, true},
   119  		{"Reads failed", 3, 0, true, false},
   120  		{"Writes failed", 1, 3, true, false},
   121  		{"Reads and writes failed", 3, 3, true, false},
   122  	}
   123  
   124  	for _, tst := range testList {
   125  		t.Run(tst.title, func(t *testing.T) {
   126  			res, err := isTestPassed("/tmp", tst.readErrs, tst.writeErrs, tst.avail)
   127  			if res != tst.result {
   128  				t.Errorf("%s failed: expected %v, got %v (%v)", tst.title, tst.result, res, err)
   129  			}
   130  		})
   131  	}
   132  }
   133  
   134  func TestFSCheckerTryReadFile(t *testing.T) {
   135  	setupTests(t)
   136  
   137  	var (
   138  		mpath    = fsCheckerTmpDir + "/1"
   139  		filePath = mpath + "/smth.txt"
   140  	)
   141  
   142  	// Create file with some content inside.
   143  	file, err := cos.CreateFile(filePath)
   144  	tassert.CheckFatal(t, err)
   145  	err = cos.FloodWriter(file, cos.MiB)
   146  	file.Close()
   147  	tassert.CheckFatal(t, err)
   148  
   149  	err = tryReadFile(filePath)
   150  	tassert.CheckFatal(t, err)
   151  }
   152  
   153  func TestFSCheckerTryWriteFile(t *testing.T) {
   154  	setupTests(t)
   155  	mpath := fsCheckerTmpDir + "/1"
   156  	err := tryWriteFile(mpath, cos.KiB)
   157  	tassert.CheckFatal(t, err)
   158  }