github.com/rohankumardubey/proxyfs@v0.0.0-20210108201508-653efa9ab00e/inode/setup_teardown_test.go (about)

     1  package inode
     2  
     3  import (
     4  	"io/ioutil"
     5  	"os"
     6  	"runtime"
     7  	"sync"
     8  	"syscall"
     9  	"testing"
    10  
    11  	"golang.org/x/sys/unix"
    12  
    13  	"github.com/swiftstack/ProxyFS/conf"
    14  	"github.com/swiftstack/ProxyFS/ramswift"
    15  	"github.com/swiftstack/ProxyFS/transitions"
    16  )
    17  
    18  var (
    19  	ramswiftDoneChan chan bool
    20  	testConfMap      conf.ConfMap
    21  )
    22  
    23  func testSetup(t *testing.T, starvationMode bool) {
    24  	var (
    25  		err                           error
    26  		rLimit                        syscall.Rlimit
    27  		rLimitMinimum                 uint64
    28  		signalHandlerIsArmedWG        sync.WaitGroup
    29  		testChunkedConnectionPoolSize uint64
    30  		testConfStrings               []string
    31  		testConfUpdateStrings         []string
    32  		testDir                       string
    33  	)
    34  
    35  	testDir, err = ioutil.TempDir(os.TempDir(), "ProxyFS_test_inode_")
    36  	if nil != err {
    37  		t.Fatalf("ioutil.TempDir() failed: %v", err)
    38  	}
    39  
    40  	err = os.Chdir(testDir)
    41  	if nil != err {
    42  		t.Fatalf("os.Chdir() failed: %v", err)
    43  	}
    44  
    45  	testConfStrings = []string{
    46  		"Stats.IPAddr=localhost",
    47  		"Stats.UDPPort=52184",
    48  		"Stats.BufferLength=100",
    49  		"Stats.MaxLatency=1s",
    50  		"Logging.LogFilePath=/dev/null",
    51  		"Logging.LogToConsole=false",
    52  		"SwiftClient.NoAuthIPAddr=127.0.0.1",
    53  		"SwiftClient.NoAuthTCPPort=35262",
    54  		"SwiftClient.Timeout=10s",
    55  		"SwiftClient.RetryLimit=3",
    56  		"SwiftClient.RetryLimitObject=3",
    57  		"SwiftClient.RetryDelay=10ms",
    58  		"SwiftClient.RetryDelayObject=10ms",
    59  		"SwiftClient.RetryExpBackoff=1.2",
    60  		"SwiftClient.RetryExpBackoffObject=1.0",
    61  		"PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerStoragePolicy=silver",
    62  		"PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerNamePrefix=Replicated3Way_",
    63  		"PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainersPerPeer=10",
    64  		"PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.MaxObjectsPerContainer=1000000",
    65  		"Peer:Peer0.PublicIPAddr=127.0.0.1",
    66  		"Peer:Peer0.PrivateIPAddr=127.0.0.1",
    67  		"Peer:Peer0.ReadCacheQuotaFraction=0.20",
    68  		"Cluster.Peers=Peer0",
    69  		"Cluster.WhoAmI=Peer0",
    70  		"Volume:TestVolume.FSID=1",
    71  		"Volume:TestVolume.AccountName=AUTH_test",
    72  		"Volume:TestVolume.AutoFormat=true",
    73  		"Volume:TestVolume.CheckpointContainerName=.__checkpoint__",
    74  		"Volume:TestVolume.CheckpointContainerStoragePolicy=gold",
    75  		"Volume:TestVolume.CheckpointInterval=10s",
    76  		"Volume:TestVolume.DefaultPhysicalContainerLayout=PhysicalContainerLayoutReplicated3Way",
    77  		"Volume:TestVolume.MaxFlushSize=10485760",
    78  		"Volume:TestVolume.MaxFlushTime=10s",
    79  		"Volume:TestVolume.FileDefragmentChunkSize=10485760",
    80  		"Volume:TestVolume.FileDefragmentChunkDelay=10ms",
    81  		"Volume:TestVolume.NonceValuesToReserve=100",
    82  		"Volume:TestVolume.MaxEntriesPerDirNode=32",
    83  		"Volume:TestVolume.MaxExtentsPerFileNode=32",
    84  		"Volume:TestVolume.MaxInodesPerMetadataNode=32",
    85  		"Volume:TestVolume.MaxLogSegmentsPerMetadataNode=64",
    86  		"Volume:TestVolume.MaxDirFileNodesPerMetadataNode=16",
    87  		"Volume:TestVolume.MaxBytesInodeCache=100000",
    88  		"Volume:TestVolume.InodeCacheEvictInterval=1s",
    89  		"Volume:TestVolume.ActiveLeaseEvictLowLimit=5000",
    90  		"Volume:TestVolume.ActiveLeaseEvictHighLimit=5010",
    91  		"VolumeGroup:TestVolumeGroup.VolumeList=TestVolume",
    92  		"VolumeGroup:TestVolumeGroup.VirtualIPAddr=",
    93  		"VolumeGroup:TestVolumeGroup.PrimaryPeer=Peer0",
    94  		"VolumeGroup:TestVolumeGroup.ReadCacheLineSize=1000000",
    95  		"VolumeGroup:TestVolumeGroup.ReadCacheWeight=100",
    96  		"FSGlobals.VolumeGroupList=TestVolumeGroup",
    97  		"FSGlobals.CheckpointHeaderConsensusAttempts=5",
    98  		"FSGlobals.MountRetryLimit=6",
    99  		"FSGlobals.MountRetryDelay=1s",
   100  		"FSGlobals.MountRetryExpBackoff=2",
   101  		"FSGlobals.LogCheckpointHeaderPosts=true",
   102  		"FSGlobals.TryLockBackoffMin=10ms",
   103  		"FSGlobals.TryLockBackoffMax=50ms",
   104  		"FSGlobals.TryLockSerializationThreshhold=5",
   105  		"FSGlobals.SymlinkMax=32",
   106  		"FSGlobals.CoalesceElementChunkSize=16",
   107  		"FSGlobals.InodeRecCacheEvictLowLimit=10000",
   108  		"FSGlobals.InodeRecCacheEvictHighLimit=10010",
   109  		"FSGlobals.LogSegmentRecCacheEvictLowLimit=10000",
   110  		"FSGlobals.LogSegmentRecCacheEvictHighLimit=10010",
   111  		"FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000",
   112  		"FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010",
   113  		"FSGlobals.DirEntryCacheEvictLowLimit=10000",
   114  		"FSGlobals.DirEntryCacheEvictHighLimit=10010",
   115  		"FSGlobals.FileExtentMapEvictLowLimit=10000",
   116  		"FSGlobals.FileExtentMapEvictHighLimit=10010",
   117  		"FSGlobals.EtcdEnabled=false",
   118  		"RamSwiftInfo.MaxAccountNameLength=256",
   119  		"RamSwiftInfo.MaxContainerNameLength=256",
   120  		"RamSwiftInfo.MaxObjectNameLength=1024",
   121  		"RamSwiftInfo.AccountListingLimit=10000",
   122  		"RamSwiftInfo.ContainerListingLimit=10000",
   123  	}
   124  
   125  	testConfMap, err = conf.MakeConfMapFromStrings(testConfStrings)
   126  	if nil != err {
   127  		t.Fatalf("conf.MakeConfMapFromStrings() failed: %v", err)
   128  	}
   129  
   130  	if starvationMode {
   131  		testConfUpdateStrings = []string{
   132  			"SwiftClient.ChunkedConnectionPoolSize=1",
   133  			"SwiftClient.NonChunkedConnectionPoolSize=1",
   134  		}
   135  	} else {
   136  		testConfUpdateStrings = []string{
   137  			"SwiftClient.ChunkedConnectionPoolSize=256",
   138  			"SwiftClient.NonChunkedConnectionPoolSize=64",
   139  		}
   140  	}
   141  
   142  	err = testConfMap.UpdateFromStrings(testConfUpdateStrings)
   143  	if nil != err {
   144  		t.Fatalf("testConfMap.UpdateFromStrings(testConfUpdateStrings) failed: %v", err)
   145  	}
   146  
   147  	testChunkedConnectionPoolSize, err = testConfMap.FetchOptionValueUint64("SwiftClient", "ChunkedConnectionPoolSize")
   148  	if nil != err {
   149  		t.Fatalf("testConfMap.FetchOptionValueUint64(\"SwiftClient\", \"ChunkedConnectionPoolSize\") failed: %v", err)
   150  	}
   151  
   152  	err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
   153  	if nil != err {
   154  		t.Fatalf("syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)() failed: %v", err)
   155  	}
   156  	rLimitMinimum = 3 * testChunkedConnectionPoolSize
   157  	if rLimitMinimum > rLimit.Cur {
   158  		t.Errorf("RLIMIT too low... must be at least %v (was %v)", rLimitMinimum, rLimit.Cur)
   159  		t.Errorf("On Mac:")
   160  		t.Errorf("  <CHECK CURRENT KERNEL LIMITS>")
   161  		t.Errorf("    sysctl -a | grep ^kern.maxfiles")
   162  		t.Errorf("  <ADJUST KERNEL LIMITS IF NECESSARY>")
   163  		t.Errorf("    sudo /usr/libexec/PlistBuddy /Library/LaunchAgents/com.kern.maxfiles.plist -c \"add Label string com.kern.maxfiles\" -c \"add ProgramArguments array\" -c \"add ProgramArguments: string sysctl\" -c \"add ProgramArguments: string -w\" -c \"add ProgramArguments: string kern.maxfiles=20480\" -c \"add RunAtLoad bool true\"")
   164  		t.Errorf("    sudo /usr/libexec/PlistBuddy /Library/LaunchAgents/com.kern.maxfilesperproc.plist -c \"add Label string com.kern.maxfilesperproc\" -c \"add ProgramArguments array\" -c \"add ProgramArguments: string sysctl\" -c \"add ProgramArguments: string -w\" -c \"add ProgramArguments: string kern.perprocmaxfiles=10240\" -c \"add RunAtLoad bool true\"")
   165  		t.Errorf("  <ADJUST PROCESS HARD LIMITS IF NECESSARY>")
   166  		t.Errorf("    sudo /usr/libexec/PlistBuddy /Library/LaunchAgents/com.launchd.maxfiles.plist -c \"add Label string com.launchd.maxfiles\" -c \"add ProgramArguments array\" -c \"add ProgramArguments: string launchctl\" -c \"add ProgramArguments: string limit\" -c \"add ProgramArguments: string maxfiles\" -c \"add ProgramArguments: string 5120\" -c \"add ProgramArguments: string unlimited\" -c \"add RunAtLoad bool true\"")
   167  		t.Errorf("  <RESTART>")
   168  		t.Errorf("  <ADJUST PROCESS SOFT LIMITS IF NECESSARY>")
   169  		t.Errorf("    ulimit -n 2560")
   170  		t.Errorf("On Linux:")
   171  		t.Fatalf("  ulimit -n 2560")
   172  	}
   173  
   174  	signalHandlerIsArmedWG.Add(1)
   175  	ramswiftDoneChan = make(chan bool, 1)
   176  	go ramswift.Daemon("/dev/null", testConfStrings, &signalHandlerIsArmedWG, ramswiftDoneChan, unix.SIGTERM)
   177  
   178  	signalHandlerIsArmedWG.Wait()
   179  
   180  	err = transitions.Up(testConfMap)
   181  	if nil != err {
   182  		t.Fatalf("transitions.Up() failed: %v", err)
   183  	}
   184  }
   185  
   186  func testTeardown(t *testing.T) {
   187  	var (
   188  		err     error
   189  		testDir string
   190  	)
   191  
   192  	err = transitions.Down(testConfMap)
   193  	if nil != err {
   194  		t.Fatalf("transitions.Down() failed: %v", err)
   195  	}
   196  
   197  	_ = syscall.Kill(syscall.Getpid(), unix.SIGTERM)
   198  	_ = <-ramswiftDoneChan
   199  
   200  	// Run GC to reclaim memory before we proceed to next test
   201  	runtime.GC()
   202  
   203  	testDir, err = os.Getwd()
   204  	if nil != err {
   205  		t.Fatalf("os.Getwd() failed: %v", err)
   206  	}
   207  
   208  	err = os.Chdir("..")
   209  	if nil != err {
   210  		t.Fatalf("os.Chdir() failed: %v", err)
   211  	}
   212  
   213  	err = os.RemoveAll(testDir)
   214  	if nil != err {
   215  		t.Fatalf("os.RemoveAll() failed: %v", err)
   216  	}
   217  }