github.com/swiftstack/ProxyFS@v0.0.0-20210203235616-4017c267d62f/pfsagentd/setup_teardown_test.go (about)

     1  // Copyright (c) 2015-2021, NVIDIA CORPORATION.
     2  // SPDX-License-Identifier: Apache-2.0
     3  
     4  package main
     5  
     6  import (
     7  	"io/ioutil"
     8  	"net/http"
     9  	"os"
    10  	"strconv"
    11  	"sync"
    12  	"testing"
    13  	"time"
    14  
    15  	"golang.org/x/sys/unix"
    16  
    17  	"github.com/swiftstack/ProxyFS/conf"
    18  	"github.com/swiftstack/ProxyFS/proxyfsd"
    19  	"github.com/swiftstack/ProxyFS/ramswift"
    20  )
    21  
    22  const (
    23  	testAccountName             = "AUTH_test"
    24  	testAuthKey                 = "testing"
    25  	testAuthUser                = "test:tester"
    26  	testDaemonStartPollInterval = 1 * time.Second
    27  	testProxyFSDaemonHTTPPort   = "15347"
    28  	testProxyFSDaemonIPAddr     = "127.0.0.1"
    29  	testSwiftNoAuthIPAddr       = "127.0.0.1"
    30  	testSwiftNoAuthPort         = "38090"
    31  	testSwiftProxyAddr          = "localhost:38080"
    32  )
    33  
    34  type testDaemonGlobalsStruct struct {
    35  	proxyfsdErrChan  chan error
    36  	proxyfsdWG       sync.WaitGroup
    37  	ramswiftDoneChan chan bool
    38  }
    39  
    40  var testDaemonGlobals testDaemonGlobalsStruct
    41  
    42  func testSetup(t *testing.T) {
    43  	var (
    44  		err                            error
    45  		infoResponse                   *http.Response
    46  		ramswiftSignalHandlerIsArmedWG sync.WaitGroup
    47  		testConfMap                    conf.ConfMap
    48  		testConfStrings                []string
    49  		testDir                        string
    50  		testPlugInEnvValue             string
    51  		versionResponse                *http.Response
    52  	)
    53  
    54  	testDir, err = ioutil.TempDir(os.TempDir(), "pfsagentd_test_")
    55  	if nil != err {
    56  		t.Fatalf("ioutil.TempDir() failed: %v", err)
    57  	}
    58  
    59  	err = os.Chdir(testDir)
    60  	if nil != err {
    61  		t.Fatalf("os.Chdir() failed: %v", err)
    62  	}
    63  
    64  	err = os.Mkdir("ProxyFSMountPointPath", 0777) // Agent.FUSEMountPointPath
    65  	if nil != err {
    66  		t.Fatalf("os.Mkdir() failed: %v", err)
    67  	}
    68  
    69  	err = os.Mkdir("PfsAgentMountPointPath", 0777) // Volume:CommonVolume.FUSEMountPointName
    70  	if nil != err {
    71  		t.Fatalf("os.Mkdir() failed: %v", err)
    72  	}
    73  
    74  	testPlugInEnvValue = "{\"AuthURL\":\"http://"
    75  	testPlugInEnvValue += testSwiftProxyAddr
    76  	testPlugInEnvValue += "/auth/v1.0\"\\u002C\"AuthUser\":\""
    77  	testPlugInEnvValue += testAuthUser
    78  	testPlugInEnvValue += "\"\\u002C\"AuthKey\":\""
    79  	testPlugInEnvValue += testAuthKey
    80  	testPlugInEnvValue += "\"\\u002C\"Account\":\""
    81  	testPlugInEnvValue += testAccountName
    82  	testPlugInEnvValue += "\"}"
    83  
    84  	testConfStrings = []string{
    85  		"Agent.FUSEVolumeName=CommonVolume",
    86  		"Agent.FUSEMountPointPath=PfsAgentMountPointPath",
    87  		"Agent.FUSEUnMountRetryDelay=100ms",
    88  		"Agent.FUSEUnMountRetryCap=100",
    89  		"Agent.PlugInPath=/dev/null", // Using hard-coded AUTH
    90  		"Agent.PlugInEnvName=SwiftAuthBlob",
    91  		"Agent.PlugInEnvValue=" + testPlugInEnvValue,
    92  		"Agent.SwiftTimeout=20s",
    93  		"Agent.SwiftRetryLimit=10",
    94  		"Agent.SwiftRetryDelay=10ms",
    95  		"Agent.SwiftRetryDelayVariance=25",
    96  		"Agent.SwiftRetryExpBackoff=1.4",
    97  		"Agent.SwiftConnectionPoolSize=200",
    98  		"Agent.FetchExtentsFromFileOffset=32",
    99  		"Agent.FetchExtentsBeforeFileOffset=0",
   100  		"Agent.ReadCacheLineSize=1048576",
   101  		"Agent.ReadCacheLineCount=1000",
   102  		"Agent.LeaseRetryLimit=10",
   103  		"Agent.LeaseRetryDelay=10ms",
   104  		"Agent.LeaseRetryDelayVariance=25",
   105  		"Agent.LeaseRetryExpBackoff=1.4",
   106  		"Agent.SharedLeaseLimit=1000",
   107  		"Agent.ExclusiveLeaseLimit=100",
   108  		"Agent.ExtentMapEntryLimit=1048576",
   109  		"Agent.DirtyLogSegmentLimit=50",
   110  		"Agent.DirtyFileLimit=50", // TODO - obsolete this
   111  		"Agent.MaxFlushSize=10485760",
   112  		"Agent.MaxFlushTime=10s",
   113  		"Agent.LogFilePath=",
   114  		"Agent.LogToConsole=false",
   115  		"Agent.TraceEnabled=false",
   116  		"Agent.HTTPServerIPAddr=127.0.0.1",
   117  		"Agent.HTTPServerTCPPort=54323",
   118  		"Agent.ReadDirPlusEnabled=false",
   119  		"Agent.XAttrEnabled=false",
   120  		"Agent.EntryDuration=10s",
   121  		"Agent.AttrDuration=10s",
   122  		"Agent.AttrBlockSize=65536",
   123  		"Agent.ReaddirMaxEntries=1024",
   124  		"Agent.FUSEMaxBackground=100",
   125  		"Agent.FUSECongestionThreshhold=0",
   126  		"Agent.FUSEMaxWrite=131072", // Linux max... 128KiB is good enough for testing
   127  		"Agent.RetryRPCDeadlineIO=60s",
   128  		"Agent.RetryRPCKeepAlivePeriod=60s",
   129  
   130  		"Stats.IPAddr=localhost",
   131  		"Stats.UDPPort=54324",
   132  		"Stats.BufferLength=100",
   133  		"Stats.MaxLatency=1s",
   134  
   135  		"StatsLogger.Period=0m",
   136  		"StatsLogger.Verbose=false",
   137  
   138  		"Logging.LogFilePath=/dev/null",
   139  		"Logging.LogToConsole=false",
   140  
   141  		"Peer:Peer0.PublicIPAddr=" + testProxyFSDaemonIPAddr,
   142  		"Peer:Peer0.PrivateIPAddr=" + testProxyFSDaemonIPAddr,
   143  		"Peer:Peer0.ReadCacheQuotaFraction=0.20",
   144  
   145  		"Cluster.WhoAmI=Peer0",
   146  		"Cluster.Peers=Peer0",
   147  		"Cluster.ServerGuid=a66488e9-a051-4ff7-865d-87bfb84cc2ae",
   148  		"Cluster.PrivateClusterUDPPort=54325",
   149  		"Cluster.UDPPacketSendSize=1400",
   150  		"Cluster.UDPPacketRecvSize=1500",
   151  		"Cluster.UDPPacketCapPerMessage=5",
   152  		"Cluster.HeartBeatDuration=1s",
   153  		"Cluster.HeartBeatMissLimit=3",
   154  		"Cluster.MessageQueueDepthPerPeer=4",
   155  		"Cluster.MaxRequestDuration=1s",
   156  		"Cluster.LivenessCheckRedundancy=2",
   157  
   158  		"HTTPServer.TCPPort=" + testProxyFSDaemonHTTPPort,
   159  
   160  		"SwiftClient.NoAuthIPAddr=" + testSwiftNoAuthIPAddr,
   161  		"SwiftClient.NoAuthTCPPort=" + testSwiftNoAuthPort,
   162  		"SwiftClient.Timeout=10s",
   163  		"SwiftClient.RetryLimit=1",
   164  		"SwiftClient.RetryLimitObject=1",
   165  		"SwiftClient.RetryDelay=10ms",
   166  		"SwiftClient.RetryDelayObject=10ms",
   167  		"SwiftClient.RetryExpBackoff=1.2",
   168  		"SwiftClient.RetryExpBackoffObject=2.0",
   169  		"SwiftClient.ChunkedConnectionPoolSize=64",
   170  		"SwiftClient.NonChunkedConnectionPoolSize=32",
   171  		"SwiftClient.SwiftReconChecksPerConfCheck=0",
   172  
   173  		"PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerStoragePolicy=silver",
   174  		"PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerNamePrefix=Replicated3Way_",
   175  		"PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainersPerPeer=10",
   176  		"PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.MaxObjectsPerContainer=1000000",
   177  
   178  		"Volume:CommonVolume.FSID=1",
   179  		"Volume:CommonVolume.FUSEMountPointName=ProxyFSMountPointPath",
   180  		"Volume:CommonVolume.NFSExportClientMapList=CommonVolumeNFSClient0",
   181  		"Volume:CommonVolume.SMBShareName=CommonShare",
   182  		"Volume:CommonVolume.PrimaryPeer=Peer0",
   183  		"Volume:CommonVolume.AccountName=AUTH_test",
   184  		"Volume:CommonVolume.CheckpointContainerName=.__checkpoint__",
   185  		"Volume:CommonVolume.CheckpointContainerStoragePolicy=gold",
   186  		"Volume:CommonVolume.CheckpointInterval=10s",
   187  		"Volume:CommonVolume.DefaultPhysicalContainerLayout=PhysicalContainerLayoutReplicated3Way",
   188  		"Volume:CommonVolume.MaxFlushSize=10485760",
   189  		"Volume:CommonVolume.MaxFlushTime=10s",
   190  		"Volume:CommonVolume.FileDefragmentChunkSize=10485760",
   191  		"Volume:CommonVolume.FileDefragmentChunkDelay=10ms",
   192  		"Volume:CommonVolume.NonceValuesToReserve=100",
   193  		"Volume:CommonVolume.MaxEntriesPerDirNode=32",
   194  		"Volume:CommonVolume.MaxExtentsPerFileNode=32",
   195  		"Volume:CommonVolume.MaxInodesPerMetadataNode=32",
   196  		"Volume:CommonVolume.MaxLogSegmentsPerMetadataNode=64",
   197  		"Volume:CommonVolume.MaxDirFileNodesPerMetadataNode=16",
   198  		"Volume:CommonVolume.MaxBytesInodeCache=100000",
   199  		"Volume:CommonVolume.InodeCacheEvictInterval=1s",
   200  		"Volume:CommonVolume.AutoFormat=true",
   201  		"Volume:CommonVolume.ActiveLeaseEvictLowLimit=5000",
   202  		"Volume:CommonVolume.ActiveLeaseEvictHighLimit=5010",
   203  
   204  		"NFSClientMap:CommonVolumeNFSClient0.ClientPattern=*",
   205  		"NFSClientMap:CommonVolumeNFSClient0.AccessMode=rw",
   206  		"NFSClientMap:CommonVolumeNFSClient0.RootSquash=no_root_squash",
   207  		"NFSClientMap:CommonVolumeNFSClient0.Secure=insecure",
   208  
   209  		"VolumeGroup:CommonVolumeGroup.VolumeList=CommonVolume",
   210  		"VolumeGroup:CommonVolumeGroup.VirtualIPAddr=",
   211  		"VolumeGroup:CommonVolumeGroup.PrimaryPeer=Peer0",
   212  		"VolumeGroup:CommonVolumeGroup.ReadCacheLineSize=1000000",
   213  		"VolumeGroup:CommonVolumeGroup.ReadCacheWeight=100",
   214  
   215  		"FSGlobals.VolumeGroupList=CommonVolumeGroup",
   216  		"FSGlobals.CheckpointHeaderConsensusAttempts=5",
   217  		"FSGlobals.MountRetryLimit=6",
   218  		"FSGlobals.MountRetryDelay=1s",
   219  		"FSGlobals.MountRetryExpBackoff=2",
   220  		"FSGlobals.LogCheckpointHeaderPosts=true",
   221  		"FSGlobals.TryLockBackoffMin=10ms",
   222  		"FSGlobals.TryLockBackoffMax=50ms",
   223  		"FSGlobals.TryLockSerializationThreshhold=5",
   224  		"FSGlobals.SymlinkMax=32",
   225  		"FSGlobals.CoalesceElementChunkSize=16",
   226  		"FSGlobals.InodeRecCacheEvictLowLimit=10000",
   227  		"FSGlobals.InodeRecCacheEvictHighLimit=10010",
   228  		"FSGlobals.LogSegmentRecCacheEvictLowLimit=10000",
   229  		"FSGlobals.LogSegmentRecCacheEvictHighLimit=10010",
   230  		"FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000",
   231  		"FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010",
   232  		"FSGlobals.DirEntryCacheEvictLowLimit=10000",
   233  		"FSGlobals.DirEntryCacheEvictHighLimit=10010",
   234  		"FSGlobals.FileExtentMapEvictLowLimit=10000",
   235  		"FSGlobals.FileExtentMapEvictHighLimit=10010",
   236  		"FSGlobals.EtcdEnabled=false",
   237  
   238  		"JSONRPCServer.TCPPort=54326",
   239  		"JSONRPCServer.FastTCPPort=54327",
   240  		"JSONRPCServer.RetryRPCPort=54328",
   241  		"JSONRPCServer.RetryRPCTTLCompleted=10s",
   242  		"JSONRPCServer.RetryRPCAckTrim=10ms",
   243  		"JSONRPCServer.DataPathLogging=false",
   244  		"JSONRPCServer.MinLeaseDuration=250ms",
   245  		"JSONRPCServer.LeaseInterruptInterval=250ms",
   246  		"JSONRPCServer.LeaseInterruptLimit=20",
   247  	}
   248  
   249  	testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxAccountNameLength="+strconv.FormatUint(testMaxAccountNameLength, 10))
   250  	testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxContainerNameLength="+strconv.FormatUint(testMaxContainerNameLength, 10))
   251  	testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxObjectNameLength="+strconv.FormatUint(testMaxObjectNameLength, 10))
   252  	testConfStrings = append(testConfStrings, "RamSwiftInfo.AccountListingLimit="+strconv.FormatUint(testAccountListingLimit, 10))
   253  	testConfStrings = append(testConfStrings, "RamSwiftInfo.ContainerListingLimit="+strconv.FormatUint(testContainerListingLimit, 10))
   254  
   255  	ramswiftSignalHandlerIsArmedWG.Add(1)
   256  	testDaemonGlobals.ramswiftDoneChan = make(chan bool, 1)
   257  
   258  	go ramswift.Daemon("/dev/null", testConfStrings, &ramswiftSignalHandlerIsArmedWG, testDaemonGlobals.ramswiftDoneChan, unix.SIGUSR1)
   259  
   260  	ramswiftSignalHandlerIsArmedWG.Wait()
   261  
   262  	for {
   263  		infoResponse, err = http.Get("http://" + testSwiftNoAuthIPAddr + ":" + testSwiftNoAuthPort + "/info")
   264  		if (nil == err) && (http.StatusOK == infoResponse.StatusCode) {
   265  			break
   266  		}
   267  
   268  		time.Sleep(testDaemonStartPollInterval)
   269  	}
   270  
   271  	testDaemonGlobals.proxyfsdErrChan = make(chan error, 1) // Must be buffered to avoid race
   272  
   273  	go proxyfsd.Daemon("/dev/null", testConfStrings, testDaemonGlobals.proxyfsdErrChan, &testDaemonGlobals.proxyfsdWG, []string{}, unix.SIGUSR2)
   274  
   275  	err = <-testDaemonGlobals.proxyfsdErrChan
   276  	if nil != err {
   277  		t.Fatalf("proxyfsd.Daemon() startup failed: %v", err)
   278  	}
   279  
   280  	for {
   281  		versionResponse, err = http.Get("http://" + testProxyFSDaemonIPAddr + ":" + testProxyFSDaemonHTTPPort + "/version")
   282  		if (nil == err) && (http.StatusOK == versionResponse.StatusCode) {
   283  			break
   284  		}
   285  
   286  		time.Sleep(testDaemonStartPollInterval)
   287  	}
   288  
   289  	testConfMap, err = conf.MakeConfMapFromStrings(testConfStrings)
   290  	if nil != err {
   291  		t.Fatalf("conf.MakeConfMapFromStrings() failed: %v", err)
   292  	}
   293  
   294  	globals.logFile = nil
   295  	globals.config.LogFilePath = ""
   296  	globals.config.LogToConsole = true
   297  
   298  	startSwiftProxyEmulator(t, testConfMap)
   299  
   300  	initializeGlobals(testConfMap)
   301  
   302  	// Fake out that plug-in auth has already obtained AuthToken & StorageURL
   303  
   304  	globals.swiftAuthToken = testAuthToken
   305  	globals.swiftStorageURL = "http://" + testSwiftProxyAddr + "/proxyfs/" + testAccountName
   306  
   307  	go testSwallowFissionErrChan(t, globals.fissionErrChan)
   308  
   309  	doMountProxyFS()
   310  
   311  	performMountFUSE()
   312  }
   313  
   314  func testSwallowFissionErrChan(t *testing.T, fissionErrChan chan error) {
   315  	var (
   316  		err error
   317  	)
   318  
   319  	err = <-fissionErrChan
   320  	if nil != err {
   321  		t.Fatalf("fissionErrChan received err: %v", err)
   322  	}
   323  }
   324  
   325  func testTeardown(t *testing.T) {
   326  	var (
   327  		err     error
   328  		testDir string
   329  	)
   330  
   331  	performUnmountFUSE()
   332  
   333  	emptyFileInodeDirtyListAndLogSegmentChan()
   334  
   335  	doUnmountProxyFS()
   336  
   337  	uninitializeGlobals()
   338  
   339  	_ = unix.Kill(unix.Getpid(), unix.SIGUSR2)
   340  
   341  	err = <-testDaemonGlobals.proxyfsdErrChan
   342  
   343  	testDaemonGlobals.proxyfsdWG.Wait()
   344  
   345  	if nil != err {
   346  		t.Fatalf("proxyfsd.Daemon() exited with error: %v", err)
   347  	}
   348  
   349  	unix.Kill(unix.Getpid(), unix.SIGUSR1)
   350  
   351  	_ = <-testDaemonGlobals.ramswiftDoneChan
   352  
   353  	testDir, err = os.Getwd()
   354  	if nil != err {
   355  		t.Fatalf("os.Getwd() failed: %v", err)
   356  	}
   357  
   358  	err = os.Chdir("..")
   359  	if nil != err {
   360  		t.Fatalf("os.Chdir() failed: %v", err)
   361  	}
   362  
   363  	err = os.RemoveAll(testDir)
   364  	if nil != err {
   365  		t.Fatalf("os.RemoveAll() failed: %v", err)
   366  	}
   367  
   368  	stopSwiftProxyEmulator()
   369  }