github.com/swiftstack/ProxyFS@v0.0.0-20210203235616-4017c267d62f/pfsagentd/setup_teardown_test.go (about) 1 // Copyright (c) 2015-2021, NVIDIA CORPORATION. 2 // SPDX-License-Identifier: Apache-2.0 3 4 package main 5 6 import ( 7 "io/ioutil" 8 "net/http" 9 "os" 10 "strconv" 11 "sync" 12 "testing" 13 "time" 14 15 "golang.org/x/sys/unix" 16 17 "github.com/swiftstack/ProxyFS/conf" 18 "github.com/swiftstack/ProxyFS/proxyfsd" 19 "github.com/swiftstack/ProxyFS/ramswift" 20 ) 21 22 const ( 23 testAccountName = "AUTH_test" 24 testAuthKey = "testing" 25 testAuthUser = "test:tester" 26 testDaemonStartPollInterval = 1 * time.Second 27 testProxyFSDaemonHTTPPort = "15347" 28 testProxyFSDaemonIPAddr = "127.0.0.1" 29 testSwiftNoAuthIPAddr = "127.0.0.1" 30 testSwiftNoAuthPort = "38090" 31 testSwiftProxyAddr = "localhost:38080" 32 ) 33 34 type testDaemonGlobalsStruct struct { 35 proxyfsdErrChan chan error 36 proxyfsdWG sync.WaitGroup 37 ramswiftDoneChan chan bool 38 } 39 40 var testDaemonGlobals testDaemonGlobalsStruct 41 42 func testSetup(t *testing.T) { 43 var ( 44 err error 45 infoResponse *http.Response 46 ramswiftSignalHandlerIsArmedWG sync.WaitGroup 47 testConfMap conf.ConfMap 48 testConfStrings []string 49 testDir string 50 testPlugInEnvValue string 51 versionResponse *http.Response 52 ) 53 54 testDir, err = ioutil.TempDir(os.TempDir(), "pfsagentd_test_") 55 if nil != err { 56 t.Fatalf("ioutil.TempDir() failed: %v", err) 57 } 58 59 err = os.Chdir(testDir) 60 if nil != err { 61 t.Fatalf("os.Chdir() failed: %v", err) 62 } 63 64 err = os.Mkdir("ProxyFSMountPointPath", 0777) // Agent.FUSEMountPointPath 65 if nil != err { 66 t.Fatalf("os.Mkdir() failed: %v", err) 67 } 68 69 err = os.Mkdir("PfsAgentMountPointPath", 0777) // Volume:CommonVolume.FUSEMountPointName 70 if nil != err { 71 t.Fatalf("os.Mkdir() failed: %v", err) 72 } 73 74 testPlugInEnvValue = "{\"AuthURL\":\"http://" 75 testPlugInEnvValue += testSwiftProxyAddr 76 testPlugInEnvValue += "/auth/v1.0\"\\u002C\"AuthUser\":\"" 77 testPlugInEnvValue += testAuthUser 78 testPlugInEnvValue += "\"\\u002C\"AuthKey\":\"" 79 testPlugInEnvValue += testAuthKey 80 testPlugInEnvValue += "\"\\u002C\"Account\":\"" 81 testPlugInEnvValue += testAccountName 82 testPlugInEnvValue += "\"}" 83 84 testConfStrings = []string{ 85 "Agent.FUSEVolumeName=CommonVolume", 86 "Agent.FUSEMountPointPath=PfsAgentMountPointPath", 87 "Agent.FUSEUnMountRetryDelay=100ms", 88 "Agent.FUSEUnMountRetryCap=100", 89 "Agent.PlugInPath=/dev/null", // Using hard-coded AUTH 90 "Agent.PlugInEnvName=SwiftAuthBlob", 91 "Agent.PlugInEnvValue=" + testPlugInEnvValue, 92 "Agent.SwiftTimeout=20s", 93 "Agent.SwiftRetryLimit=10", 94 "Agent.SwiftRetryDelay=10ms", 95 "Agent.SwiftRetryDelayVariance=25", 96 "Agent.SwiftRetryExpBackoff=1.4", 97 "Agent.SwiftConnectionPoolSize=200", 98 "Agent.FetchExtentsFromFileOffset=32", 99 "Agent.FetchExtentsBeforeFileOffset=0", 100 "Agent.ReadCacheLineSize=1048576", 101 "Agent.ReadCacheLineCount=1000", 102 "Agent.LeaseRetryLimit=10", 103 "Agent.LeaseRetryDelay=10ms", 104 "Agent.LeaseRetryDelayVariance=25", 105 "Agent.LeaseRetryExpBackoff=1.4", 106 "Agent.SharedLeaseLimit=1000", 107 "Agent.ExclusiveLeaseLimit=100", 108 "Agent.ExtentMapEntryLimit=1048576", 109 "Agent.DirtyLogSegmentLimit=50", 110 "Agent.DirtyFileLimit=50", // TODO - obsolete this 111 "Agent.MaxFlushSize=10485760", 112 "Agent.MaxFlushTime=10s", 113 "Agent.LogFilePath=", 114 "Agent.LogToConsole=false", 115 "Agent.TraceEnabled=false", 116 "Agent.HTTPServerIPAddr=127.0.0.1", 117 "Agent.HTTPServerTCPPort=54323", 118 "Agent.ReadDirPlusEnabled=false", 119 "Agent.XAttrEnabled=false", 120 "Agent.EntryDuration=10s", 121 "Agent.AttrDuration=10s", 122 "Agent.AttrBlockSize=65536", 123 "Agent.ReaddirMaxEntries=1024", 124 "Agent.FUSEMaxBackground=100", 125 "Agent.FUSECongestionThreshhold=0", 126 "Agent.FUSEMaxWrite=131072", // Linux max... 128KiB is good enough for testing 127 "Agent.RetryRPCDeadlineIO=60s", 128 "Agent.RetryRPCKeepAlivePeriod=60s", 129 130 "Stats.IPAddr=localhost", 131 "Stats.UDPPort=54324", 132 "Stats.BufferLength=100", 133 "Stats.MaxLatency=1s", 134 135 "StatsLogger.Period=0m", 136 "StatsLogger.Verbose=false", 137 138 "Logging.LogFilePath=/dev/null", 139 "Logging.LogToConsole=false", 140 141 "Peer:Peer0.PublicIPAddr=" + testProxyFSDaemonIPAddr, 142 "Peer:Peer0.PrivateIPAddr=" + testProxyFSDaemonIPAddr, 143 "Peer:Peer0.ReadCacheQuotaFraction=0.20", 144 145 "Cluster.WhoAmI=Peer0", 146 "Cluster.Peers=Peer0", 147 "Cluster.ServerGuid=a66488e9-a051-4ff7-865d-87bfb84cc2ae", 148 "Cluster.PrivateClusterUDPPort=54325", 149 "Cluster.UDPPacketSendSize=1400", 150 "Cluster.UDPPacketRecvSize=1500", 151 "Cluster.UDPPacketCapPerMessage=5", 152 "Cluster.HeartBeatDuration=1s", 153 "Cluster.HeartBeatMissLimit=3", 154 "Cluster.MessageQueueDepthPerPeer=4", 155 "Cluster.MaxRequestDuration=1s", 156 "Cluster.LivenessCheckRedundancy=2", 157 158 "HTTPServer.TCPPort=" + testProxyFSDaemonHTTPPort, 159 160 "SwiftClient.NoAuthIPAddr=" + testSwiftNoAuthIPAddr, 161 "SwiftClient.NoAuthTCPPort=" + testSwiftNoAuthPort, 162 "SwiftClient.Timeout=10s", 163 "SwiftClient.RetryLimit=1", 164 "SwiftClient.RetryLimitObject=1", 165 "SwiftClient.RetryDelay=10ms", 166 "SwiftClient.RetryDelayObject=10ms", 167 "SwiftClient.RetryExpBackoff=1.2", 168 "SwiftClient.RetryExpBackoffObject=2.0", 169 "SwiftClient.ChunkedConnectionPoolSize=64", 170 "SwiftClient.NonChunkedConnectionPoolSize=32", 171 "SwiftClient.SwiftReconChecksPerConfCheck=0", 172 173 "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerStoragePolicy=silver", 174 "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerNamePrefix=Replicated3Way_", 175 "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainersPerPeer=10", 176 "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.MaxObjectsPerContainer=1000000", 177 178 "Volume:CommonVolume.FSID=1", 179 "Volume:CommonVolume.FUSEMountPointName=ProxyFSMountPointPath", 180 "Volume:CommonVolume.NFSExportClientMapList=CommonVolumeNFSClient0", 181 "Volume:CommonVolume.SMBShareName=CommonShare", 182 "Volume:CommonVolume.PrimaryPeer=Peer0", 183 "Volume:CommonVolume.AccountName=AUTH_test", 184 "Volume:CommonVolume.CheckpointContainerName=.__checkpoint__", 185 "Volume:CommonVolume.CheckpointContainerStoragePolicy=gold", 186 "Volume:CommonVolume.CheckpointInterval=10s", 187 "Volume:CommonVolume.DefaultPhysicalContainerLayout=PhysicalContainerLayoutReplicated3Way", 188 "Volume:CommonVolume.MaxFlushSize=10485760", 189 "Volume:CommonVolume.MaxFlushTime=10s", 190 "Volume:CommonVolume.FileDefragmentChunkSize=10485760", 191 "Volume:CommonVolume.FileDefragmentChunkDelay=10ms", 192 "Volume:CommonVolume.NonceValuesToReserve=100", 193 "Volume:CommonVolume.MaxEntriesPerDirNode=32", 194 "Volume:CommonVolume.MaxExtentsPerFileNode=32", 195 "Volume:CommonVolume.MaxInodesPerMetadataNode=32", 196 "Volume:CommonVolume.MaxLogSegmentsPerMetadataNode=64", 197 "Volume:CommonVolume.MaxDirFileNodesPerMetadataNode=16", 198 "Volume:CommonVolume.MaxBytesInodeCache=100000", 199 "Volume:CommonVolume.InodeCacheEvictInterval=1s", 200 "Volume:CommonVolume.AutoFormat=true", 201 "Volume:CommonVolume.ActiveLeaseEvictLowLimit=5000", 202 "Volume:CommonVolume.ActiveLeaseEvictHighLimit=5010", 203 204 "NFSClientMap:CommonVolumeNFSClient0.ClientPattern=*", 205 "NFSClientMap:CommonVolumeNFSClient0.AccessMode=rw", 206 "NFSClientMap:CommonVolumeNFSClient0.RootSquash=no_root_squash", 207 "NFSClientMap:CommonVolumeNFSClient0.Secure=insecure", 208 209 "VolumeGroup:CommonVolumeGroup.VolumeList=CommonVolume", 210 "VolumeGroup:CommonVolumeGroup.VirtualIPAddr=", 211 "VolumeGroup:CommonVolumeGroup.PrimaryPeer=Peer0", 212 "VolumeGroup:CommonVolumeGroup.ReadCacheLineSize=1000000", 213 "VolumeGroup:CommonVolumeGroup.ReadCacheWeight=100", 214 215 "FSGlobals.VolumeGroupList=CommonVolumeGroup", 216 "FSGlobals.CheckpointHeaderConsensusAttempts=5", 217 "FSGlobals.MountRetryLimit=6", 218 "FSGlobals.MountRetryDelay=1s", 219 "FSGlobals.MountRetryExpBackoff=2", 220 "FSGlobals.LogCheckpointHeaderPosts=true", 221 "FSGlobals.TryLockBackoffMin=10ms", 222 "FSGlobals.TryLockBackoffMax=50ms", 223 "FSGlobals.TryLockSerializationThreshhold=5", 224 "FSGlobals.SymlinkMax=32", 225 "FSGlobals.CoalesceElementChunkSize=16", 226 "FSGlobals.InodeRecCacheEvictLowLimit=10000", 227 "FSGlobals.InodeRecCacheEvictHighLimit=10010", 228 "FSGlobals.LogSegmentRecCacheEvictLowLimit=10000", 229 "FSGlobals.LogSegmentRecCacheEvictHighLimit=10010", 230 "FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000", 231 "FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010", 232 "FSGlobals.DirEntryCacheEvictLowLimit=10000", 233 "FSGlobals.DirEntryCacheEvictHighLimit=10010", 234 "FSGlobals.FileExtentMapEvictLowLimit=10000", 235 "FSGlobals.FileExtentMapEvictHighLimit=10010", 236 "FSGlobals.EtcdEnabled=false", 237 238 "JSONRPCServer.TCPPort=54326", 239 "JSONRPCServer.FastTCPPort=54327", 240 "JSONRPCServer.RetryRPCPort=54328", 241 "JSONRPCServer.RetryRPCTTLCompleted=10s", 242 "JSONRPCServer.RetryRPCAckTrim=10ms", 243 "JSONRPCServer.DataPathLogging=false", 244 "JSONRPCServer.MinLeaseDuration=250ms", 245 "JSONRPCServer.LeaseInterruptInterval=250ms", 246 "JSONRPCServer.LeaseInterruptLimit=20", 247 } 248 249 testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxAccountNameLength="+strconv.FormatUint(testMaxAccountNameLength, 10)) 250 testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxContainerNameLength="+strconv.FormatUint(testMaxContainerNameLength, 10)) 251 testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxObjectNameLength="+strconv.FormatUint(testMaxObjectNameLength, 10)) 252 testConfStrings = append(testConfStrings, "RamSwiftInfo.AccountListingLimit="+strconv.FormatUint(testAccountListingLimit, 10)) 253 testConfStrings = append(testConfStrings, "RamSwiftInfo.ContainerListingLimit="+strconv.FormatUint(testContainerListingLimit, 10)) 254 255 ramswiftSignalHandlerIsArmedWG.Add(1) 256 testDaemonGlobals.ramswiftDoneChan = make(chan bool, 1) 257 258 go ramswift.Daemon("/dev/null", testConfStrings, &ramswiftSignalHandlerIsArmedWG, testDaemonGlobals.ramswiftDoneChan, unix.SIGUSR1) 259 260 ramswiftSignalHandlerIsArmedWG.Wait() 261 262 for { 263 infoResponse, err = http.Get("http://" + testSwiftNoAuthIPAddr + ":" + testSwiftNoAuthPort + "/info") 264 if (nil == err) && (http.StatusOK == infoResponse.StatusCode) { 265 break 266 } 267 268 time.Sleep(testDaemonStartPollInterval) 269 } 270 271 testDaemonGlobals.proxyfsdErrChan = make(chan error, 1) // Must be buffered to avoid race 272 273 go proxyfsd.Daemon("/dev/null", testConfStrings, testDaemonGlobals.proxyfsdErrChan, &testDaemonGlobals.proxyfsdWG, []string{}, unix.SIGUSR2) 274 275 err = <-testDaemonGlobals.proxyfsdErrChan 276 if nil != err { 277 t.Fatalf("proxyfsd.Daemon() startup failed: %v", err) 278 } 279 280 for { 281 versionResponse, err = http.Get("http://" + testProxyFSDaemonIPAddr + ":" + testProxyFSDaemonHTTPPort + "/version") 282 if (nil == err) && (http.StatusOK == versionResponse.StatusCode) { 283 break 284 } 285 286 time.Sleep(testDaemonStartPollInterval) 287 } 288 289 testConfMap, err = conf.MakeConfMapFromStrings(testConfStrings) 290 if nil != err { 291 t.Fatalf("conf.MakeConfMapFromStrings() failed: %v", err) 292 } 293 294 globals.logFile = nil 295 globals.config.LogFilePath = "" 296 globals.config.LogToConsole = true 297 298 startSwiftProxyEmulator(t, testConfMap) 299 300 initializeGlobals(testConfMap) 301 302 // Fake out that plug-in auth has already obtained AuthToken & StorageURL 303 304 globals.swiftAuthToken = testAuthToken 305 globals.swiftStorageURL = "http://" + testSwiftProxyAddr + "/proxyfs/" + testAccountName 306 307 go testSwallowFissionErrChan(t, globals.fissionErrChan) 308 309 doMountProxyFS() 310 311 performMountFUSE() 312 } 313 314 func testSwallowFissionErrChan(t *testing.T, fissionErrChan chan error) { 315 var ( 316 err error 317 ) 318 319 err = <-fissionErrChan 320 if nil != err { 321 t.Fatalf("fissionErrChan received err: %v", err) 322 } 323 } 324 325 func testTeardown(t *testing.T) { 326 var ( 327 err error 328 testDir string 329 ) 330 331 performUnmountFUSE() 332 333 emptyFileInodeDirtyListAndLogSegmentChan() 334 335 doUnmountProxyFS() 336 337 uninitializeGlobals() 338 339 _ = unix.Kill(unix.Getpid(), unix.SIGUSR2) 340 341 err = <-testDaemonGlobals.proxyfsdErrChan 342 343 testDaemonGlobals.proxyfsdWG.Wait() 344 345 if nil != err { 346 t.Fatalf("proxyfsd.Daemon() exited with error: %v", err) 347 } 348 349 unix.Kill(unix.Getpid(), unix.SIGUSR1) 350 351 _ = <-testDaemonGlobals.ramswiftDoneChan 352 353 testDir, err = os.Getwd() 354 if nil != err { 355 t.Fatalf("os.Getwd() failed: %v", err) 356 } 357 358 err = os.Chdir("..") 359 if nil != err { 360 t.Fatalf("os.Chdir() failed: %v", err) 361 } 362 363 err = os.RemoveAll(testDir) 364 if nil != err { 365 t.Fatalf("os.RemoveAll() failed: %v", err) 366 } 367 368 stopSwiftProxyEmulator() 369 }