storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/tree-walk_test.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2016 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "context" 21 "fmt" 22 "io/ioutil" 23 "os" 24 "reflect" 25 "sort" 26 "strings" 27 "testing" 28 "time" 29 ) 30 31 // Fixed volume name that could be used across tests 32 const volume = "testvolume" 33 34 // Test for filterMatchingPrefix. 35 func TestFilterMatchingPrefix(t *testing.T) { 36 entries := []string{"a", "aab", "ab", "abbbb", "zzz"} 37 testCases := []struct { 38 prefixEntry string 39 result []string 40 }{ 41 { 42 // Empty prefix should match all entries. 43 "", 44 []string{"a", "aab", "ab", "abbbb", "zzz"}, 45 }, 46 { 47 "a", 48 []string{"a", "aab", "ab", "abbbb"}, 49 }, 50 { 51 "aa", 52 []string{"aab"}, 53 }, 54 { 55 // Does not match any of the entries. 56 "c", 57 []string{}, 58 }, 59 } 60 for i, testCase := range testCases { 61 expected := testCase.result 62 got := filterMatchingPrefix(entries, testCase.prefixEntry) 63 if !reflect.DeepEqual(expected, got) { 64 t.Errorf("Test %d : expected %v, got %v", i+1, expected, got) 65 } 66 } 67 } 68 69 // Helper function that creates a volume and files in it. 70 func createNamespace(disk StorageAPI, volume string, files []string) error { 71 // Make a volume. 72 err := disk.MakeVol(context.Background(), volume) 73 if err != nil { 74 return err 75 } 76 77 // Create files. 78 for _, file := range files { 79 err = disk.AppendFile(context.Background(), volume, file, []byte{}) 80 if err != nil { 81 return err 82 } 83 } 84 return err 85 } 86 87 // Returns function "listDir" of the type listDirFunc. 88 // disks - used for doing disk.ListDir() 89 func listDirFactory(ctx context.Context, disk StorageAPI, isLeaf IsLeafFunc) ListDirFunc { 90 return func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) { 91 entries, err := disk.ListDir(ctx, volume, dirPath, -1) 92 if err != nil { 93 return false, nil, false 94 } 95 if len(entries) == 0 { 96 return true, nil, false 97 } 98 entries, delayIsLeaf = filterListEntries(volume, dirPath, entries, dirEntry, isLeaf) 99 return false, entries, delayIsLeaf 100 } 101 } 102 103 // Test if tree walker returns entries matching prefix alone are received 104 // when a non empty prefix is supplied. 105 func testTreeWalkPrefix(t *testing.T, listDir ListDirFunc, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc) { 106 // Start the tree walk go-routine. 107 prefix := "d/" 108 endWalkCh := make(chan struct{}) 109 twResultCh := startTreeWalk(context.Background(), volume, prefix, "", true, listDir, isLeaf, isLeafDir, endWalkCh) 110 111 // Check if all entries received on the channel match the prefix. 112 for res := range twResultCh { 113 if !HasPrefix(res.entry, prefix) { 114 t.Errorf("Entry %s doesn't match prefix %s", res.entry, prefix) 115 } 116 } 117 } 118 119 // Test if entries received on tree walk's channel appear after the supplied marker. 120 func testTreeWalkMarker(t *testing.T, listDir ListDirFunc, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc) { 121 // Start the tree walk go-routine. 122 prefix := "" 123 endWalkCh := make(chan struct{}) 124 twResultCh := startTreeWalk(context.Background(), volume, prefix, "d/g", true, listDir, isLeaf, isLeafDir, endWalkCh) 125 126 // Check if only 3 entries, namely d/g/h, i/j/k, lmn are received on the channel. 127 expectedCount := 3 128 actualCount := 0 129 for range twResultCh { 130 actualCount++ 131 } 132 if expectedCount != actualCount { 133 t.Errorf("Expected %d entries, actual no. of entries were %d", expectedCount, actualCount) 134 } 135 } 136 137 // Test tree-walk. 138 func TestTreeWalk(t *testing.T) { 139 fsDir, err := ioutil.TempDir(globalTestTmpDir, "minio-") 140 if err != nil { 141 t.Fatalf("Unable to create tmp directory: %s", err) 142 } 143 endpoints := mustGetNewEndpoints(fsDir) 144 disk, err := newStorageAPI(endpoints[0]) 145 if err != nil { 146 t.Fatalf("Unable to create StorageAPI: %s", err) 147 } 148 149 var files = []string{ 150 "d/e", 151 "d/f", 152 "d/g/h", 153 "i/j/k", 154 "lmn", 155 } 156 err = createNamespace(disk, volume, files) 157 if err != nil { 158 t.Fatal(err) 159 } 160 161 isLeaf := func(bucket, leafPath string) bool { 162 return !strings.HasSuffix(leafPath, slashSeparator) 163 } 164 165 isLeafDir := func(bucket, leafPath string) bool { 166 entries, _ := disk.ListDir(context.Background(), bucket, leafPath, 1) 167 return len(entries) == 0 168 } 169 170 listDir := listDirFactory(context.Background(), disk, isLeaf) 171 172 // Simple test for prefix based walk. 173 testTreeWalkPrefix(t, listDir, isLeaf, isLeafDir) 174 175 // Simple test when marker is set. 176 testTreeWalkMarker(t, listDir, isLeaf, isLeafDir) 177 178 err = os.RemoveAll(fsDir) 179 if err != nil { 180 t.Fatal(err) 181 } 182 } 183 184 // Test if tree walk go-routine exits cleanly if tree walk is aborted because of timeout. 185 func TestTreeWalkTimeout(t *testing.T) { 186 fsDir, err := ioutil.TempDir(globalTestTmpDir, "minio-") 187 if err != nil { 188 t.Fatalf("Unable to create tmp directory: %s", err) 189 } 190 endpoints := mustGetNewEndpoints(fsDir) 191 disk, err := newStorageAPI(endpoints[0]) 192 if err != nil { 193 t.Fatalf("Unable to create StorageAPI: %s", err) 194 } 195 var myfiles []string 196 // Create maxObjectsList+1 number of entries. 197 for i := 0; i < maxObjectList+1; i++ { 198 myfiles = append(myfiles, fmt.Sprintf("file.%d", i)) 199 } 200 err = createNamespace(disk, volume, myfiles) 201 if err != nil { 202 t.Fatal(err) 203 } 204 205 isLeaf := func(bucket, leafPath string) bool { 206 return !strings.HasSuffix(leafPath, slashSeparator) 207 } 208 209 isLeafDir := func(bucket, leafPath string) bool { 210 entries, _ := disk.ListDir(context.Background(), bucket, leafPath, 1) 211 return len(entries) == 0 212 } 213 214 listDir := listDirFactory(context.Background(), disk, isLeaf) 215 216 // TreeWalk pool with 2 seconds timeout for tree-walk go routines. 217 pool := NewTreeWalkPool(2 * time.Second) 218 219 endWalkCh := make(chan struct{}) 220 prefix := "" 221 marker := "" 222 recursive := true 223 resultCh := startTreeWalk(context.Background(), volume, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh) 224 225 params := listParams{ 226 bucket: volume, 227 recursive: recursive, 228 } 229 // Add Treewalk to the pool. 230 pool.Set(params, resultCh, endWalkCh) 231 232 // Wait for the Treewalk to timeout. 233 <-time.After(3 * time.Second) 234 235 // Read maxObjectList number of entries from the channel. 236 // maxObjectsList number of entries would have been filled into the resultCh 237 // buffered channel. After the timeout resultCh would get closed and hence the 238 // maxObjectsList+1 entry would not be sent in the channel. 239 i := 0 240 for range resultCh { 241 i++ 242 if i == maxObjectList { 243 break 244 } 245 } 246 247 // The last entry will not be received as the Treewalk goroutine would have exited. 248 _, ok := <-resultCh 249 if ok { 250 t.Error("Tree-walk go routine has not exited after timeout.") 251 } 252 err = os.RemoveAll(fsDir) 253 if err != nil { 254 t.Error(err) 255 } 256 } 257 258 // TestRecursiveWalk - tests if treeWalk returns entries correctly with and 259 // without recursively traversing prefixes. 260 func TestRecursiveTreeWalk(t *testing.T) { 261 // Create a backend directories fsDir1. 262 fsDir1, err := ioutil.TempDir(globalTestTmpDir, "minio-") 263 if err != nil { 264 t.Fatalf("Unable to create tmp directory: %s", err) 265 } 266 267 endpoints := mustGetNewEndpoints(fsDir1) 268 disk1, err := newStorageAPI(endpoints[0]) 269 if err != nil { 270 t.Fatalf("Unable to create StorageAPI: %s", err) 271 } 272 273 isLeaf := func(bucket, leafPath string) bool { 274 return !strings.HasSuffix(leafPath, slashSeparator) 275 } 276 277 isLeafDir := func(bucket, leafPath string) bool { 278 entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1) 279 return len(entries) == 0 280 } 281 282 // Create listDir function. 283 listDir := listDirFactory(context.Background(), disk1, isLeaf) 284 285 // Create the namespace. 286 var files = []string{ 287 "d/e", 288 "d/f", 289 "d/g/h", 290 "i/j/k", 291 "lmn", 292 } 293 err = createNamespace(disk1, volume, files) 294 if err != nil { 295 t.Fatal(err) 296 } 297 298 endWalkCh := make(chan struct{}) 299 testCases := []struct { 300 prefix string 301 marker string 302 recursive bool 303 expected map[string]struct{} 304 }{ 305 // with no prefix, no marker and no recursive traversal 306 {"", "", false, map[string]struct{}{ 307 "d/": {}, 308 "i/": {}, 309 "lmn": {}, 310 }}, 311 // with no prefix, no marker and recursive traversal 312 {"", "", true, map[string]struct{}{ 313 "d/f": {}, 314 "d/g/h": {}, 315 "d/e": {}, 316 "i/j/k": {}, 317 "lmn": {}, 318 }}, 319 // with no prefix, marker and no recursive traversal 320 {"", "d/e", false, map[string]struct{}{ 321 "d/f": {}, 322 "d/g/": {}, 323 "i/": {}, 324 "lmn": {}, 325 }}, 326 // with no prefix, marker and recursive traversal 327 {"", "d/e", true, map[string]struct{}{ 328 "d/f": {}, 329 "d/g/h": {}, 330 "i/j/k": {}, 331 "lmn": {}, 332 }}, 333 // with prefix, no marker and no recursive traversal 334 {"d/", "", false, map[string]struct{}{ 335 "d/e": {}, 336 "d/f": {}, 337 "d/g/": {}, 338 }}, 339 // with prefix, no marker and no recursive traversal 340 {"d/", "", true, map[string]struct{}{ 341 "d/e": {}, 342 "d/f": {}, 343 "d/g/h": {}, 344 }}, 345 // with prefix, marker and no recursive traversal 346 {"d/", "d/e", false, map[string]struct{}{ 347 "d/f": {}, 348 "d/g/": {}, 349 }}, 350 // with prefix, marker and recursive traversal 351 {"d/", "d/e", true, map[string]struct{}{ 352 "d/f": {}, 353 "d/g/h": {}, 354 }}, 355 } 356 for i, testCase := range testCases { 357 testCase := testCase 358 t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) { 359 for entry := range startTreeWalk(context.Background(), volume, 360 testCase.prefix, testCase.marker, testCase.recursive, 361 listDir, isLeaf, isLeafDir, endWalkCh) { 362 if _, found := testCase.expected[entry.entry]; !found { 363 t.Errorf("Expected %s, but couldn't find", entry.entry) 364 } 365 } 366 }) 367 } 368 err = os.RemoveAll(fsDir1) 369 if err != nil { 370 t.Error(err) 371 } 372 } 373 374 func TestSortedness(t *testing.T) { 375 // Create a backend directories fsDir1. 376 fsDir1, err := ioutil.TempDir(globalTestTmpDir, "minio-") 377 if err != nil { 378 t.Errorf("Unable to create tmp directory: %s", err) 379 } 380 381 endpoints := mustGetNewEndpoints(fsDir1) 382 disk1, err := newStorageAPI(endpoints[0]) 383 if err != nil { 384 t.Fatalf("Unable to create StorageAPI: %s", err) 385 } 386 387 isLeaf := func(bucket, leafPath string) bool { 388 return !strings.HasSuffix(leafPath, slashSeparator) 389 } 390 391 isLeafDir := func(bucket, leafPath string) bool { 392 entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1) 393 return len(entries) == 0 394 } 395 396 // Create listDir function. 397 listDir := listDirFactory(context.Background(), disk1, isLeaf) 398 399 // Create the namespace. 400 var files = []string{ 401 "d/e", 402 "d/f", 403 "d/g/h", 404 "i/j/k", 405 "lmn", 406 } 407 err = createNamespace(disk1, volume, files) 408 if err != nil { 409 t.Fatal(err) 410 } 411 412 endWalkCh := make(chan struct{}) 413 testCases := []struct { 414 prefix string 415 marker string 416 recursive bool 417 }{ 418 // with no prefix, no marker and no recursive traversal 419 {"", "", false}, 420 // with no prefix, no marker and recursive traversal 421 {"", "", true}, 422 // with no prefix, marker and no recursive traversal 423 {"", "d/e", false}, 424 // with no prefix, marker and recursive traversal 425 {"", "d/e", true}, 426 // with prefix, no marker and no recursive traversal 427 {"d/", "", false}, 428 // with prefix, no marker and no recursive traversal 429 {"d/", "", true}, 430 // with prefix, marker and no recursive traversal 431 {"d/", "d/e", false}, 432 // with prefix, marker and recursive traversal 433 {"d/", "d/e", true}, 434 } 435 for i, test := range testCases { 436 var actualEntries []string 437 for entry := range startTreeWalk(context.Background(), volume, 438 test.prefix, test.marker, test.recursive, 439 listDir, isLeaf, isLeafDir, endWalkCh) { 440 actualEntries = append(actualEntries, entry.entry) 441 } 442 if !sort.IsSorted(sort.StringSlice(actualEntries)) { 443 t.Error(i+1, "Expected entries to be sort, but it wasn't") 444 } 445 } 446 447 // Remove directory created for testing 448 err = os.RemoveAll(fsDir1) 449 if err != nil { 450 t.Error(err) 451 } 452 } 453 454 func TestTreeWalkIsEnd(t *testing.T) { 455 // Create a backend directories fsDir1. 456 fsDir1, err := ioutil.TempDir(globalTestTmpDir, "minio-") 457 if err != nil { 458 t.Errorf("Unable to create tmp directory: %s", err) 459 } 460 461 endpoints := mustGetNewEndpoints(fsDir1) 462 disk1, err := newStorageAPI(endpoints[0]) 463 if err != nil { 464 t.Fatalf("Unable to create StorageAPI: %s", err) 465 } 466 467 isLeaf := func(bucket, leafPath string) bool { 468 return !strings.HasSuffix(leafPath, slashSeparator) 469 } 470 471 isLeafDir := func(bucket, leafPath string) bool { 472 entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1) 473 return len(entries) == 0 474 } 475 476 // Create listDir function. 477 listDir := listDirFactory(context.Background(), disk1, isLeaf) 478 479 // Create the namespace. 480 var files = []string{ 481 "d/e", 482 "d/f", 483 "d/g/h", 484 "i/j/k", 485 "lmn", 486 } 487 err = createNamespace(disk1, volume, files) 488 if err != nil { 489 t.Fatal(err) 490 } 491 492 endWalkCh := make(chan struct{}) 493 testCases := []struct { 494 prefix string 495 marker string 496 recursive bool 497 expectedEntry string 498 }{ 499 // with no prefix, no marker and no recursive traversal 500 {"", "", false, "lmn"}, 501 // with no prefix, no marker and recursive traversal 502 {"", "", true, "lmn"}, 503 // with no prefix, marker and no recursive traversal 504 {"", "d/e", false, "lmn"}, 505 // with no prefix, marker and recursive traversal 506 {"", "d/e", true, "lmn"}, 507 // with prefix, no marker and no recursive traversal 508 {"d/", "", false, "d/g/"}, 509 // with prefix, no marker and no recursive traversal 510 {"d/", "", true, "d/g/h"}, 511 // with prefix, marker and no recursive traversal 512 {"d/", "d/e", false, "d/g/"}, 513 // with prefix, marker and recursive traversal 514 {"d/", "d/e", true, "d/g/h"}, 515 } 516 for i, test := range testCases { 517 var entry TreeWalkResult 518 for entry = range startTreeWalk(context.Background(), volume, test.prefix, 519 test.marker, test.recursive, listDir, isLeaf, isLeafDir, endWalkCh) { 520 } 521 if entry.entry != test.expectedEntry { 522 t.Errorf("Test %d: Expected entry %s, but received %s with the EOF marker", i, test.expectedEntry, entry.entry) 523 } 524 if !entry.end { 525 t.Errorf("Test %d: Last entry %s, doesn't have EOF marker set", i, entry.entry) 526 } 527 } 528 529 // Remove directory created for testing 530 err = os.RemoveAll(fsDir1) 531 if err != nil { 532 t.Error(err) 533 } 534 }