storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/tree-walk_test.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"io/ioutil"
    23  	"os"
    24  	"reflect"
    25  	"sort"
    26  	"strings"
    27  	"testing"
    28  	"time"
    29  )
    30  
    31  // Fixed volume name that could be used across tests
    32  const volume = "testvolume"
    33  
    34  // Test for filterMatchingPrefix.
    35  func TestFilterMatchingPrefix(t *testing.T) {
    36  	entries := []string{"a", "aab", "ab", "abbbb", "zzz"}
    37  	testCases := []struct {
    38  		prefixEntry string
    39  		result      []string
    40  	}{
    41  		{
    42  			// Empty prefix should match all entries.
    43  			"",
    44  			[]string{"a", "aab", "ab", "abbbb", "zzz"},
    45  		},
    46  		{
    47  			"a",
    48  			[]string{"a", "aab", "ab", "abbbb"},
    49  		},
    50  		{
    51  			"aa",
    52  			[]string{"aab"},
    53  		},
    54  		{
    55  			// Does not match any of the entries.
    56  			"c",
    57  			[]string{},
    58  		},
    59  	}
    60  	for i, testCase := range testCases {
    61  		expected := testCase.result
    62  		got := filterMatchingPrefix(entries, testCase.prefixEntry)
    63  		if !reflect.DeepEqual(expected, got) {
    64  			t.Errorf("Test %d : expected %v, got %v", i+1, expected, got)
    65  		}
    66  	}
    67  }
    68  
    69  // Helper function that creates a volume and files in it.
    70  func createNamespace(disk StorageAPI, volume string, files []string) error {
    71  	// Make a volume.
    72  	err := disk.MakeVol(context.Background(), volume)
    73  	if err != nil {
    74  		return err
    75  	}
    76  
    77  	// Create files.
    78  	for _, file := range files {
    79  		err = disk.AppendFile(context.Background(), volume, file, []byte{})
    80  		if err != nil {
    81  			return err
    82  		}
    83  	}
    84  	return err
    85  }
    86  
    87  // Returns function "listDir" of the type listDirFunc.
    88  // disks - used for doing disk.ListDir()
    89  func listDirFactory(ctx context.Context, disk StorageAPI, isLeaf IsLeafFunc) ListDirFunc {
    90  	return func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
    91  		entries, err := disk.ListDir(ctx, volume, dirPath, -1)
    92  		if err != nil {
    93  			return false, nil, false
    94  		}
    95  		if len(entries) == 0 {
    96  			return true, nil, false
    97  		}
    98  		entries, delayIsLeaf = filterListEntries(volume, dirPath, entries, dirEntry, isLeaf)
    99  		return false, entries, delayIsLeaf
   100  	}
   101  }
   102  
   103  // Test if tree walker returns entries matching prefix alone are received
   104  // when a non empty prefix is supplied.
   105  func testTreeWalkPrefix(t *testing.T, listDir ListDirFunc, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc) {
   106  	// Start the tree walk go-routine.
   107  	prefix := "d/"
   108  	endWalkCh := make(chan struct{})
   109  	twResultCh := startTreeWalk(context.Background(), volume, prefix, "", true, listDir, isLeaf, isLeafDir, endWalkCh)
   110  
   111  	// Check if all entries received on the channel match the prefix.
   112  	for res := range twResultCh {
   113  		if !HasPrefix(res.entry, prefix) {
   114  			t.Errorf("Entry %s doesn't match prefix %s", res.entry, prefix)
   115  		}
   116  	}
   117  }
   118  
   119  // Test if entries received on tree walk's channel appear after the supplied marker.
   120  func testTreeWalkMarker(t *testing.T, listDir ListDirFunc, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc) {
   121  	// Start the tree walk go-routine.
   122  	prefix := ""
   123  	endWalkCh := make(chan struct{})
   124  	twResultCh := startTreeWalk(context.Background(), volume, prefix, "d/g", true, listDir, isLeaf, isLeafDir, endWalkCh)
   125  
   126  	// Check if only 3 entries, namely d/g/h, i/j/k, lmn are received on the channel.
   127  	expectedCount := 3
   128  	actualCount := 0
   129  	for range twResultCh {
   130  		actualCount++
   131  	}
   132  	if expectedCount != actualCount {
   133  		t.Errorf("Expected %d entries, actual no. of entries were %d", expectedCount, actualCount)
   134  	}
   135  }
   136  
   137  // Test tree-walk.
   138  func TestTreeWalk(t *testing.T) {
   139  	fsDir, err := ioutil.TempDir(globalTestTmpDir, "minio-")
   140  	if err != nil {
   141  		t.Fatalf("Unable to create tmp directory: %s", err)
   142  	}
   143  	endpoints := mustGetNewEndpoints(fsDir)
   144  	disk, err := newStorageAPI(endpoints[0])
   145  	if err != nil {
   146  		t.Fatalf("Unable to create StorageAPI: %s", err)
   147  	}
   148  
   149  	var files = []string{
   150  		"d/e",
   151  		"d/f",
   152  		"d/g/h",
   153  		"i/j/k",
   154  		"lmn",
   155  	}
   156  	err = createNamespace(disk, volume, files)
   157  	if err != nil {
   158  		t.Fatal(err)
   159  	}
   160  
   161  	isLeaf := func(bucket, leafPath string) bool {
   162  		return !strings.HasSuffix(leafPath, slashSeparator)
   163  	}
   164  
   165  	isLeafDir := func(bucket, leafPath string) bool {
   166  		entries, _ := disk.ListDir(context.Background(), bucket, leafPath, 1)
   167  		return len(entries) == 0
   168  	}
   169  
   170  	listDir := listDirFactory(context.Background(), disk, isLeaf)
   171  
   172  	// Simple test for prefix based walk.
   173  	testTreeWalkPrefix(t, listDir, isLeaf, isLeafDir)
   174  
   175  	// Simple test when marker is set.
   176  	testTreeWalkMarker(t, listDir, isLeaf, isLeafDir)
   177  
   178  	err = os.RemoveAll(fsDir)
   179  	if err != nil {
   180  		t.Fatal(err)
   181  	}
   182  }
   183  
   184  // Test if tree walk go-routine exits cleanly if tree walk is aborted because of timeout.
   185  func TestTreeWalkTimeout(t *testing.T) {
   186  	fsDir, err := ioutil.TempDir(globalTestTmpDir, "minio-")
   187  	if err != nil {
   188  		t.Fatalf("Unable to create tmp directory: %s", err)
   189  	}
   190  	endpoints := mustGetNewEndpoints(fsDir)
   191  	disk, err := newStorageAPI(endpoints[0])
   192  	if err != nil {
   193  		t.Fatalf("Unable to create StorageAPI: %s", err)
   194  	}
   195  	var myfiles []string
   196  	// Create maxObjectsList+1 number of entries.
   197  	for i := 0; i < maxObjectList+1; i++ {
   198  		myfiles = append(myfiles, fmt.Sprintf("file.%d", i))
   199  	}
   200  	err = createNamespace(disk, volume, myfiles)
   201  	if err != nil {
   202  		t.Fatal(err)
   203  	}
   204  
   205  	isLeaf := func(bucket, leafPath string) bool {
   206  		return !strings.HasSuffix(leafPath, slashSeparator)
   207  	}
   208  
   209  	isLeafDir := func(bucket, leafPath string) bool {
   210  		entries, _ := disk.ListDir(context.Background(), bucket, leafPath, 1)
   211  		return len(entries) == 0
   212  	}
   213  
   214  	listDir := listDirFactory(context.Background(), disk, isLeaf)
   215  
   216  	// TreeWalk pool with 2 seconds timeout for tree-walk go routines.
   217  	pool := NewTreeWalkPool(2 * time.Second)
   218  
   219  	endWalkCh := make(chan struct{})
   220  	prefix := ""
   221  	marker := ""
   222  	recursive := true
   223  	resultCh := startTreeWalk(context.Background(), volume, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
   224  
   225  	params := listParams{
   226  		bucket:    volume,
   227  		recursive: recursive,
   228  	}
   229  	// Add Treewalk to the pool.
   230  	pool.Set(params, resultCh, endWalkCh)
   231  
   232  	// Wait for the Treewalk to timeout.
   233  	<-time.After(3 * time.Second)
   234  
   235  	// Read maxObjectList number of entries from the channel.
   236  	// maxObjectsList number of entries would have been filled into the resultCh
   237  	// buffered channel. After the timeout resultCh would get closed and hence the
   238  	// maxObjectsList+1 entry would not be sent in the channel.
   239  	i := 0
   240  	for range resultCh {
   241  		i++
   242  		if i == maxObjectList {
   243  			break
   244  		}
   245  	}
   246  
   247  	// The last entry will not be received as the Treewalk goroutine would have exited.
   248  	_, ok := <-resultCh
   249  	if ok {
   250  		t.Error("Tree-walk go routine has not exited after timeout.")
   251  	}
   252  	err = os.RemoveAll(fsDir)
   253  	if err != nil {
   254  		t.Error(err)
   255  	}
   256  }
   257  
   258  // TestRecursiveWalk - tests if treeWalk returns entries correctly with and
   259  // without recursively traversing prefixes.
   260  func TestRecursiveTreeWalk(t *testing.T) {
   261  	// Create a backend directories fsDir1.
   262  	fsDir1, err := ioutil.TempDir(globalTestTmpDir, "minio-")
   263  	if err != nil {
   264  		t.Fatalf("Unable to create tmp directory: %s", err)
   265  	}
   266  
   267  	endpoints := mustGetNewEndpoints(fsDir1)
   268  	disk1, err := newStorageAPI(endpoints[0])
   269  	if err != nil {
   270  		t.Fatalf("Unable to create StorageAPI: %s", err)
   271  	}
   272  
   273  	isLeaf := func(bucket, leafPath string) bool {
   274  		return !strings.HasSuffix(leafPath, slashSeparator)
   275  	}
   276  
   277  	isLeafDir := func(bucket, leafPath string) bool {
   278  		entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1)
   279  		return len(entries) == 0
   280  	}
   281  
   282  	// Create listDir function.
   283  	listDir := listDirFactory(context.Background(), disk1, isLeaf)
   284  
   285  	// Create the namespace.
   286  	var files = []string{
   287  		"d/e",
   288  		"d/f",
   289  		"d/g/h",
   290  		"i/j/k",
   291  		"lmn",
   292  	}
   293  	err = createNamespace(disk1, volume, files)
   294  	if err != nil {
   295  		t.Fatal(err)
   296  	}
   297  
   298  	endWalkCh := make(chan struct{})
   299  	testCases := []struct {
   300  		prefix    string
   301  		marker    string
   302  		recursive bool
   303  		expected  map[string]struct{}
   304  	}{
   305  		// with no prefix, no marker and no recursive traversal
   306  		{"", "", false, map[string]struct{}{
   307  			"d/":  {},
   308  			"i/":  {},
   309  			"lmn": {},
   310  		}},
   311  		// with no prefix, no marker and recursive traversal
   312  		{"", "", true, map[string]struct{}{
   313  			"d/f":   {},
   314  			"d/g/h": {},
   315  			"d/e":   {},
   316  			"i/j/k": {},
   317  			"lmn":   {},
   318  		}},
   319  		// with no prefix, marker and no recursive traversal
   320  		{"", "d/e", false, map[string]struct{}{
   321  			"d/f":  {},
   322  			"d/g/": {},
   323  			"i/":   {},
   324  			"lmn":  {},
   325  		}},
   326  		// with no prefix, marker and recursive traversal
   327  		{"", "d/e", true, map[string]struct{}{
   328  			"d/f":   {},
   329  			"d/g/h": {},
   330  			"i/j/k": {},
   331  			"lmn":   {},
   332  		}},
   333  		// with prefix, no marker and no recursive traversal
   334  		{"d/", "", false, map[string]struct{}{
   335  			"d/e":  {},
   336  			"d/f":  {},
   337  			"d/g/": {},
   338  		}},
   339  		// with prefix, no marker and no recursive traversal
   340  		{"d/", "", true, map[string]struct{}{
   341  			"d/e":   {},
   342  			"d/f":   {},
   343  			"d/g/h": {},
   344  		}},
   345  		// with prefix, marker and no recursive traversal
   346  		{"d/", "d/e", false, map[string]struct{}{
   347  			"d/f":  {},
   348  			"d/g/": {},
   349  		}},
   350  		// with prefix, marker and recursive traversal
   351  		{"d/", "d/e", true, map[string]struct{}{
   352  			"d/f":   {},
   353  			"d/g/h": {},
   354  		}},
   355  	}
   356  	for i, testCase := range testCases {
   357  		testCase := testCase
   358  		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
   359  			for entry := range startTreeWalk(context.Background(), volume,
   360  				testCase.prefix, testCase.marker, testCase.recursive,
   361  				listDir, isLeaf, isLeafDir, endWalkCh) {
   362  				if _, found := testCase.expected[entry.entry]; !found {
   363  					t.Errorf("Expected %s, but couldn't find", entry.entry)
   364  				}
   365  			}
   366  		})
   367  	}
   368  	err = os.RemoveAll(fsDir1)
   369  	if err != nil {
   370  		t.Error(err)
   371  	}
   372  }
   373  
   374  func TestSortedness(t *testing.T) {
   375  	// Create a backend directories fsDir1.
   376  	fsDir1, err := ioutil.TempDir(globalTestTmpDir, "minio-")
   377  	if err != nil {
   378  		t.Errorf("Unable to create tmp directory: %s", err)
   379  	}
   380  
   381  	endpoints := mustGetNewEndpoints(fsDir1)
   382  	disk1, err := newStorageAPI(endpoints[0])
   383  	if err != nil {
   384  		t.Fatalf("Unable to create StorageAPI: %s", err)
   385  	}
   386  
   387  	isLeaf := func(bucket, leafPath string) bool {
   388  		return !strings.HasSuffix(leafPath, slashSeparator)
   389  	}
   390  
   391  	isLeafDir := func(bucket, leafPath string) bool {
   392  		entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1)
   393  		return len(entries) == 0
   394  	}
   395  
   396  	// Create listDir function.
   397  	listDir := listDirFactory(context.Background(), disk1, isLeaf)
   398  
   399  	// Create the namespace.
   400  	var files = []string{
   401  		"d/e",
   402  		"d/f",
   403  		"d/g/h",
   404  		"i/j/k",
   405  		"lmn",
   406  	}
   407  	err = createNamespace(disk1, volume, files)
   408  	if err != nil {
   409  		t.Fatal(err)
   410  	}
   411  
   412  	endWalkCh := make(chan struct{})
   413  	testCases := []struct {
   414  		prefix    string
   415  		marker    string
   416  		recursive bool
   417  	}{
   418  		// with no prefix, no marker and no recursive traversal
   419  		{"", "", false},
   420  		// with no prefix, no marker and recursive traversal
   421  		{"", "", true},
   422  		// with no prefix, marker and no recursive traversal
   423  		{"", "d/e", false},
   424  		// with no prefix, marker and recursive traversal
   425  		{"", "d/e", true},
   426  		// with prefix, no marker and no recursive traversal
   427  		{"d/", "", false},
   428  		// with prefix, no marker and no recursive traversal
   429  		{"d/", "", true},
   430  		// with prefix, marker and no recursive traversal
   431  		{"d/", "d/e", false},
   432  		// with prefix, marker and recursive traversal
   433  		{"d/", "d/e", true},
   434  	}
   435  	for i, test := range testCases {
   436  		var actualEntries []string
   437  		for entry := range startTreeWalk(context.Background(), volume,
   438  			test.prefix, test.marker, test.recursive,
   439  			listDir, isLeaf, isLeafDir, endWalkCh) {
   440  			actualEntries = append(actualEntries, entry.entry)
   441  		}
   442  		if !sort.IsSorted(sort.StringSlice(actualEntries)) {
   443  			t.Error(i+1, "Expected entries to be sort, but it wasn't")
   444  		}
   445  	}
   446  
   447  	// Remove directory created for testing
   448  	err = os.RemoveAll(fsDir1)
   449  	if err != nil {
   450  		t.Error(err)
   451  	}
   452  }
   453  
   454  func TestTreeWalkIsEnd(t *testing.T) {
   455  	// Create a backend directories fsDir1.
   456  	fsDir1, err := ioutil.TempDir(globalTestTmpDir, "minio-")
   457  	if err != nil {
   458  		t.Errorf("Unable to create tmp directory: %s", err)
   459  	}
   460  
   461  	endpoints := mustGetNewEndpoints(fsDir1)
   462  	disk1, err := newStorageAPI(endpoints[0])
   463  	if err != nil {
   464  		t.Fatalf("Unable to create StorageAPI: %s", err)
   465  	}
   466  
   467  	isLeaf := func(bucket, leafPath string) bool {
   468  		return !strings.HasSuffix(leafPath, slashSeparator)
   469  	}
   470  
   471  	isLeafDir := func(bucket, leafPath string) bool {
   472  		entries, _ := disk1.ListDir(context.Background(), bucket, leafPath, 1)
   473  		return len(entries) == 0
   474  	}
   475  
   476  	// Create listDir function.
   477  	listDir := listDirFactory(context.Background(), disk1, isLeaf)
   478  
   479  	// Create the namespace.
   480  	var files = []string{
   481  		"d/e",
   482  		"d/f",
   483  		"d/g/h",
   484  		"i/j/k",
   485  		"lmn",
   486  	}
   487  	err = createNamespace(disk1, volume, files)
   488  	if err != nil {
   489  		t.Fatal(err)
   490  	}
   491  
   492  	endWalkCh := make(chan struct{})
   493  	testCases := []struct {
   494  		prefix        string
   495  		marker        string
   496  		recursive     bool
   497  		expectedEntry string
   498  	}{
   499  		// with no prefix, no marker and no recursive traversal
   500  		{"", "", false, "lmn"},
   501  		// with no prefix, no marker and recursive traversal
   502  		{"", "", true, "lmn"},
   503  		// with no prefix, marker and no recursive traversal
   504  		{"", "d/e", false, "lmn"},
   505  		// with no prefix, marker and recursive traversal
   506  		{"", "d/e", true, "lmn"},
   507  		// with prefix, no marker and no recursive traversal
   508  		{"d/", "", false, "d/g/"},
   509  		// with prefix, no marker and no recursive traversal
   510  		{"d/", "", true, "d/g/h"},
   511  		// with prefix, marker and no recursive traversal
   512  		{"d/", "d/e", false, "d/g/"},
   513  		// with prefix, marker and recursive traversal
   514  		{"d/", "d/e", true, "d/g/h"},
   515  	}
   516  	for i, test := range testCases {
   517  		var entry TreeWalkResult
   518  		for entry = range startTreeWalk(context.Background(), volume, test.prefix,
   519  			test.marker, test.recursive, listDir, isLeaf, isLeafDir, endWalkCh) {
   520  		}
   521  		if entry.entry != test.expectedEntry {
   522  			t.Errorf("Test %d: Expected entry %s, but received %s with the EOF marker", i, test.expectedEntry, entry.entry)
   523  		}
   524  		if !entry.end {
   525  			t.Errorf("Test %d: Last entry %s, doesn't have EOF marker set", i, entry.entry)
   526  		}
   527  	}
   528  
   529  	// Remove directory created for testing
   530  	err = os.RemoveAll(fsDir1)
   531  	if err != nil {
   532  		t.Error(err)
   533  	}
   534  }