storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/metacache_test.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"testing"
    21  	"time"
    22  )
    23  
    24  var metaCacheTestsetTimestamp = time.Now()
    25  
    26  var metaCacheTestset = []metacache{
    27  	0: {
    28  		id:           "case-1-normal",
    29  		bucket:       "bucket",
    30  		root:         "folder/prefix",
    31  		recursive:    false,
    32  		status:       scanStateSuccess,
    33  		fileNotFound: false,
    34  		error:        "",
    35  		started:      metaCacheTestsetTimestamp,
    36  		ended:        metaCacheTestsetTimestamp.Add(time.Minute),
    37  		lastUpdate:   metaCacheTestsetTimestamp.Add(time.Minute),
    38  		lastHandout:  metaCacheTestsetTimestamp,
    39  		startedCycle: 10,
    40  		endedCycle:   10,
    41  		dataVersion:  metacacheStreamVersion,
    42  	},
    43  	1: {
    44  		id:           "case-2-recursive",
    45  		bucket:       "bucket",
    46  		root:         "folder/prefix",
    47  		recursive:    true,
    48  		status:       scanStateSuccess,
    49  		fileNotFound: false,
    50  		error:        "",
    51  		started:      metaCacheTestsetTimestamp,
    52  		ended:        metaCacheTestsetTimestamp.Add(time.Minute),
    53  		lastUpdate:   metaCacheTestsetTimestamp.Add(time.Minute),
    54  		lastHandout:  metaCacheTestsetTimestamp,
    55  		startedCycle: 10,
    56  		endedCycle:   10,
    57  		dataVersion:  metacacheStreamVersion,
    58  	},
    59  	2: {
    60  		id:           "case-3-older",
    61  		bucket:       "bucket",
    62  		root:         "folder/prefix",
    63  		recursive:    false,
    64  		status:       scanStateSuccess,
    65  		fileNotFound: true,
    66  		error:        "",
    67  		started:      metaCacheTestsetTimestamp.Add(-time.Minute),
    68  		ended:        metaCacheTestsetTimestamp,
    69  		lastUpdate:   metaCacheTestsetTimestamp,
    70  		lastHandout:  metaCacheTestsetTimestamp,
    71  		startedCycle: 10,
    72  		endedCycle:   10,
    73  		dataVersion:  metacacheStreamVersion,
    74  	},
    75  	3: {
    76  		id:           "case-4-error",
    77  		bucket:       "bucket",
    78  		root:         "folder/prefix",
    79  		recursive:    false,
    80  		status:       scanStateError,
    81  		fileNotFound: false,
    82  		error:        "an error lol",
    83  		started:      metaCacheTestsetTimestamp.Add(-20 * time.Minute),
    84  		ended:        metaCacheTestsetTimestamp.Add(-20 * time.Minute),
    85  		lastUpdate:   metaCacheTestsetTimestamp.Add(-20 * time.Minute),
    86  		lastHandout:  metaCacheTestsetTimestamp.Add(-20 * time.Minute),
    87  		startedCycle: 10,
    88  		endedCycle:   10,
    89  		dataVersion:  metacacheStreamVersion,
    90  	},
    91  	4: {
    92  		id:           "case-5-noupdate",
    93  		bucket:       "bucket",
    94  		root:         "folder/prefix",
    95  		recursive:    false,
    96  		status:       scanStateStarted,
    97  		fileNotFound: false,
    98  		error:        "",
    99  		started:      metaCacheTestsetTimestamp.Add(-time.Minute),
   100  		ended:        time.Time{},
   101  		lastUpdate:   metaCacheTestsetTimestamp.Add(-time.Minute),
   102  		lastHandout:  metaCacheTestsetTimestamp,
   103  		startedCycle: 10,
   104  		endedCycle:   10,
   105  		dataVersion:  metacacheStreamVersion,
   106  	},
   107  	5: {
   108  		id:           "case-6-404notfound",
   109  		bucket:       "bucket",
   110  		root:         "folder/notfound",
   111  		recursive:    true,
   112  		status:       scanStateSuccess,
   113  		fileNotFound: true,
   114  		error:        "",
   115  		started:      metaCacheTestsetTimestamp,
   116  		ended:        metaCacheTestsetTimestamp.Add(time.Minute),
   117  		lastUpdate:   metaCacheTestsetTimestamp.Add(time.Minute),
   118  		lastHandout:  metaCacheTestsetTimestamp,
   119  		startedCycle: 10,
   120  		endedCycle:   10,
   121  		dataVersion:  metacacheStreamVersion,
   122  	},
   123  	6: {
   124  		id:           "case-7-oldcycle",
   125  		bucket:       "bucket",
   126  		root:         "folder/prefix",
   127  		recursive:    true,
   128  		status:       scanStateSuccess,
   129  		fileNotFound: false,
   130  		error:        "",
   131  		started:      metaCacheTestsetTimestamp.Add(-10 * time.Minute),
   132  		ended:        metaCacheTestsetTimestamp.Add(-8 * time.Minute),
   133  		lastUpdate:   metaCacheTestsetTimestamp.Add(-8 * time.Minute),
   134  		lastHandout:  metaCacheTestsetTimestamp,
   135  		startedCycle: 6,
   136  		endedCycle:   8,
   137  		dataVersion:  metacacheStreamVersion,
   138  	},
   139  	7: {
   140  		id:           "case-8-running",
   141  		bucket:       "bucket",
   142  		root:         "folder/running",
   143  		recursive:    false,
   144  		status:       scanStateStarted,
   145  		fileNotFound: false,
   146  		error:        "",
   147  		started:      metaCacheTestsetTimestamp.Add(-1 * time.Minute),
   148  		ended:        time.Time{},
   149  		lastUpdate:   metaCacheTestsetTimestamp.Add(-1 * time.Minute),
   150  		lastHandout:  metaCacheTestsetTimestamp,
   151  		startedCycle: 10,
   152  		endedCycle:   0,
   153  		dataVersion:  metacacheStreamVersion,
   154  	},
   155  	8: {
   156  		id:           "case-8-finished-a-week-ago",
   157  		bucket:       "bucket",
   158  		root:         "folder/finished",
   159  		recursive:    false,
   160  		status:       scanStateSuccess,
   161  		fileNotFound: false,
   162  		error:        "",
   163  		started:      metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour),
   164  		ended:        metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour),
   165  		lastUpdate:   metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour),
   166  		lastHandout:  metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour),
   167  		startedCycle: 10,
   168  		endedCycle:   10,
   169  		dataVersion:  metacacheStreamVersion,
   170  	},
   171  }
   172  
   173  func Test_baseDirFromPrefix(t *testing.T) {
   174  	tests := []struct {
   175  		name   string
   176  		prefix string
   177  		want   string
   178  	}{
   179  		{
   180  			name:   "root",
   181  			prefix: "object.ext",
   182  			want:   "",
   183  		},
   184  		{
   185  			name:   "rootdotslash",
   186  			prefix: "./object.ext",
   187  			want:   "",
   188  		},
   189  		{
   190  			name:   "rootslash",
   191  			prefix: "/",
   192  			want:   "",
   193  		},
   194  		{
   195  			name:   "folder",
   196  			prefix: "prefix/",
   197  			want:   "prefix/",
   198  		},
   199  		{
   200  			name:   "folderobj",
   201  			prefix: "prefix/obj.ext",
   202  			want:   "prefix/",
   203  		},
   204  		{
   205  			name:   "folderfolderobj",
   206  			prefix: "prefix/prefix2/obj.ext",
   207  			want:   "prefix/prefix2/",
   208  		},
   209  		{
   210  			name:   "folderfolder",
   211  			prefix: "prefix/prefix2/",
   212  			want:   "prefix/prefix2/",
   213  		},
   214  	}
   215  	for _, tt := range tests {
   216  		t.Run(tt.name, func(t *testing.T) {
   217  			if got := baseDirFromPrefix(tt.prefix); got != tt.want {
   218  				t.Errorf("baseDirFromPrefix() = %v, want %v", got, tt.want)
   219  			}
   220  		})
   221  	}
   222  }
   223  
   224  func Test_metacache_canBeReplacedBy(t *testing.T) {
   225  	testAgainst := metacache{
   226  		id:           "case-1-modified",
   227  		bucket:       "bucket",
   228  		root:         "folder/prefix",
   229  		recursive:    true,
   230  		status:       scanStateSuccess,
   231  		fileNotFound: false,
   232  		error:        "",
   233  		started:      metaCacheTestsetTimestamp.Add(time.Minute),
   234  		ended:        metaCacheTestsetTimestamp.Add(2 * time.Minute),
   235  		lastUpdate:   metaCacheTestsetTimestamp.Add(2 * time.Minute),
   236  		lastHandout:  metaCacheTestsetTimestamp.Add(time.Minute),
   237  		startedCycle: 10,
   238  		endedCycle:   10,
   239  		dataVersion:  metacacheStreamVersion,
   240  	}
   241  	wantResults := []bool{0: true, 1: true, 2: true, 3: true, 4: true, 5: false, 6: true, 7: false, 8: false}
   242  
   243  	for i, tt := range metaCacheTestset {
   244  		t.Run(tt.id, func(t *testing.T) {
   245  			var want bool
   246  			if i >= len(wantResults) {
   247  				t.Logf("no expected result for test #%d", i)
   248  			} else {
   249  				want = wantResults[i]
   250  			}
   251  			// Add an hour, otherwise it will never be replaced.
   252  			// We operated on a copy.
   253  			tt.lastHandout = tt.lastHandout.Add(-2 * time.Hour)
   254  			tt.lastUpdate = tt.lastHandout.Add(-2 * time.Hour)
   255  			got := tt.canBeReplacedBy(&testAgainst)
   256  			if got != want {
   257  				t.Errorf("#%d: want %v, got %v", i, want, got)
   258  			}
   259  		})
   260  	}
   261  }
   262  
   263  func Test_metacache_finished(t *testing.T) {
   264  	wantResults := []bool{0: true, 1: true, 2: true, 3: true, 4: false, 5: true, 6: true, 7: false, 8: true}
   265  
   266  	for i, tt := range metaCacheTestset {
   267  		t.Run(tt.id, func(t *testing.T) {
   268  			var want bool
   269  			if i >= len(wantResults) {
   270  				t.Logf("no expected result for test #%d", i)
   271  			} else {
   272  				want = wantResults[i]
   273  			}
   274  
   275  			got := tt.finished()
   276  			if got != want {
   277  				t.Errorf("#%d: want %v, got %v", i, want, got)
   278  			}
   279  		})
   280  	}
   281  }
   282  
   283  func Test_metacache_worthKeeping(t *testing.T) {
   284  	wantResults := []bool{0: true, 1: true, 2: true, 3: false, 4: false, 5: true, 6: false, 7: false, 8: false}
   285  
   286  	for i, tt := range metaCacheTestset {
   287  		t.Run(tt.id, func(t *testing.T) {
   288  			var want bool
   289  			if i >= len(wantResults) {
   290  				t.Logf("no expected result for test #%d", i)
   291  			} else {
   292  				want = wantResults[i]
   293  			}
   294  
   295  			got := tt.worthKeeping(7 + dataUsageUpdateDirCycles)
   296  			if got != want {
   297  				t.Errorf("#%d: want %v, got %v", i, want, got)
   298  			}
   299  		})
   300  	}
   301  }