storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/metacache_test.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2020 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "testing" 21 "time" 22 ) 23 24 var metaCacheTestsetTimestamp = time.Now() 25 26 var metaCacheTestset = []metacache{ 27 0: { 28 id: "case-1-normal", 29 bucket: "bucket", 30 root: "folder/prefix", 31 recursive: false, 32 status: scanStateSuccess, 33 fileNotFound: false, 34 error: "", 35 started: metaCacheTestsetTimestamp, 36 ended: metaCacheTestsetTimestamp.Add(time.Minute), 37 lastUpdate: metaCacheTestsetTimestamp.Add(time.Minute), 38 lastHandout: metaCacheTestsetTimestamp, 39 startedCycle: 10, 40 endedCycle: 10, 41 dataVersion: metacacheStreamVersion, 42 }, 43 1: { 44 id: "case-2-recursive", 45 bucket: "bucket", 46 root: "folder/prefix", 47 recursive: true, 48 status: scanStateSuccess, 49 fileNotFound: false, 50 error: "", 51 started: metaCacheTestsetTimestamp, 52 ended: metaCacheTestsetTimestamp.Add(time.Minute), 53 lastUpdate: metaCacheTestsetTimestamp.Add(time.Minute), 54 lastHandout: metaCacheTestsetTimestamp, 55 startedCycle: 10, 56 endedCycle: 10, 57 dataVersion: metacacheStreamVersion, 58 }, 59 2: { 60 id: "case-3-older", 61 bucket: "bucket", 62 root: "folder/prefix", 63 recursive: false, 64 status: scanStateSuccess, 65 fileNotFound: true, 66 error: "", 67 started: metaCacheTestsetTimestamp.Add(-time.Minute), 68 ended: metaCacheTestsetTimestamp, 69 lastUpdate: metaCacheTestsetTimestamp, 70 lastHandout: metaCacheTestsetTimestamp, 71 startedCycle: 10, 72 endedCycle: 10, 73 dataVersion: metacacheStreamVersion, 74 }, 75 3: { 76 id: "case-4-error", 77 bucket: "bucket", 78 root: "folder/prefix", 79 recursive: false, 80 status: scanStateError, 81 fileNotFound: false, 82 error: "an error lol", 83 started: metaCacheTestsetTimestamp.Add(-20 * time.Minute), 84 ended: metaCacheTestsetTimestamp.Add(-20 * time.Minute), 85 lastUpdate: metaCacheTestsetTimestamp.Add(-20 * time.Minute), 86 lastHandout: metaCacheTestsetTimestamp.Add(-20 * time.Minute), 87 startedCycle: 10, 88 endedCycle: 10, 89 dataVersion: metacacheStreamVersion, 90 }, 91 4: { 92 id: "case-5-noupdate", 93 bucket: "bucket", 94 root: "folder/prefix", 95 recursive: false, 96 status: scanStateStarted, 97 fileNotFound: false, 98 error: "", 99 started: metaCacheTestsetTimestamp.Add(-time.Minute), 100 ended: time.Time{}, 101 lastUpdate: metaCacheTestsetTimestamp.Add(-time.Minute), 102 lastHandout: metaCacheTestsetTimestamp, 103 startedCycle: 10, 104 endedCycle: 10, 105 dataVersion: metacacheStreamVersion, 106 }, 107 5: { 108 id: "case-6-404notfound", 109 bucket: "bucket", 110 root: "folder/notfound", 111 recursive: true, 112 status: scanStateSuccess, 113 fileNotFound: true, 114 error: "", 115 started: metaCacheTestsetTimestamp, 116 ended: metaCacheTestsetTimestamp.Add(time.Minute), 117 lastUpdate: metaCacheTestsetTimestamp.Add(time.Minute), 118 lastHandout: metaCacheTestsetTimestamp, 119 startedCycle: 10, 120 endedCycle: 10, 121 dataVersion: metacacheStreamVersion, 122 }, 123 6: { 124 id: "case-7-oldcycle", 125 bucket: "bucket", 126 root: "folder/prefix", 127 recursive: true, 128 status: scanStateSuccess, 129 fileNotFound: false, 130 error: "", 131 started: metaCacheTestsetTimestamp.Add(-10 * time.Minute), 132 ended: metaCacheTestsetTimestamp.Add(-8 * time.Minute), 133 lastUpdate: metaCacheTestsetTimestamp.Add(-8 * time.Minute), 134 lastHandout: metaCacheTestsetTimestamp, 135 startedCycle: 6, 136 endedCycle: 8, 137 dataVersion: metacacheStreamVersion, 138 }, 139 7: { 140 id: "case-8-running", 141 bucket: "bucket", 142 root: "folder/running", 143 recursive: false, 144 status: scanStateStarted, 145 fileNotFound: false, 146 error: "", 147 started: metaCacheTestsetTimestamp.Add(-1 * time.Minute), 148 ended: time.Time{}, 149 lastUpdate: metaCacheTestsetTimestamp.Add(-1 * time.Minute), 150 lastHandout: metaCacheTestsetTimestamp, 151 startedCycle: 10, 152 endedCycle: 0, 153 dataVersion: metacacheStreamVersion, 154 }, 155 8: { 156 id: "case-8-finished-a-week-ago", 157 bucket: "bucket", 158 root: "folder/finished", 159 recursive: false, 160 status: scanStateSuccess, 161 fileNotFound: false, 162 error: "", 163 started: metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour), 164 ended: metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour), 165 lastUpdate: metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour), 166 lastHandout: metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour), 167 startedCycle: 10, 168 endedCycle: 10, 169 dataVersion: metacacheStreamVersion, 170 }, 171 } 172 173 func Test_baseDirFromPrefix(t *testing.T) { 174 tests := []struct { 175 name string 176 prefix string 177 want string 178 }{ 179 { 180 name: "root", 181 prefix: "object.ext", 182 want: "", 183 }, 184 { 185 name: "rootdotslash", 186 prefix: "./object.ext", 187 want: "", 188 }, 189 { 190 name: "rootslash", 191 prefix: "/", 192 want: "", 193 }, 194 { 195 name: "folder", 196 prefix: "prefix/", 197 want: "prefix/", 198 }, 199 { 200 name: "folderobj", 201 prefix: "prefix/obj.ext", 202 want: "prefix/", 203 }, 204 { 205 name: "folderfolderobj", 206 prefix: "prefix/prefix2/obj.ext", 207 want: "prefix/prefix2/", 208 }, 209 { 210 name: "folderfolder", 211 prefix: "prefix/prefix2/", 212 want: "prefix/prefix2/", 213 }, 214 } 215 for _, tt := range tests { 216 t.Run(tt.name, func(t *testing.T) { 217 if got := baseDirFromPrefix(tt.prefix); got != tt.want { 218 t.Errorf("baseDirFromPrefix() = %v, want %v", got, tt.want) 219 } 220 }) 221 } 222 } 223 224 func Test_metacache_canBeReplacedBy(t *testing.T) { 225 testAgainst := metacache{ 226 id: "case-1-modified", 227 bucket: "bucket", 228 root: "folder/prefix", 229 recursive: true, 230 status: scanStateSuccess, 231 fileNotFound: false, 232 error: "", 233 started: metaCacheTestsetTimestamp.Add(time.Minute), 234 ended: metaCacheTestsetTimestamp.Add(2 * time.Minute), 235 lastUpdate: metaCacheTestsetTimestamp.Add(2 * time.Minute), 236 lastHandout: metaCacheTestsetTimestamp.Add(time.Minute), 237 startedCycle: 10, 238 endedCycle: 10, 239 dataVersion: metacacheStreamVersion, 240 } 241 wantResults := []bool{0: true, 1: true, 2: true, 3: true, 4: true, 5: false, 6: true, 7: false, 8: false} 242 243 for i, tt := range metaCacheTestset { 244 t.Run(tt.id, func(t *testing.T) { 245 var want bool 246 if i >= len(wantResults) { 247 t.Logf("no expected result for test #%d", i) 248 } else { 249 want = wantResults[i] 250 } 251 // Add an hour, otherwise it will never be replaced. 252 // We operated on a copy. 253 tt.lastHandout = tt.lastHandout.Add(-2 * time.Hour) 254 tt.lastUpdate = tt.lastHandout.Add(-2 * time.Hour) 255 got := tt.canBeReplacedBy(&testAgainst) 256 if got != want { 257 t.Errorf("#%d: want %v, got %v", i, want, got) 258 } 259 }) 260 } 261 } 262 263 func Test_metacache_finished(t *testing.T) { 264 wantResults := []bool{0: true, 1: true, 2: true, 3: true, 4: false, 5: true, 6: true, 7: false, 8: true} 265 266 for i, tt := range metaCacheTestset { 267 t.Run(tt.id, func(t *testing.T) { 268 var want bool 269 if i >= len(wantResults) { 270 t.Logf("no expected result for test #%d", i) 271 } else { 272 want = wantResults[i] 273 } 274 275 got := tt.finished() 276 if got != want { 277 t.Errorf("#%d: want %v, got %v", i, want, got) 278 } 279 }) 280 } 281 } 282 283 func Test_metacache_worthKeeping(t *testing.T) { 284 wantResults := []bool{0: true, 1: true, 2: true, 3: false, 4: false, 5: true, 6: false, 7: false, 8: false} 285 286 for i, tt := range metaCacheTestset { 287 t.Run(tt.id, func(t *testing.T) { 288 var want bool 289 if i >= len(wantResults) { 290 t.Logf("no expected result for test #%d", i) 291 } else { 292 want = wantResults[i] 293 } 294 295 got := tt.worthKeeping(7 + dataUsageUpdateDirCycles) 296 if got != want { 297 t.Errorf("#%d: want %v, got %v", i, want, got) 298 } 299 }) 300 } 301 }