github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/b2/b2_internal_test.go (about) 1 package b2 2 3 import ( 4 "context" 5 "crypto/sha1" 6 "fmt" 7 "path" 8 "strings" 9 "testing" 10 "time" 11 12 "github.com/rclone/rclone/backend/b2/api" 13 "github.com/rclone/rclone/fs" 14 "github.com/rclone/rclone/fs/cache" 15 "github.com/rclone/rclone/fs/hash" 16 "github.com/rclone/rclone/fstest" 17 "github.com/rclone/rclone/fstest/fstests" 18 "github.com/rclone/rclone/lib/bucket" 19 "github.com/rclone/rclone/lib/random" 20 "github.com/rclone/rclone/lib/version" 21 "github.com/stretchr/testify/assert" 22 "github.com/stretchr/testify/require" 23 ) 24 25 // Test b2 string encoding 26 // https://www.backblaze.com/b2/docs/string_encoding.html 27 28 var encodeTest = []struct { 29 fullyEncoded string 30 minimallyEncoded string 31 plainText string 32 }{ 33 {fullyEncoded: "%20", minimallyEncoded: "+", plainText: " "}, 34 {fullyEncoded: "%21", minimallyEncoded: "!", plainText: "!"}, 35 {fullyEncoded: "%22", minimallyEncoded: "%22", plainText: "\""}, 36 {fullyEncoded: "%23", minimallyEncoded: "%23", plainText: "#"}, 37 {fullyEncoded: "%24", minimallyEncoded: "$", plainText: "$"}, 38 {fullyEncoded: "%25", minimallyEncoded: "%25", plainText: "%"}, 39 {fullyEncoded: "%26", minimallyEncoded: "%26", plainText: "&"}, 40 {fullyEncoded: "%27", minimallyEncoded: "'", plainText: "'"}, 41 {fullyEncoded: "%28", minimallyEncoded: "(", plainText: "("}, 42 {fullyEncoded: "%29", minimallyEncoded: ")", plainText: ")"}, 43 {fullyEncoded: "%2A", minimallyEncoded: "*", plainText: "*"}, 44 {fullyEncoded: "%2B", minimallyEncoded: "%2B", plainText: "+"}, 45 {fullyEncoded: "%2C", minimallyEncoded: "%2C", plainText: ","}, 46 {fullyEncoded: "%2D", minimallyEncoded: "-", plainText: "-"}, 47 {fullyEncoded: "%2E", minimallyEncoded: ".", plainText: "."}, 48 {fullyEncoded: "%2F", minimallyEncoded: "/", plainText: "/"}, 49 {fullyEncoded: "%30", minimallyEncoded: "0", plainText: "0"}, 50 {fullyEncoded: "%31", minimallyEncoded: "1", plainText: "1"}, 51 {fullyEncoded: "%32", minimallyEncoded: "2", plainText: "2"}, 52 {fullyEncoded: "%33", minimallyEncoded: "3", plainText: "3"}, 53 {fullyEncoded: "%34", minimallyEncoded: "4", plainText: "4"}, 54 {fullyEncoded: "%35", minimallyEncoded: "5", plainText: "5"}, 55 {fullyEncoded: "%36", minimallyEncoded: "6", plainText: "6"}, 56 {fullyEncoded: "%37", minimallyEncoded: "7", plainText: "7"}, 57 {fullyEncoded: "%38", minimallyEncoded: "8", plainText: "8"}, 58 {fullyEncoded: "%39", minimallyEncoded: "9", plainText: "9"}, 59 {fullyEncoded: "%3A", minimallyEncoded: ":", plainText: ":"}, 60 {fullyEncoded: "%3B", minimallyEncoded: ";", plainText: ";"}, 61 {fullyEncoded: "%3C", minimallyEncoded: "%3C", plainText: "<"}, 62 {fullyEncoded: "%3D", minimallyEncoded: "=", plainText: "="}, 63 {fullyEncoded: "%3E", minimallyEncoded: "%3E", plainText: ">"}, 64 {fullyEncoded: "%3F", minimallyEncoded: "%3F", plainText: "?"}, 65 {fullyEncoded: "%40", minimallyEncoded: "@", plainText: "@"}, 66 {fullyEncoded: "%41", minimallyEncoded: "A", plainText: "A"}, 67 {fullyEncoded: "%42", minimallyEncoded: "B", plainText: "B"}, 68 {fullyEncoded: "%43", minimallyEncoded: "C", plainText: "C"}, 69 {fullyEncoded: "%44", minimallyEncoded: "D", plainText: "D"}, 70 {fullyEncoded: "%45", minimallyEncoded: "E", plainText: "E"}, 71 {fullyEncoded: "%46", minimallyEncoded: "F", plainText: "F"}, 72 {fullyEncoded: "%47", minimallyEncoded: "G", plainText: "G"}, 73 {fullyEncoded: "%48", minimallyEncoded: "H", plainText: "H"}, 74 {fullyEncoded: "%49", minimallyEncoded: "I", plainText: "I"}, 75 {fullyEncoded: "%4A", minimallyEncoded: "J", plainText: "J"}, 76 {fullyEncoded: "%4B", minimallyEncoded: "K", plainText: "K"}, 77 {fullyEncoded: "%4C", minimallyEncoded: "L", plainText: "L"}, 78 {fullyEncoded: "%4D", minimallyEncoded: "M", plainText: "M"}, 79 {fullyEncoded: "%4E", minimallyEncoded: "N", plainText: "N"}, 80 {fullyEncoded: "%4F", minimallyEncoded: "O", plainText: "O"}, 81 {fullyEncoded: "%50", minimallyEncoded: "P", plainText: "P"}, 82 {fullyEncoded: "%51", minimallyEncoded: "Q", plainText: "Q"}, 83 {fullyEncoded: "%52", minimallyEncoded: "R", plainText: "R"}, 84 {fullyEncoded: "%53", minimallyEncoded: "S", plainText: "S"}, 85 {fullyEncoded: "%54", minimallyEncoded: "T", plainText: "T"}, 86 {fullyEncoded: "%55", minimallyEncoded: "U", plainText: "U"}, 87 {fullyEncoded: "%56", minimallyEncoded: "V", plainText: "V"}, 88 {fullyEncoded: "%57", minimallyEncoded: "W", plainText: "W"}, 89 {fullyEncoded: "%58", minimallyEncoded: "X", plainText: "X"}, 90 {fullyEncoded: "%59", minimallyEncoded: "Y", plainText: "Y"}, 91 {fullyEncoded: "%5A", minimallyEncoded: "Z", plainText: "Z"}, 92 {fullyEncoded: "%5B", minimallyEncoded: "%5B", plainText: "["}, 93 {fullyEncoded: "%5C", minimallyEncoded: "%5C", plainText: "\\"}, 94 {fullyEncoded: "%5D", minimallyEncoded: "%5D", plainText: "]"}, 95 {fullyEncoded: "%5E", minimallyEncoded: "%5E", plainText: "^"}, 96 {fullyEncoded: "%5F", minimallyEncoded: "_", plainText: "_"}, 97 {fullyEncoded: "%60", minimallyEncoded: "%60", plainText: "`"}, 98 {fullyEncoded: "%61", minimallyEncoded: "a", plainText: "a"}, 99 {fullyEncoded: "%62", minimallyEncoded: "b", plainText: "b"}, 100 {fullyEncoded: "%63", minimallyEncoded: "c", plainText: "c"}, 101 {fullyEncoded: "%64", minimallyEncoded: "d", plainText: "d"}, 102 {fullyEncoded: "%65", minimallyEncoded: "e", plainText: "e"}, 103 {fullyEncoded: "%66", minimallyEncoded: "f", plainText: "f"}, 104 {fullyEncoded: "%67", minimallyEncoded: "g", plainText: "g"}, 105 {fullyEncoded: "%68", minimallyEncoded: "h", plainText: "h"}, 106 {fullyEncoded: "%69", minimallyEncoded: "i", plainText: "i"}, 107 {fullyEncoded: "%6A", minimallyEncoded: "j", plainText: "j"}, 108 {fullyEncoded: "%6B", minimallyEncoded: "k", plainText: "k"}, 109 {fullyEncoded: "%6C", minimallyEncoded: "l", plainText: "l"}, 110 {fullyEncoded: "%6D", minimallyEncoded: "m", plainText: "m"}, 111 {fullyEncoded: "%6E", minimallyEncoded: "n", plainText: "n"}, 112 {fullyEncoded: "%6F", minimallyEncoded: "o", plainText: "o"}, 113 {fullyEncoded: "%70", minimallyEncoded: "p", plainText: "p"}, 114 {fullyEncoded: "%71", minimallyEncoded: "q", plainText: "q"}, 115 {fullyEncoded: "%72", minimallyEncoded: "r", plainText: "r"}, 116 {fullyEncoded: "%73", minimallyEncoded: "s", plainText: "s"}, 117 {fullyEncoded: "%74", minimallyEncoded: "t", plainText: "t"}, 118 {fullyEncoded: "%75", minimallyEncoded: "u", plainText: "u"}, 119 {fullyEncoded: "%76", minimallyEncoded: "v", plainText: "v"}, 120 {fullyEncoded: "%77", minimallyEncoded: "w", plainText: "w"}, 121 {fullyEncoded: "%78", minimallyEncoded: "x", plainText: "x"}, 122 {fullyEncoded: "%79", minimallyEncoded: "y", plainText: "y"}, 123 {fullyEncoded: "%7A", minimallyEncoded: "z", plainText: "z"}, 124 {fullyEncoded: "%7B", minimallyEncoded: "%7B", plainText: "{"}, 125 {fullyEncoded: "%7C", minimallyEncoded: "%7C", plainText: "|"}, 126 {fullyEncoded: "%7D", minimallyEncoded: "%7D", plainText: "}"}, 127 {fullyEncoded: "%7E", minimallyEncoded: "~", plainText: "~"}, 128 {fullyEncoded: "%7F", minimallyEncoded: "%7F", plainText: "\u007f"}, 129 {fullyEncoded: "%E8%87%AA%E7%94%B1", minimallyEncoded: "%E8%87%AA%E7%94%B1", plainText: "自由"}, 130 {fullyEncoded: "%F0%90%90%80", minimallyEncoded: "%F0%90%90%80", plainText: "𐐀"}, 131 } 132 133 func TestUrlEncode(t *testing.T) { 134 for _, test := range encodeTest { 135 got := urlEncode(test.plainText) 136 if got != test.minimallyEncoded && got != test.fullyEncoded { 137 t.Errorf("urlEncode(%q) got %q wanted %q or %q", test.plainText, got, test.minimallyEncoded, test.fullyEncoded) 138 } 139 } 140 } 141 142 func TestTimeString(t *testing.T) { 143 for _, test := range []struct { 144 in time.Time 145 want string 146 }{ 147 {fstest.Time("1970-01-01T00:00:00.000000000Z"), "0"}, 148 {fstest.Time("2001-02-03T04:05:10.123123123Z"), "981173110123"}, 149 {fstest.Time("2001-02-03T05:05:10.123123123+01:00"), "981173110123"}, 150 } { 151 got := timeString(test.in) 152 if test.want != got { 153 t.Logf("%v: want %v got %v", test.in, test.want, got) 154 } 155 } 156 157 } 158 159 func TestParseTimeString(t *testing.T) { 160 for _, test := range []struct { 161 in string 162 want time.Time 163 wantError string 164 }{ 165 {"0", fstest.Time("1970-01-01T00:00:00.000000000Z"), ""}, 166 {"981173110123", fstest.Time("2001-02-03T04:05:10.123000000Z"), ""}, 167 {"", time.Time{}, ""}, 168 {"potato", time.Time{}, `strconv.ParseInt: parsing "potato": invalid syntax`}, 169 } { 170 o := Object{} 171 err := o.parseTimeString(test.in) 172 got := o.modTime 173 var gotError string 174 if err != nil { 175 gotError = err.Error() 176 } 177 if test.want != got { 178 t.Logf("%v: want %v got %v", test.in, test.want, got) 179 } 180 if test.wantError != gotError { 181 t.Logf("%v: want error %v got error %v", test.in, test.wantError, gotError) 182 } 183 } 184 185 } 186 187 // This is adapted from the s3 equivalent. 188 func (f *Fs) InternalTestMetadata(t *testing.T) { 189 ctx := context.Background() 190 original := random.String(1000) 191 contents := fstest.Gz(t, original) 192 mimeType := "text/html" 193 194 item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499Z")) 195 btime := time.Now() 196 obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, contents, true, mimeType, nil) 197 defer func() { 198 assert.NoError(t, obj.Remove(ctx)) 199 }() 200 o := obj.(*Object) 201 gotMetadata, err := o.getMetaData(ctx) 202 require.NoError(t, err) 203 204 // We currently have a limited amount of metadata to test with B2 205 assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type") 206 207 // Modification time from the x-bz-info-src_last_modified_millis header 208 var mtime api.Timestamp 209 err = mtime.UnmarshalJSON([]byte(gotMetadata.Info[timeKey])) 210 if err != nil { 211 fs.Debugf(o, "Bad "+timeHeader+" header: %v", err) 212 } 213 assert.Equal(t, item.ModTime, time.Time(mtime), "Modification time") 214 215 // Upload time 216 gotBtime := time.Time(gotMetadata.UploadTimestamp) 217 dt := gotBtime.Sub(btime) 218 assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("btime more than 1 minute out want %v got %v delta %v", btime, gotBtime, dt)) 219 220 t.Run("GzipEncoding", func(t *testing.T) { 221 // Test that the gzipped file we uploaded can be 222 // downloaded 223 checkDownload := func(wantContents string, wantSize int64, wantHash string) { 224 gotContents := fstests.ReadObject(ctx, t, o, -1) 225 assert.Equal(t, wantContents, gotContents) 226 assert.Equal(t, wantSize, o.Size()) 227 gotHash, err := o.Hash(ctx, hash.SHA1) 228 require.NoError(t, err) 229 assert.Equal(t, wantHash, gotHash) 230 } 231 232 t.Run("NoDecompress", func(t *testing.T) { 233 checkDownload(contents, int64(len(contents)), sha1Sum(t, contents)) 234 }) 235 }) 236 } 237 238 func sha1Sum(t *testing.T, s string) string { 239 hash := sha1.Sum([]byte(s)) 240 return fmt.Sprintf("%x", hash) 241 } 242 243 // This is adapted from the s3 equivalent. 244 func (f *Fs) InternalTestVersions(t *testing.T) { 245 ctx := context.Background() 246 247 // Small pause to make the LastModified different since AWS 248 // only seems to track them to 1 second granularity 249 time.Sleep(2 * time.Second) 250 251 // Create an object 252 const dirName = "versions" 253 const fileName = dirName + "/" + "test-versions.txt" 254 contents := random.String(100) 255 item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) 256 obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) 257 defer func() { 258 assert.NoError(t, obj.Remove(ctx)) 259 }() 260 objMetadata, err := obj.(*Object).getMetaData(ctx) 261 require.NoError(t, err) 262 263 // Small pause 264 time.Sleep(2 * time.Second) 265 266 // Remove it 267 assert.NoError(t, obj.Remove(ctx)) 268 269 // Small pause to make the LastModified different since AWS only seems to track them to 1 second granularity 270 time.Sleep(2 * time.Second) 271 272 // And create it with different size and contents 273 newContents := random.String(101) 274 newItem := fstest.NewItem(fileName, newContents, fstest.Time("2002-05-06T04:05:06.499999999Z")) 275 newObj := fstests.PutTestContents(ctx, t, f, &newItem, newContents, true) 276 newObjMetadata, err := newObj.(*Object).getMetaData(ctx) 277 require.NoError(t, err) 278 279 t.Run("Versions", func(t *testing.T) { 280 // Set --b2-versions for this test 281 f.opt.Versions = true 282 defer func() { 283 f.opt.Versions = false 284 }() 285 286 // Read the contents 287 entries, err := f.List(ctx, dirName) 288 require.NoError(t, err) 289 tests := 0 290 var fileNameVersion string 291 for _, entry := range entries { 292 t.Log(entry) 293 remote := entry.Remote() 294 if remote == fileName { 295 t.Run("ReadCurrent", func(t *testing.T) { 296 assert.Equal(t, newContents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1)) 297 }) 298 tests++ 299 } else if versionTime, p := version.Remove(remote); !versionTime.IsZero() && p == fileName { 300 t.Run("ReadVersion", func(t *testing.T) { 301 assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1)) 302 }) 303 assert.WithinDuration(t, time.Time(objMetadata.UploadTimestamp), versionTime, time.Second, "object time must be with 1 second of version time") 304 fileNameVersion = remote 305 tests++ 306 } 307 } 308 assert.Equal(t, 2, tests, "object missing from listing") 309 310 // Check we can read the object with a version suffix 311 t.Run("NewObject", func(t *testing.T) { 312 o, err := f.NewObject(ctx, fileNameVersion) 313 require.NoError(t, err) 314 require.NotNil(t, o) 315 assert.Equal(t, int64(100), o.Size(), o.Remote()) 316 }) 317 318 // Check we can make a NewFs from that object with a version suffix 319 t.Run("NewFs", func(t *testing.T) { 320 newPath := bucket.Join(fs.ConfigStringFull(f), fileNameVersion) 321 // Make sure --b2-versions is set in the config of the new remote 322 fs.Debugf(nil, "oldPath = %q", newPath) 323 lastColon := strings.LastIndex(newPath, ":") 324 require.True(t, lastColon >= 0) 325 newPath = newPath[:lastColon] + ",versions" + newPath[lastColon:] 326 fs.Debugf(nil, "newPath = %q", newPath) 327 fNew, err := cache.Get(ctx, newPath) 328 // This should return pointing to a file 329 require.Equal(t, fs.ErrorIsFile, err) 330 require.NotNil(t, fNew) 331 // With the directory above 332 assert.Equal(t, dirName, path.Base(fs.ConfigStringFull(fNew))) 333 }) 334 }) 335 336 t.Run("VersionAt", func(t *testing.T) { 337 // We set --b2-version-at for this test so make sure we reset it at the end 338 defer func() { 339 f.opt.VersionAt = fs.Time{} 340 }() 341 342 var ( 343 firstObjectTime = time.Time(objMetadata.UploadTimestamp) 344 secondObjectTime = time.Time(newObjMetadata.UploadTimestamp) 345 ) 346 347 for _, test := range []struct { 348 what string 349 at time.Time 350 want []fstest.Item 351 wantErr error 352 wantSize int64 353 }{ 354 { 355 what: "Before", 356 at: firstObjectTime.Add(-time.Second), 357 want: fstests.InternalTestFiles, 358 wantErr: fs.ErrorObjectNotFound, 359 }, 360 { 361 what: "AfterOne", 362 at: firstObjectTime.Add(time.Second), 363 want: append([]fstest.Item{item}, fstests.InternalTestFiles...), 364 wantSize: 100, 365 }, 366 { 367 what: "AfterDelete", 368 at: secondObjectTime.Add(-time.Second), 369 want: fstests.InternalTestFiles, 370 wantErr: fs.ErrorObjectNotFound, 371 }, 372 { 373 what: "AfterTwo", 374 at: secondObjectTime.Add(time.Second), 375 want: append([]fstest.Item{newItem}, fstests.InternalTestFiles...), 376 wantSize: 101, 377 }, 378 } { 379 t.Run(test.what, func(t *testing.T) { 380 f.opt.VersionAt = fs.Time(test.at) 381 t.Run("List", func(t *testing.T) { 382 fstest.CheckListing(t, f, test.want) 383 }) 384 // b2 NewObject doesn't work with VersionAt 385 //t.Run("NewObject", func(t *testing.T) { 386 // gotObj, gotErr := f.NewObject(ctx, fileName) 387 // assert.Equal(t, test.wantErr, gotErr) 388 // if gotErr == nil { 389 // assert.Equal(t, test.wantSize, gotObj.Size()) 390 // } 391 //}) 392 }) 393 } 394 }) 395 396 t.Run("Cleanup", func(t *testing.T) { 397 require.NoError(t, f.cleanUp(ctx, true, false, 0)) 398 items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...) 399 fstest.CheckListing(t, f, items) 400 // Set --b2-versions for this test 401 f.opt.Versions = true 402 defer func() { 403 f.opt.Versions = false 404 }() 405 fstest.CheckListing(t, f, items) 406 }) 407 408 // Purge gets tested later 409 } 410 411 // -run TestIntegration/FsMkdir/FsPutFiles/Internal 412 func (f *Fs) InternalTest(t *testing.T) { 413 t.Run("Metadata", f.InternalTestMetadata) 414 t.Run("Versions", f.InternalTestVersions) 415 } 416 417 var _ fstests.InternalTester = (*Fs)(nil)