github.com/artpar/rclone@v1.67.3/backend/s3/s3_internal_test.go (about) 1 package s3 2 3 import ( 4 "bytes" 5 "compress/gzip" 6 "context" 7 "crypto/md5" 8 "fmt" 9 "path" 10 "strings" 11 "testing" 12 "time" 13 14 "github.com/artpar/rclone/fs" 15 "github.com/artpar/rclone/fs/cache" 16 "github.com/artpar/rclone/fs/hash" 17 "github.com/artpar/rclone/fstest" 18 "github.com/artpar/rclone/fstest/fstests" 19 "github.com/artpar/rclone/lib/bucket" 20 "github.com/artpar/rclone/lib/random" 21 "github.com/artpar/rclone/lib/version" 22 "github.com/aws/aws-sdk-go/aws" 23 "github.com/aws/aws-sdk-go/aws/awserr" 24 "github.com/aws/aws-sdk-go/service/s3" 25 "github.com/stretchr/testify/assert" 26 "github.com/stretchr/testify/require" 27 ) 28 29 func gz(t *testing.T, s string) string { 30 var buf bytes.Buffer 31 zw := gzip.NewWriter(&buf) 32 _, err := zw.Write([]byte(s)) 33 require.NoError(t, err) 34 err = zw.Close() 35 require.NoError(t, err) 36 return buf.String() 37 } 38 39 func md5sum(t *testing.T, s string) string { 40 hash := md5.Sum([]byte(s)) 41 return fmt.Sprintf("%x", hash) 42 } 43 44 func (f *Fs) InternalTestMetadata(t *testing.T) { 45 ctx := context.Background() 46 original := random.String(1000) 47 contents := gz(t, original) 48 49 item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) 50 btime := time.Now() 51 metadata := fs.Metadata{ 52 "cache-control": "no-cache", 53 "content-disposition": "inline", 54 "content-encoding": "gzip", 55 "content-language": "en-US", 56 "content-type": "text/plain", 57 "mtime": "2009-05-06T04:05:06.499999999Z", 58 // "tier" - read only 59 // "btime" - read only 60 } 61 obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, contents, true, "text/html", metadata) 62 defer func() { 63 assert.NoError(t, obj.Remove(ctx)) 64 }() 65 o := obj.(*Object) 66 gotMetadata, err := o.Metadata(ctx) 67 require.NoError(t, err) 68 for k, v := range metadata { 69 got := gotMetadata[k] 70 switch k { 71 case "mtime": 72 assert.True(t, fstest.Time(v).Equal(fstest.Time(got))) 73 case "btime": 74 gotBtime := fstest.Time(got) 75 dt := gotBtime.Sub(btime) 76 assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("btime more than 1 minute out want %v got %v delta %v", btime, gotBtime, dt)) 77 assert.True(t, fstest.Time(v).Equal(fstest.Time(got))) 78 case "tier": 79 assert.NotEqual(t, "", got) 80 default: 81 assert.Equal(t, v, got, k) 82 } 83 } 84 85 t.Run("GzipEncoding", func(t *testing.T) { 86 // Test that the gzipped file we uploaded can be 87 // downloaded with and without decompression 88 checkDownload := func(wantContents string, wantSize int64, wantHash string) { 89 gotContents := fstests.ReadObject(ctx, t, o, -1) 90 assert.Equal(t, wantContents, gotContents) 91 assert.Equal(t, wantSize, o.Size()) 92 gotHash, err := o.Hash(ctx, hash.MD5) 93 require.NoError(t, err) 94 assert.Equal(t, wantHash, gotHash) 95 } 96 97 t.Run("NoDecompress", func(t *testing.T) { 98 checkDownload(contents, int64(len(contents)), md5sum(t, contents)) 99 }) 100 t.Run("Decompress", func(t *testing.T) { 101 f.opt.Decompress = true 102 defer func() { 103 f.opt.Decompress = false 104 }() 105 checkDownload(original, -1, "") 106 }) 107 108 }) 109 } 110 111 func (f *Fs) InternalTestNoHead(t *testing.T) { 112 ctx := context.Background() 113 // Set NoHead for this test 114 f.opt.NoHead = true 115 defer func() { 116 f.opt.NoHead = false 117 }() 118 contents := random.String(1000) 119 item := fstest.NewItem("test-no-head", contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) 120 obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) 121 defer func() { 122 assert.NoError(t, obj.Remove(ctx)) 123 }() 124 // PutTestcontents checks the received object 125 126 } 127 128 func TestVersionLess(t *testing.T) { 129 key1 := "key1" 130 key2 := "key2" 131 t1 := fstest.Time("2022-01-21T12:00:00+01:00") 132 t2 := fstest.Time("2022-01-21T12:00:01+01:00") 133 for n, test := range []struct { 134 a, b *s3.ObjectVersion 135 want bool 136 }{ 137 {a: nil, b: nil, want: true}, 138 {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: nil, want: false}, 139 {a: nil, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true}, 140 {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false}, 141 {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t2}, want: false}, 142 {a: &s3.ObjectVersion{Key: &key1, LastModified: &t2}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true}, 143 {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key2, LastModified: &t1}, want: true}, 144 {a: &s3.ObjectVersion{Key: &key2, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false}, 145 {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false}, 146 {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true}, 147 {a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, want: false}, 148 } { 149 got := versionLess(test.a, test.b) 150 assert.Equal(t, test.want, got, fmt.Sprintf("%d: %+v", n, test)) 151 } 152 } 153 154 func TestMergeDeleteMarkers(t *testing.T) { 155 key1 := "key1" 156 key2 := "key2" 157 t1 := fstest.Time("2022-01-21T12:00:00+01:00") 158 t2 := fstest.Time("2022-01-21T12:00:01+01:00") 159 for n, test := range []struct { 160 versions []*s3.ObjectVersion 161 markers []*s3.DeleteMarkerEntry 162 want []*s3.ObjectVersion 163 }{ 164 { 165 versions: []*s3.ObjectVersion{}, 166 markers: []*s3.DeleteMarkerEntry{}, 167 want: []*s3.ObjectVersion{}, 168 }, 169 { 170 versions: []*s3.ObjectVersion{ 171 { 172 Key: &key1, 173 LastModified: &t1, 174 }, 175 }, 176 markers: []*s3.DeleteMarkerEntry{}, 177 want: []*s3.ObjectVersion{ 178 { 179 Key: &key1, 180 LastModified: &t1, 181 }, 182 }, 183 }, 184 { 185 versions: []*s3.ObjectVersion{}, 186 markers: []*s3.DeleteMarkerEntry{ 187 { 188 Key: &key1, 189 LastModified: &t1, 190 }, 191 }, 192 want: []*s3.ObjectVersion{ 193 { 194 Key: &key1, 195 LastModified: &t1, 196 Size: isDeleteMarker, 197 }, 198 }, 199 }, 200 { 201 versions: []*s3.ObjectVersion{ 202 { 203 Key: &key1, 204 LastModified: &t2, 205 }, 206 { 207 Key: &key2, 208 LastModified: &t2, 209 }, 210 }, 211 markers: []*s3.DeleteMarkerEntry{ 212 { 213 Key: &key1, 214 LastModified: &t1, 215 }, 216 }, 217 want: []*s3.ObjectVersion{ 218 { 219 Key: &key1, 220 LastModified: &t2, 221 }, 222 { 223 Key: &key1, 224 LastModified: &t1, 225 Size: isDeleteMarker, 226 }, 227 { 228 Key: &key2, 229 LastModified: &t2, 230 }, 231 }, 232 }, 233 } { 234 got := mergeDeleteMarkers(test.versions, test.markers) 235 assert.Equal(t, test.want, got, fmt.Sprintf("%d: %+v", n, test)) 236 } 237 } 238 239 func (f *Fs) InternalTestVersions(t *testing.T) { 240 ctx := context.Background() 241 242 // Enable versioning for this bucket during this test 243 _, err := f.setGetVersioning(ctx, "Enabled") 244 if err != nil { 245 t.Skipf("Couldn't enable versioning: %v", err) 246 } 247 defer func() { 248 // Disable versioning for this bucket 249 _, err := f.setGetVersioning(ctx, "Suspended") 250 assert.NoError(t, err) 251 }() 252 253 // Small pause to make the LastModified different since AWS 254 // only seems to track them to 1 second granularity 255 time.Sleep(2 * time.Second) 256 257 // Create an object 258 const dirName = "versions" 259 const fileName = dirName + "/" + "test-versions.txt" 260 contents := random.String(100) 261 item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z")) 262 obj := fstests.PutTestContents(ctx, t, f, &item, contents, true) 263 defer func() { 264 assert.NoError(t, obj.Remove(ctx)) 265 }() 266 267 // Small pause 268 time.Sleep(2 * time.Second) 269 270 // Remove it 271 assert.NoError(t, obj.Remove(ctx)) 272 273 // Small pause to make the LastModified different since AWS only seems to track them to 1 second granularity 274 time.Sleep(2 * time.Second) 275 276 // And create it with different size and contents 277 newContents := random.String(101) 278 newItem := fstest.NewItem(fileName, newContents, fstest.Time("2002-05-06T04:05:06.499999999Z")) 279 newObj := fstests.PutTestContents(ctx, t, f, &newItem, newContents, true) 280 281 t.Run("Versions", func(t *testing.T) { 282 // Set --s3-versions for this test 283 f.opt.Versions = true 284 defer func() { 285 f.opt.Versions = false 286 }() 287 288 // Read the contents 289 entries, err := f.List(ctx, dirName) 290 require.NoError(t, err) 291 tests := 0 292 var fileNameVersion string 293 for _, entry := range entries { 294 t.Log(entry) 295 remote := entry.Remote() 296 if remote == fileName { 297 t.Run("ReadCurrent", func(t *testing.T) { 298 assert.Equal(t, newContents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1)) 299 }) 300 tests++ 301 } else if versionTime, p := version.Remove(remote); !versionTime.IsZero() && p == fileName { 302 t.Run("ReadVersion", func(t *testing.T) { 303 assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1)) 304 }) 305 assert.WithinDuration(t, obj.(*Object).lastModified, versionTime, time.Second, "object time must be with 1 second of version time") 306 fileNameVersion = remote 307 tests++ 308 } 309 } 310 assert.Equal(t, 2, tests, "object missing from listing") 311 312 // Check we can read the object with a version suffix 313 t.Run("NewObject", func(t *testing.T) { 314 o, err := f.NewObject(ctx, fileNameVersion) 315 require.NoError(t, err) 316 require.NotNil(t, o) 317 assert.Equal(t, int64(100), o.Size(), o.Remote()) 318 }) 319 320 // Check we can make a NewFs from that object with a version suffix 321 t.Run("NewFs", func(t *testing.T) { 322 newPath := bucket.Join(fs.ConfigStringFull(f), fileNameVersion) 323 // Make sure --s3-versions is set in the config of the new remote 324 fs.Debugf(nil, "oldPath = %q", newPath) 325 lastColon := strings.LastIndex(newPath, ":") 326 require.True(t, lastColon >= 0) 327 newPath = newPath[:lastColon] + ",versions" + newPath[lastColon:] 328 fs.Debugf(nil, "newPath = %q", newPath) 329 fNew, err := cache.Get(ctx, newPath) 330 // This should return pointing to a file 331 require.Equal(t, fs.ErrorIsFile, err) 332 require.NotNil(t, fNew) 333 // With the directory the directory above 334 assert.Equal(t, dirName, path.Base(fs.ConfigStringFull(fNew))) 335 }) 336 }) 337 338 t.Run("VersionAt", func(t *testing.T) { 339 // We set --s3-version-at for this test so make sure we reset it at the end 340 defer func() { 341 f.opt.VersionAt = fs.Time{} 342 }() 343 344 var ( 345 firstObjectTime = obj.(*Object).lastModified 346 secondObjectTime = newObj.(*Object).lastModified 347 ) 348 349 for _, test := range []struct { 350 what string 351 at time.Time 352 want []fstest.Item 353 wantErr error 354 wantSize int64 355 }{ 356 { 357 what: "Before", 358 at: firstObjectTime.Add(-time.Second), 359 want: fstests.InternalTestFiles, 360 wantErr: fs.ErrorObjectNotFound, 361 }, 362 { 363 what: "AfterOne", 364 at: firstObjectTime.Add(time.Second), 365 want: append([]fstest.Item{item}, fstests.InternalTestFiles...), 366 wantSize: 100, 367 }, 368 { 369 what: "AfterDelete", 370 at: secondObjectTime.Add(-time.Second), 371 want: fstests.InternalTestFiles, 372 wantErr: fs.ErrorObjectNotFound, 373 }, 374 { 375 what: "AfterTwo", 376 at: secondObjectTime.Add(time.Second), 377 want: append([]fstest.Item{newItem}, fstests.InternalTestFiles...), 378 wantSize: 101, 379 }, 380 } { 381 t.Run(test.what, func(t *testing.T) { 382 f.opt.VersionAt = fs.Time(test.at) 383 t.Run("List", func(t *testing.T) { 384 fstest.CheckListing(t, f, test.want) 385 }) 386 t.Run("NewObject", func(t *testing.T) { 387 gotObj, gotErr := f.NewObject(ctx, fileName) 388 assert.Equal(t, test.wantErr, gotErr) 389 if gotErr == nil { 390 assert.Equal(t, test.wantSize, gotObj.Size()) 391 } 392 }) 393 }) 394 } 395 }) 396 397 t.Run("Mkdir", func(t *testing.T) { 398 // Test what happens when we create a bucket we already own and see whether the 399 // quirk is set correctly 400 req := s3.CreateBucketInput{ 401 Bucket: &f.rootBucket, 402 ACL: stringPointerOrNil(f.opt.BucketACL), 403 } 404 if f.opt.LocationConstraint != "" { 405 req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{ 406 LocationConstraint: &f.opt.LocationConstraint, 407 } 408 } 409 err := f.pacer.Call(func() (bool, error) { 410 _, err := f.c.CreateBucketWithContext(ctx, &req) 411 return f.shouldRetry(ctx, err) 412 }) 413 var errString string 414 if err == nil { 415 errString = "No Error" 416 } else if awsErr, ok := err.(awserr.Error); ok { 417 errString = awsErr.Code() 418 } else { 419 assert.Fail(t, "Unknown error %T %v", err, err) 420 } 421 t.Logf("Creating a bucket we already have created returned code: %s", errString) 422 switch errString { 423 case "BucketAlreadyExists": 424 assert.False(t, f.opt.UseAlreadyExists.Value, "Need to clear UseAlreadyExists quirk") 425 case "No Error", "BucketAlreadyOwnedByYou": 426 assert.True(t, f.opt.UseAlreadyExists.Value, "Need to set UseAlreadyExists quirk") 427 default: 428 assert.Fail(t, "Unknown error string %q", errString) 429 } 430 }) 431 432 t.Run("Cleanup", func(t *testing.T) { 433 require.NoError(t, f.CleanUpHidden(ctx)) 434 items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...) 435 fstest.CheckListing(t, f, items) 436 // Set --s3-versions for this test 437 f.opt.Versions = true 438 defer func() { 439 f.opt.Versions = false 440 }() 441 fstest.CheckListing(t, f, items) 442 }) 443 444 // Purge gets tested later 445 } 446 447 func (f *Fs) InternalTest(t *testing.T) { 448 t.Run("Metadata", f.InternalTestMetadata) 449 t.Run("NoHead", f.InternalTestNoHead) 450 t.Run("Versions", f.InternalTestVersions) 451 } 452 453 var _ fstests.InternalTester = (*Fs)(nil)