cloud.google.com/go/storage@v1.40.0/integration_test.go (about) 1 // Copyright 2014 Google LLC 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package storage 16 17 import ( 18 "bytes" 19 "compress/gzip" 20 "context" 21 "crypto" 22 "crypto/md5" 23 cryptorand "crypto/rand" 24 "crypto/rsa" 25 "crypto/sha256" 26 "encoding/base64" 27 "encoding/json" 28 "errors" 29 "flag" 30 "fmt" 31 "hash/crc32" 32 "io" 33 "io/ioutil" 34 "log" 35 "math" 36 "math/rand" 37 "mime/multipart" 38 "net/http" 39 "net/http/httputil" 40 "os" 41 "sort" 42 "strconv" 43 "strings" 44 "testing" 45 "time" 46 47 "cloud.google.com/go/httpreplay" 48 "cloud.google.com/go/iam" 49 "cloud.google.com/go/iam/apiv1/iampb" 50 "cloud.google.com/go/internal/testutil" 51 "cloud.google.com/go/internal/uid" 52 "github.com/google/go-cmp/cmp" 53 "github.com/google/go-cmp/cmp/cmpopts" 54 "github.com/googleapis/gax-go/v2/apierror" 55 "golang.org/x/oauth2/google" 56 "google.golang.org/api/googleapi" 57 "google.golang.org/api/iterator" 58 itesting "google.golang.org/api/iterator/testing" 59 "google.golang.org/api/option" 60 raw "google.golang.org/api/storage/v1" 61 "google.golang.org/api/transport" 62 "google.golang.org/grpc" 63 "google.golang.org/grpc/codes" 64 "google.golang.org/grpc/status" 65 ) 66 67 type skipTransportTestKey string 68 69 const ( 70 testPrefix = "go-integration-test" 71 replayFilename = "storage.replay" 72 // TODO(jba): move to testutil, factor out from firestore/integration_test.go. 73 envFirestoreProjID = "GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID" 74 envFirestorePrivateKey = "GCLOUD_TESTS_GOLANG_FIRESTORE_KEY" 75 grpcTestPrefix = "golang-grpc-test" 76 ) 77 78 var ( 79 record = flag.Bool("record", false, "record RPCs") 80 81 uidSpace *uid.Space 82 uidSpaceObjects *uid.Space 83 bucketName string 84 grpcBucketName string 85 // Use our own random number generator to isolate the sequence of random numbers from 86 // other packages. This makes it possible to use HTTP replay and draw the same sequence 87 // of numbers as during recording. 88 rng *rand.Rand 89 newTestClient func(ctx context.Context, opts ...option.ClientOption) (*Client, error) 90 replaying bool 91 testTime time.Time 92 ) 93 94 func TestMain(m *testing.M) { 95 grpc.EnableTracing = true 96 cleanup := initIntegrationTest() 97 cleanupEmulatorClients := initEmulatorClients() 98 exit := m.Run() 99 if err := cleanup(); err != nil { 100 // Don't fail the test if cleanup fails. 101 log.Printf("Post-test cleanup failed: %v", err) 102 } 103 if err := cleanupEmulatorClients(); err != nil { 104 // Don't fail the test if cleanup fails. 105 log.Printf("Post-test cleanup failed for emulator clients: %v", err) 106 } 107 108 os.Exit(exit) 109 } 110 111 // If integration tests will be run, create a unique bucket for them. 112 // Also, set newTestClient to handle record/replay. 113 // Return a cleanup function. 114 func initIntegrationTest() func() error { 115 flag.Parse() // needed for testing.Short() 116 switch { 117 case testing.Short() && *record: 118 log.Fatal("cannot combine -short and -record") 119 return nil 120 121 case testing.Short() && httpreplay.Supported() && testutil.CanReplay(replayFilename) && testutil.ProjID() != "": 122 // go test -short with a replay file will replay the integration tests, if 123 // the appropriate environment variables have been set. 124 replaying = true 125 httpreplay.DebugHeaders() 126 replayer, err := httpreplay.NewReplayer(replayFilename) 127 if err != nil { 128 log.Fatal(err) 129 } 130 var t time.Time 131 if err := json.Unmarshal(replayer.Initial(), &t); err != nil { 132 log.Fatal(err) 133 } 134 initUIDsAndRand(t) 135 newTestClient = func(ctx context.Context, _ ...option.ClientOption) (*Client, error) { 136 hc, err := replayer.Client(ctx) // no creds needed 137 if err != nil { 138 return nil, err 139 } 140 return NewClient(ctx, option.WithHTTPClient(hc)) 141 } 142 log.Printf("replaying from %s", replayFilename) 143 return func() error { return replayer.Close() } 144 145 case testing.Short(): 146 // go test -short without a replay file skips the integration tests. 147 if testutil.CanReplay(replayFilename) && testutil.ProjID() != "" { 148 log.Print("replay not supported for Go versions before 1.8") 149 } 150 newTestClient = nil 151 return func() error { return nil } 152 153 default: // Run integration tests against a real backend. 154 now := time.Now().UTC() 155 initUIDsAndRand(now) 156 var cleanup func() error 157 if *record && httpreplay.Supported() { 158 // Remember the time for replay. 159 nowBytes, err := json.Marshal(now) 160 if err != nil { 161 log.Fatal(err) 162 } 163 recorder, err := httpreplay.NewRecorder(replayFilename, nowBytes) 164 if err != nil { 165 log.Fatalf("could not record: %v", err) 166 } 167 newTestClient = func(ctx context.Context, opts ...option.ClientOption) (*Client, error) { 168 hc, err := recorder.Client(ctx, opts...) 169 if err != nil { 170 return nil, err 171 } 172 return NewClient(ctx, option.WithHTTPClient(hc)) 173 } 174 cleanup = func() error { 175 err1 := cleanupBuckets() 176 err2 := recorder.Close() 177 if err1 != nil { 178 return err1 179 } 180 return err2 181 } 182 log.Printf("recording to %s", replayFilename) 183 } else { 184 if *record { 185 log.Print("record not supported for Go versions before 1.8") 186 } 187 newTestClient = NewClient 188 cleanup = cleanupBuckets 189 } 190 ctx := context.Background() 191 client, err := newTestClient(ctx) 192 if err != nil { 193 log.Fatalf("NewClient: %v", err) 194 } 195 if client == nil { 196 return func() error { return nil } 197 } 198 defer client.Close() 199 if err := client.Bucket(bucketName).Create(ctx, testutil.ProjID(), nil); err != nil { 200 log.Fatalf("creating bucket %q: %v", bucketName, err) 201 } 202 if err := client.Bucket(grpcBucketName).Create(ctx, testutil.ProjID(), nil); err != nil { 203 log.Fatalf("creating bucket %q: %v", grpcBucketName, err) 204 } 205 return cleanup 206 } 207 } 208 209 func initUIDsAndRand(t time.Time) { 210 uidSpace = uid.NewSpace("", &uid.Options{Time: t, Short: true}) 211 bucketName = testPrefix + uidSpace.New() 212 uidSpaceObjects = uid.NewSpace("obj", &uid.Options{Time: t}) 213 grpcBucketName = grpcTestPrefix + uidSpace.New() 214 // Use our own random source, to avoid other parts of the program taking 215 // random numbers from the global source and putting record and replay 216 // out of sync. 217 rng = testutil.NewRand(t) 218 testTime = t 219 } 220 221 // testConfig returns the Client used to access GCS. testConfig skips 222 // the current test if credentials are not available or when being run 223 // in Short mode. 224 func testConfig(ctx context.Context, t *testing.T, opts ...option.ClientOption) *Client { 225 if testing.Short() && !replaying { 226 t.Skip("Integration tests skipped in short mode") 227 } 228 client, err := newTestClient(ctx, opts...) 229 if err != nil { 230 t.Fatalf("NewClient: %v", err) 231 } 232 if client == nil { 233 t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") 234 } 235 return client 236 } 237 238 // testConfigGPRC returns a gRPC-based client to access GCS. testConfigGRPC 239 // skips the curent test when being run in Short mode. 240 func testConfigGRPC(ctx context.Context, t *testing.T, opts ...option.ClientOption) (gc *Client) { 241 if testing.Short() { 242 t.Skip("Integration tests skipped in short mode") 243 } 244 245 gc, err := NewGRPCClient(ctx, opts...) 246 if err != nil { 247 t.Fatalf("NewGRPCClient: %v", err) 248 } 249 250 return 251 } 252 253 // initTransportClients initializes Storage clients for each supported transport. 254 func initTransportClients(ctx context.Context, t *testing.T, opts ...option.ClientOption) map[string]*Client { 255 withJSON := append(opts, WithJSONReads()) 256 return map[string]*Client{ 257 "http": testConfig(ctx, t, opts...), 258 "grpc": testConfigGRPC(ctx, t, opts...), 259 // TODO: remove jsonReads when support for XML reads is dropped 260 "jsonReads": testConfig(ctx, t, withJSON...), 261 } 262 } 263 264 // multiTransportTest initializes fresh clients for each transport, then runs 265 // given testing function using each transport-specific client, supplying the 266 // test function with the sub-test instance, the context it was given, the name 267 // of an existing bucket to use, a bucket name to use for bucket creation, and 268 // the client to use. 269 func multiTransportTest(ctx context.Context, t *testing.T, 270 test func(*testing.T, context.Context, string, string, *Client), 271 opts ...option.ClientOption) { 272 for transport, client := range initTransportClients(ctx, t, opts...) { 273 t.Run(transport, func(t *testing.T) { 274 t.Cleanup(func() { 275 client.Close() 276 }) 277 278 if reason := ctx.Value(skipTransportTestKey(transport)); reason != nil { 279 t.Skip("transport", fmt.Sprintf("%q", transport), "explicitly skipped:", reason) 280 } 281 282 bucket := bucketName 283 prefix := testPrefix 284 if transport == "grpc" { 285 bucket = grpcBucketName 286 prefix = grpcTestPrefix 287 } 288 289 test(t, ctx, bucket, prefix, client) 290 }) 291 } 292 } 293 294 func TestIntegration_BucketCreateDelete(t *testing.T) { 295 ctx := skipJSONReads(context.Background(), "no reads in test") 296 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 297 projectID := testutil.ProjID() 298 299 labels := map[string]string{ 300 "l1": "v1", 301 "empty": "", 302 } 303 304 lifecycle := Lifecycle{ 305 Rules: []LifecycleRule{{ 306 Action: LifecycleAction{ 307 Type: SetStorageClassAction, 308 StorageClass: "NEARLINE", 309 }, 310 Condition: LifecycleCondition{ 311 AgeInDays: 10, 312 Liveness: Archived, 313 CreatedBefore: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC), 314 MatchesStorageClasses: []string{"STANDARD"}, 315 NumNewerVersions: 3, 316 }, 317 }, { 318 Action: LifecycleAction{ 319 Type: SetStorageClassAction, 320 StorageClass: "ARCHIVE", 321 }, 322 Condition: LifecycleCondition{ 323 CustomTimeBefore: time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC), 324 DaysSinceCustomTime: 20, 325 Liveness: Live, 326 MatchesStorageClasses: []string{"STANDARD"}, 327 }, 328 }, { 329 Action: LifecycleAction{ 330 Type: DeleteAction, 331 }, 332 Condition: LifecycleCondition{ 333 DaysSinceNoncurrentTime: 30, 334 Liveness: Live, 335 NoncurrentTimeBefore: time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC), 336 MatchesStorageClasses: []string{"NEARLINE"}, 337 NumNewerVersions: 10, 338 }, 339 }, { 340 Action: LifecycleAction{ 341 Type: DeleteAction, 342 }, 343 Condition: LifecycleCondition{ 344 AgeInDays: 10, 345 MatchesPrefix: []string{"testPrefix"}, 346 MatchesSuffix: []string{"testSuffix"}, 347 NumNewerVersions: 3, 348 }, 349 }, { 350 Action: LifecycleAction{ 351 Type: DeleteAction, 352 }, 353 Condition: LifecycleCondition{ 354 AllObjects: true, 355 }, 356 }}, 357 } 358 359 // testedAttrs are the bucket attrs directly compared in this test 360 type testedAttrs struct { 361 StorageClass string 362 VersioningEnabled bool 363 LocationType string 364 Labels map[string]string 365 Location string 366 Lifecycle Lifecycle 367 CustomPlacementConfig *CustomPlacementConfig 368 } 369 370 for _, test := range []struct { 371 name string 372 attrs *BucketAttrs 373 wantAttrs testedAttrs 374 }{ 375 { 376 name: "no attrs", 377 attrs: nil, 378 wantAttrs: testedAttrs{ 379 StorageClass: "STANDARD", 380 VersioningEnabled: false, 381 LocationType: "multi-region", 382 Location: "US", 383 }, 384 }, 385 { 386 name: "with attrs", 387 attrs: &BucketAttrs{ 388 StorageClass: "NEARLINE", 389 VersioningEnabled: true, 390 Labels: labels, 391 Lifecycle: lifecycle, 392 Location: "SOUTHAMERICA-EAST1", 393 }, 394 wantAttrs: testedAttrs{ 395 StorageClass: "NEARLINE", 396 VersioningEnabled: true, 397 Labels: labels, 398 Location: "SOUTHAMERICA-EAST1", 399 LocationType: "region", 400 Lifecycle: lifecycle, 401 }, 402 }, 403 { 404 name: "dual-region", 405 attrs: &BucketAttrs{ 406 Location: "US", 407 CustomPlacementConfig: &CustomPlacementConfig{ 408 DataLocations: []string{"US-EAST1", "US-WEST1"}, 409 }, 410 }, 411 wantAttrs: testedAttrs{ 412 Location: "US", 413 LocationType: "dual-region", 414 StorageClass: "STANDARD", 415 CustomPlacementConfig: &CustomPlacementConfig{ 416 DataLocations: []string{"US-EAST1", "US-WEST1"}, 417 }, 418 }, 419 }, 420 } { 421 t.Run(test.name, func(t *testing.T) { 422 newBucketName := prefix + uidSpace.New() 423 b := client.Bucket(newBucketName) 424 425 if err := b.Create(ctx, projectID, test.attrs); err != nil { 426 t.Fatalf("bucket create: %v", err) 427 } 428 429 gotAttrs, err := b.Attrs(ctx) 430 if err != nil { 431 t.Fatalf("bucket attrs: %v", err) 432 } 433 434 // All newly created buckets should conform to the following: 435 if gotAttrs.MetaGeneration != 1 { 436 t.Errorf("metageneration: got %d, should be 1", gotAttrs.MetaGeneration) 437 } 438 if gotAttrs.ProjectNumber == 0 { 439 t.Errorf("got a zero ProjectNumber") 440 } 441 442 // Test specific wanted bucket attrs 443 if gotAttrs.VersioningEnabled != test.wantAttrs.VersioningEnabled { 444 t.Errorf("versioning enabled: got %t, want %t", gotAttrs.VersioningEnabled, test.wantAttrs.VersioningEnabled) 445 } 446 if got, want := gotAttrs.Labels, test.wantAttrs.Labels; !testutil.Equal(got, want) { 447 t.Errorf("labels: got %v, want %v", got, want) 448 } 449 if diff := cmp.Diff(gotAttrs.Lifecycle, test.wantAttrs.Lifecycle); diff != "" { 450 t.Errorf("lifecycle: diff got vs. want: %v", diff) 451 } 452 if gotAttrs.LocationType != test.wantAttrs.LocationType { 453 t.Errorf("location type: got %s, want %s", gotAttrs.LocationType, test.wantAttrs.LocationType) 454 } 455 if gotAttrs.StorageClass != test.wantAttrs.StorageClass { 456 t.Errorf("storage class: got %s, want %s", gotAttrs.StorageClass, test.wantAttrs.StorageClass) 457 } 458 if gotAttrs.Location != test.wantAttrs.Location { 459 t.Errorf("location: got %s, want %s", gotAttrs.Location, test.wantAttrs.Location) 460 } 461 if got, want := gotAttrs.CustomPlacementConfig, test.wantAttrs.CustomPlacementConfig; !testutil.Equal(got, want) { 462 t.Errorf("customPlacementConfig: \ngot\t%v\nwant\t%v", got, want) 463 } 464 465 // Delete the bucket and check that the deletion was succesful 466 if err := b.Delete(ctx); err != nil { 467 t.Fatalf("bucket delete: %v", err) 468 } 469 _, err = b.Attrs(ctx) 470 if err != ErrBucketNotExist { 471 t.Fatalf("expected ErrBucketNotExist, got %v", err) 472 } 473 }) 474 } 475 }) 476 } 477 478 func TestIntegration_BucketLifecycle(t *testing.T) { 479 ctx := skipJSONReads(context.Background(), "no reads in test") 480 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 481 h := testHelper{t} 482 483 wantLifecycle := Lifecycle{ 484 Rules: []LifecycleRule{ 485 { 486 Action: LifecycleAction{Type: AbortIncompleteMPUAction}, 487 Condition: LifecycleCondition{AgeInDays: 30}, 488 }, 489 { 490 Action: LifecycleAction{Type: DeleteAction}, 491 Condition: LifecycleCondition{AllObjects: true}, 492 }, 493 }, 494 } 495 496 bucket := client.Bucket(prefix + uidSpace.New()) 497 498 // Create bucket with lifecycle rules 499 h.mustCreate(bucket, testutil.ProjID(), &BucketAttrs{ 500 Lifecycle: wantLifecycle, 501 }) 502 defer h.mustDeleteBucket(bucket) 503 504 attrs := h.mustBucketAttrs(bucket) 505 if !testutil.Equal(attrs.Lifecycle, wantLifecycle) { 506 t.Fatalf("got %v, want %v", attrs.Lifecycle, wantLifecycle) 507 } 508 509 // Remove lifecycle rules 510 ua := BucketAttrsToUpdate{Lifecycle: &Lifecycle{}} 511 attrs = h.mustUpdateBucket(bucket, ua, attrs.MetaGeneration) 512 if !testutil.Equal(attrs.Lifecycle, Lifecycle{}) { 513 t.Fatalf("got %v, want %v", attrs.Lifecycle, Lifecycle{}) 514 } 515 516 // Update bucket with a lifecycle rule 517 ua = BucketAttrsToUpdate{Lifecycle: &wantLifecycle} 518 attrs = h.mustUpdateBucket(bucket, ua, attrs.MetaGeneration) 519 if !testutil.Equal(attrs.Lifecycle, wantLifecycle) { 520 t.Fatalf("got %v, want %v", attrs.Lifecycle, wantLifecycle) 521 } 522 }) 523 } 524 525 func TestIntegration_BucketUpdate(t *testing.T) { 526 ctx := skipJSONReads(context.Background(), "no reads in test") 527 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 528 h := testHelper{t} 529 530 b := client.Bucket(prefix + uidSpace.New()) 531 h.mustCreate(b, testutil.ProjID(), nil) 532 defer h.mustDeleteBucket(b) 533 534 attrs := h.mustBucketAttrs(b) 535 if attrs.VersioningEnabled { 536 t.Fatal("bucket should not have versioning by default") 537 } 538 if len(attrs.Labels) > 0 { 539 t.Fatal("bucket should not have labels initially") 540 } 541 542 // Turn on versioning, add some labels. 543 ua := BucketAttrsToUpdate{VersioningEnabled: true} 544 ua.SetLabel("l1", "v1") 545 ua.SetLabel("empty", "") 546 attrs = h.mustUpdateBucket(b, ua, attrs.MetaGeneration) 547 if !attrs.VersioningEnabled { 548 t.Fatal("should have versioning now") 549 } 550 wantLabels := map[string]string{ 551 "l1": "v1", 552 "empty": "", 553 } 554 if !testutil.Equal(attrs.Labels, wantLabels) { 555 t.Fatalf("add labels: got %v, want %v", attrs.Labels, wantLabels) 556 } 557 558 // Turn off versioning again; add and remove some more labels. 559 ua = BucketAttrsToUpdate{VersioningEnabled: false} 560 ua.SetLabel("l1", "v2") // update 561 ua.SetLabel("new", "new") // create 562 ua.DeleteLabel("empty") // delete 563 ua.DeleteLabel("absent") // delete non-existent 564 attrs = h.mustUpdateBucket(b, ua, attrs.MetaGeneration) 565 if attrs.VersioningEnabled { 566 t.Fatal("should have versioning off") 567 } 568 wantLabels = map[string]string{ 569 "l1": "v2", 570 "new": "new", 571 } 572 if !testutil.Equal(attrs.Labels, wantLabels) { 573 t.Fatalf("got %v, want %v", attrs.Labels, wantLabels) 574 } 575 576 // Configure a lifecycle 577 wantLifecycle := Lifecycle{ 578 Rules: []LifecycleRule{ 579 { 580 Action: LifecycleAction{Type: "Delete"}, 581 Condition: LifecycleCondition{ 582 AgeInDays: 30, 583 MatchesPrefix: []string{"testPrefix"}, 584 MatchesSuffix: []string{"testSuffix"}, 585 }, 586 }, 587 }, 588 } 589 ua = BucketAttrsToUpdate{Lifecycle: &wantLifecycle} 590 attrs = h.mustUpdateBucket(b, ua, attrs.MetaGeneration) 591 if !testutil.Equal(attrs.Lifecycle, wantLifecycle) { 592 t.Fatalf("got %v, want %v", attrs.Lifecycle, wantLifecycle) 593 } 594 // Check that StorageClass has "STANDARD" value for unset field by default 595 // before passing new value. 596 wantStorageClass := "STANDARD" 597 if !testutil.Equal(attrs.StorageClass, wantStorageClass) { 598 t.Fatalf("got %v, want %v", attrs.StorageClass, wantStorageClass) 599 } 600 wantStorageClass = "NEARLINE" 601 ua = BucketAttrsToUpdate{StorageClass: wantStorageClass} 602 attrs = h.mustUpdateBucket(b, ua, attrs.MetaGeneration) 603 if !testutil.Equal(attrs.StorageClass, wantStorageClass) { 604 t.Fatalf("got %v, want %v", attrs.StorageClass, wantStorageClass) 605 } 606 }) 607 } 608 609 func TestIntegration_BucketPolicyOnly(t *testing.T) { 610 ctx := skipJSONReads(context.Background(), "no reads in test") 611 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 612 h := testHelper{t} 613 614 bkt := client.Bucket(prefix + uidSpace.New()) 615 h.mustCreate(bkt, testutil.ProjID(), nil) 616 defer h.mustDeleteBucket(bkt) 617 618 // Insert an object with custom ACL. 619 o := bkt.Object("bucketPolicyOnly") 620 defer func() { 621 if err := o.Delete(ctx); err != nil { 622 log.Printf("failed to delete test object: %v", err) 623 } 624 }() 625 wc := o.NewWriter(ctx) 626 wc.ContentType = "text/plain" 627 h.mustWrite(wc, []byte("test")) 628 a := o.ACL() 629 aclEntity := ACLEntity("user-test@example.com") 630 err := a.Set(ctx, aclEntity, RoleReader) 631 if err != nil { 632 t.Fatalf("set ACL failed: %v", err) 633 } 634 635 // Enable BucketPolicyOnly. 636 ua := BucketAttrsToUpdate{BucketPolicyOnly: &BucketPolicyOnly{Enabled: true}} 637 attrs := h.mustUpdateBucket(bkt, ua, h.mustBucketAttrs(bkt).MetaGeneration) 638 if got, want := attrs.BucketPolicyOnly.Enabled, true; got != want { 639 t.Fatalf("got %v, want %v", got, want) 640 } 641 if got := attrs.BucketPolicyOnly.LockedTime; got.IsZero() { 642 t.Fatal("got a zero time value, want a populated value") 643 } 644 645 // Confirm BucketAccessControl returns error, since we cannot get legacy ACL 646 // for a bucket that has uniform bucket-level access. 647 648 // Metadata updates may be delayed up to 10s. Since we expect an error from 649 // this call, we retry on a nil error until we get the non-retryable error 650 // that we are expecting. 651 ctxWithTimeout, cancelCtx := context.WithTimeout(ctx, time.Second*10) 652 b := bkt.Retryer(WithErrorFunc(retryOnNilAndTransientErrs)) 653 _, err = b.ACL().List(ctxWithTimeout) 654 cancelCtx() 655 if err == nil { 656 t.Errorf("ACL.List: expected bucket ACL list to fail") 657 } 658 659 // Confirm ObjectAccessControl returns error, for same reason as above. 660 ctxWithTimeout, cancelCtx = context.WithTimeout(ctx, time.Second*10) 661 _, err = o.Retryer(WithErrorFunc(retryOnNilAndTransientErrs)).ACL().List(ctxWithTimeout) 662 cancelCtx() 663 if err == nil { 664 t.Errorf("ACL.List: expected object ACL list to fail") 665 } 666 667 // Disable BucketPolicyOnly. 668 ua = BucketAttrsToUpdate{BucketPolicyOnly: &BucketPolicyOnly{Enabled: false}} 669 attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration) 670 if got, want := attrs.BucketPolicyOnly.Enabled, false; got != want { 671 t.Fatalf("attrs.BucketPolicyOnly.Enabled: got %v, want %v", got, want) 672 } 673 674 // Check that the object ACL rules are the same. 675 676 // Metadata updates may be delayed up to 10s. Before that, we can get a 400 677 // indicating that uniform bucket-level access is still enabled in HTTP. 678 // We need to retry manually as GRPC will not error but provide empty ACL. 679 var acl []ACLRule 680 err = retry(ctx, func() error { 681 var err error 682 acl, err = o.ACL().List(ctx) 683 if err != nil { 684 return fmt.Errorf("ACL.List: object ACL list failed: %v", err) 685 } 686 return nil 687 }, func() error { 688 if !containsACLRule(acl, entityRoleACL{aclEntity, RoleReader}) { 689 return fmt.Errorf("containsACL: expected ACL %v to include custom ACL entity %v", acl, entityRoleACL{aclEntity, RoleReader}) 690 } 691 return nil 692 }) 693 if err != nil { 694 t.Fatal(err) 695 } 696 }) 697 } 698 699 func TestIntegration_UniformBucketLevelAccess(t *testing.T) { 700 ctx := skipJSONReads(context.Background(), "no reads in test") 701 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 702 h := testHelper{t} 703 bkt := client.Bucket(prefix + uidSpace.New()) 704 h.mustCreate(bkt, testutil.ProjID(), nil) 705 defer h.mustDeleteBucket(bkt) 706 707 // Insert an object with custom ACL. 708 o := bkt.Object("uniformBucketLevelAccess") 709 defer func() { 710 if err := o.Delete(ctx); err != nil { 711 log.Printf("failed to delete test object: %v", err) 712 } 713 }() 714 wc := o.NewWriter(ctx) 715 wc.ContentType = "text/plain" 716 h.mustWrite(wc, []byte("test")) 717 a := o.ACL() 718 aclEntity := ACLEntity("user-test@example.com") 719 err := a.Set(ctx, aclEntity, RoleReader) 720 if err != nil { 721 t.Fatalf("set ACL failed: %v", err) 722 } 723 724 // Enable UniformBucketLevelAccess. 725 ua := BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: true}} 726 attrs := h.mustUpdateBucket(bkt, ua, h.mustBucketAttrs(bkt).MetaGeneration) 727 if got, want := attrs.UniformBucketLevelAccess.Enabled, true; got != want { 728 t.Fatalf("got %v, want %v", got, want) 729 } 730 if got := attrs.UniformBucketLevelAccess.LockedTime; got.IsZero() { 731 t.Fatal("got a zero time value, want a populated value") 732 } 733 734 // Confirm BucketAccessControl returns error. 735 // We retry on nil to account for propagation delay in metadata update. 736 ctxWithTimeout, cancelCtx := context.WithTimeout(ctx, time.Second*10) 737 b := bkt.Retryer(WithErrorFunc(retryOnNilAndTransientErrs)) 738 _, err = b.ACL().List(ctxWithTimeout) 739 cancelCtx() 740 if err == nil { 741 t.Errorf("ACL.List: expected bucket ACL list to fail") 742 } 743 744 // Confirm ObjectAccessControl returns error. 745 ctxWithTimeout, cancelCtx = context.WithTimeout(ctx, time.Second*10) 746 _, err = o.Retryer(WithErrorFunc(retryOnNilAndTransientErrs)).ACL().List(ctxWithTimeout) 747 cancelCtx() 748 if err == nil { 749 t.Errorf("ACL.List: expected object ACL list to fail") 750 } 751 752 // Disable UniformBucketLevelAccess. 753 ua = BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: false}} 754 attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration) 755 if got, want := attrs.UniformBucketLevelAccess.Enabled, false; got != want { 756 t.Fatalf("got %v, want %v", got, want) 757 } 758 759 // Metadata updates may be delayed up to 10s. Before that, we can get a 400 760 // indicating that uniform bucket-level access is still enabled in HTTP. 761 // We need to retry manually as GRPC will not error but provide empty ACL. 762 var acl []ACLRule 763 err = retry(ctx, func() error { 764 var err error 765 acl, err = o.ACL().List(ctx) 766 if err != nil { 767 return fmt.Errorf("ACL.List: object ACL list failed: %v", err) 768 } 769 return nil 770 }, func() error { 771 if !containsACLRule(acl, entityRoleACL{aclEntity, RoleReader}) { 772 return fmt.Errorf("containsACL: expected ACL %v to include custom ACL entity %v", acl, entityRoleACL{aclEntity, RoleReader}) 773 } 774 return nil 775 }) 776 if err != nil { 777 t.Fatal(err) 778 } 779 }) 780 } 781 782 func TestIntegration_PublicAccessPrevention(t *testing.T) { 783 ctx := skipJSONReads(context.Background(), "no reads in test") 784 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 785 h := testHelper{t} 786 787 // Create a bucket with PublicAccessPrevention enforced. 788 bkt := client.Bucket(prefix + uidSpace.New()) 789 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{PublicAccessPrevention: PublicAccessPreventionEnforced}) 790 defer h.mustDeleteBucket(bkt) 791 792 // Making bucket public should fail. 793 policy, err := bkt.IAM().V3().Policy(ctx) 794 if err != nil { 795 t.Fatalf("fetching bucket IAM policy: %v", err) 796 } 797 policy.Bindings = append(policy.Bindings, &iampb.Binding{ 798 Role: "roles/storage.objectViewer", 799 Members: []string{iam.AllUsers}, 800 }) 801 if err := bkt.IAM().V3().SetPolicy(ctx, policy); err == nil { 802 t.Error("SetPolicy: expected adding AllUsers policy to bucket should fail") 803 } 804 805 // Making object public via ACL should fail. 806 o := bkt.Object("publicAccessPrevention") 807 defer func() { 808 if err := o.Delete(ctx); err != nil { 809 log.Printf("failed to delete test object: %v", err) 810 } 811 }() 812 wc := o.NewWriter(ctx) 813 wc.ContentType = "text/plain" 814 h.mustWrite(wc, []byte("test")) 815 a := o.ACL() 816 if err := a.Set(ctx, AllUsers, RoleReader); err == nil { 817 t.Error("ACL.Set: expected adding AllUsers ACL to object should fail") 818 } 819 820 // Update PAP setting to inherited should work and not affect UBLA setting. 821 attrs, err := bkt.Update(ctx, BucketAttrsToUpdate{PublicAccessPrevention: PublicAccessPreventionInherited}) 822 if err != nil { 823 t.Fatalf("updating PublicAccessPrevention failed: %v", err) 824 } 825 if attrs.PublicAccessPrevention != PublicAccessPreventionInherited { 826 t.Errorf("updating PublicAccessPrevention: got %s, want %s", attrs.PublicAccessPrevention, PublicAccessPreventionInherited) 827 } 828 if attrs.UniformBucketLevelAccess.Enabled || attrs.BucketPolicyOnly.Enabled { 829 t.Error("updating PublicAccessPrevention changed UBLA setting") 830 } 831 832 // Now, making object public or making bucket public should succeed. Run with 833 // retry because ACL settings may take time to propagate. 834 retrier := func(err error) bool { 835 // Once ACL settings propagate, PAP should no longer be enforced and the call will succeed. 836 // In the meantime, while PAP is enforced, trying to set ACL results in: 837 // - FailedPrecondition for gRPC 838 // - condition not met (412) for HTTP 839 return ShouldRetry(err) || status.Code(err) == codes.FailedPrecondition || extractErrCode(err) == http.StatusPreconditionFailed 840 } 841 842 ctxWithTimeout, cancelCtx := context.WithTimeout(ctx, time.Second*10) 843 a = o.Retryer(WithErrorFunc(retrier), WithPolicy(RetryAlways)).ACL() 844 err = a.Set(ctxWithTimeout, AllUsers, RoleReader) 845 cancelCtx() 846 if err != nil { 847 t.Errorf("ACL.Set: making object public failed: %v", err) 848 } 849 850 policy, err = bkt.IAM().V3().Policy(ctx) 851 if err != nil { 852 t.Fatalf("fetching bucket IAM policy: %v", err) 853 } 854 policy.Bindings = append(policy.Bindings, &iampb.Binding{ 855 Role: "roles/storage.objectViewer", 856 Members: []string{iam.AllUsers}, 857 }) 858 if err := bkt.IAM().V3().SetPolicy(ctx, policy); err != nil { 859 t.Errorf("SetPolicy: making bucket public failed: %v", err) 860 } 861 862 // Updating UBLA should not affect PAP setting. 863 attrs, err = bkt.Update(ctx, BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: true}}) 864 if err != nil { 865 t.Fatalf("updating UBLA failed: %v", err) 866 } 867 if !attrs.UniformBucketLevelAccess.Enabled { 868 t.Error("updating UBLA: got UBLA not enabled, want enabled") 869 } 870 if attrs.PublicAccessPrevention != PublicAccessPreventionInherited { 871 t.Errorf("updating UBLA: got %s, want %s", attrs.PublicAccessPrevention, PublicAccessPreventionInherited) 872 } 873 }) 874 } 875 876 func TestIntegration_Autoclass(t *testing.T) { 877 ctx := skipJSONReads(context.Background(), "no reads in test") 878 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 879 h := testHelper{t} 880 881 // Create a bucket with Autoclass enabled. 882 bkt := client.Bucket(prefix + uidSpace.New()) 883 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{Autoclass: &Autoclass{Enabled: true}}) 884 defer h.mustDeleteBucket(bkt) 885 886 // Get Autoclass configuration from bucket attrs. 887 // Autoclass.TerminalStorageClass is defaulted to NEARLINE if not specified. 888 attrs, err := bkt.Attrs(ctx) 889 if err != nil { 890 t.Fatalf("get bucket attrs failed: %v", err) 891 } 892 var toggleTime time.Time 893 var tscUpdateTime time.Time 894 if attrs != nil && attrs.Autoclass != nil { 895 if got, want := attrs.Autoclass.Enabled, true; got != want { 896 t.Errorf("attr.Autoclass.Enabled = %v, want %v", got, want) 897 } 898 if toggleTime = attrs.Autoclass.ToggleTime; toggleTime.IsZero() { 899 t.Error("got a zero time value, want a populated value") 900 } 901 if got, want := attrs.Autoclass.TerminalStorageClass, "NEARLINE"; got != want { 902 t.Errorf("attr.Autoclass.TerminalStorageClass = %v, want %v", got, want) 903 } 904 if tscUpdateTime := attrs.Autoclass.TerminalStorageClassUpdateTime; tscUpdateTime.IsZero() { 905 t.Error("got a zero time value, want a populated value") 906 } 907 } 908 909 // Update TerminalStorageClass on the bucket. 910 ua := BucketAttrsToUpdate{Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"}} 911 attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration) 912 if got, want := attrs.Autoclass.Enabled, true; got != want { 913 t.Errorf("attr.Autoclass.Enabled = %v, want %v", got, want) 914 } 915 if got, want := attrs.Autoclass.TerminalStorageClass, "ARCHIVE"; got != want { 916 t.Errorf("attr.Autoclass.TerminalStorageClass = %v, want %v", got, want) 917 } 918 latestTSCUpdateTime := attrs.Autoclass.TerminalStorageClassUpdateTime 919 if latestTSCUpdateTime.IsZero() { 920 t.Error("got a zero time value, want a populated value") 921 } 922 if !latestTSCUpdateTime.After(tscUpdateTime) { 923 t.Error("latestTSCUpdateTime should be newer than bucket creation tscUpdateTime") 924 } 925 926 // Disable Autoclass on the bucket. 927 ua = BucketAttrsToUpdate{Autoclass: &Autoclass{Enabled: false}} 928 attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration) 929 if got, want := attrs.Autoclass.Enabled, false; got != want { 930 t.Errorf("attr.Autoclass.Enabled = %v, want %v", got, want) 931 } 932 latestToggleTime := attrs.Autoclass.ToggleTime 933 if latestToggleTime.IsZero() { 934 t.Error("got a zero time value, want a populated value") 935 } 936 if !latestToggleTime.After(toggleTime) { 937 t.Error("latestToggleTime should be newer than bucket creation toggleTime") 938 } 939 }) 940 } 941 942 func TestIntegration_ConditionalDelete(t *testing.T) { 943 ctx := skipJSONReads(context.Background(), "no reads in test") 944 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 945 h := testHelper{t} 946 947 o := client.Bucket(bucket).Object("conddel") 948 949 wc := o.NewWriter(ctx) 950 wc.ContentType = "text/plain" 951 h.mustWrite(wc, []byte("foo")) 952 953 gen := wc.Attrs().Generation 954 metaGen := wc.Attrs().Metageneration 955 956 if err := o.Generation(gen - 1).Delete(ctx); err == nil { 957 t.Fatalf("Unexpected successful delete with Generation") 958 } 959 if err := o.If(Conditions{MetagenerationMatch: metaGen + 1}).Delete(ctx); err == nil { 960 t.Fatalf("Unexpected successful delete with IfMetaGenerationMatch") 961 } 962 if err := o.If(Conditions{MetagenerationNotMatch: metaGen}).Delete(ctx); err == nil { 963 t.Fatalf("Unexpected successful delete with IfMetaGenerationNotMatch") 964 } 965 if err := o.Generation(gen).Delete(ctx); err != nil { 966 t.Fatalf("final delete failed: %v", err) 967 } 968 }) 969 } 970 971 func TestIntegration_ObjectsRangeReader(t *testing.T) { 972 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 973 bkt := client.Bucket(bucket) 974 975 objName := uidSpaceObjects.New() 976 obj := bkt.Object(objName) 977 contents := []byte("Hello, world this is a range request") 978 979 w := obj.If(Conditions{DoesNotExist: true}).NewWriter(ctx) 980 if _, err := w.Write(contents); err != nil { 981 t.Errorf("Failed to write contents: %v", err) 982 } 983 if err := w.Close(); err != nil { 984 t.Errorf("Failed to close writer: %v", err) 985 } 986 987 last5s := []struct { 988 name string 989 start int64 990 length int64 991 }{ 992 {name: "negative offset", start: -5, length: -1}, 993 {name: "offset with specified length", start: int64(len(contents)) - 5, length: 5}, 994 {name: "offset and read till end", start: int64(len(contents)) - 5, length: -1}, 995 } 996 997 for _, last5 := range last5s { 998 t.Run(last5.name, func(t *testing.T) { 999 wantBuf := contents[len(contents)-5:] 1000 r, err := obj.NewRangeReader(ctx, last5.start, last5.length) 1001 if err != nil { 1002 t.Fatalf("Failed to make range read: %v", err) 1003 } 1004 defer r.Close() 1005 1006 if got, want := r.Attrs.StartOffset, int64(len(contents))-5; got != want { 1007 t.Errorf("StartOffset mismatch, got %d want %d", got, want) 1008 } 1009 1010 gotBuf := &bytes.Buffer{} 1011 nr, _ := io.Copy(gotBuf, r) 1012 if got, want := nr, int64(5); got != want { 1013 t.Errorf("Body length mismatch, got %d want %d", got, want) 1014 } else if diff := cmp.Diff(gotBuf.String(), string(wantBuf)); diff != "" { 1015 t.Errorf("Content read does not match - got(-),want(+):\n%s", diff) 1016 } 1017 }) 1018 } 1019 }) 1020 } 1021 1022 func TestIntegration_ObjectReadChunksGRPC(t *testing.T) { 1023 multiTransportTest(skipHTTP("gRPC implementation specific test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 1024 h := testHelper{t} 1025 // Use a larger blob to test chunking logic. This is a little over 5MB. 1026 content := make([]byte, 5<<20) 1027 rand.New(rand.NewSource(0)).Read(content) 1028 1029 // Upload test data. 1030 obj := client.Bucket(bucket).Object(uidSpaceObjects.New()) 1031 if err := writeObject(ctx, obj, "text/plain", content); err != nil { 1032 t.Fatal(err) 1033 } 1034 defer h.mustDeleteObject(obj) 1035 1036 r, err := obj.NewReader(ctx) 1037 if err != nil { 1038 t.Fatal(err) 1039 } 1040 defer r.Close() 1041 1042 if size := r.Size(); size != int64(len(content)) { 1043 t.Errorf("got size = %v, want %v", size, len(content)) 1044 } 1045 if rem := r.Remain(); rem != int64(len(content)) { 1046 t.Errorf("got %v bytes remaining, want %v", rem, len(content)) 1047 } 1048 1049 bufSize := len(content) 1050 buf := make([]byte, bufSize) 1051 1052 // Read in smaller chunks, offset to provoke reading across a Recv boundary. 1053 chunk := 4<<10 + 1234 1054 offset := 0 1055 for { 1056 end := math.Min(float64(offset+chunk), float64(bufSize)) 1057 n, err := r.Read(buf[offset:int(end)]) 1058 if err == io.EOF { 1059 break 1060 } 1061 if err != nil { 1062 t.Fatal(err) 1063 } 1064 offset += n 1065 } 1066 1067 if rem := r.Remain(); rem != 0 { 1068 t.Errorf("got %v bytes remaining, want 0", rem) 1069 } 1070 if !bytes.Equal(buf, content) { 1071 t.Errorf("content mismatch") 1072 } 1073 }) 1074 } 1075 1076 func TestIntegration_MultiMessageWriteGRPC(t *testing.T) { 1077 multiTransportTest(skipHTTP("gRPC implementation specific test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 1078 h := testHelper{t} 1079 1080 name := uidSpaceObjects.New() 1081 obj := client.Bucket(bucket).Object(name).Retryer(WithPolicy(RetryAlways)) 1082 defer h.mustDeleteObject(obj) 1083 1084 // Use a larger blob to test multi-message logic. This is a little over 5MB. 1085 content := bytes.Repeat([]byte("a"), 5<<20) 1086 1087 crc32c := crc32.Checksum(content, crc32cTable) 1088 w := obj.NewWriter(ctx) 1089 w.ProgressFunc = func(p int64) { 1090 t.Logf("%s: committed %d\n", t.Name(), p) 1091 } 1092 w.SendCRC32C = true 1093 w.CRC32C = crc32c 1094 got, err := w.Write(content) 1095 if err != nil { 1096 t.Fatalf("Writer.Write: %v", err) 1097 } 1098 // Flush the buffer to finish the upload. 1099 if err := w.Close(); err != nil { 1100 t.Fatalf("Writer.Close: %v", err) 1101 } 1102 1103 want := len(content) 1104 if got != want { 1105 t.Errorf("While writing got: %d want %d", got, want) 1106 } 1107 1108 // Read back the Object for verification. 1109 reader, err := client.Bucket(bucket).Object(name).NewReader(ctx) 1110 if err != nil { 1111 t.Fatal(err) 1112 } 1113 defer reader.Close() 1114 1115 buf := make([]byte, want+4<<10) 1116 b := bytes.NewBuffer(buf) 1117 gotr, err := io.Copy(b, reader) 1118 if err != nil { 1119 t.Fatal(err) 1120 } 1121 if gotr != int64(want) { 1122 t.Errorf("While reading got: %d want %d", gotr, want) 1123 } 1124 }) 1125 } 1126 1127 func TestIntegration_MultiChunkWrite(t *testing.T) { 1128 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 1129 h := testHelper{t} 1130 obj := client.Bucket(bucket).Object(uidSpaceObjects.New()).Retryer(WithPolicy(RetryAlways)) 1131 defer h.mustDeleteObject(obj) 1132 1133 // Use a larger blob to test multi-message logic. This is a little over 5MB. 1134 content := bytes.Repeat([]byte("a"), 5<<20) 1135 crc32c := crc32.Checksum(content, crc32cTable) 1136 1137 w := obj.NewWriter(ctx) 1138 w.SendCRC32C = true 1139 w.CRC32C = crc32c 1140 // Use a 1 MB chunk size. 1141 w.ChunkSize = 1 << 20 1142 w.ProgressFunc = func(p int64) { 1143 t.Logf("%s: committed %d\n", t.Name(), p) 1144 } 1145 got, err := w.Write(content) 1146 if err != nil { 1147 t.Fatalf("Writer.Write: %v", err) 1148 } 1149 // Flush the buffer to finish the upload. 1150 if err := w.Close(); err != nil { 1151 t.Fatalf("Writer.Close: %v", err) 1152 } 1153 1154 want := len(content) 1155 if got != want { 1156 t.Errorf("While writing got: %d want %d", got, want) 1157 } 1158 1159 r, err := obj.NewReader(ctx) 1160 if err != nil { 1161 t.Fatal(err) 1162 } 1163 defer r.Close() 1164 1165 buf := make([]byte, want+4<<10) 1166 b := bytes.NewBuffer(buf) 1167 gotr, err := io.Copy(b, r) 1168 if err != nil { 1169 t.Fatal(err) 1170 } 1171 if gotr != int64(want) { 1172 t.Errorf("While reading got: %d want %d", gotr, want) 1173 } 1174 }) 1175 } 1176 1177 func TestIntegration_ConditionalDownload(t *testing.T) { 1178 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 1179 h := testHelper{t} 1180 1181 o := client.Bucket(bucket).Object("condread") 1182 defer o.Delete(ctx) 1183 1184 wc := o.NewWriter(ctx) 1185 wc.ContentType = "text/plain" 1186 h.mustWrite(wc, []byte("foo")) 1187 1188 gen := wc.Attrs().Generation 1189 metaGen := wc.Attrs().Metageneration 1190 1191 if _, err := o.Generation(gen + 1).NewReader(ctx); err == nil { 1192 t.Fatalf("Unexpected successful download with nonexistent Generation") 1193 } 1194 if _, err := o.If(Conditions{MetagenerationMatch: metaGen + 1}).NewReader(ctx); err == nil { 1195 t.Fatalf("Unexpected successful download with failed preconditions IfMetaGenerationMatch") 1196 } 1197 if _, err := o.If(Conditions{GenerationMatch: gen + 1}).NewReader(ctx); err == nil { 1198 t.Fatalf("Unexpected successful download with failed preconditions IfGenerationMatch") 1199 } 1200 if _, err := o.If(Conditions{GenerationMatch: gen}).NewReader(ctx); err != nil { 1201 t.Fatalf("Download failed: %v", err) 1202 } 1203 }) 1204 } 1205 1206 func TestIntegration_ObjectIteration(t *testing.T) { 1207 ctx := skipJSONReads(context.Background(), "no reads in test") 1208 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 1209 // Reset testTime, 'cause object last modification time should be within 5 min 1210 // from test (test iteration if -count passed) start time. 1211 testTime = time.Now().UTC() 1212 newBucketName := prefix + uidSpace.New() 1213 h := testHelper{t} 1214 bkt := client.Bucket(newBucketName).Retryer(WithPolicy(RetryAlways)) 1215 1216 h.mustCreate(bkt, testutil.ProjID(), nil) 1217 defer func() { 1218 if err := killBucket(ctx, client, newBucketName); err != nil { 1219 log.Printf("deleting %q: %v", newBucketName, err) 1220 } 1221 }() 1222 const defaultType = "text/plain" 1223 1224 // Populate object names and make a map for their contents. 1225 objects := []string{ 1226 "obj1", 1227 "obj2", 1228 "obj/with/slashes", 1229 "obj/", 1230 } 1231 contents := make(map[string][]byte) 1232 1233 // Test Writer. 1234 for _, obj := range objects { 1235 c := randomContents() 1236 if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil { 1237 t.Errorf("Write for %v failed with %v", obj, err) 1238 } 1239 contents[obj] = c 1240 } 1241 1242 testObjectIterator(t, bkt, objects) 1243 testObjectsIterateSelectedAttrs(t, bkt, objects) 1244 testObjectsIterateAllSelectedAttrs(t, bkt, objects) 1245 testObjectIteratorWithOffset(t, bkt, objects) 1246 testObjectsIterateWithProjection(t, bkt) 1247 t.Run("testObjectsIterateSelectedAttrsDelimiter", func(t *testing.T) { 1248 query := &Query{Prefix: "", Delimiter: "/"} 1249 if err := query.SetAttrSelection([]string{"Name"}); err != nil { 1250 t.Fatalf("selecting query attrs: %v", err) 1251 } 1252 1253 var gotNames []string 1254 var gotPrefixes []string 1255 it := bkt.Objects(context.Background(), query) 1256 for { 1257 attrs, err := it.Next() 1258 if err == iterator.Done { 1259 break 1260 } 1261 if err != nil { 1262 t.Fatalf("iterator.Next: %v", err) 1263 } 1264 if attrs.Name != "" { 1265 gotNames = append(gotNames, attrs.Name) 1266 } else if attrs.Prefix != "" { 1267 gotPrefixes = append(gotPrefixes, attrs.Prefix) 1268 } 1269 1270 if attrs.Bucket != "" { 1271 t.Errorf("Bucket field not selected, want empty, got = %v", attrs.Bucket) 1272 } 1273 } 1274 1275 sortedNames := []string{"obj1", "obj2"} 1276 if !cmp.Equal(sortedNames, gotNames) { 1277 t.Errorf("names = %v, want %v", gotNames, sortedNames) 1278 } 1279 sortedPrefixes := []string{"obj/"} 1280 if !cmp.Equal(sortedPrefixes, gotPrefixes) { 1281 t.Errorf("prefixes = %v, want %v", gotPrefixes, sortedPrefixes) 1282 } 1283 }) 1284 t.Run("testObjectsIterateSelectedAttrsDelimiterIncludeTrailingDelimiter", func(t *testing.T) { 1285 query := &Query{Prefix: "", Delimiter: "/", IncludeTrailingDelimiter: true} 1286 if err := query.SetAttrSelection([]string{"Name"}); err != nil { 1287 t.Fatalf("selecting query attrs: %v", err) 1288 } 1289 1290 var gotNames []string 1291 var gotPrefixes []string 1292 it := bkt.Objects(context.Background(), query) 1293 for { 1294 attrs, err := it.Next() 1295 if err == iterator.Done { 1296 break 1297 } 1298 if err != nil { 1299 t.Fatalf("iterator.Next: %v", err) 1300 } 1301 if attrs.Name != "" { 1302 gotNames = append(gotNames, attrs.Name) 1303 } else if attrs.Prefix != "" { 1304 gotPrefixes = append(gotPrefixes, attrs.Prefix) 1305 } 1306 1307 if attrs.Bucket != "" { 1308 t.Errorf("Bucket field not selected, want empty, got = %v", attrs.Bucket) 1309 } 1310 } 1311 1312 sortedNames := []string{"obj/", "obj1", "obj2"} 1313 if !cmp.Equal(sortedNames, gotNames) { 1314 t.Errorf("names = %v, want %v", gotNames, sortedNames) 1315 } 1316 sortedPrefixes := []string{"obj/"} 1317 if !cmp.Equal(sortedPrefixes, gotPrefixes) { 1318 t.Errorf("prefixes = %v, want %v", gotPrefixes, sortedPrefixes) 1319 } 1320 }) 1321 }) 1322 } 1323 1324 func TestIntegration_ObjectIterationMatchGlob(t *testing.T) { 1325 multiTransportTest(skipJSONReads(context.Background(), "no reads in test"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 1326 // Reset testTime, 'cause object last modification time should be within 5 min 1327 // from test (test iteration if -count passed) start time. 1328 testTime = time.Now().UTC() 1329 newBucketName := prefix + uidSpace.New() 1330 h := testHelper{t} 1331 bkt := client.Bucket(newBucketName).Retryer(WithPolicy(RetryAlways)) 1332 1333 h.mustCreate(bkt, testutil.ProjID(), nil) 1334 defer func() { 1335 if err := killBucket(ctx, client, newBucketName); err != nil { 1336 log.Printf("deleting %q: %v", newBucketName, err) 1337 } 1338 }() 1339 const defaultType = "text/plain" 1340 1341 // Populate object names and make a map for their contents. 1342 objects := []string{ 1343 "obj1", 1344 "obj2", 1345 "obj/with/slashes", 1346 "obj/", 1347 "other/obj1", 1348 } 1349 contents := make(map[string][]byte) 1350 1351 // Test Writer. 1352 for _, obj := range objects { 1353 c := randomContents() 1354 if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil { 1355 t.Errorf("Write for %v failed with %v", obj, err) 1356 } 1357 contents[obj] = c 1358 } 1359 query := &Query{MatchGlob: "**obj1"} 1360 1361 var gotNames []string 1362 it := bkt.Objects(context.Background(), query) 1363 for { 1364 attrs, err := it.Next() 1365 if err == iterator.Done { 1366 break 1367 } 1368 if err != nil { 1369 t.Fatalf("iterator.Next: %v", err) 1370 } 1371 if attrs.Name != "" { 1372 gotNames = append(gotNames, attrs.Name) 1373 } 1374 } 1375 1376 sortedNames := []string{"obj1", "other/obj1"} 1377 if !cmp.Equal(sortedNames, gotNames) { 1378 t.Errorf("names = %v, want %v", gotNames, sortedNames) 1379 } 1380 }) 1381 } 1382 1383 func TestIntegration_ObjectIterationManagedFolder(t *testing.T) { 1384 ctx := skipGRPC("not yet implemented in gRPC") 1385 multiTransportTest(skipJSONReads(ctx, "no reads in test"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 1386 newBucketName := prefix + uidSpace.New() 1387 h := testHelper{t} 1388 bkt := client.Bucket(newBucketName).Retryer(WithPolicy(RetryAlways)) 1389 1390 // Create bucket with UBLA enabled as this is necessary for managed folders. 1391 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ 1392 UniformBucketLevelAccess: UniformBucketLevelAccess{ 1393 Enabled: true, 1394 }, 1395 }) 1396 1397 t.Cleanup(func() { 1398 if err := killBucket(ctx, client, newBucketName); err != nil { 1399 log.Printf("deleting %q: %v", newBucketName, err) 1400 } 1401 }) 1402 const defaultType = "text/plain" 1403 1404 // Populate object names and make a map for their contents. 1405 objects := []string{ 1406 "obj1", 1407 "obj2", 1408 "obj/with/slashes", 1409 "obj/", 1410 "other/obj1", 1411 } 1412 contents := make(map[string][]byte) 1413 1414 // Test Writer. 1415 for _, obj := range objects { 1416 c := randomContents() 1417 if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil { 1418 t.Errorf("Write for %v failed with %v", obj, err) 1419 } 1420 contents[obj] = c 1421 } 1422 1423 // Create a managed folder. This requires using the Apiary client as this is not available 1424 // in the veneer layer. 1425 // TODO: change to use storage control client once available. 1426 call := client.raw.ManagedFolders.Insert(newBucketName, &raw.ManagedFolder{Name: "mf"}) 1427 mf, err := call.Context(ctx).Do() 1428 if err != nil { 1429 t.Fatalf("creating managed folder: %v", err) 1430 } 1431 1432 t.Cleanup(func() { 1433 // TODO: add this cleanup logic to killBucket as well once gRPC support is available. 1434 call := client.raw.ManagedFolders.Delete(newBucketName, mf.Name) 1435 call.Context(ctx).Do() 1436 }) 1437 1438 // Test that managed folders are only included when IncludeFoldersAsPrefixes is set. 1439 cases := []struct { 1440 name string 1441 query *Query 1442 want []string 1443 }{ 1444 { 1445 name: "include folders", 1446 query: &Query{Delimiter: "/", IncludeFoldersAsPrefixes: true}, 1447 want: []string{"mf/", "obj/", "other/"}, 1448 }, 1449 { 1450 name: "no folders", 1451 query: &Query{Delimiter: "/"}, 1452 want: []string{"obj/", "other/"}, 1453 }, 1454 } 1455 1456 for _, c := range cases { 1457 t.Run(c.name, func(t *testing.T) { 1458 var gotNames []string 1459 var gotPrefixes []string 1460 it := bkt.Objects(context.Background(), c.query) 1461 for { 1462 attrs, err := it.Next() 1463 if err == iterator.Done { 1464 break 1465 } 1466 if err != nil { 1467 t.Fatalf("iterator.Next: %v", err) 1468 } 1469 if attrs.Name != "" { 1470 gotNames = append(gotNames, attrs.Name) 1471 } 1472 if attrs.Prefix != "" { 1473 gotPrefixes = append(gotPrefixes, attrs.Prefix) 1474 } 1475 } 1476 1477 sortedNames := []string{"obj1", "obj2"} 1478 if !cmp.Equal(sortedNames, gotNames) { 1479 t.Errorf("names = %v, want %v", gotNames, sortedNames) 1480 } 1481 1482 if !cmp.Equal(c.want, gotPrefixes) { 1483 t.Errorf("prefixes = %v, want %v", gotPrefixes, c.want) 1484 } 1485 }) 1486 } 1487 }) 1488 } 1489 1490 func TestIntegration_ObjectUpdate(t *testing.T) { 1491 ctx := skipJSONReads(context.Background(), "no reads in test") 1492 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 1493 b := client.Bucket(bucket) 1494 1495 o := b.Object("update-obj" + uidSpaceObjects.New()) 1496 w := o.NewWriter(ctx) 1497 _, err := io.Copy(w, bytes.NewReader(randomContents())) 1498 if err != nil { 1499 t.Fatalf("io.Copy: %v", err) 1500 } 1501 if err := w.Close(); err != nil { 1502 t.Fatalf("w.Close: %v", err) 1503 } 1504 defer func() { 1505 if err := o.Delete(ctx); err != nil { 1506 t.Errorf("o.Delete : %v", err) 1507 } 1508 }() 1509 1510 attrs, err := o.Attrs(ctx) 1511 if err != nil { 1512 t.Fatalf("o.Attrs: %v", err) 1513 } 1514 1515 // Test UpdateAttrs. 1516 metadata := map[string]string{"key": "value"} 1517 1518 updated, err := o.If(Conditions{MetagenerationMatch: attrs.Metageneration}).Update(ctx, ObjectAttrsToUpdate{ 1519 ContentType: "text/html", 1520 ContentLanguage: "en", 1521 Metadata: metadata, 1522 ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}, 1523 }) 1524 if err != nil { 1525 t.Fatalf("o.Update: %v", err) 1526 } 1527 1528 if got, want := updated.ContentType, "text/html"; got != want { 1529 t.Errorf("updated.ContentType == %q; want %q", got, want) 1530 } 1531 if got, want := updated.ContentLanguage, "en"; got != want { 1532 t.Errorf("updated.ContentLanguage == %q; want %q", updated.ContentLanguage, want) 1533 } 1534 if got, want := updated.Metadata, metadata; !testutil.Equal(got, want) { 1535 t.Errorf("updated.Metadata == %+v; want %+v", updated.Metadata, want) 1536 } 1537 if got, want := updated.Created, attrs.Created; got != want { 1538 t.Errorf("updated.Created == %q; want %q", got, want) 1539 } 1540 if !updated.Created.Before(updated.Updated) { 1541 t.Errorf("updated.Updated should be newer than update.Created") 1542 } 1543 1544 // Add another metadata key 1545 anotherKey := map[string]string{"key2": "value2"} 1546 metadata["key2"] = "value2" 1547 1548 updated, err = o.Update(ctx, ObjectAttrsToUpdate{ 1549 Metadata: anotherKey, 1550 }) 1551 if err != nil { 1552 t.Fatalf("o.Update: %v", err) 1553 } 1554 1555 if got, want := updated.Metadata, metadata; !testutil.Equal(got, want) { 1556 t.Errorf("updated.Metadata == %+v; want %+v", updated.Metadata, want) 1557 } 1558 1559 // Delete ContentType and ContentLanguage and Metadata. 1560 updated, err = o.If(Conditions{MetagenerationMatch: updated.Metageneration}).Update(ctx, ObjectAttrsToUpdate{ 1561 ContentType: "", 1562 ContentLanguage: "", 1563 Metadata: map[string]string{}, 1564 ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}, 1565 }) 1566 if err != nil { 1567 t.Fatalf("o.Update: %v", err) 1568 } 1569 1570 if got, want := updated.ContentType, ""; got != want { 1571 t.Errorf("updated.ContentType == %q; want %q", got, want) 1572 } 1573 if got, want := updated.ContentLanguage, ""; got != want { 1574 t.Errorf("updated.ContentLanguage == %q; want %q", updated.ContentLanguage, want) 1575 } 1576 if updated.Metadata != nil { 1577 t.Errorf("updated.Metadata == %+v; want nil", updated.Metadata) 1578 } 1579 if got, want := updated.Created, attrs.Created; got != want { 1580 t.Errorf("updated.Created == %q; want %q", got, want) 1581 } 1582 if !updated.Created.Before(updated.Updated) { 1583 t.Errorf("updated.Updated should be newer than update.Created") 1584 } 1585 }) 1586 } 1587 1588 func TestIntegration_ObjectChecksums(t *testing.T) { 1589 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 1590 b := client.Bucket(bucket) 1591 checksumCases := []struct { 1592 name string 1593 contents [][]byte 1594 size int64 1595 md5 string 1596 crc32c uint32 1597 }{ 1598 { 1599 name: "checksum-object", 1600 contents: [][]byte{[]byte("hello"), []byte("world")}, 1601 size: 10, 1602 md5: "fc5e038d38a57032085441e7fe7010b0", 1603 crc32c: 1456190592, 1604 }, 1605 { 1606 name: "zero-object", 1607 contents: [][]byte{}, 1608 size: 0, 1609 md5: "d41d8cd98f00b204e9800998ecf8427e", 1610 crc32c: 0, 1611 }, 1612 } 1613 for _, c := range checksumCases { 1614 wc := b.Object(c.name + uidSpaceObjects.New()).NewWriter(ctx) 1615 for _, data := range c.contents { 1616 if _, err := wc.Write(data); err != nil { 1617 t.Fatalf("Write(%q) failed with %q", data, err) 1618 } 1619 } 1620 if err := wc.Close(); err != nil { 1621 t.Fatalf("%q: close failed with %q", c.name, err) 1622 } 1623 obj := wc.Attrs() 1624 if got, want := obj.Size, c.size; got != want { 1625 t.Errorf("Object (%q) Size = %v; want %v", c.name, got, want) 1626 } 1627 if got, want := fmt.Sprintf("%x", obj.MD5), c.md5; got != want { 1628 t.Errorf("Object (%q) MD5 = %q; want %q", c.name, got, want) 1629 } 1630 if got, want := obj.CRC32C, c.crc32c; got != want { 1631 t.Errorf("Object (%q) CRC32C = %v; want %v", c.name, got, want) 1632 } 1633 } 1634 }) 1635 } 1636 1637 func TestIntegration_ObjectCompose(t *testing.T) { 1638 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 1639 b := client.Bucket(bucket) 1640 1641 objects := []*ObjectHandle{ 1642 b.Object("obj1" + uidSpaceObjects.New()), 1643 b.Object("obj2" + uidSpaceObjects.New()), 1644 b.Object("obj/with/slashes" + uidSpaceObjects.New()), 1645 b.Object("obj/" + uidSpaceObjects.New()), 1646 } 1647 var compSrcs []*ObjectHandle 1648 wantContents := make([]byte, 0) 1649 1650 // Write objects to compose 1651 for _, obj := range objects { 1652 c := randomContents() 1653 if err := writeObject(ctx, obj, "text/plain", c); err != nil { 1654 t.Errorf("Write for %v failed with %v", obj, err) 1655 } 1656 compSrcs = append(compSrcs, obj) 1657 wantContents = append(wantContents, c...) 1658 defer obj.Delete(ctx) 1659 } 1660 1661 checkCompose := func(obj *ObjectHandle, contentTypeSet *string) { 1662 r, err := obj.NewReader(ctx) 1663 if err != nil { 1664 t.Fatalf("new reader: %v", err) 1665 } 1666 1667 slurp, err := ioutil.ReadAll(r) 1668 if err != nil { 1669 t.Fatalf("ioutil.ReadAll: %v", err) 1670 } 1671 defer r.Close() 1672 if !bytes.Equal(slurp, wantContents) { 1673 t.Errorf("Composed object contents\ngot: %q\nwant: %q", slurp, wantContents) 1674 } 1675 got := r.ContentType() 1676 // Accept both an empty string and octet-stream if the content type was not set; 1677 // HTTP will set the content type as octet-stream whilst GRPC will not set it all. 1678 if !(contentTypeSet == nil && (got == "" || got == "application/octet-stream")) && got != *contentTypeSet { 1679 t.Errorf("Composed object content-type = %q, want %q", got, *contentTypeSet) 1680 } 1681 } 1682 1683 // Compose should work even if the user sets no destination attributes. 1684 compDst := b.Object("composed1") 1685 c := compDst.ComposerFrom(compSrcs...) 1686 attrs, err := c.Run(ctx) 1687 if err != nil { 1688 t.Fatalf("ComposeFrom error: %v", err) 1689 } 1690 if attrs.ComponentCount != int64(len(objects)) { 1691 t.Errorf("mismatching ComponentCount: got %v, want %v", attrs.ComponentCount, int64(len(objects))) 1692 } 1693 checkCompose(compDst, nil) 1694 1695 // It should also work if we do. 1696 contentType := "text/json" 1697 compDst = b.Object("composed2") 1698 c = compDst.ComposerFrom(compSrcs...) 1699 c.ContentType = contentType 1700 attrs, err = c.Run(ctx) 1701 if err != nil { 1702 t.Fatalf("ComposeFrom error: %v", err) 1703 } 1704 if attrs.ComponentCount != int64(len(objects)) { 1705 t.Errorf("mismatching ComponentCount: got %v, want %v", attrs.ComponentCount, int64(len(objects))) 1706 } 1707 checkCompose(compDst, &contentType) 1708 }) 1709 } 1710 1711 func TestIntegration_Copy(t *testing.T) { 1712 ctx := skipJSONReads(context.Background(), "no reads in test") 1713 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, prefix string, client *Client) { 1714 h := testHelper{t} 1715 1716 bucketFrom := client.Bucket(bucket) 1717 bucketInSameRegion := client.Bucket(prefix + uidSpace.New()) 1718 bucketInDifferentRegion := client.Bucket(prefix + uidSpace.New()) 1719 1720 // Create new bucket 1721 if err := bucketInSameRegion.Create(ctx, testutil.ProjID(), nil); err != nil { 1722 t.Fatalf("bucket.Create: %v", err) 1723 } 1724 t.Cleanup(func() { 1725 h.mustDeleteBucket(bucketInSameRegion) 1726 }) 1727 1728 // Create new bucket 1729 if err := bucketInDifferentRegion.Create(ctx, testutil.ProjID(), &BucketAttrs{Location: "NORTHAMERICA-NORTHEAST2"}); err != nil { 1730 t.Fatalf("bucket.Create: %v", err) 1731 } 1732 t.Cleanup(func() { 1733 h.mustDeleteBucket(bucketInDifferentRegion) 1734 }) 1735 1736 // We use a larger object size to be able to trigger multiple rewrite calls 1737 minObjectSize := 2500000 // 2.5 Mb 1738 obj := bucketFrom.Object("copy-object-original" + uidSpaceObjects.New()) 1739 1740 // Create an object to copy from 1741 w := obj.NewWriter(ctx) 1742 c := randomContents() 1743 for written := 0; written < minObjectSize; { 1744 n, err := w.Write(c) 1745 if err != nil { 1746 t.Fatalf("w.Write: %v", err) 1747 } 1748 written += n 1749 } 1750 if err := w.Close(); err != nil { 1751 t.Fatalf("w.Close: %v", err) 1752 } 1753 t.Cleanup(func() { 1754 h.mustDeleteObject(obj) 1755 }) 1756 1757 attrs, err := obj.Attrs(ctx) 1758 if err != nil { 1759 t.Fatalf("obj.Attrs: %v", err) 1760 } 1761 1762 crc32c := attrs.CRC32C 1763 1764 type copierAttrs struct { 1765 contentEncoding string 1766 maxBytesPerCall int64 1767 } 1768 1769 for _, test := range []struct { 1770 desc string 1771 toObj string 1772 toBucket *BucketHandle 1773 copierAttrs *copierAttrs 1774 numExpectedRewriteCalls int 1775 }{ 1776 { 1777 desc: "copy within bucket", 1778 toObj: "copy-within-bucket", 1779 toBucket: bucketFrom, 1780 numExpectedRewriteCalls: 1, 1781 }, 1782 { 1783 desc: "copy to new bucket", 1784 toObj: "copy-new-bucket", 1785 toBucket: bucketInSameRegion, 1786 numExpectedRewriteCalls: 1, 1787 }, 1788 { 1789 desc: "copy with attributes", 1790 toObj: "copy-with-attributes", 1791 toBucket: bucketInSameRegion, 1792 copierAttrs: &copierAttrs{contentEncoding: "identity"}, 1793 numExpectedRewriteCalls: 1, 1794 }, 1795 { 1796 // this test should trigger multiple re-write calls and may fail 1797 // with a rate limit error if those calls are stuck in an infinite loop 1798 desc: "copy to new region", 1799 toObj: "copy-new-region", 1800 toBucket: bucketInDifferentRegion, 1801 copierAttrs: &copierAttrs{maxBytesPerCall: 1048576}, 1802 numExpectedRewriteCalls: 3, 1803 }, 1804 } { 1805 t.Run(test.desc, func(t *testing.T) { 1806 copyObj := test.toBucket.Object(test.toObj) 1807 copier := copyObj.CopierFrom(obj) 1808 1809 if attrs := test.copierAttrs; attrs != nil { 1810 if attrs.contentEncoding != "" { 1811 copier.ContentEncoding = attrs.contentEncoding 1812 } 1813 if attrs.maxBytesPerCall != 0 { 1814 copier.maxBytesRewrittenPerCall = attrs.maxBytesPerCall 1815 } 1816 } 1817 1818 rewriteCallsCount := 0 1819 copier.ProgressFunc = func(_, _ uint64) { 1820 rewriteCallsCount++ 1821 } 1822 1823 attrs, err = copier.Run(ctx) 1824 if err != nil { 1825 t.Fatalf("Copier.Run failed with %v", err) 1826 } 1827 t.Cleanup(func() { 1828 h.mustDeleteObject(copyObj) 1829 }) 1830 1831 // Check copied object is in the correct bucket with the correct name 1832 if attrs.Bucket != test.toBucket.name || attrs.Name != test.toObj { 1833 t.Errorf("unexpected copy behaviour: got: %s in bucket %s, want: %s in bucket %s", attrs.Name, attrs.Bucket, attrs.Name, test.toBucket.name) 1834 } 1835 1836 // Check attrs 1837 if test.copierAttrs != nil { 1838 if attrs.ContentEncoding != test.copierAttrs.contentEncoding { 1839 t.Errorf("unexpected ContentEncoding; got: %s, want: %s", attrs.ContentEncoding, test.copierAttrs.contentEncoding) 1840 } 1841 } 1842 1843 // Check the copied contents 1844 if attrs.CRC32C != crc32c { 1845 t.Errorf("mismatching checksum: got %v, want %v", attrs.CRC32C, crc32c) 1846 } 1847 1848 // Check that the number of requests made is as expected 1849 if rewriteCallsCount != test.numExpectedRewriteCalls { 1850 t.Errorf("unexpected number of rewrite calls: got %v, want %v", rewriteCallsCount, test.numExpectedRewriteCalls) 1851 } 1852 }) 1853 } 1854 }) 1855 } 1856 1857 func TestIntegration_Encoding(t *testing.T) { 1858 multiTransportTest(skipGRPC("gzip transcoding not supported"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 1859 bkt := client.Bucket(bucket) 1860 1861 // Test content encoding 1862 const zeroCount = 20 << 1 // TODO: should be 20 << 20 1863 obj := bkt.Object("gzip-test") 1864 w := obj.NewWriter(ctx) 1865 w.ContentEncoding = "gzip" 1866 gw := gzip.NewWriter(w) 1867 if _, err := io.Copy(gw, io.LimitReader(zeros{}, zeroCount)); err != nil { 1868 t.Fatalf("io.Copy, upload: %v", err) 1869 } 1870 if err := gw.Close(); err != nil { 1871 t.Errorf("gzip.Close(): %v", err) 1872 } 1873 if err := w.Close(); err != nil { 1874 t.Errorf("w.Close(): %v", err) 1875 } 1876 r, err := obj.NewReader(ctx) 1877 if err != nil { 1878 t.Fatalf("NewReader(gzip-test): %v", err) 1879 } 1880 n, err := io.Copy(ioutil.Discard, r) 1881 if err != nil { 1882 t.Errorf("io.Copy, download: %v", err) 1883 } 1884 if n != zeroCount { 1885 t.Errorf("downloaded bad data: got %d bytes, want %d", n, zeroCount) 1886 } 1887 1888 // Test NotFound. 1889 _, err = bkt.Object("obj-not-exists").NewReader(ctx) 1890 if err != ErrObjectNotExist { 1891 t.Errorf("Object should not exist, err found to be %v", err) 1892 } 1893 }) 1894 } 1895 1896 func testObjectIterator(t *testing.T, bkt *BucketHandle, objects []string) { 1897 ctx := context.Background() 1898 h := testHelper{t} 1899 // Collect the list of items we expect: ObjectAttrs in lexical order by name. 1900 names := make([]string, len(objects)) 1901 copy(names, objects) 1902 sort.Strings(names) 1903 var attrs []*ObjectAttrs 1904 for _, name := range names { 1905 attrs = append(attrs, h.mustObjectAttrs(bkt.Object(name))) 1906 } 1907 msg, ok := itesting.TestIterator(attrs, 1908 func() interface{} { return bkt.Objects(ctx, &Query{Prefix: "obj"}) }, 1909 func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() }) 1910 if !ok { 1911 t.Errorf("ObjectIterator.Next: %s", msg) 1912 } 1913 // TODO(jba): test query.Delimiter != "" 1914 } 1915 1916 func testObjectIteratorWithOffset(t *testing.T, bkt *BucketHandle, objects []string) { 1917 ctx := context.Background() 1918 h := testHelper{t} 1919 // Collect the list of items we expect: ObjectAttrs in lexical order by name. 1920 names := make([]string, len(objects)) 1921 copy(names, objects) 1922 sort.Strings(names) 1923 var attrs []*ObjectAttrs 1924 for _, name := range names { 1925 attrs = append(attrs, h.mustObjectAttrs(bkt.Object(name))) 1926 } 1927 m := make(map[string][]*ObjectAttrs) 1928 for i, name := range names { 1929 // StartOffset takes the value of object names, the result must be for: 1930 // ― obj/with/slashes: obj/with/slashes, obj1, obj2 1931 // ― obj1: obj1, obj2 1932 // ― obj2: obj2. 1933 m[name] = attrs[i:] 1934 msg, ok := itesting.TestIterator(m[name], 1935 func() interface{} { return bkt.Objects(ctx, &Query{StartOffset: name}) }, 1936 func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() }) 1937 if !ok { 1938 t.Errorf("ObjectIterator.Next: %s", msg) 1939 } 1940 // EndOffset takes the value of object names, the result must be for: 1941 // ― obj/with/slashes: "" 1942 // ― obj1: obj/with/slashes 1943 // ― obj2: obj/with/slashes, obj1. 1944 m[name] = attrs[:i] 1945 msg, ok = itesting.TestIterator(m[name], 1946 func() interface{} { return bkt.Objects(ctx, &Query{EndOffset: name}) }, 1947 func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() }) 1948 if !ok { 1949 t.Errorf("ObjectIterator.Next: %s", msg) 1950 } 1951 } 1952 } 1953 1954 func testObjectsIterateSelectedAttrs(t *testing.T, bkt *BucketHandle, objects []string) { 1955 // Create a query that will only select the "Name" attr of objects, and 1956 // invoke object listing. 1957 query := &Query{Prefix: ""} 1958 query.SetAttrSelection([]string{"Name"}) 1959 1960 var gotNames []string 1961 it := bkt.Objects(context.Background(), query) 1962 for { 1963 attrs, err := it.Next() 1964 if err == iterator.Done { 1965 break 1966 } 1967 if err != nil { 1968 t.Fatalf("iterator.Next: %v", err) 1969 } 1970 gotNames = append(gotNames, attrs.Name) 1971 1972 if len(attrs.Bucket) > 0 { 1973 t.Errorf("Bucket field not selected, want empty, got = %v", attrs.Bucket) 1974 } 1975 } 1976 1977 sortedNames := make([]string, len(objects)) 1978 copy(sortedNames, objects) 1979 sort.Strings(sortedNames) 1980 sort.Strings(gotNames) 1981 1982 if !cmp.Equal(sortedNames, gotNames) { 1983 t.Errorf("names = %v, want %v", gotNames, sortedNames) 1984 } 1985 } 1986 1987 func testObjectsIterateAllSelectedAttrs(t *testing.T, bkt *BucketHandle, objects []string) { 1988 // Tests that all selected attributes work - query succeeds (without actually 1989 // verifying the returned results). 1990 query := &Query{ 1991 Prefix: "", 1992 StartOffset: "obj/", 1993 EndOffset: "obj2", 1994 } 1995 var selectedAttrs []string 1996 for k := range attrToFieldMap { 1997 selectedAttrs = append(selectedAttrs, k) 1998 } 1999 query.SetAttrSelection(selectedAttrs) 2000 2001 count := 0 2002 it := bkt.Objects(context.Background(), query) 2003 for { 2004 _, err := it.Next() 2005 if err == iterator.Done { 2006 break 2007 } 2008 if err != nil { 2009 t.Fatalf("iterator.Next: %v", err) 2010 } 2011 count++ 2012 } 2013 2014 if count != len(objects)-1 { 2015 t.Errorf("count = %v, want %v", count, len(objects)-1) 2016 } 2017 } 2018 2019 func testObjectsIterateWithProjection(t *testing.T, bkt *BucketHandle) { 2020 projections := map[Projection]bool{ 2021 ProjectionDefault: true, 2022 ProjectionFull: true, 2023 ProjectionNoACL: false, 2024 } 2025 2026 for projection, expectACL := range projections { 2027 query := &Query{Projection: projection} 2028 it := bkt.Objects(context.Background(), query) 2029 attrs, err := it.Next() 2030 if err == iterator.Done { 2031 t.Fatalf("iterator: no objects") 2032 } 2033 if err != nil { 2034 t.Fatalf("iterator.Next: %v", err) 2035 } 2036 2037 if expectACL { 2038 if attrs.Owner == "" { 2039 t.Errorf("projection %q: Owner is empty, want nonempty Owner", projection) 2040 } 2041 if len(attrs.ACL) == 0 { 2042 t.Errorf("projection %q: ACL is empty, want at least one ACL rule", projection) 2043 } 2044 } else { 2045 if attrs.Owner != "" { 2046 t.Errorf("projection %q: got Owner = %q, want empty Owner", projection, attrs.Owner) 2047 } 2048 if len(attrs.ACL) != 0 { 2049 t.Errorf("projection %q: got %d ACL rules, want empty ACL", projection, len(attrs.ACL)) 2050 } 2051 } 2052 } 2053 } 2054 2055 func TestIntegration_SignedURL(t *testing.T) { 2056 multiTransportTest(skipJSONReads(context.Background(), "no reads in test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 2057 // To test SignedURL, we need a real user email and private key. Extract them 2058 // from the JSON key file. 2059 jwtConf, err := testutil.JWTConfig() 2060 if err != nil { 2061 t.Fatal(err) 2062 } 2063 if jwtConf == nil { 2064 t.Skip("JSON key file is not present") 2065 } 2066 2067 bkt := client.Bucket(bucket) 2068 obj := "signedURL" 2069 contents := []byte("This is a test of SignedURL.\n") 2070 md5 := "Jyxvgwm9n2MsrGTMPbMeYA==" // base64-encoded MD5 of contents 2071 if err := writeObject(ctx, bkt.Object(obj), "text/plain", contents); err != nil { 2072 t.Fatalf("writing: %v", err) 2073 } 2074 for _, test := range []struct { 2075 desc string 2076 opts SignedURLOptions 2077 headers map[string][]string 2078 fail bool 2079 }{ 2080 { 2081 desc: "basic v2", 2082 }, 2083 { 2084 desc: "basic v4", 2085 opts: SignedURLOptions{Scheme: SigningSchemeV4}, 2086 }, 2087 { 2088 desc: "MD5 sent and matches", 2089 opts: SignedURLOptions{MD5: md5}, 2090 headers: map[string][]string{"Content-MD5": {md5}}, 2091 }, 2092 { 2093 desc: "MD5 not sent", 2094 opts: SignedURLOptions{MD5: md5}, 2095 fail: true, 2096 }, 2097 { 2098 desc: "Content-Type sent and matches", 2099 opts: SignedURLOptions{ContentType: "text/plain"}, 2100 headers: map[string][]string{"Content-Type": {"text/plain"}}, 2101 }, 2102 { 2103 desc: "Content-Type sent but does not match", 2104 opts: SignedURLOptions{ContentType: "text/plain"}, 2105 headers: map[string][]string{"Content-Type": {"application/json"}}, 2106 fail: true, 2107 }, 2108 { 2109 desc: "Canonical headers sent and match", 2110 opts: SignedURLOptions{Headers: []string{ 2111 " X-Goog-Foo: Bar baz ", 2112 "X-Goog-Novalue", // ignored: no value 2113 "X-Google-Foo", // ignored: wrong prefix 2114 "x-goog-meta-start-time: 2023-02-10T02:00:00Z", // with colons 2115 }}, 2116 headers: map[string][]string{"X-Goog-foo": {"Bar baz "}, "x-goog-meta-start-time": {"2023-02-10T02:00:00Z"}}, 2117 }, 2118 { 2119 desc: "Canonical headers sent and match using V4", 2120 opts: SignedURLOptions{Headers: []string{ 2121 "x-goog-meta-start-time: 2023-02-10T02:", // with colons 2122 " X-Goog-Foo: Bar baz ", 2123 "X-Goog-Novalue", // ignored: no value 2124 "X-Google-Foo", // ignored: wrong prefix 2125 }, 2126 Scheme: SigningSchemeV4, 2127 }, 2128 headers: map[string][]string{"x-goog-meta-start-time": {"2023-02-10T02:"}, "X-Goog-foo": {"Bar baz "}}, 2129 }, 2130 { 2131 desc: "Canonical headers sent but don't match", 2132 opts: SignedURLOptions{Headers: []string{" X-Goog-Foo: Bar baz"}}, 2133 headers: map[string][]string{"X-Goog-Foo": {"bar baz"}}, 2134 fail: true, 2135 }, 2136 { 2137 desc: "Virtual hosted style with custom hostname", 2138 opts: SignedURLOptions{ 2139 Style: VirtualHostedStyle(), 2140 Hostname: "storage.googleapis.com:443", 2141 }, 2142 fail: false, 2143 }, 2144 { 2145 desc: "Hostname v4", 2146 opts: SignedURLOptions{ 2147 Hostname: "storage.googleapis.com:443", 2148 Scheme: SigningSchemeV4, 2149 }, 2150 fail: false, 2151 }, 2152 } { 2153 opts := test.opts 2154 opts.GoogleAccessID = jwtConf.Email 2155 opts.PrivateKey = jwtConf.PrivateKey 2156 opts.Method = "GET" 2157 opts.Expires = time.Now().Add(time.Hour) 2158 2159 u, err := bkt.SignedURL(obj, &opts) 2160 if err != nil { 2161 t.Errorf("%s: SignedURL: %v", test.desc, err) 2162 continue 2163 } 2164 2165 err = verifySignedURL(u, test.headers, contents) 2166 if err != nil && !test.fail { 2167 t.Errorf("%s: wanted success but got error:\n%v", test.desc, err) 2168 } else if err == nil && test.fail { 2169 t.Errorf("%s: wanted failure but test succeeded", test.desc) 2170 } 2171 } 2172 }) 2173 } 2174 2175 func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) { 2176 multiTransportTest(skipJSONReads(context.Background(), "no reads in test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 2177 2178 // To test SignedURL, we need a real user email and private key. Extract 2179 // them from the JSON key file. 2180 jwtConf, err := testutil.JWTConfig() 2181 if err != nil { 2182 t.Fatal(err) 2183 } 2184 if jwtConf == nil { 2185 t.Skip("JSON key file is not present") 2186 } 2187 2188 bkt := client.Bucket(bucket) 2189 2190 // TODO(deklerk): document how these were generated and their significance 2191 encryptionKey := "AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI=" 2192 encryptionKeySha256 := "QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k=" 2193 headers := map[string][]string{ 2194 "x-goog-encryption-algorithm": {"AES256"}, 2195 "x-goog-encryption-key": {encryptionKey}, 2196 "x-goog-encryption-key-sha256": {encryptionKeySha256}, 2197 } 2198 contents := []byte(`{"message":"encryption with csek works"}`) 2199 tests := []struct { 2200 desc string 2201 opts *SignedURLOptions 2202 }{ 2203 { 2204 desc: "v4 URL with customer supplied encryption keys for PUT", 2205 opts: &SignedURLOptions{ 2206 Method: "PUT", 2207 Headers: []string{ 2208 "x-goog-encryption-algorithm:AES256", 2209 "x-goog-encryption-key:AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI=", 2210 "x-goog-encryption-key-sha256:QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k=", 2211 }, 2212 Scheme: SigningSchemeV4, 2213 }, 2214 }, 2215 { 2216 desc: "v4 URL with customer supplied encryption keys for GET", 2217 opts: &SignedURLOptions{ 2218 Method: "GET", 2219 Headers: []string{ 2220 "x-goog-encryption-algorithm:AES256", 2221 fmt.Sprintf("x-goog-encryption-key:%s", encryptionKey), 2222 fmt.Sprintf("x-goog-encryption-key-sha256:%s", encryptionKeySha256), 2223 }, 2224 Scheme: SigningSchemeV4, 2225 }, 2226 }, 2227 } 2228 defer func() { 2229 // Delete encrypted object. 2230 err := bkt.Object("csek.json").Delete(ctx) 2231 if err != nil { 2232 log.Printf("failed to deleted encrypted file: %v", err) 2233 } 2234 }() 2235 2236 for _, test := range tests { 2237 opts := test.opts 2238 opts.GoogleAccessID = jwtConf.Email 2239 opts.PrivateKey = jwtConf.PrivateKey 2240 opts.Expires = time.Now().Add(time.Hour) 2241 2242 u, err := bkt.SignedURL("csek.json", test.opts) 2243 if err != nil { 2244 t.Fatalf("%s: %v", test.desc, err) 2245 } 2246 2247 if test.opts.Method == "PUT" { 2248 if _, err := putURL(u, headers, bytes.NewReader(contents)); err != nil { 2249 t.Fatalf("%s: %v", test.desc, err) 2250 } 2251 } 2252 2253 if test.opts.Method == "GET" { 2254 if err := verifySignedURL(u, headers, contents); err != nil { 2255 t.Fatalf("%s: %v", test.desc, err) 2256 } 2257 } 2258 } 2259 }) 2260 } 2261 2262 func TestIntegration_SignedURL_EmptyStringObjectName(t *testing.T) { 2263 multiTransportTest(skipJSONReads(context.Background(), "no reads in test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 2264 2265 // To test SignedURL, we need a real user email and private key. Extract them 2266 // from the JSON key file. 2267 jwtConf, err := testutil.JWTConfig() 2268 if err != nil { 2269 t.Fatal(err) 2270 } 2271 if jwtConf == nil { 2272 t.Skip("JSON key file is not present") 2273 } 2274 2275 opts := &SignedURLOptions{ 2276 Scheme: SigningSchemeV4, 2277 Method: "GET", 2278 GoogleAccessID: jwtConf.Email, 2279 PrivateKey: jwtConf.PrivateKey, 2280 Expires: time.Now().Add(time.Hour), 2281 } 2282 2283 bkt := client.Bucket(bucket) 2284 u, err := bkt.SignedURL("", opts) 2285 if err != nil { 2286 t.Fatal(err) 2287 } 2288 2289 // Should be some ListBucketResult response. 2290 _, err = getURL(u, nil) 2291 if err != nil { 2292 t.Fatal(err) 2293 } 2294 }) 2295 2296 } 2297 2298 func TestIntegration_BucketACL(t *testing.T) { 2299 ctx := skipJSONReads(context.Background(), "no reads in test") 2300 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 2301 h := testHelper{t} 2302 2303 bucket := prefix + uidSpace.New() 2304 bkt := client.Bucket(bucket) 2305 h.mustCreate(bkt, testutil.ProjID(), nil) 2306 defer h.mustDeleteBucket(bkt) 2307 2308 entity := ACLEntity("domain-google.com") 2309 rule := ACLRule{Entity: entity, Role: RoleReader, Domain: "google.com"} 2310 2311 if err := bkt.DefaultObjectACL().Set(ctx, entity, RoleReader); err != nil { 2312 t.Errorf("Can't put default ACL rule for the bucket, errored with %v", err) 2313 } 2314 2315 acl, err := bkt.DefaultObjectACL().List(ctx) 2316 if err != nil { 2317 t.Errorf("DefaultObjectACL.List for bucket %q: %v", bucket, err) 2318 } 2319 if !containsACLRule(acl, testACLRule(rule)) { 2320 t.Fatalf("default ACL rule missing; want: %#v, got rules: %+v", rule, acl) 2321 } 2322 2323 o := bkt.Object("acl1") 2324 defer h.mustDeleteObject(o) 2325 2326 // Retry to account for propagation delay in metadata update. 2327 err = retry(ctx, func() error { 2328 if err := writeObject(ctx, o, "", randomContents()); err != nil { 2329 return fmt.Errorf("Write for %v failed with %v", o.ObjectName(), err) 2330 } 2331 acl, err = o.ACL().List(ctx) 2332 return err 2333 }, func() error { 2334 if !containsACLRule(acl, testACLRule(rule)) { 2335 return fmt.Errorf("object ACL rule missing %+v from ACL \n%+v", rule, acl) 2336 } 2337 return nil 2338 }) 2339 if err != nil { 2340 t.Error(err) 2341 } 2342 2343 if err := o.ACL().Delete(ctx, entity); err != nil { 2344 t.Errorf("object ACL: could not delete entity %s", entity) 2345 } 2346 // Delete the default ACL rule. We can't move this code earlier in the 2347 // test, because the test depends on the fact that the object ACL inherits 2348 // it. 2349 if err := bkt.DefaultObjectACL().Delete(ctx, entity); err != nil { 2350 t.Errorf("default ACL: could not delete entity %s", entity) 2351 } 2352 2353 entity2 := AllAuthenticatedUsers 2354 rule2 := ACLRule{Entity: entity2, Role: RoleReader} 2355 if err := bkt.ACL().Set(ctx, entity2, RoleReader); err != nil { 2356 t.Errorf("Error while putting bucket ACL rule: %v", err) 2357 } 2358 2359 var bACL []ACLRule 2360 2361 // Retry to account for propagation delay in metadata update. 2362 err = retry(ctx, func() error { 2363 bACL, err = bkt.ACL().List(ctx) 2364 return err 2365 }, func() error { 2366 if !containsACLRule(bACL, testACLRule(rule2)) { 2367 return fmt.Errorf("bucket ACL missing %+v", rule2) 2368 } 2369 return nil 2370 }) 2371 if err != nil { 2372 t.Error(err) 2373 } 2374 2375 if err := bkt.ACL().Delete(ctx, entity2); err != nil { 2376 t.Errorf("Error while deleting bucket ACL rule: %v", err) 2377 } 2378 }) 2379 } 2380 2381 func TestIntegration_ValidObjectNames(t *testing.T) { 2382 ctx := skipJSONReads(context.Background(), "no reads in test") 2383 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 2384 bkt := client.Bucket(bucket) 2385 2386 validNames := []string{ 2387 "gopher", 2388 "Гоферови", 2389 "a", 2390 strings.Repeat("a", 1024), 2391 } 2392 for _, name := range validNames { 2393 if err := writeObject(ctx, bkt.Object(name), "", []byte("data")); err != nil { 2394 t.Errorf("Object %q write failed: %v. Want success", name, err) 2395 continue 2396 } 2397 defer bkt.Object(name).Delete(ctx) 2398 } 2399 2400 invalidNames := []string{ 2401 "", // Too short. 2402 strings.Repeat("a", 1025), // Too long. 2403 "new\nlines", 2404 "bad\xffunicode", 2405 } 2406 for _, name := range invalidNames { 2407 // Invalid object names will either cause failure during Write or Close. 2408 if err := writeObject(ctx, bkt.Object(name), "", []byte("data")); err != nil { 2409 continue 2410 } 2411 defer bkt.Object(name).Delete(ctx) 2412 t.Errorf("%q should have failed. Didn't", name) 2413 } 2414 }) 2415 } 2416 2417 func TestIntegration_WriterContentType(t *testing.T) { 2418 ctx := skipJSONReads(context.Background(), "no reads in test") 2419 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 2420 obj := client.Bucket(bucket).Object("content") 2421 testCases := []struct { 2422 content string 2423 setType, wantType string 2424 forceEmptyContentType bool 2425 }{ 2426 { 2427 // Sniffed content type. 2428 content: "It was the best of times, it was the worst of times.", 2429 wantType: "text/plain; charset=utf-8", 2430 }, 2431 { 2432 // Sniffed content type. 2433 content: "<html><head><title>My first page</title></head></html>", 2434 wantType: "text/html; charset=utf-8", 2435 }, 2436 { 2437 content: "<html><head><title>My first page</title></head></html>", 2438 setType: "text/html", 2439 wantType: "text/html", 2440 }, 2441 { 2442 content: "<html><head><title>My first page</title></head></html>", 2443 setType: "image/jpeg", 2444 wantType: "image/jpeg", 2445 }, 2446 { 2447 // Content type sniffing disabled. 2448 content: "<html><head><title>My first page</title></head></html>", 2449 setType: "", 2450 wantType: "", 2451 forceEmptyContentType: true, 2452 }, 2453 } 2454 for i, tt := range testCases { 2455 writer := newWriter(ctx, obj, tt.setType, tt.forceEmptyContentType) 2456 if err := writeContents(writer, []byte(tt.content)); err != nil { 2457 t.Errorf("writing #%d: %v", i, err) 2458 } 2459 attrs, err := obj.Attrs(ctx) 2460 if err != nil { 2461 t.Errorf("obj.Attrs: %v", err) 2462 continue 2463 } 2464 if got := attrs.ContentType; got != tt.wantType { 2465 t.Errorf("Content-Type = %q; want %q\nContent: %q\nSet Content-Type: %q", got, tt.wantType, tt.content, tt.setType) 2466 } 2467 } 2468 }) 2469 } 2470 2471 func TestIntegration_WriterChunksize(t *testing.T) { 2472 ctx := skipJSONReads(context.Background(), "no reads in test") 2473 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 2474 obj := client.Bucket(bucket).Object("writer-chunksize-test" + uidSpaceObjects.New()) 2475 objSize := 1<<10<<10 + 1 // 1 Mib + 1 byte 2476 contents := bytes.Repeat([]byte("a"), objSize) 2477 2478 for _, test := range []struct { 2479 desc string 2480 chunksize int 2481 wantBytesPerCall int64 2482 wantCallbacks int 2483 }{ 2484 { 2485 desc: "default chunksize", 2486 chunksize: 16 << 10 << 10, 2487 wantBytesPerCall: 16 << 10 << 10, 2488 wantCallbacks: 0, 2489 }, 2490 { 2491 desc: "small chunksize rounds up to 256kib", 2492 chunksize: 1, 2493 wantBytesPerCall: 256 << 10, 2494 wantCallbacks: 5, 2495 }, 2496 { 2497 desc: "chunksize of 256kib", 2498 chunksize: 256 << 10, 2499 wantBytesPerCall: 256 << 10, 2500 wantCallbacks: 5, 2501 }, 2502 { 2503 desc: "chunksize of just over 256kib rounds up", 2504 chunksize: 256<<10 + 1, 2505 wantBytesPerCall: 256 * 2 << 10, 2506 wantCallbacks: 3, 2507 }, 2508 { 2509 desc: "multiple of 256kib", 2510 chunksize: 256 * 3 << 10, 2511 wantBytesPerCall: 256 * 3 << 10, 2512 wantCallbacks: 2, 2513 }, 2514 { 2515 desc: "chunksize 0 uploads everything", 2516 chunksize: 0, 2517 wantBytesPerCall: int64(objSize), 2518 wantCallbacks: 0, 2519 }, 2520 } { 2521 t.Run(test.desc, func(t *testing.T) { 2522 t.Cleanup(func() { obj.Delete(ctx) }) 2523 2524 w := obj.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx) 2525 w.ChunkSize = test.chunksize 2526 2527 bytesWrittenSoFar := int64(0) 2528 callbacks := 0 2529 2530 w.ProgressFunc = func(i int64) { 2531 bytesWrittenByCall := i - bytesWrittenSoFar 2532 2533 // Error if this is not the last call and we don't write exactly wantBytesPerCall 2534 if i != int64(objSize) && bytesWrittenByCall != test.wantBytesPerCall { 2535 t.Errorf("unexpected number of bytes written by call; wanted: %d, written: %d", test.wantBytesPerCall, bytesWrittenByCall) 2536 } 2537 2538 bytesWrittenSoFar = i 2539 callbacks++ 2540 } 2541 2542 if _, err := w.Write(contents); err != nil { 2543 _ = w.Close() 2544 t.Fatalf("writer.Write: %v", err) 2545 } 2546 if err := w.Close(); err != nil { 2547 t.Fatalf("writer.Close: %v", err) 2548 } 2549 2550 if callbacks != test.wantCallbacks { 2551 t.Errorf("ProgressFunc was called %d times, expected %d", callbacks, test.wantCallbacks) 2552 } 2553 2554 // Confirm all bytes were uploaded. 2555 attrs, err := obj.Attrs(ctx) 2556 if err != nil { 2557 t.Fatalf("obj.Attrs: %v", err) 2558 } 2559 if attrs.Size != int64(objSize) { 2560 t.Errorf("incorrect number of bytes written; got %v, want %v", attrs.Size, objSize) 2561 } 2562 }) 2563 } 2564 }) 2565 } 2566 2567 func TestIntegration_ZeroSizedObject(t *testing.T) { 2568 t.Parallel() 2569 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 2570 h := testHelper{t} 2571 2572 obj := client.Bucket(bucket).Object("zero") 2573 2574 // Check writing it works as expected. 2575 w := obj.NewWriter(ctx) 2576 if err := w.Close(); err != nil { 2577 t.Fatalf("Writer.Close: %v", err) 2578 } 2579 defer obj.Delete(ctx) 2580 2581 // Check we can read it too. 2582 body := h.mustRead(obj) 2583 if len(body) != 0 { 2584 t.Errorf("Body is %v, want empty []byte{}", body) 2585 } 2586 }) 2587 } 2588 2589 func TestIntegration_Encryption(t *testing.T) { 2590 // This function tests customer-supplied encryption keys for all operations 2591 // involving objects. Bucket and ACL operations aren't tested because they 2592 // aren't affected by customer encryption. Neither is deletion. 2593 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 2594 h := testHelper{t} 2595 2596 obj := client.Bucket(bucket).Object("customer-encryption") 2597 key := []byte("my-secret-AES-256-encryption-key") 2598 keyHash := sha256.Sum256(key) 2599 keyHashB64 := base64.StdEncoding.EncodeToString(keyHash[:]) 2600 key2 := []byte("My-Secret-AES-256-Encryption-Key") 2601 contents := "top secret." 2602 2603 checkMetadataCall := func(msg string, f func(o *ObjectHandle) (*ObjectAttrs, error)) { 2604 // Performing a metadata operation without the key should succeed. 2605 attrs, err := f(obj) 2606 if err != nil { 2607 t.Fatalf("%s: %v", msg, err) 2608 } 2609 // The key hash should match... 2610 if got, want := attrs.CustomerKeySHA256, keyHashB64; got != want { 2611 t.Errorf("%s: key hash: got %q, want %q", msg, got, want) 2612 } 2613 // ...but CRC and MD5 should not be present. 2614 if attrs.CRC32C != 0 { 2615 t.Errorf("%s: CRC: got %v, want 0", msg, attrs.CRC32C) 2616 } 2617 if len(attrs.MD5) > 0 { 2618 t.Errorf("%s: MD5: got %v, want len == 0", msg, attrs.MD5) 2619 } 2620 2621 // Performing a metadata operation with the key should succeed. 2622 attrs, err = f(obj.Key(key)) 2623 if err != nil { 2624 t.Fatalf("%s: %v", msg, err) 2625 } 2626 // Check the key and content hashes. 2627 if got, want := attrs.CustomerKeySHA256, keyHashB64; got != want { 2628 t.Errorf("%s: key hash: got %q, want %q", msg, got, want) 2629 } 2630 if attrs.CRC32C == 0 { 2631 t.Errorf("%s: CRC: got 0, want non-zero", msg) 2632 } 2633 if len(attrs.MD5) == 0 { 2634 t.Errorf("%s: MD5: got len == 0, want len > 0", msg) 2635 } 2636 } 2637 2638 checkRead := func(msg string, o *ObjectHandle, k []byte, wantContents string) { 2639 // Reading the object without the key should fail. 2640 if _, err := readObject(ctx, o); err == nil { 2641 t.Errorf("%s: reading without key: want error, got nil", msg) 2642 } 2643 // Reading the object with the key should succeed. 2644 got := h.mustRead(o.Key(k)) 2645 gotContents := string(got) 2646 // And the contents should match what we wrote. 2647 if gotContents != wantContents { 2648 t.Errorf("%s: contents: got %q, want %q", msg, gotContents, wantContents) 2649 } 2650 } 2651 2652 checkReadUnencrypted := func(msg string, obj *ObjectHandle, wantContents string) { 2653 got := h.mustRead(obj) 2654 gotContents := string(got) 2655 if gotContents != wantContents { 2656 t.Errorf("%s: got %q, want %q", msg, gotContents, wantContents) 2657 } 2658 } 2659 2660 // Write to obj using our own encryption key, which is a valid 32-byte 2661 // AES-256 key. 2662 h.mustWrite(obj.Key(key).NewWriter(ctx), []byte(contents)) 2663 2664 checkMetadataCall("Attrs", func(o *ObjectHandle) (*ObjectAttrs, error) { 2665 return o.Attrs(ctx) 2666 }) 2667 2668 checkMetadataCall("Update", func(o *ObjectHandle) (*ObjectAttrs, error) { 2669 return o.Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"}) 2670 }) 2671 2672 checkRead("first object", obj, key, contents) 2673 2674 // We create 2 objects here and we can interleave operations to get around 2675 // the rate limit for object mutation operations (create, update, and delete). 2676 obj2 := client.Bucket(bucket).Object("customer-encryption-2") 2677 obj4 := client.Bucket(bucket).Object("customer-encryption-4") 2678 2679 // Copying an object without the key should fail. 2680 if _, err := obj4.CopierFrom(obj).Run(ctx); err == nil { 2681 t.Fatal("want error, got nil") 2682 } 2683 // Copying an object with the key should succeed. 2684 if _, err := obj2.CopierFrom(obj.Key(key)).Run(ctx); err != nil { 2685 t.Fatal(err) 2686 } 2687 // The destination object is not encrypted; we can read it without a key. 2688 checkReadUnencrypted("copy dest", obj2, contents) 2689 2690 // Providing a key on the destination but not the source should fail, 2691 // since the source is encrypted. 2692 if _, err := obj2.Key(key2).CopierFrom(obj).Run(ctx); err == nil { 2693 t.Fatal("want error, got nil") 2694 } 2695 2696 // But copying with keys for both source and destination should succeed. 2697 if _, err := obj2.Key(key2).CopierFrom(obj.Key(key)).Run(ctx); err != nil { 2698 t.Fatal(err) 2699 } 2700 // And the destination should be encrypted, meaning we can only read it 2701 // with a key. 2702 checkRead("copy destination", obj2, key2, contents) 2703 2704 // Change obj2's key to prepare for compose, where all objects must have 2705 // the same key. Also illustrates key rotation: copy an object to itself 2706 // with a different key. 2707 if _, err := obj2.Key(key).CopierFrom(obj2.Key(key2)).Run(ctx); err != nil { 2708 t.Fatal(err) 2709 } 2710 obj3 := client.Bucket(bucket).Object("customer-encryption-3") 2711 // Composing without keys should fail. 2712 if _, err := obj3.ComposerFrom(obj, obj2).Run(ctx); err == nil { 2713 t.Fatal("want error, got nil") 2714 } 2715 // Keys on the source objects result in an error. 2716 if _, err := obj3.ComposerFrom(obj.Key(key), obj2).Run(ctx); err == nil { 2717 t.Fatal("want error, got nil") 2718 } 2719 // A key on the destination object both decrypts the source objects 2720 // and encrypts the destination. 2721 if _, err := obj3.Key(key).ComposerFrom(obj, obj2).Run(ctx); err != nil { 2722 t.Fatalf("got %v, want nil", err) 2723 } 2724 // Check that the destination in encrypted. 2725 checkRead("compose destination", obj3, key, contents+contents) 2726 2727 // You can't compose one or more unencrypted source objects into an 2728 // encrypted destination object. 2729 _, err := obj4.CopierFrom(obj2.Key(key)).Run(ctx) // unencrypt obj2 2730 if err != nil { 2731 t.Fatal(err) 2732 } 2733 if _, err := obj3.Key(key).ComposerFrom(obj4).Run(ctx); err == nil { 2734 t.Fatal("got nil, want error") 2735 } 2736 }) 2737 } 2738 2739 func TestIntegration_NonexistentObjectRead(t *testing.T) { 2740 t.Parallel() 2741 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 2742 _, err := client.Bucket(bucket).Object("object-does-not-exist").NewReader(ctx) 2743 if !errors.Is(err, ErrObjectNotExist) { 2744 t.Errorf("Objects: got %v, want ErrObjectNotExist", err) 2745 } 2746 }) 2747 } 2748 2749 func TestIntegration_NonexistentBucket(t *testing.T) { 2750 t.Parallel() 2751 ctx := skipJSONReads(context.Background(), "no reads in test") 2752 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) { 2753 bkt := client.Bucket(prefix + uidSpace.New()) 2754 if _, err := bkt.Attrs(ctx); err != ErrBucketNotExist { 2755 t.Errorf("Attrs: got %v, want ErrBucketNotExist", err) 2756 } 2757 it := bkt.Objects(ctx, nil) 2758 if _, err := it.Next(); err != ErrBucketNotExist { 2759 t.Errorf("Objects: got %v, want ErrBucketNotExist", err) 2760 } 2761 }) 2762 } 2763 2764 func TestIntegration_PerObjectStorageClass(t *testing.T) { 2765 const ( 2766 defaultStorageClass = "STANDARD" 2767 newStorageClass = "NEARLINE" 2768 ) 2769 ctx := skipJSONReads(context.Background(), "no reads in test") 2770 2771 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 2772 h := testHelper{t} 2773 2774 bkt := client.Bucket(bucket) 2775 2776 // The bucket should have the default storage class. 2777 battrs := h.mustBucketAttrs(bkt) 2778 if battrs.StorageClass != defaultStorageClass { 2779 t.Fatalf("bucket storage class: got %q, want %q", 2780 battrs.StorageClass, defaultStorageClass) 2781 } 2782 // Write an object; it should start with the bucket's storage class. 2783 obj := bkt.Object("posc") 2784 h.mustWrite(obj.NewWriter(ctx), []byte("foo")) 2785 oattrs, err := obj.Attrs(ctx) 2786 if err != nil { 2787 t.Fatal(err) 2788 } 2789 if oattrs.StorageClass != defaultStorageClass { 2790 t.Fatalf("object storage class: got %q, want %q", 2791 oattrs.StorageClass, defaultStorageClass) 2792 } 2793 // Now use Copy to change the storage class. 2794 copier := obj.CopierFrom(obj) 2795 copier.StorageClass = newStorageClass 2796 oattrs2, err := copier.Run(ctx) 2797 if err != nil { 2798 log.Fatal(err) 2799 } 2800 if oattrs2.StorageClass != newStorageClass { 2801 t.Fatalf("new object storage class: got %q, want %q", 2802 oattrs2.StorageClass, newStorageClass) 2803 } 2804 2805 // We can also write a new object using a non-default storage class. 2806 obj2 := bkt.Object("posc2") 2807 w := obj2.NewWriter(ctx) 2808 w.StorageClass = newStorageClass 2809 h.mustWrite(w, []byte("xxx")) 2810 if w.Attrs().StorageClass != newStorageClass { 2811 t.Fatalf("new object storage class: got %q, want %q", 2812 w.Attrs().StorageClass, newStorageClass) 2813 } 2814 }) 2815 } 2816 2817 func TestIntegration_NoUnicodeNormalization(t *testing.T) { 2818 t.Parallel() 2819 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 2820 bkt := client.Bucket(bucket) 2821 h := testHelper{t} 2822 2823 for _, tst := range []struct { 2824 nameQuoted, content string 2825 }{ 2826 {`"Caf\u00e9"`, "Normalization Form C"}, 2827 {`"Cafe\u0301"`, "Normalization Form D"}, 2828 } { 2829 name, err := strconv.Unquote(tst.nameQuoted) 2830 w := bkt.Object(name).NewWriter(ctx) 2831 h.mustWrite(w, []byte(tst.content)) 2832 if err != nil { 2833 t.Fatalf("invalid name: %s: %v", tst.nameQuoted, err) 2834 } 2835 if got := string(h.mustRead(bkt.Object(name))); got != tst.content { 2836 t.Errorf("content of %s is %q, want %q", tst.nameQuoted, got, tst.content) 2837 } 2838 } 2839 }) 2840 } 2841 2842 func TestIntegration_HashesOnUpload(t *testing.T) { 2843 // Check that the user can provide hashes on upload, and that these are checked. 2844 ctx := skipJSONReads(context.Background(), "no reads in test") 2845 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 2846 obj := client.Bucket(bucket).Object("hashesOnUpload-1") 2847 data := []byte("I can't wait to be verified") 2848 2849 write := func(w *Writer) error { 2850 if _, err := w.Write(data); err != nil { 2851 _ = w.Close() 2852 return err 2853 } 2854 return w.Close() 2855 } 2856 2857 crc32c := crc32.Checksum(data, crc32cTable) 2858 // The correct CRC should succeed. 2859 w := obj.NewWriter(ctx) 2860 w.CRC32C = crc32c 2861 w.SendCRC32C = true 2862 if err := write(w); err != nil { 2863 t.Error(err) 2864 } 2865 2866 // If we change the CRC, validation should fail. 2867 w = obj.NewWriter(ctx) 2868 w.CRC32C = crc32c + 1 2869 w.SendCRC32C = true 2870 if err := write(w); err == nil { 2871 t.Error("write with bad CRC32c: want error, got nil") 2872 } 2873 2874 // If we have the wrong CRC but forget to send it, we succeed. 2875 w = obj.NewWriter(ctx) 2876 w.CRC32C = crc32c + 1 2877 if err := write(w); err != nil { 2878 t.Error(err) 2879 } 2880 2881 // MD5 2882 md5 := md5.Sum(data) 2883 // The correct MD5 should succeed. 2884 w = obj.NewWriter(ctx) 2885 w.MD5 = md5[:] 2886 if err := write(w); err != nil { 2887 t.Error(err) 2888 } 2889 2890 // If we change the MD5, validation should fail. 2891 w = obj.NewWriter(ctx) 2892 w.MD5 = append([]byte(nil), md5[:]...) 2893 w.MD5[0]++ 2894 if err := write(w); err == nil { 2895 t.Error("write with bad MD5: want error, got nil") 2896 } 2897 }) 2898 } 2899 2900 func TestIntegration_BucketIAM(t *testing.T) { 2901 ctx := skipJSONReads(context.Background(), "no reads in test") 2902 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) { 2903 h := testHelper{t} 2904 bkt := client.Bucket(prefix + uidSpace.New()) 2905 h.mustCreate(bkt, testutil.ProjID(), nil) 2906 defer h.mustDeleteBucket(bkt) 2907 // This bucket is unique to this test run. So we don't have 2908 // to worry about other runs interfering with our IAM policy 2909 // changes. 2910 2911 member := "projectViewer:" + testutil.ProjID() 2912 role := iam.RoleName("roles/storage.objectViewer") 2913 // Get the bucket's IAM policy. 2914 policy, err := bkt.IAM().Policy(ctx) 2915 if err != nil { 2916 t.Fatalf("Getting policy: %v", err) 2917 } 2918 // The member should not have the role. 2919 if policy.HasRole(member, role) { 2920 t.Errorf("member %q has role %q", member, role) 2921 } 2922 // Change the policy. 2923 policy.Add(member, role) 2924 if err := bkt.IAM().SetPolicy(ctx, policy); err != nil { 2925 t.Fatalf("SetPolicy: %v", err) 2926 } 2927 // Confirm that the binding was added. 2928 policy, err = bkt.IAM().Policy(ctx) 2929 if err != nil { 2930 t.Fatalf("Getting policy: %v", err) 2931 } 2932 if !policy.HasRole(member, role) { 2933 t.Errorf("member %q does not have role %q", member, role) 2934 } 2935 2936 // Check TestPermissions. 2937 // This client should have all these permissions (and more). 2938 perms := []string{"storage.buckets.get", "storage.buckets.delete"} 2939 got, err := bkt.IAM().TestPermissions(ctx, perms) 2940 if err != nil { 2941 t.Fatalf("TestPermissions: %v", err) 2942 } 2943 sort.Strings(perms) 2944 sort.Strings(got) 2945 if !testutil.Equal(got, perms) { 2946 t.Errorf("got %v, want %v", got, perms) 2947 } 2948 }) 2949 } 2950 2951 // This test tests only possibilities where the user making the request is an 2952 // owner on the project that owns the requester pays bucket. Therefore, we don't 2953 // need a second project for this test. 2954 // 2955 // There are up to three entities involved in a requester-pays call: 2956 // 2957 // 1. The user making the request. Here, we use the account used as credentials 2958 // for most of our integration tests. The following must hold for this test: 2959 // - this user must have resourcemanager.projects.createBillingAssignment 2960 // permission (Owner role) on (2) (the project, not the bucket) 2961 // - this user must NOT have that permission on (3b). 2962 // 2. The project that owns the requester-pays bucket. Here, that 2963 // is the test project ID (see testutil.ProjID). 2964 // 3. The project provided as the userProject parameter of the request; 2965 // the project to be billed. This test uses: 2966 // a. The project that owns the requester-pays bucket (same as (2)) 2967 // b. Another project (the Firestore project). 2968 func TestIntegration_RequesterPaysOwner(t *testing.T) { 2969 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) { 2970 jwt, err := testutil.JWTConfig() 2971 if err != nil { 2972 t.Fatalf("testutil.JWTConfig: %v", err) 2973 } 2974 // an account that has permissions on the project that owns the bucket 2975 mainUserEmail := jwt.Email 2976 2977 // the project that owns the requester-pays bucket 2978 mainProjectID := testutil.ProjID() 2979 2980 client.SetRetry(WithPolicy(RetryAlways)) 2981 2982 // Secondary project: a project that does not own the bucket. 2983 // The "main" user should not have permission on this. 2984 secondaryProject := os.Getenv(envFirestoreProjID) 2985 if secondaryProject == "" { 2986 t.Fatalf("need a second project (env var %s)", envFirestoreProjID) 2987 } 2988 2989 for _, test := range []struct { 2990 desc string 2991 userProject *string // to set on bucket, nil if it should not be set 2992 expectSuccess bool 2993 }{ 2994 { 2995 desc: "user is Owner on the project that owns the bucket", 2996 userProject: nil, 2997 expectSuccess: true, // by the rule permitting access by owners of the containing bucket 2998 }, 2999 { 3000 desc: "userProject is unnecessary but allowed", 3001 userProject: &mainProjectID, 3002 expectSuccess: true, // by the rule permitting access by owners of the containing bucket 3003 }, 3004 { 3005 desc: "cannot use someone else's project for billing", 3006 userProject: &secondaryProject, 3007 expectSuccess: false, // we cannot use a project we don't have access to for billing 3008 }, 3009 } { 3010 t.Run(test.desc, func(t *testing.T) { 3011 h := testHelper{t} 3012 ctx, cancel := context.WithTimeout(ctx, 30*time.Second) 3013 defer cancel() 3014 3015 printTestCase := func() string { 3016 userProject := "none" 3017 if test.userProject != nil { 3018 userProject = *test.userProject 3019 } 3020 return fmt.Sprintf("user: %s\n\t\tcontaining project: %s\n\t\tUserProject: %s", mainUserEmail, mainProjectID, userProject) 3021 } 3022 3023 checkforErrors := func(desc string, err error) { 3024 if err != nil && test.expectSuccess { 3025 t.Errorf("%s: got unexpected error:%v\n\t\t%s", desc, err, printTestCase()) 3026 } else if err == nil && !test.expectSuccess { 3027 t.Errorf("%s: got unexpected success\n\t\t%s", desc, printTestCase()) 3028 } 3029 } 3030 3031 bucketName := prefix + uidSpace.New() 3032 requesterPaysBucket := client.Bucket(bucketName) 3033 3034 // Create a requester-pays bucket 3035 h.mustCreate(requesterPaysBucket, mainProjectID, &BucketAttrs{RequesterPays: true}) 3036 t.Cleanup(func() { h.mustDeleteBucket(requesterPaysBucket) }) 3037 3038 // Make sure the object exists, so we don't get confused by ErrObjectNotExist. 3039 // The later write we perform may fail so we always write to the object as the user 3040 // with permissions on the containing bucket (mainUser). 3041 // The storage service may perform validation in any order (perhaps in parallel), 3042 // so if we delete or update an object that doesn't exist and for which we lack permission, 3043 // we could see either of those two errors. (See Google-internal bug 78341001.) 3044 objectName := "acl-go-test" + uidSpaceObjects.New() 3045 h.mustWrite(requesterPaysBucket.Object(objectName).NewWriter(ctx), []byte("hello")) 3046 3047 // Set up the bucket to use depending on the test case 3048 bucket := client.Bucket(bucketName) 3049 if test.userProject != nil { 3050 bucket = bucket.UserProject(*test.userProject) 3051 } 3052 3053 // Get bucket attrs 3054 attrs, err := bucket.Attrs(ctx) 3055 checkforErrors("get bucket attrs", err) 3056 if attrs != nil { 3057 if got, want := attrs.RequesterPays, true; got != want { 3058 t.Fatalf("attr.RequesterPays = %t, want %t", got, want) 3059 } 3060 } 3061 3062 // Bucket ACL operations 3063 entity := ACLEntity("domain-google.com") 3064 3065 checkforErrors("bucket acl set", bucket.ACL().Set(ctx, entity, RoleReader)) 3066 _, err = bucket.ACL().List(ctx) 3067 checkforErrors("bucket acl list", err) 3068 checkforErrors("bucket acl delete", bucket.ACL().Delete(ctx, entity)) 3069 3070 // Object operations (except for delete) 3071 // Retry to account for propagation delay to objects in metadata update 3072 // (we updated the metadata to add the otherUserEmail as owner on the bucket) 3073 o := bucket.Object(objectName) 3074 ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Second*10) 3075 defer cancel() 3076 // Only retry when we expect success to avoid retrying for 10 seconds 3077 // when we know it will fail 3078 if test.expectSuccess { 3079 o = o.Retryer(WithErrorFunc(retryOnTransient400and403)) 3080 } 3081 checkforErrors("write object", writeObject(ctxWithTimeout, o, "text/plain", []byte("hello"))) 3082 _, err = readObject(ctx, bucket.Object(objectName)) 3083 checkforErrors("read object", err) 3084 _, err = bucket.Object(objectName).Attrs(ctx) 3085 checkforErrors("get object attrs", err) 3086 _, err = bucket.Object(objectName).Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"}) 3087 checkforErrors("update object", err) 3088 3089 // Object ACL operations 3090 checkforErrors("object acl set", bucket.Object(objectName).ACL().Set(ctx, entity, RoleReader)) 3091 _, err = bucket.Object(objectName).ACL().List(ctx) 3092 checkforErrors("object acl list", err) 3093 checkforErrors("object acl list", bucket.Object(objectName).ACL().Delete(ctx, entity)) 3094 3095 // Default object ACL operations 3096 // Once again, we interleave buckets to avoid rate limits 3097 checkforErrors("default object acl set", bucket.DefaultObjectACL().Set(ctx, entity, RoleReader)) 3098 _, err = bucket.DefaultObjectACL().List(ctx) 3099 checkforErrors("default object acl list", err) 3100 checkforErrors("default object acl delete", bucket.DefaultObjectACL().Delete(ctx, entity)) 3101 3102 // Copy 3103 _, err = bucket.Object("copy").CopierFrom(bucket.Object(objectName)).Run(ctx) 3104 checkforErrors("copy", err) 3105 // Delete "copy" object, if created 3106 if err == nil { 3107 t.Cleanup(func() { 3108 h.mustDeleteObject(bucket.Object("copy")) 3109 }) 3110 } 3111 3112 // Compose 3113 _, err = bucket.Object("compose").ComposerFrom(bucket.Object(objectName), bucket.Object("copy")).Run(ctx) 3114 checkforErrors("compose", err) 3115 // Delete "compose" object, if created 3116 if err == nil { 3117 t.Cleanup(func() { 3118 h.mustDeleteObject(bucket.Object("compose")) 3119 }) 3120 } 3121 3122 // Delete object 3123 if err = bucket.Object(objectName).Delete(ctx); err != nil { 3124 // We still want to delete object if the test errors 3125 h.mustDeleteObject(requesterPaysBucket.Object(objectName)) 3126 } 3127 checkforErrors("delete object", err) 3128 }) 3129 } 3130 }) 3131 } 3132 3133 // This test needs a second project and user to test all possibilities. Since we 3134 // need these things for Firestore already, we use them here. 3135 // 3136 // There are up to three entities involved in a requester-pays call: 3137 // 1. The user making the request. Here, we use the account used for the 3138 // Firestore tests. The following must hold for this test to work: 3139 // - this user must NOT have resourcemanager.projects.createBillingAssignment 3140 // on the project that owns the bucket (2). 3141 // - this user must have serviceusage.services.use permission on the Firestore 3142 // project (3b). 3143 // - this user must NOT have that serviceusage.services.use permission on 3144 // the project that owns the bucket (3a). 3145 // 2. The project that owns the requester-pays bucket. Here, that 3146 // is the test project ID (see testutil.ProjID). 3147 // 3. The project provided as the userProject parameter of the request; 3148 // the project to be billed. This test uses: 3149 // a. The project that owns the requester-pays bucket (same as (2)) 3150 // b. Another project (the Firestore project). 3151 func TestIntegration_RequesterPaysNonOwner(t *testing.T) { 3152 if testing.Short() && !replaying { 3153 t.Skip("Integration tests skipped in short mode") 3154 } 3155 ctx := context.Background() 3156 3157 // Main project: the project that owns the requester-pays bucket. 3158 mainProject := testutil.ProjID() 3159 3160 // Secondary project: a project that does not own the bucket. 3161 // The "main" user does not have permission on this. 3162 // This project should have billing enabled. 3163 secondaryProject := os.Getenv(envFirestoreProjID) 3164 if secondaryProject == "" { 3165 t.Fatalf("need a second project (env var %s)", envFirestoreProjID) 3166 } 3167 3168 // Secondary email: an account with permissions on the secondary project, 3169 // but not on the main project. 3170 // We will grant this email permissions to the bucket created under the main 3171 // project, but it must provide a user project to make requests 3172 // against that bucket (since it's a requester-pays bucket). 3173 secondaryUserEmail, err := keyFileEmail(os.Getenv(envFirestorePrivateKey)) 3174 if err != nil { 3175 t.Fatalf("keyFileEmail error getting second account (env var %s): %v", envFirestorePrivateKey, err) 3176 } 3177 3178 // Token source from secondary email to authenticate to client 3179 ts := testutil.TokenSourceEnv(ctx, envFirestorePrivateKey, ScopeFullControl) 3180 if ts == nil { 3181 t.Fatalf("need a second account (env var %s)", envFirestorePrivateKey) 3182 } 3183 3184 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) { 3185 client.SetRetry(WithPolicy(RetryAlways)) 3186 3187 for _, test := range []struct { 3188 desc string 3189 userProject *string // to set on bucket, nil if it should not be set 3190 expectSuccess bool 3191 wantErrorCode int 3192 wantErrorCodeGRPC codes.Code 3193 }{ 3194 { 3195 desc: "no UserProject", 3196 userProject: nil, 3197 expectSuccess: false, // by the standard requester-pays rule 3198 }, 3199 { 3200 desc: "user is an Editor on UserProject", 3201 userProject: &secondaryProject, 3202 expectSuccess: true, // by the standard requester-pays rule 3203 }, 3204 { 3205 desc: "user is not an Editor on UserProject", 3206 userProject: &mainProject, 3207 expectSuccess: false, // we cannot use a project we don't have access to for billing 3208 wantErrorCode: 403, 3209 wantErrorCodeGRPC: codes.PermissionDenied, 3210 }, 3211 } { 3212 t.Run(test.desc, func(t *testing.T) { 3213 ctx, cancel := context.WithTimeout(ctx, 30*time.Second) 3214 t.Cleanup(cancel) 3215 3216 printTestCase := func() string { 3217 userProject := "none" 3218 if test.userProject != nil { 3219 userProject = *test.userProject 3220 } 3221 return fmt.Sprintf("user: %s\n\t\tcontaining project: %s\n\t\tUserProject: %s", secondaryUserEmail, mainProject, userProject) 3222 } 3223 3224 checkforErrors := func(desc string, err error) { 3225 errCode := extractErrCode(err) 3226 if err != nil && test.expectSuccess { 3227 t.Errorf("%s: got unexpected error:%v\n\t\t%s", desc, err, printTestCase()) 3228 } else if err == nil && !test.expectSuccess { 3229 t.Errorf("%s: got unexpected success\n\t\t%s", desc, printTestCase()) 3230 } else if !test.expectSuccess && test.wantErrorCode != 0 { 3231 if (status.Code(err) != codes.OK && status.Code(err) != codes.Unknown && status.Code(err) != test.wantErrorCodeGRPC) || (errCode > 0 && errCode != test.wantErrorCode) { 3232 fmt.Println(status.Code(err), " ", status.Code(err) != test.wantErrorCodeGRPC) 3233 t.Errorf("%s: mismatched errors; want error code: %d or grpc error: %s, got error: %v \n\t\t%s\n", 3234 desc, test.wantErrorCode, test.wantErrorCodeGRPC, err, printTestCase()) 3235 } 3236 } 3237 } 3238 3239 bucketName := prefix + uidSpace.New() 3240 objectName := "acl-go-test" + uidSpaceObjects.New() 3241 3242 setUpRequesterPaysBucket(ctx, t, bucketName, objectName, secondaryUserEmail) 3243 3244 // Set up the bucket to use depending on the test case 3245 bucket := client.Bucket(bucketName) 3246 if test.userProject != nil { 3247 bucket = bucket.UserProject(*test.userProject) 3248 } 3249 3250 // Get bucket attrs 3251 attrs, err := bucket.Attrs(ctx) 3252 checkforErrors("get bucket attrs", err) 3253 if attrs != nil { 3254 if got, want := attrs.RequesterPays, true; got != want { 3255 t.Fatalf("attr.RequesterPays = %t, want %t", got, want) 3256 } 3257 } 3258 3259 // Bucket ACL operations 3260 entity := ACLEntity("domain-google.com") 3261 3262 checkforErrors("bucket acl set", bucket.ACL().Set(ctx, entity, RoleReader)) 3263 _, err = bucket.ACL().List(ctx) 3264 checkforErrors("bucket acl list", err) 3265 checkforErrors("bucket acl delete", bucket.ACL().Delete(ctx, entity)) 3266 3267 // Object operations (except for delete) 3268 // Retry to account for propagation delay to objects in metadata update 3269 // (we updated the metadata to add the otherUserEmail as owner on the bucket) 3270 o := bucket.Object(objectName) 3271 ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Second*15) 3272 defer cancel() 3273 // Only retry when we expect success to avoid retrying 3274 // when we know it will fail 3275 if test.expectSuccess { 3276 o = o.Retryer(WithErrorFunc(retryOnTransient400and403)) 3277 } 3278 checkforErrors("write object", writeObject(ctxWithTimeout, o, "text/plain", []byte("hello"))) 3279 _, err = readObject(ctx, bucket.Object(objectName)) 3280 checkforErrors("read object", err) 3281 _, err = bucket.Object(objectName).Attrs(ctx) 3282 checkforErrors("get object attrs", err) 3283 _, err = bucket.Object(objectName).Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"}) 3284 checkforErrors("update object", err) 3285 3286 // Object ACL operations 3287 checkforErrors("object acl set", bucket.Object(objectName).ACL().Set(ctx, entity, RoleReader)) 3288 _, err = bucket.Object(objectName).ACL().List(ctx) 3289 checkforErrors("object acl list", err) 3290 checkforErrors("object acl list", bucket.Object(objectName).ACL().Delete(ctx, entity)) 3291 3292 // Default object ACL operations 3293 // Once again, we interleave buckets to avoid rate limits 3294 checkforErrors("default object acl set", bucket.DefaultObjectACL().Set(ctx, entity, RoleReader)) 3295 _, err = bucket.DefaultObjectACL().List(ctx) 3296 checkforErrors("default object acl list", err) 3297 checkforErrors("default object acl delete", bucket.DefaultObjectACL().Delete(ctx, entity)) 3298 3299 // Copy 3300 copyObj := bucket.Object("copy") 3301 _, err = copyObj.CopierFrom(bucket.Object(objectName)).Run(ctx) 3302 checkforErrors("copy", err) 3303 // Delete "copy" object, if created 3304 if err == nil { 3305 t.Cleanup(func() { 3306 if err := deleteObjectIfExists(copyObj, WithErrorFunc(retryOnTransient400and403)); err != nil { 3307 t.Error(err) 3308 } 3309 }) 3310 } 3311 3312 // Compose 3313 composeObj := bucket.Object("compose") 3314 _, err = composeObj.ComposerFrom(bucket.Object(objectName), bucket.Object("copy")).Run(ctx) 3315 checkforErrors("compose", err) 3316 // Delete "compose" object, if created 3317 if err == nil { 3318 t.Cleanup(func() { 3319 if err := deleteObjectIfExists(composeObj, WithErrorFunc(retryOnTransient400and403)); err != nil { 3320 t.Error(err) 3321 } 3322 }) 3323 } 3324 3325 // Delete object 3326 checkforErrors("delete object", bucket.Object(objectName).Delete(ctx)) 3327 }) 3328 } 3329 }, option.WithTokenSource(ts)) 3330 } 3331 3332 func TestIntegration_Notifications(t *testing.T) { 3333 multiTransportTest(skipGRPC("notifications not implemented"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 3334 bkt := client.Bucket(bucket) 3335 3336 checkNotifications := func(msg string, want map[string]*Notification) { 3337 got, err := bkt.Notifications(ctx) 3338 if err != nil { 3339 t.Fatal(err) 3340 } 3341 if diff := testutil.Diff(got, want); diff != "" { 3342 t.Errorf("%s: got=-, want=+:\n%s", msg, diff) 3343 } 3344 } 3345 checkNotifications("initial", map[string]*Notification{}) 3346 3347 nArg := &Notification{ 3348 TopicProjectID: testutil.ProjID(), 3349 TopicID: "go-storage-notification-test", 3350 PayloadFormat: NoPayload, 3351 } 3352 n, err := bkt.AddNotification(ctx, nArg) 3353 if err != nil { 3354 t.Fatal(err) 3355 } 3356 if n.ID == "" { 3357 t.Fatal("expected created Notification to have non-empty ID") 3358 } 3359 nArg.ID = n.ID 3360 if !testutil.Equal(n, nArg) { 3361 t.Errorf("got %+v, want %+v", n, nArg) 3362 } 3363 checkNotifications("after add", map[string]*Notification{n.ID: n}) 3364 3365 if err := bkt.DeleteNotification(ctx, n.ID); err != nil { 3366 t.Fatal(err) 3367 } 3368 checkNotifications("after delete", map[string]*Notification{}) 3369 }) 3370 } 3371 3372 func TestIntegration_PublicBucket(t *testing.T) { 3373 // Confirm that an unauthenticated client can access a public bucket. 3374 // See https://cloud.google.com/storage/docs/public-datasets/landsat 3375 3376 multiTransportTest(skipGRPC("no public buckets for gRPC"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 3377 const landsatBucket = "gcp-public-data-landsat" 3378 const landsatPrefix = "LC08/01/001/002/LC08_L1GT_001002_20160817_20170322_01_T2/" 3379 const landsatObject = landsatPrefix + "LC08_L1GT_001002_20160817_20170322_01_T2_ANG.txt" 3380 3381 h := testHelper{t} 3382 bkt := client.Bucket(landsatBucket) 3383 obj := bkt.Object(landsatObject) 3384 3385 // Read a public object. 3386 bytes := h.mustRead(obj) 3387 if got, want := len(bytes), 117255; got != want { 3388 t.Errorf("len(bytes) = %d, want %d", got, want) 3389 } 3390 3391 // List objects in a public bucket. 3392 iter := bkt.Objects(ctx, &Query{Prefix: landsatPrefix}) 3393 gotCount := 0 3394 for { 3395 _, err := iter.Next() 3396 if err == iterator.Done { 3397 break 3398 } 3399 if err != nil { 3400 t.Fatal(err) 3401 } 3402 gotCount++ 3403 } 3404 if wantCount := 14; gotCount != wantCount { 3405 t.Errorf("object count: got %d, want %d", gotCount, wantCount) 3406 } 3407 3408 errCode := func(err error) int { 3409 var err2 *googleapi.Error 3410 if ok := errors.As(err, &err2); !ok { 3411 return -1 3412 } 3413 return err2.Code 3414 } 3415 3416 // Reading from or writing to a non-public bucket fails. 3417 c := testConfig(ctx, t) 3418 defer c.Close() 3419 nonPublicObj := client.Bucket(bucket).Object("noauth") 3420 // XML API calls return 403 but the JSON API returns 401. Either is 3421 // acceptable for reads. 3422 _, err := readObject(ctx, nonPublicObj) 3423 if got := errCode(err); got != 403 && got != 401 { 3424 t.Errorf("got code %d; want %v\nerror: %v", got, "401 or 403", err) 3425 } 3426 err = writeObject(ctx, nonPublicObj, "text/plain", []byte("b")) 3427 if got, want := errCode(err), 401; got != want { 3428 t.Errorf("got code %d; want %d\nerror: %v", got, want, err) 3429 } 3430 }, option.WithoutAuthentication()) 3431 } 3432 3433 func TestIntegration_PublicObject(t *testing.T) { 3434 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 3435 publicObj := client.Bucket(bucket).Object("public-obj" + uidSpaceObjects.New()) 3436 contents := randomContents() 3437 3438 w := publicObj.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx) 3439 if _, err := w.Write(contents); err != nil { 3440 t.Fatalf("writer.Write: %v", err) 3441 } 3442 if err := w.Close(); err != nil { 3443 t.Errorf("writer.Close: %v", err) 3444 } 3445 3446 // Set object ACL to public read. 3447 if err := publicObj.ACL().Set(ctx, AllUsers, RoleReader); err != nil { 3448 t.Fatalf("PutACLEntry failed with %v", err) 3449 } 3450 3451 // Create unauthenticated client. 3452 publicClient, err := newTestClient(ctx, option.WithoutAuthentication()) 3453 if err != nil { 3454 t.Fatalf("newTestClient: %v", err) 3455 } 3456 3457 // Test can read public object. 3458 publicObjUnauthenticated := publicClient.Bucket(bucket).Object(publicObj.ObjectName()) 3459 data, err := readObject(context.Background(), publicObjUnauthenticated) 3460 if err != nil { 3461 t.Fatalf("readObject: %v", err) 3462 } 3463 3464 if !bytes.Equal(data, contents) { 3465 t.Errorf("Public object's content: got %q, want %q", data, contents) 3466 } 3467 3468 // Test cannot write to read-only object without authentication. 3469 wc := publicObjUnauthenticated.NewWriter(ctx) 3470 if _, err := wc.Write([]byte("hello")); err != nil { 3471 t.Errorf("Write unexpectedly failed with %v", err) 3472 } 3473 if err = wc.Close(); err == nil { 3474 t.Error("Close expected an error, found none") 3475 } 3476 }) 3477 } 3478 3479 func TestIntegration_ReadCRC(t *testing.T) { 3480 // Test that the checksum is handled correctly when reading files. 3481 // For gzipped files, see https://github.com/GoogleCloudPlatform/google-cloud-dotnet/issues/1641. 3482 ctx := skipJSONReads(skipGRPC("transcoding not supported"), "https://github.com/googleapis/google-cloud-go/issues/7786") 3483 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 3484 const ( 3485 // This is an uncompressed file. 3486 // See https://cloud.google.com/storage/docs/public-datasets/landsat 3487 uncompressedBucket = "gcp-public-data-landsat" 3488 uncompressedObject = "LC08/01/001/002/LC08_L1GT_001002_20160817_20170322_01_T2/LC08_L1GT_001002_20160817_20170322_01_T2_ANG.txt" 3489 3490 gzippedObject = "gzipped-text.txt" 3491 ) 3492 3493 h := testHelper{t} 3494 3495 // Create gzipped object. 3496 var buf bytes.Buffer 3497 zw := gzip.NewWriter(&buf) 3498 zw.Name = gzippedObject 3499 if _, err := zw.Write([]byte("gzipped object data")); err != nil { 3500 t.Fatalf("creating gzip: %v", err) 3501 } 3502 if err := zw.Close(); err != nil { 3503 t.Fatalf("closing gzip writer: %v", err) 3504 } 3505 w := client.Bucket(bucket).Object(gzippedObject).NewWriter(ctx) 3506 w.ContentEncoding = "gzip" 3507 w.ContentType = "text/plain" 3508 h.mustWrite(w, buf.Bytes()) 3509 3510 for _, test := range []struct { 3511 desc string 3512 obj *ObjectHandle 3513 offset, length int64 3514 readCompressed bool // don't decompress a gzipped file 3515 3516 wantErr bool 3517 wantCheck bool // Should Reader try to check the CRC? 3518 }{ 3519 { 3520 desc: "uncompressed, entire file", 3521 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 3522 offset: 0, 3523 length: -1, 3524 readCompressed: false, 3525 wantCheck: true, 3526 }, 3527 { 3528 desc: "uncompressed, entire file, don't decompress", 3529 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 3530 offset: 0, 3531 length: -1, 3532 readCompressed: true, 3533 wantCheck: true, 3534 }, 3535 { 3536 desc: "uncompressed, suffix", 3537 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 3538 offset: 1, 3539 length: -1, 3540 readCompressed: false, 3541 wantCheck: false, 3542 }, 3543 { 3544 desc: "uncompressed, prefix", 3545 obj: client.Bucket(uncompressedBucket).Object(uncompressedObject), 3546 offset: 0, 3547 length: 18, 3548 readCompressed: false, 3549 wantCheck: false, 3550 }, 3551 { 3552 // When a gzipped file is unzipped on read, we can't verify the checksum 3553 // because it was computed against the zipped contents. We can detect 3554 // this case using http.Response.Uncompressed. 3555 desc: "compressed, entire file, unzipped", 3556 obj: client.Bucket(bucket).Object(gzippedObject), 3557 offset: 0, 3558 length: -1, 3559 readCompressed: false, 3560 wantCheck: false, 3561 }, 3562 { 3563 // When we read a gzipped file uncompressed, it's like reading a regular file: 3564 // the served content and the CRC match. 3565 desc: "compressed, entire file, read compressed", 3566 obj: client.Bucket(bucket).Object(gzippedObject), 3567 offset: 0, 3568 length: -1, 3569 readCompressed: true, 3570 wantCheck: true, 3571 }, 3572 { 3573 desc: "compressed, partial, server unzips", 3574 obj: client.Bucket(bucket).Object(gzippedObject), 3575 offset: 1, 3576 length: 8, 3577 readCompressed: false, 3578 wantErr: true, // GCS can't serve part of a gzipped object 3579 wantCheck: false, 3580 }, 3581 { 3582 desc: "compressed, partial, read compressed", 3583 obj: client.Bucket(bucket).Object(gzippedObject), 3584 offset: 1, 3585 length: 8, 3586 readCompressed: true, 3587 wantCheck: false, 3588 }, 3589 } { 3590 obj := test.obj.ReadCompressed(test.readCompressed) 3591 r, err := obj.NewRangeReader(ctx, test.offset, test.length) 3592 if err != nil { 3593 if test.wantErr { 3594 continue 3595 } 3596 t.Fatalf("%s: %v", test.desc, err) 3597 } 3598 if got, want := r.checkCRC, test.wantCheck; got != want { 3599 t.Errorf("%s, checkCRC: got %t, want %t", test.desc, got, want) 3600 } 3601 _, err = ioutil.ReadAll(r) 3602 _ = r.Close() 3603 if err != nil { 3604 t.Fatalf("%s: %v", test.desc, err) 3605 } 3606 } 3607 }) 3608 3609 } 3610 3611 func TestIntegration_CancelWrite(t *testing.T) { 3612 // Verify that canceling the writer's context immediately stops uploading an object 3613 ctx := skipJSONReads(context.Background(), "no reads in test") 3614 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 3615 bkt := client.Bucket(bucket) 3616 3617 cctx, cancel := context.WithCancel(ctx) 3618 defer cancel() 3619 obj := bkt.Object("cancel-write") 3620 w := obj.NewWriter(cctx) 3621 w.ChunkSize = googleapi.MinUploadChunkSize 3622 buf := make([]byte, w.ChunkSize) 3623 // Write the first chunk. This is read in its entirety before sending the request 3624 // (see google.golang.org/api/gensupport.PrepareUpload), so we expect it to return 3625 // without error. 3626 _, err := w.Write(buf) 3627 if err != nil { 3628 t.Fatal(err) 3629 } 3630 // Now cancel the context. 3631 cancel() 3632 // The next Write should return context.Canceled. 3633 _, err = w.Write(buf) 3634 if !errors.Is(err, context.Canceled) { 3635 t.Fatalf("got %v, wanted context.Canceled", err) 3636 } 3637 // The Close should too. 3638 err = w.Close() 3639 if !errors.Is(err, context.Canceled) { 3640 t.Fatalf("got %v, wanted context.Canceled", err) 3641 } 3642 }) 3643 } 3644 3645 func TestIntegration_UpdateCORS(t *testing.T) { 3646 ctx := skipJSONReads(context.Background(), "no reads in test") 3647 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 3648 initialSettings := []CORS{ 3649 { 3650 MaxAge: time.Hour, 3651 Methods: []string{"POST"}, 3652 Origins: []string{"some-origin.com"}, 3653 ResponseHeaders: []string{"foo-bar"}, 3654 }, 3655 } 3656 3657 for _, test := range []struct { 3658 desc string 3659 input []CORS 3660 want []CORS 3661 }{ 3662 { 3663 desc: "set new values", 3664 input: []CORS{ 3665 { 3666 MaxAge: time.Hour, 3667 Methods: []string{"GET"}, 3668 Origins: []string{"*"}, 3669 ResponseHeaders: []string{"some-header"}, 3670 }, 3671 }, 3672 want: []CORS{ 3673 { 3674 MaxAge: time.Hour, 3675 Methods: []string{"GET"}, 3676 Origins: []string{"*"}, 3677 ResponseHeaders: []string{"some-header"}, 3678 }, 3679 }, 3680 }, 3681 { 3682 desc: "set to empty to remove existing policies", 3683 input: []CORS{}, 3684 want: nil, 3685 }, 3686 { 3687 desc: "do not set to keep existing policies", 3688 input: nil, 3689 want: []CORS{ 3690 { 3691 MaxAge: time.Hour, 3692 Methods: []string{"POST"}, 3693 Origins: []string{"some-origin.com"}, 3694 ResponseHeaders: []string{"foo-bar"}, 3695 }, 3696 }, 3697 }, 3698 } { 3699 t.Run(test.desc, func(t *testing.T) { 3700 h := testHelper{t} 3701 3702 bkt := client.Bucket(prefix + uidSpace.New()) 3703 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{CORS: initialSettings}) 3704 defer h.mustDeleteBucket(bkt) 3705 // Set VersioningEnabled so that we don't send an empty update/patch request, which is invalid for gRPC 3706 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{CORS: test.input, VersioningEnabled: false}, h.mustBucketAttrs(bkt).MetaGeneration) 3707 attrs := h.mustBucketAttrs(bkt) 3708 if diff := testutil.Diff(attrs.CORS, test.want); diff != "" { 3709 t.Errorf("input: %v\ngot=-, want=+:\n%s", test.input, diff) 3710 } 3711 }) 3712 } 3713 }) 3714 } 3715 3716 func TestIntegration_UpdateDefaultEventBasedHold(t *testing.T) { 3717 ctx := skipJSONReads(context.Background(), "no reads in test") 3718 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 3719 h := testHelper{t} 3720 3721 bkt := client.Bucket(prefix + uidSpace.New()) 3722 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{}) 3723 defer h.mustDeleteBucket(bkt) 3724 attrs := h.mustBucketAttrs(bkt) 3725 if attrs.DefaultEventBasedHold != false { 3726 t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, false) 3727 } 3728 3729 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{DefaultEventBasedHold: true}, attrs.MetaGeneration) 3730 attrs = h.mustBucketAttrs(bkt) 3731 if attrs.DefaultEventBasedHold != true { 3732 t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, true) 3733 } 3734 3735 // Omitting it should leave the value unchanged. 3736 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RequesterPays: true}, attrs.MetaGeneration) 3737 attrs = h.mustBucketAttrs(bkt) 3738 if attrs.DefaultEventBasedHold != true { 3739 t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, true) 3740 } 3741 }) 3742 } 3743 3744 func TestIntegration_UpdateEventBasedHold(t *testing.T) { 3745 ctx := skipJSONReads(context.Background(), "no reads in test") 3746 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 3747 h := testHelper{t} 3748 3749 obj := client.Bucket(bucket).Object("some-obj") 3750 h.mustWrite(obj.NewWriter(ctx), randomContents()) 3751 3752 defer func() { 3753 h.mustUpdateObject(obj, ObjectAttrsToUpdate{EventBasedHold: false}, h.mustObjectAttrs(obj).Metageneration) 3754 h.mustDeleteObject(obj) 3755 }() 3756 3757 attrs := h.mustObjectAttrs(obj) 3758 if attrs.EventBasedHold != false { 3759 t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, false) 3760 } 3761 3762 h.mustUpdateObject(obj, ObjectAttrsToUpdate{EventBasedHold: true}, attrs.Metageneration) 3763 attrs = h.mustObjectAttrs(obj) 3764 if attrs.EventBasedHold != true { 3765 t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, true) 3766 } 3767 3768 // Omitting it should leave the value unchanged. 3769 h.mustUpdateObject(obj, ObjectAttrsToUpdate{ContentType: "foo"}, attrs.Metageneration) 3770 attrs = h.mustObjectAttrs(obj) 3771 if attrs.EventBasedHold != true { 3772 t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, true) 3773 } 3774 }) 3775 } 3776 3777 func TestIntegration_UpdateTemporaryHold(t *testing.T) { 3778 ctx := skipJSONReads(context.Background(), "no reads in test") 3779 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 3780 h := testHelper{t} 3781 3782 obj := client.Bucket(bucket).Object("updatetemporaryhold-obj") 3783 h.mustWrite(obj.NewWriter(ctx), randomContents()) 3784 3785 defer func() { 3786 h.mustUpdateObject(obj, ObjectAttrsToUpdate{TemporaryHold: false}, h.mustObjectAttrs(obj).Metageneration) 3787 h.mustDeleteObject(obj) 3788 }() 3789 3790 attrs := h.mustObjectAttrs(obj) 3791 if attrs.TemporaryHold != false { 3792 t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, false) 3793 } 3794 3795 h.mustUpdateObject(obj, ObjectAttrsToUpdate{TemporaryHold: true}, attrs.Metageneration) 3796 attrs = h.mustObjectAttrs(obj) 3797 if attrs.TemporaryHold != true { 3798 t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, true) 3799 } 3800 3801 // Omitting it should leave the value unchanged. 3802 h.mustUpdateObject(obj, ObjectAttrsToUpdate{ContentType: "foo"}, attrs.Metageneration) 3803 attrs = h.mustObjectAttrs(obj) 3804 if attrs.TemporaryHold != true { 3805 t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, true) 3806 } 3807 }) 3808 } 3809 3810 func TestIntegration_UpdateRetentionExpirationTime(t *testing.T) { 3811 ctx := skipJSONReads(context.Background(), "no reads in test") 3812 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 3813 h := testHelper{t} 3814 3815 bkt := client.Bucket(prefix + uidSpace.New()) 3816 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour}}) 3817 obj := bkt.Object("some-obj") 3818 h.mustWrite(obj.NewWriter(ctx), randomContents()) 3819 3820 defer func() { 3821 t.Helper() 3822 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{RetentionPeriod: 0}}, h.mustBucketAttrs(bkt).MetaGeneration) 3823 3824 // RetentionPeriod of less than a day is explicitly called out 3825 // as best effort and not guaranteed, so let's log problems deleting 3826 // objects instead of failing. 3827 if err := obj.Delete(context.Background()); err != nil { 3828 t.Logf("object delete: %v", err) 3829 } 3830 if err := bkt.Delete(context.Background()); err != nil { 3831 t.Logf("bucket delete: %v", err) 3832 } 3833 }() 3834 3835 attrs := h.mustObjectAttrs(obj) 3836 if attrs.RetentionExpirationTime == (time.Time{}) { 3837 t.Fatalf("got=%v, wanted a non-zero value", attrs.RetentionExpirationTime) 3838 } 3839 }) 3840 } 3841 3842 func TestIntegration_CustomTime(t *testing.T) { 3843 ctx := skipJSONReads(context.Background(), "no reads in test") 3844 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 3845 h := testHelper{t} 3846 3847 // Create object with CustomTime. 3848 bkt := client.Bucket(bucket) 3849 obj := bkt.Object("custom-time-obj") 3850 w := obj.NewWriter(ctx) 3851 ct := time.Date(2020, 8, 25, 12, 12, 12, 0, time.UTC) 3852 w.ObjectAttrs.CustomTime = ct 3853 h.mustWrite(w, randomContents()) 3854 3855 // Validate that CustomTime has been set 3856 checkCustomTime := func(want time.Time) error { 3857 attrs, err := obj.Attrs(ctx) 3858 if err != nil { 3859 return fmt.Errorf("failed to get object attrs: %v", err) 3860 } 3861 if got := attrs.CustomTime; got != want { 3862 return fmt.Errorf("CustomTime not set correctly: got %+v, want %+v", got, ct) 3863 } 3864 return nil 3865 } 3866 3867 if err := checkCustomTime(ct); err != nil { 3868 t.Fatalf("checking CustomTime: %v", err) 3869 } 3870 3871 // Update CustomTime to the future should succeed. 3872 laterTime := ct.Add(10 * time.Hour) 3873 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{CustomTime: laterTime}); err != nil { 3874 t.Fatalf("updating CustomTime: %v", err) 3875 } 3876 3877 // Update CustomTime to the past should give error. 3878 earlierTime := ct.Add(5 * time.Hour) 3879 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{CustomTime: earlierTime}); err == nil { 3880 t.Fatalf("backdating CustomTime: expected error, got none") 3881 } 3882 3883 // Zero value for CustomTime should be ignored. Set TemporaryHold so that 3884 // we don't send an empty update request, which is invalid for gRPC. 3885 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{TemporaryHold: false}); err != nil { 3886 t.Fatalf("empty update: %v", err) 3887 } 3888 if err := checkCustomTime(laterTime); err != nil { 3889 t.Fatalf("after sending zero value: %v", err) 3890 } 3891 }) 3892 } 3893 3894 func TestIntegration_UpdateRetentionPolicy(t *testing.T) { 3895 ctx := skipJSONReads(context.Background(), "no reads in test") 3896 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 3897 initial := &RetentionPolicy{RetentionPeriod: time.Minute} 3898 3899 for _, test := range []struct { 3900 desc string 3901 input *RetentionPolicy 3902 want *RetentionPolicy 3903 }{ 3904 { 3905 desc: "update", 3906 input: &RetentionPolicy{RetentionPeriod: time.Hour}, 3907 want: &RetentionPolicy{RetentionPeriod: time.Hour}, 3908 }, 3909 { 3910 desc: "update even with timestamp (EffectiveTime should be ignored)", 3911 input: &RetentionPolicy{RetentionPeriod: time.Hour, EffectiveTime: time.Now()}, 3912 want: &RetentionPolicy{RetentionPeriod: time.Hour}, 3913 }, 3914 { 3915 desc: "remove", 3916 input: &RetentionPolicy{}, 3917 want: nil, 3918 }, 3919 { 3920 desc: "remove even with timestamp (EffectiveTime should be ignored)", 3921 input: &RetentionPolicy{EffectiveTime: time.Now().Add(time.Hour)}, 3922 want: nil, 3923 }, 3924 { 3925 desc: "ignore", 3926 input: nil, 3927 want: initial, 3928 }, 3929 } { 3930 t.Run(test.desc, func(t *testing.T) { 3931 h := testHelper{t} 3932 bkt := client.Bucket(prefix + uidSpace.New()) 3933 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: initial}) 3934 defer h.mustDeleteBucket(bkt) 3935 // Set VersioningEnabled so that we don't send an empty update request, which is invalid for gRPC 3936 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: test.input, VersioningEnabled: false}, h.mustBucketAttrs(bkt).MetaGeneration) 3937 3938 attrs := h.mustBucketAttrs(bkt) 3939 if attrs.RetentionPolicy != nil && attrs.RetentionPolicy.EffectiveTime.Unix() == 0 { 3940 // Should be set by the server and parsed by the client 3941 t.Fatal("EffectiveTime should be set, but it was not") 3942 } 3943 if diff := testutil.Diff(attrs.RetentionPolicy, test.want, cmpopts.IgnoreTypes(time.Time{})); diff != "" { 3944 t.Errorf("input: %v\ngot=-, want=+:\n%s", test.input, diff) 3945 } 3946 }) 3947 } 3948 }) 3949 } 3950 3951 func TestIntegration_DeleteObjectInBucketWithRetentionPolicy(t *testing.T) { 3952 ctx := skipJSONReads(context.Background(), "no reads in test") 3953 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 3954 h := testHelper{t} 3955 3956 bkt := client.Bucket(prefix + uidSpace.New()) 3957 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: 25 * time.Hour}}) 3958 defer h.mustDeleteBucket(bkt) 3959 3960 o := bkt.Object("some-object") 3961 if err := writeObject(ctx, o, "text/plain", []byte("hello world")); err != nil { 3962 t.Fatal(err) 3963 } 3964 3965 if err := o.Delete(ctx); err == nil { 3966 t.Fatal("expected to err deleting an object in a bucket with retention period, but got nil") 3967 } 3968 3969 // Remove the retention period 3970 h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{}}, h.mustBucketAttrs(bkt).MetaGeneration) 3971 3972 // Delete with retry, as bucket metadata changes 3973 // can take some time to propagate. 3974 retry := func(err error) bool { return err != nil } 3975 ctx, cancel := context.WithTimeout(ctx, time.Second*10) 3976 defer cancel() 3977 3978 o = o.Retryer(WithErrorFunc(retry), WithPolicy(RetryAlways)) 3979 if err := o.Delete(ctx); err != nil { 3980 t.Fatalf("object delete: %v", err) 3981 } 3982 }) 3983 } 3984 3985 func TestIntegration_LockBucket(t *testing.T) { 3986 ctx := skipJSONReads(context.Background(), "no reads in test") 3987 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 3988 h := testHelper{t} 3989 3990 bkt := client.Bucket(prefix + uidSpace.New()) 3991 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour * 25}}) 3992 attrs := h.mustBucketAttrs(bkt) 3993 if attrs.RetentionPolicy.IsLocked { 3994 t.Fatal("Expected bucket to begin unlocked, but it was not") 3995 } 3996 err := bkt.If(BucketConditions{MetagenerationMatch: attrs.MetaGeneration}).LockRetentionPolicy(ctx) 3997 if err != nil { 3998 t.Fatal("could not lock", err) 3999 } 4000 4001 attrs = h.mustBucketAttrs(bkt) 4002 if !attrs.RetentionPolicy.IsLocked { 4003 t.Fatal("Expected bucket to be locked, but it was not") 4004 } 4005 4006 _, err = bkt.Update(ctx, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour}}) 4007 if err == nil { 4008 t.Fatal("Expected error updating locked bucket, got nil") 4009 } 4010 }) 4011 } 4012 4013 func TestIntegration_LockBucket_MetagenerationRequired(t *testing.T) { 4014 ctx := skipJSONReads(context.Background(), "no reads in test") 4015 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 4016 h := testHelper{t} 4017 4018 bkt := client.Bucket(prefix + uidSpace.New()) 4019 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ 4020 RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour * 25}, 4021 }) 4022 err := bkt.LockRetentionPolicy(ctx) 4023 if err == nil { 4024 t.Fatal("expected error locking bucket without metageneration condition, got nil") 4025 } 4026 }) 4027 } 4028 4029 func TestIntegration_BucketObjectRetention(t *testing.T) { 4030 ctx := skipJSONReads(skipGRPC("not yet available in gRPC - b/308194853"), "no reads in test") 4031 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 4032 setTrue, setFalse := true, false 4033 4034 for _, test := range []struct { 4035 desc string 4036 enable *bool 4037 wantRetentionMode string 4038 }{ 4039 { 4040 desc: "ObjectRetentionMode is not enabled by default", 4041 wantRetentionMode: "", 4042 }, 4043 { 4044 desc: "Enable retention", 4045 enable: &setTrue, 4046 wantRetentionMode: "Enabled", 4047 }, 4048 { 4049 desc: "Set object retention to false", 4050 enable: &setFalse, 4051 wantRetentionMode: "", 4052 }, 4053 } { 4054 t.Run(test.desc, func(t *testing.T) { 4055 b := client.Bucket(prefix + uidSpace.New()) 4056 if test.enable != nil { 4057 b = b.SetObjectRetention(*test.enable) 4058 } 4059 4060 err := b.Create(ctx, testutil.ProjID(), nil) 4061 if err != nil { 4062 t.Fatalf("error creating bucket: %v", err) 4063 } 4064 t.Cleanup(func() { b.Delete(ctx) }) 4065 4066 attrs, err := b.Attrs(ctx) 4067 if err != nil { 4068 t.Fatalf("b.Attrs: %v", err) 4069 } 4070 if got, want := attrs.ObjectRetentionMode, test.wantRetentionMode; got != want { 4071 t.Errorf("expected ObjectRetentionMode to be %q, got %q", want, got) 4072 } 4073 }) 4074 } 4075 }) 4076 } 4077 4078 func TestIntegration_ObjectRetention(t *testing.T) { 4079 ctx := skipJSONReads(skipGRPC("not yet available in gRPC - b/308194853"), "no reads in test") 4080 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 4081 h := testHelper{t} 4082 4083 b := client.Bucket(prefix + uidSpace.New()).SetObjectRetention(true) 4084 4085 if err := b.Create(ctx, testutil.ProjID(), nil); err != nil { 4086 t.Fatalf("error creating bucket: %v", err) 4087 } 4088 t.Cleanup(func() { h.mustDeleteBucket(b) }) 4089 4090 retentionUnlocked := &ObjectRetention{ 4091 Mode: "Unlocked", 4092 RetainUntil: time.Now().Add(time.Minute * 20).Truncate(time.Second), 4093 } 4094 retentionUnlockedExtended := &ObjectRetention{ 4095 Mode: "Unlocked", 4096 RetainUntil: time.Now().Add(time.Hour).Truncate(time.Second), 4097 } 4098 4099 // Create an object with future retain until time 4100 o := b.Object("retention-on-create" + uidSpaceObjects.New()) 4101 w := o.NewWriter(ctx) 4102 w.Retention = retentionUnlocked 4103 h.mustWrite(w, []byte("contents")) 4104 t.Cleanup(func() { 4105 if _, err := o.OverrideUnlockedRetention(true).Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{}}); err != nil { 4106 t.Fatalf("failed to remove retention from object: %v", err) 4107 } 4108 h.mustDeleteObject(o) 4109 }) 4110 4111 if got, want := w.Attrs().Retention, retentionUnlocked; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) { 4112 t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want) 4113 } 4114 4115 // Delete object under retention returns 403 4116 if err := o.Delete(ctx); err == nil || extractErrCode(err) != http.StatusForbidden { 4117 t.Fatalf("delete should have failed with: %v, instead got:%v", http.StatusForbidden, err) 4118 } 4119 4120 // Extend retain until time of Unlocked object is possible 4121 attrs, err := o.Update(ctx, ObjectAttrsToUpdate{Retention: retentionUnlockedExtended}) 4122 if err != nil { 4123 t.Fatalf("failed to add retention to object: %v", err) 4124 } 4125 4126 if got, want := attrs.Retention, retentionUnlockedExtended; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) { 4127 t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want) 4128 } 4129 4130 // Reduce retain until time of Unlocked object without 4131 // override_unlocked_retention=True returns 403 4132 _, err = o.Update(ctx, ObjectAttrsToUpdate{Retention: retentionUnlocked}) 4133 if err == nil || extractErrCode(err) != http.StatusForbidden { 4134 t.Fatalf("o.Update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) 4135 } 4136 4137 // Remove retention of Unlocked object without 4138 // override_unlocked_retention=True returns 403 4139 _, err = o.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{}}) 4140 if err == nil || extractErrCode(err) != http.StatusForbidden { 4141 t.Fatalf("o.Update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) 4142 } 4143 4144 // Reduce retain until time of Unlocked object with override_unlocked_retention=True 4145 attrs, err = o.OverrideUnlockedRetention(true).Update(ctx, ObjectAttrsToUpdate{ 4146 Retention: retentionUnlocked, 4147 }) 4148 if err != nil { 4149 t.Fatalf("failed to add retention to object: %v", err) 4150 } 4151 4152 if got, want := attrs.Retention, retentionUnlocked; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) { 4153 t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want) 4154 } 4155 4156 // Create a new object 4157 objectWithRetentionOnUpdate := b.Object("retention-on-update" + uidSpaceObjects.New()) 4158 w = objectWithRetentionOnUpdate.NewWriter(ctx) 4159 h.mustWrite(w, []byte("contents")) 4160 4161 // Retention should not be set 4162 if got := w.Attrs().Retention; got != nil { 4163 t.Errorf("expected no ObjectRetention, got: %+v", got) 4164 } 4165 4166 // Update object with only one of (retain until time, retention mode) returns 400 4167 _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{Mode: "Locked"}}) 4168 if err == nil || extractErrCode(err) != http.StatusBadRequest { 4169 t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) 4170 } 4171 4172 _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{RetainUntil: time.Now().Add(time.Second)}}) 4173 if err == nil || extractErrCode(err) != http.StatusBadRequest { 4174 t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) 4175 } 4176 4177 // Update object with future retain until time 4178 attrs, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: retentionUnlocked}) 4179 if err != nil { 4180 t.Errorf("o.Update: %v", err) 4181 } 4182 4183 if got, want := attrs.Retention, retentionUnlocked; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) { 4184 t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want) 4185 } 4186 4187 // Update/Patch object with retain until time in the past returns 400 4188 _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{RetainUntil: time.Now().Add(-time.Second)}}) 4189 if err == nil || extractErrCode(err) != http.StatusBadRequest { 4190 t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) 4191 } 4192 4193 // Update object with only one of (retain until time, retention mode) returns 400 4194 _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{Mode: "Locked"}}) 4195 if err == nil || extractErrCode(err) != http.StatusBadRequest { 4196 t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) 4197 } 4198 4199 _, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{RetainUntil: time.Now().Add(time.Second)}}) 4200 if err == nil || extractErrCode(err) != http.StatusBadRequest { 4201 t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err) 4202 } 4203 4204 // Remove retention of Unlocked object with override_unlocked_retention=True 4205 attrs, err = objectWithRetentionOnUpdate.OverrideUnlockedRetention(true).Update(ctx, ObjectAttrsToUpdate{ 4206 Retention: &ObjectRetention{}, 4207 }) 4208 if err != nil { 4209 t.Fatalf("failed to remove retention from object: %v", err) 4210 } 4211 4212 if got := attrs.Retention; got != nil { 4213 t.Errorf("mismatching retention config, got: %+v, wanted nil", got) 4214 } 4215 4216 // We should be able to delete the object as normal since retention was removed 4217 if err := objectWithRetentionOnUpdate.Delete(ctx); err != nil { 4218 t.Errorf("object.Delete:%v", err) 4219 } 4220 }) 4221 } 4222 4223 func TestIntegration_KMS(t *testing.T) { 4224 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, prefix string, client *Client) { 4225 h := testHelper{t} 4226 4227 keyRingName := os.Getenv("GCLOUD_TESTS_GOLANG_KEYRING") 4228 if keyRingName == "" { 4229 t.Fatal("GCLOUD_TESTS_GOLANG_KEYRING must be set. See CONTRIBUTING.md for details") 4230 } 4231 keyName1 := keyRingName + "/cryptoKeys/key1" 4232 keyName2 := keyRingName + "/cryptoKeys/key2" 4233 contents := []byte("my secret") 4234 4235 write := func(obj *ObjectHandle, setKey bool) { 4236 w := obj.NewWriter(ctx) 4237 if setKey { 4238 w.KMSKeyName = keyName1 4239 } 4240 h.mustWrite(w, contents) 4241 } 4242 4243 checkRead := func(obj *ObjectHandle) { 4244 got := h.mustRead(obj) 4245 if !bytes.Equal(got, contents) { 4246 t.Errorf("got %v, want %v", got, contents) 4247 } 4248 attrs := h.mustObjectAttrs(obj) 4249 if len(attrs.KMSKeyName) < len(keyName1) || attrs.KMSKeyName[:len(keyName1)] != keyName1 { 4250 t.Errorf("got %q, want %q", attrs.KMSKeyName, keyName1) 4251 } 4252 } 4253 4254 // Write an object with a key, then read it to verify its contents and the presence of the key name. 4255 bkt := client.Bucket(bucket) 4256 obj := bkt.Object("kms") 4257 write(obj, true) 4258 checkRead(obj) 4259 h.mustDeleteObject(obj) 4260 4261 // Encrypt an object with a CSEK, then copy it using a CMEK. 4262 src := bkt.Object("csek").Key(testEncryptionKey) 4263 if err := writeObject(ctx, src, "text/plain", contents); err != nil { 4264 t.Fatal(err) 4265 } 4266 dest := bkt.Object("cmek") 4267 c := dest.CopierFrom(src) 4268 c.DestinationKMSKeyName = keyName1 4269 if _, err := c.Run(ctx); err != nil { 4270 t.Fatal(err) 4271 } 4272 checkRead(dest) 4273 src.Delete(ctx) 4274 dest.Delete(ctx) 4275 4276 // Create a bucket with a default key, then write and read an object. 4277 bkt = client.Bucket(prefix + uidSpace.New()) 4278 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ 4279 Location: "US", 4280 Encryption: &BucketEncryption{DefaultKMSKeyName: keyName1}, 4281 }) 4282 defer h.mustDeleteBucket(bkt) 4283 4284 attrs := h.mustBucketAttrs(bkt) 4285 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName1; got != want { 4286 t.Fatalf("got %q, want %q", got, want) 4287 } 4288 obj = bkt.Object("kms") 4289 write(obj, false) 4290 checkRead(obj) 4291 h.mustDeleteObject(obj) 4292 4293 // Update the bucket's default key to a different name. 4294 // (This key doesn't have to exist.) 4295 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{Encryption: &BucketEncryption{DefaultKMSKeyName: keyName2}}, attrs.MetaGeneration) 4296 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName2; got != want { 4297 t.Fatalf("got %q, want %q", got, want) 4298 } 4299 attrs = h.mustBucketAttrs(bkt) 4300 if got, want := attrs.Encryption.DefaultKMSKeyName, keyName2; got != want { 4301 t.Fatalf("got %q, want %q", got, want) 4302 } 4303 4304 // Remove the default KMS key. 4305 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{Encryption: &BucketEncryption{DefaultKMSKeyName: ""}}, attrs.MetaGeneration) 4306 if attrs.Encryption != nil { 4307 t.Fatalf("got %#v, want nil", attrs.Encryption) 4308 } 4309 }) 4310 } 4311 4312 func TestIntegration_PredefinedACLs(t *testing.T) { 4313 projectOwners := prefixRoleACL{prefix: "project-owners", role: RoleOwner} 4314 userOwner := prefixRoleACL{prefix: "user", role: RoleOwner} 4315 authenticatedRead := entityRoleACL{entity: AllAuthenticatedUsers, role: RoleReader} 4316 4317 ctx := skipJSONReads(context.Background(), "no reads in test") 4318 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 4319 h := testHelper{t} 4320 4321 bkt := client.Bucket(prefix + uidSpace.New()) 4322 h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{ 4323 PredefinedACL: "authenticatedRead", 4324 PredefinedDefaultObjectACL: "publicRead", 4325 }) 4326 defer h.mustDeleteBucket(bkt) 4327 attrs := h.mustBucketAttrs(bkt) 4328 4329 if acl, want := attrs.ACL, projectOwners; !containsACLRule(acl, want) { 4330 t.Fatalf("Bucket.ACL: expected acl to contain: %+v, got acl: %+v", want, acl) 4331 } 4332 if acl, want := attrs.ACL, authenticatedRead; !containsACLRule(acl, want) { 4333 t.Fatalf("Bucket.ACL: expected acl to contain: %+v, got acl: %+v", want, acl) 4334 } 4335 if acl := attrs.DefaultObjectACL; !containsACLRule(acl, entityRoleACL{AllUsers, RoleReader}) { 4336 t.Fatalf("DefaultObjectACL: expected acl to contain: %+v, got acl: %+v", entityRoleACL{AllUsers, RoleReader}, acl) 4337 } 4338 4339 // Bucket update 4340 attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{ 4341 PredefinedACL: "private", 4342 PredefinedDefaultObjectACL: "authenticatedRead", 4343 }, attrs.MetaGeneration) 4344 if acl, want := attrs.ACL, projectOwners; !containsACLRule(acl, want) { 4345 t.Fatalf("Bucket.ACL update: expected acl to contain: %+v, got acl: %+v", want, acl) 4346 } 4347 if acl, want := attrs.DefaultObjectACL, authenticatedRead; !containsACLRule(acl, want) { 4348 t.Fatalf("DefaultObjectACL update: expected acl to contain: %+v, got acl: %+v", want, acl) 4349 } 4350 4351 // Object creation 4352 obj := bkt.Object("private") 4353 w := obj.NewWriter(ctx) 4354 w.PredefinedACL = "authenticatedRead" 4355 h.mustWrite(w, []byte("hello")) 4356 defer h.mustDeleteObject(obj) 4357 var acl []ACLRule 4358 err := retry(ctx, func() error { 4359 attrs, err := obj.Attrs(ctx) 4360 if err != nil { 4361 return fmt.Errorf("Object.Attrs: object metadata get failed: %v", err) 4362 } 4363 acl = attrs.ACL 4364 return nil 4365 }, func() error { 4366 if want := userOwner; !containsACLRule(acl, want) { 4367 return fmt.Errorf("Object.ACL: expected acl to contain: %+v, got acl: %+v", want, acl) 4368 } 4369 return nil 4370 }) 4371 if err != nil { 4372 t.Fatal(err) 4373 } 4374 err = retry(ctx, func() error { 4375 attrs, err := obj.Attrs(ctx) 4376 if err != nil { 4377 return fmt.Errorf("Object.Attrs: object metadata get failed: %v", err) 4378 } 4379 acl = attrs.ACL 4380 return nil 4381 }, func() error { 4382 if want := authenticatedRead; !containsACLRule(acl, want) { 4383 return fmt.Errorf("Object.ACL: expected acl to contain: %+v, got acl: %+v", want, acl) 4384 } 4385 return nil 4386 }) 4387 if err != nil { 4388 t.Fatal(err) 4389 } 4390 4391 // Object update 4392 oattrs := h.mustUpdateObject(obj, ObjectAttrsToUpdate{PredefinedACL: "private"}, h.mustObjectAttrs(obj).Metageneration) 4393 if acl, want := oattrs.ACL, userOwner; !containsACLRule(acl, want) { 4394 t.Fatalf("Object.ACL update: expected acl to contain: %+v, got acl: %+v", want, acl) 4395 } 4396 if got := len(oattrs.ACL); got != 1 { 4397 t.Errorf("got %d ACL rules, want 1", got) 4398 } 4399 4400 // Copy 4401 dst := bkt.Object("dst") 4402 copier := dst.CopierFrom(obj) 4403 copier.PredefinedACL = "publicRead" 4404 oattrs, err = copier.Run(ctx) 4405 if err != nil { 4406 t.Fatal(err) 4407 } 4408 defer h.mustDeleteObject(dst) 4409 // The copied object still retains the "private" ACL of the source object. 4410 if acl, want := oattrs.ACL, userOwner; !containsACLRule(acl, want) { 4411 t.Fatalf("copy dest: expected acl to contain: %+v, got acl: %+v", want, acl) 4412 } 4413 if !containsACLRule(oattrs.ACL, entityRoleACL{AllUsers, RoleReader}) { 4414 t.Fatalf("copy dest: expected acl to contain: %+v, got acl: %+v", entityRoleACL{AllUsers, RoleReader}, oattrs.ACL) 4415 } 4416 4417 // Compose 4418 comp := bkt.Object("comp") 4419 4420 composer := comp.ComposerFrom(obj, dst) 4421 composer.PredefinedACL = "authenticatedRead" 4422 oattrs, err = composer.Run(ctx) 4423 if err != nil { 4424 t.Fatal(err) 4425 } 4426 defer h.mustDeleteObject(comp) 4427 // The composed object still retains the "private" ACL. 4428 if acl, want := oattrs.ACL, userOwner; !containsACLRule(acl, want) { 4429 t.Fatalf("compose: expected acl to contain: %+v, got acl: %+v", want, acl) 4430 } 4431 if acl, want := oattrs.ACL, authenticatedRead; !containsACLRule(acl, want) { 4432 t.Fatalf("compose: expected acl to contain: %+v, got acl: %+v", want, acl) 4433 } 4434 }) 4435 } 4436 4437 func TestIntegration_ServiceAccount(t *testing.T) { 4438 ctx := skipJSONReads(context.Background(), "no reads in test") 4439 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, _ string, client *Client) { 4440 s, err := client.ServiceAccount(ctx, testutil.ProjID()) 4441 if err != nil { 4442 t.Fatal(err) 4443 } 4444 want := "@gs-project-accounts.iam.gserviceaccount.com" 4445 if !strings.Contains(s, want) { 4446 t.Fatalf("got %v, want to contain %v", s, want) 4447 } 4448 }) 4449 } 4450 4451 func TestIntegration_Reader(t *testing.T) { 4452 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 4453 b := client.Bucket(bucket) 4454 const defaultType = "text/plain" 4455 4456 // Populate object names and make a map for their contents. 4457 objects := []string{ 4458 "obj1", 4459 "obj2", 4460 "obj/with/slashes", 4461 "obj/", 4462 "./obj", 4463 "!#$&'()*+,/:;=,?@,[] and spaces", 4464 } 4465 contents := make(map[string][]byte) 4466 4467 // Write objects. 4468 for _, obj := range objects { 4469 c := randomContents() 4470 if err := writeObject(ctx, b.Object(obj), defaultType, c); err != nil { 4471 t.Errorf("Write for %v failed with %v", obj, err) 4472 } 4473 contents[obj] = c 4474 } 4475 // Test Reader. Cache control and last-modified are tested separately, as 4476 // the JSON and XML APIs return different values for these. 4477 for _, obj := range objects { 4478 rc, err := b.Object(obj).NewReader(ctx) 4479 if err != nil { 4480 t.Errorf("Can't create a reader for %v, errored with %v", obj, err) 4481 continue 4482 } 4483 if !rc.checkCRC { 4484 t.Errorf("%v: not checking CRC", obj) 4485 } 4486 4487 slurp, err := ioutil.ReadAll(rc) 4488 if err != nil { 4489 t.Errorf("Can't ReadAll object %v, errored with %v", obj, err) 4490 } 4491 if got, want := slurp, contents[obj]; !bytes.Equal(got, want) { 4492 t.Errorf("Contents (%q) = %q; want %q", obj, got, want) 4493 } 4494 if got, want := rc.Size(), len(contents[obj]); got != int64(want) { 4495 t.Errorf("Size (%q) = %d; want %d", obj, got, want) 4496 } 4497 if got, want := rc.ContentType(), "text/plain"; got != want { 4498 t.Errorf("ContentType (%q) = %q; want %q", obj, got, want) 4499 } 4500 rc.Close() 4501 4502 // Check early close. 4503 buf := make([]byte, 1) 4504 rc, err = b.Object(obj).NewReader(ctx) 4505 if err != nil { 4506 t.Fatalf("%v: %v", obj, err) 4507 } 4508 _, err = rc.Read(buf) 4509 if err != nil { 4510 t.Fatalf("%v: %v", obj, err) 4511 } 4512 if got, want := buf, contents[obj][:1]; !bytes.Equal(got, want) { 4513 t.Errorf("Contents[0] (%q) = %q; want %q", obj, got, want) 4514 } 4515 if err := rc.Close(); err != nil { 4516 t.Errorf("%v Close: %v", obj, err) 4517 } 4518 } 4519 4520 obj := objects[0] 4521 objlen := int64(len(contents[obj])) 4522 4523 // Test Range Reader. 4524 for _, r := range []struct { 4525 desc string 4526 offset, length, want int64 4527 }{ 4528 {"entire object", 0, objlen, objlen}, 4529 {"first half of object", 0, objlen / 2, objlen / 2}, 4530 {"second half of object", objlen / 2, objlen, objlen / 2}, 4531 {"no bytes - start at beginning", 0, 0, 0}, 4532 {"no bytes - start halfway through", objlen / 2, 0, 0}, 4533 {"start halfway through - use negative to get rest of obj", objlen / 2, -1, objlen / 2}, 4534 {"2 times object length", 0, objlen * 2, objlen}, 4535 {"-2 offset", -2, -1, 2}, 4536 {"-object length offset", -objlen, -1, objlen}, 4537 {"-half of object length offset", -(objlen / 2), -1, objlen / 2}, 4538 } { 4539 rc, err := b.Object(obj).NewRangeReader(ctx, r.offset, r.length) 4540 if err != nil { 4541 t.Errorf("%+v: Can't create a range reader for %v, errored with %v", r.desc, obj, err) 4542 continue 4543 } 4544 if rc.Size() != objlen { 4545 t.Errorf("%+v: Reader has a content-size of %d, want %d", r.desc, rc.Size(), objlen) 4546 } 4547 if rc.Remain() != r.want { 4548 t.Errorf("%+v: Reader's available bytes reported as %d, want %d", r.desc, rc.Remain(), r.want) 4549 } 4550 slurp, err := ioutil.ReadAll(rc) 4551 if err != nil { 4552 t.Errorf("%+v: can't ReadAll object %v, errored with %v", r, obj, err) 4553 continue 4554 } 4555 if len(slurp) != int(r.want) { 4556 t.Errorf("%+v: RangeReader (%d, %d): Read %d bytes, wanted %d bytes", r.desc, r.offset, r.length, len(slurp), r.want) 4557 continue 4558 } 4559 4560 switch { 4561 case r.offset < 0: // The case of reading the last N bytes. 4562 start := objlen + r.offset 4563 if got, want := slurp, contents[obj][start:]; !bytes.Equal(got, want) { 4564 t.Errorf("RangeReader (%d, %d) = %q; want %q", r.offset, r.length, got, want) 4565 } 4566 4567 default: 4568 if got, want := slurp, contents[obj][r.offset:r.offset+r.want]; !bytes.Equal(got, want) { 4569 t.Errorf("RangeReader (%d, %d) = %q; want %q", r.offset, r.length, got, want) 4570 } 4571 } 4572 rc.Close() 4573 } 4574 4575 objName := objects[0] 4576 4577 // Test NewReader googleapi.Error. 4578 // Since a 429 or 5xx is hard to cause, we trigger a 416 (InvalidRange). 4579 realLen := len(contents[objName]) 4580 _, err := b.Object(objName).NewRangeReader(ctx, int64(realLen*2), 10) 4581 4582 var e *googleapi.Error 4583 if !errors.As(err, &e) { 4584 // Check if it is the correct GRPC error 4585 if !(status.Code(err) == codes.OutOfRange) { 4586 t.Errorf("NewRangeReader did not return a googleapi.Error nor GRPC OutOfRange error; got: %v", err) 4587 } 4588 } else { 4589 if e.Code != 416 { 4590 t.Errorf("Code = %d; want %d", e.Code, 416) 4591 } 4592 if len(e.Header) == 0 { 4593 t.Error("Missing googleapi.Error.Header") 4594 } 4595 if len(e.Body) == 0 { 4596 t.Error("Missing googleapi.Error.Body") 4597 } 4598 } 4599 }) 4600 } 4601 4602 func TestIntegration_ReaderAttrs(t *testing.T) { 4603 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 4604 bkt := client.Bucket(bucket) 4605 4606 const defaultType = "text/plain" 4607 o := bkt.Object("reader-attrs-obj") 4608 c := randomContents() 4609 if err := writeObject(ctx, o, defaultType, c); err != nil { 4610 t.Errorf("Write for %v failed with %v", o.ObjectName(), err) 4611 } 4612 defer func() { 4613 if err := o.Delete(ctx); err != nil { 4614 log.Printf("failed to delete test object: %v", err) 4615 } 4616 }() 4617 4618 rc, err := o.NewReader(ctx) 4619 if err != nil { 4620 t.Fatal(err) 4621 } 4622 4623 attrs, err := o.Attrs(ctx) 4624 if err != nil { 4625 t.Fatal(err) 4626 } 4627 4628 got := rc.Attrs 4629 want := ReaderObjectAttrs{ 4630 Size: attrs.Size, 4631 ContentType: attrs.ContentType, 4632 ContentEncoding: attrs.ContentEncoding, 4633 CacheControl: got.CacheControl, // ignored, tested separately 4634 LastModified: got.LastModified, // ignored, tested separately 4635 Generation: attrs.Generation, 4636 Metageneration: attrs.Metageneration, 4637 } 4638 if got != want { 4639 t.Fatalf("got\t%v,\nwanted\t%v", got, want) 4640 } 4641 }) 4642 } 4643 4644 func TestIntegration_ReaderLastModified(t *testing.T) { 4645 ctx := skipJSONReads(context.Background(), "LastModified not populated by json response") 4646 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 4647 testStart := time.Now() 4648 b := client.Bucket(bucket) 4649 o := b.Object("reader-lm-obj" + uidSpaceObjects.New()) 4650 4651 if err := writeObject(ctx, o, "text/plain", randomContents()); err != nil { 4652 t.Errorf("Write for %v failed with %v", o.ObjectName(), err) 4653 } 4654 defer func() { 4655 if err := o.Delete(ctx); err != nil { 4656 log.Printf("failed to delete test object: %v", err) 4657 } 4658 }() 4659 4660 r, err := o.NewReader(ctx) 4661 if err != nil { 4662 t.Fatal(err) 4663 } 4664 4665 lm := r.Attrs.LastModified 4666 if lm.IsZero() { 4667 t.Fatal("LastModified is 0, should be >0") 4668 } 4669 4670 // We just wrote this object, so it should have a recent last-modified time. 4671 // Accept a time within the start + variance of the test, to account for natural 4672 // variation. 4673 expectedVariance := time.Minute 4674 4675 if lm.After(testStart.Add(expectedVariance)) { 4676 t.Errorf("LastModified (%q): got %s, which is not within %v from test start (%v)", o.ObjectName(), lm, expectedVariance, testStart) 4677 } 4678 }) 4679 } 4680 4681 func TestIntegration_ReaderCacheControl(t *testing.T) { 4682 ctx := skipJSONReads(context.Background(), "Cache control header is populated differently by the json api") 4683 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 4684 b := client.Bucket(bucket) 4685 o := b.Object("reader-cc" + uidSpaceObjects.New()) 4686 4687 cacheControl := "public, max-age=60" 4688 4689 // Write object. 4690 w := o.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx) 4691 w.CacheControl = cacheControl 4692 if _, err := w.Write(randomContents()); err != nil { 4693 t.Fatalf("Write for %v failed with %v", o.ObjectName(), err) 4694 } 4695 if err := w.Close(); err != nil { 4696 t.Fatalf("Write close for %v failed with %v", o.ObjectName(), err) 4697 } 4698 defer func() { 4699 if err := o.Delete(ctx); err != nil { 4700 log.Printf("failed to delete test object: %v", err) 4701 } 4702 }() 4703 4704 // Check cache control on reader attrs. 4705 r, err := o.NewReader(ctx) 4706 if err != nil { 4707 t.Fatal(err) 4708 } 4709 4710 if got, want := r.Attrs.CacheControl, cacheControl; got != want { 4711 t.Fatalf("cache control; got: %s, want: %s", got, want) 4712 } 4713 }) 4714 } 4715 4716 func TestIntegration_ReaderErrObjectNotExist(t *testing.T) { 4717 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 4718 o := client.Bucket(bucket).Object("non-existing") 4719 4720 _, err := o.NewReader(ctx) 4721 if !errors.Is(err, ErrObjectNotExist) { 4722 t.Fatalf("expected ErrObjectNotExist, got %v", err) 4723 } 4724 }) 4725 } 4726 4727 // TestIntegration_JSONReaderConditions tests only JSON reads as some conditions 4728 // do not work with XML. 4729 func TestIntegration_JSONReaderConditions(t *testing.T) { 4730 ctx := skipXMLReads(skipGRPC("json-only test"), "json-only test") 4731 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 4732 b := client.Bucket(bucket) 4733 o := b.Object("reader-conditions" + uidSpaceObjects.New()) 4734 4735 // Write object. 4736 w := o.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx) 4737 if _, err := w.Write(randomContents()); err != nil { 4738 t.Fatalf("Write for %v failed with %v", o.ObjectName(), err) 4739 } 4740 if err := w.Close(); err != nil { 4741 t.Fatalf("Write close for %v failed with %v", o.ObjectName(), err) 4742 } 4743 4744 t.Cleanup(func() { 4745 if err := o.Delete(ctx); err != nil { 4746 log.Printf("failed to delete test object: %v", err) 4747 } 4748 }) 4749 4750 // Get current gens. 4751 attrs, err := o.Attrs(ctx) 4752 if err != nil { 4753 t.Fatalf("o.Attrs(%s): %v", o.ObjectName(), err) 4754 } 4755 currGen := attrs.Generation 4756 currMetagen := attrs.Metageneration 4757 4758 // Test each condition to make sure it is passed through correctly. 4759 for _, test := range []struct { 4760 desc string 4761 conds Conditions 4762 wantErrCode int 4763 }{ 4764 { 4765 desc: "GenerationMatch incorrect gen", 4766 conds: Conditions{GenerationMatch: currGen + 2}, 4767 wantErrCode: 412, 4768 }, 4769 { 4770 desc: "GenerationNotMatch current gen", 4771 conds: Conditions{GenerationNotMatch: currGen}, 4772 wantErrCode: 304, 4773 }, 4774 { 4775 desc: "DoesNotExist set to true", 4776 conds: Conditions{DoesNotExist: true}, 4777 wantErrCode: 412, 4778 }, 4779 { 4780 desc: "MetagenerationMatch incorrect gen", 4781 conds: Conditions{MetagenerationMatch: currMetagen + 1}, 4782 wantErrCode: 412, 4783 }, 4784 { 4785 desc: "MetagenerationNotMatch current gen", 4786 conds: Conditions{MetagenerationNotMatch: currMetagen}, 4787 wantErrCode: 304, 4788 }, 4789 } { 4790 t.Run(test.desc, func(t *testing.T) { 4791 o := o.If(test.conds) 4792 _, err := o.NewReader(ctx) 4793 4794 got := extractErrCode(err) 4795 if test.wantErrCode != got { 4796 t.Errorf("want err code: %v, got err: %v", test.wantErrCode, err) 4797 } 4798 }) 4799 } 4800 }) 4801 } 4802 4803 // Test that context cancellation correctly stops a download before completion. 4804 func TestIntegration_ReaderCancel(t *testing.T) { 4805 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 4806 ctx, close := context.WithDeadline(ctx, time.Now().Add(time.Second*30)) 4807 defer close() 4808 4809 bkt := client.Bucket(bucket) 4810 obj := bkt.Object("reader-cancel-obj") 4811 4812 minObjectSize := 5000000 // 5 Mb 4813 4814 w := obj.NewWriter(ctx) 4815 c := randomContents() 4816 for written := 0; written < minObjectSize; { 4817 n, err := w.Write(c) 4818 if err != nil { 4819 t.Fatalf("w.Write: %v", err) 4820 } 4821 written += n 4822 } 4823 4824 if err := w.Close(); err != nil { 4825 t.Fatalf("writer close: %v", err) 4826 } 4827 defer func() { 4828 if err := obj.Delete(ctx); err != nil { 4829 log.Printf("failed to delete test object: %v", err) 4830 } 4831 }() 4832 4833 // Create a reader (which makes a GET request to GCS and opens the body to 4834 // read the object) and then cancel the context before reading. 4835 readerCtx, cancel := context.WithCancel(ctx) 4836 r, err := obj.NewReader(readerCtx) 4837 if err != nil { 4838 t.Fatalf("obj.NewReader: %v", err) 4839 } 4840 defer func() { 4841 if err := r.Close(); err != nil { 4842 log.Printf("r.Close(): %v", err) 4843 } 4844 }() 4845 4846 cancel() 4847 4848 _, err = io.Copy(io.Discard, r) 4849 if err == nil || !errors.Is(err, context.Canceled) && !(status.Code(err) == codes.Canceled) { 4850 t.Fatalf("r.Read: got error %v, want context.Canceled", err) 4851 } 4852 }) 4853 } 4854 4855 // Ensures that a file stored with a: 4856 // * Content-Encoding of "gzip" 4857 // * Content-Type of "text/plain" 4858 // will be properly served back. 4859 // See: 4860 // - https://cloud.google.com/storage/docs/transcoding#transcoding_and_gzip 4861 // - https://github.com/googleapis/google-cloud-go/issues/1800 4862 func TestIntegration_NewReaderWithContentEncodingGzip(t *testing.T) { 4863 multiTransportTest(skipGRPC("gzip transcoding not supported"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { 4864 h := testHelper{t} 4865 4866 projectID := testutil.ProjID() 4867 bkt := client.Bucket(prefix + uidSpace.New()) 4868 h.mustCreate(bkt, projectID, nil) 4869 defer h.mustDeleteBucket(bkt) 4870 obj := bkt.Object("decompressive-transcoding") 4871 original := bytes.Repeat([]byte("a"), 4<<10) 4872 4873 // Firstly upload the gzip compressed file. 4874 w := obj.If(Conditions{DoesNotExist: true}).NewWriter(ctx) 4875 // Compress and upload the content. 4876 gzw := gzip.NewWriter(w) 4877 if _, err := gzw.Write(original); err != nil { 4878 t.Fatalf("Failed to compress content: %v", err) 4879 } 4880 if err := gzw.Close(); err != nil { 4881 t.Errorf("Failed to compress content: %v", err) 4882 } 4883 if err := w.Close(); err != nil { 4884 t.Errorf("Failed to finish uploading the file: %v", err) 4885 } 4886 4887 defer h.mustDeleteObject(obj) 4888 4889 // Now update the Content-Encoding and Content-Type to enable 4890 // decompressive transcoding. 4891 updatedAttrs, err := obj.Update(ctx, ObjectAttrsToUpdate{ 4892 ContentEncoding: "gzip", 4893 ContentType: "text/plain", 4894 }) 4895 if err != nil { 4896 t.Fatalf("Attribute update failure: %v", err) 4897 } 4898 if g, w := updatedAttrs.ContentEncoding, "gzip"; g != w { 4899 t.Fatalf("ContentEncoding mismtach:\nGot: %q\nWant: %q", g, w) 4900 } 4901 if g, w := updatedAttrs.ContentType, "text/plain"; g != w { 4902 t.Fatalf("ContentType mismtach:\nGot: %q\nWant: %q", g, w) 4903 } 4904 4905 rWhole, err := obj.NewReader(ctx) 4906 if err != nil { 4907 t.Fatalf("Failed to create wholesome reader: %v", err) 4908 } 4909 blobWhole, err := ioutil.ReadAll(rWhole) 4910 rWhole.Close() 4911 if err != nil { 4912 t.Fatalf("Failed to read the whole body: %v", err) 4913 } 4914 if g, w := blobWhole, original; !bytes.Equal(g, w) { 4915 t.Fatalf("Body mismatch\nGot:\n%s\n\nWant:\n%s", g, w) 4916 } 4917 4918 // Now try a range read, which should return the whole body anyways since 4919 // for decompressive transcoding, range requests ARE IGNORED by Cloud Storage. 4920 r2kBTo3kB, err := obj.NewRangeReader(ctx, 2<<10, 3<<10) 4921 if err != nil { 4922 t.Fatalf("Failed to create range reader: %v", err) 4923 } 4924 blob2kBTo3kB, err := ioutil.ReadAll(r2kBTo3kB) 4925 r2kBTo3kB.Close() 4926 if err != nil { 4927 t.Fatalf("Failed to read with the 2kB to 3kB range request: %v", err) 4928 } 4929 // The ENTIRE body MUST be served back regardless of the requested range. 4930 if g, w := blob2kBTo3kB, original; !bytes.Equal(g, w) { 4931 t.Fatalf("Body mismatch\nGot:\n%s\n\nWant:\n%s", g, w) 4932 } 4933 }) 4934 } 4935 4936 func TestIntegration_HMACKey(t *testing.T) { 4937 ctx := skipJSONReads(context.Background(), "no reads in test") 4938 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, _ string, client *Client) { 4939 client.SetRetry(WithPolicy(RetryAlways)) 4940 4941 projectID := testutil.ProjID() 4942 4943 // Use the service account email from the user's credentials. Requires that the 4944 // credentials are set via a JSON credentials file. 4945 // Note that a service account may only have up to 5 active HMAC keys at once; if 4946 // we see flakes because of this, we should consider switching to using a project 4947 // pool. 4948 credentials := testutil.CredentialsEnv(ctx, "GCLOUD_TESTS_GOLANG_KEY") 4949 if credentials == nil { 4950 t.Fatal("credentials could not be determined, is GCLOUD_TESTS_GOLANG_KEY set correctly?") 4951 } 4952 if credentials.JSON == nil { 4953 t.Fatal("could not read the JSON key file, is GCLOUD_TESTS_GOLANG_KEY set correctly?") 4954 } 4955 conf, err := google.JWTConfigFromJSON(credentials.JSON) 4956 if err != nil { 4957 t.Fatal(err) 4958 } 4959 serviceAccountEmail := conf.Email 4960 4961 hmacKey, err := client.CreateHMACKey(ctx, projectID, serviceAccountEmail) 4962 if err != nil { 4963 t.Fatalf("Failed to create HMACKey: %v", err) 4964 } 4965 if hmacKey == nil { 4966 t.Fatal("Unexpectedly got back a nil HMAC key") 4967 } 4968 4969 if hmacKey.State != Active { 4970 t.Fatalf("Unexpected state %q, expected %q", hmacKey.State, Active) 4971 } 4972 4973 hkh := client.HMACKeyHandle(projectID, hmacKey.AccessID) 4974 // 1. Ensure that we CANNOT delete an ACTIVE key. 4975 if err := hkh.Delete(ctx); err == nil { 4976 t.Fatal("Unexpectedly deleted key whose state is ACTIVE: No error from Delete.") 4977 } 4978 4979 invalidStates := []HMACState{"", Deleted, "active", "inactive", "foo_bar"} 4980 for _, invalidState := range invalidStates { 4981 t.Run("invalid-"+string(invalidState), func(t *testing.T) { 4982 _, err := hkh.Update(ctx, HMACKeyAttrsToUpdate{ 4983 State: invalidState, 4984 }) 4985 if err == nil { 4986 t.Fatal("Unexpectedly succeeded") 4987 } 4988 invalidStateMsg := fmt.Sprintf(`storage: invalid state %q for update, must be either "ACTIVE" or "INACTIVE"`, invalidState) 4989 if err.Error() != invalidStateMsg { 4990 t.Fatalf("Mismatched error: got: %q\nwant: %q", err, invalidStateMsg) 4991 } 4992 }) 4993 } 4994 4995 // 2.1. Setting the State to Inactive should succeed. 4996 hu, err := hkh.Update(ctx, HMACKeyAttrsToUpdate{ 4997 State: Inactive, 4998 }) 4999 if err != nil { 5000 t.Fatalf("Unexpected Update failure: %v", err) 5001 } 5002 if got, want := hu.State, Inactive; got != want { 5003 t.Fatalf("Unexpected updated state %q, expected %q", got, want) 5004 } 5005 5006 // 2.2. Setting the State back to Active should succeed. 5007 hu, err = hkh.Update(ctx, HMACKeyAttrsToUpdate{ 5008 State: Active, 5009 }) 5010 if err != nil { 5011 t.Fatalf("Unexpected Update failure: %v", err) 5012 } 5013 if got, want := hu.State, Active; got != want { 5014 t.Fatalf("Unexpected updated state %q, expected %q", got, want) 5015 } 5016 5017 // 3. Verify that keys are listed as expected. 5018 iter := client.ListHMACKeys(ctx, projectID) 5019 count := 0 5020 for ; ; count++ { 5021 _, err := iter.Next() 5022 if err == iterator.Done { 5023 break 5024 } 5025 if err != nil { 5026 t.Fatalf("Failed to ListHMACKeys: %v", err) 5027 } 5028 } 5029 if count == 0 { 5030 t.Fatal("Failed to list any HMACKeys") 5031 } 5032 5033 // 4. Finally set it to back to Inactive and 5034 // then retry the deletion which should now succeed. 5035 _, _ = hkh.Update(ctx, HMACKeyAttrsToUpdate{ 5036 State: Inactive, 5037 }) 5038 if err := hkh.Delete(ctx); err != nil { 5039 t.Fatalf("Unexpected deletion failure: %v", err) 5040 } 5041 5042 _, err = hkh.Get(ctx) 5043 if err != nil && !strings.Contains(err.Error(), "404") { 5044 // If the deleted key has already been garbage collected, a 404 is expected. 5045 // Other errors should cause a failure and are not expected. 5046 t.Fatalf("Unexpected error: %v", err) 5047 } 5048 }) 5049 } 5050 5051 func TestIntegration_PostPolicyV4(t *testing.T) { 5052 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) { 5053 jwtConf, err := testutil.JWTConfig() 5054 if err != nil { 5055 t.Fatal(err) 5056 } 5057 if jwtConf == nil { 5058 t.Skip("JSON key file is not present") 5059 } 5060 5061 projectID := testutil.ProjID() 5062 newBucketName := prefix + uidSpace.New() 5063 b := client.Bucket(newBucketName) 5064 h := testHelper{t} 5065 h.mustCreate(b, projectID, nil) 5066 defer h.mustDeleteBucket(b) 5067 5068 statusCodeToRespond := 200 5069 opts := &PostPolicyV4Options{ 5070 GoogleAccessID: jwtConf.Email, 5071 PrivateKey: jwtConf.PrivateKey, 5072 5073 Expires: time.Now().Add(30 * time.Minute), 5074 5075 Fields: &PolicyV4Fields{ 5076 StatusCodeOnSuccess: statusCodeToRespond, 5077 ContentType: "text/plain", 5078 ACL: "public-read", 5079 }, 5080 5081 // The conditions that the uploaded file will be expected to conform to. 5082 Conditions: []PostPolicyV4Condition{ 5083 // Make the file a maximum of 10mB. 5084 ConditionContentLengthRange(0, 10<<20), 5085 ConditionStartsWith("$acl", "public"), 5086 }, 5087 } 5088 5089 objectName := uidSpaceObjects.New() 5090 object := b.Object(objectName) 5091 defer h.mustDeleteObject(object) 5092 5093 pv4, err := b.GenerateSignedPostPolicyV4(objectName, opts) 5094 if err != nil { 5095 t.Fatal(err) 5096 } 5097 5098 if err := verifyPostPolicy(pv4, object, bytes.Repeat([]byte("a"), 25), statusCodeToRespond); err != nil { 5099 t.Fatal(err) 5100 } 5101 }) 5102 } 5103 5104 // Verify that custom scopes passed in by the user are applied correctly. 5105 func TestIntegration_Scopes(t *testing.T) { 5106 ctx := skipJSONReads(context.Background(), "no reads in test") 5107 5108 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 5109 bkt := client.Bucket(bucket) 5110 obj := bkt.Object("test-scopes") 5111 contents := []byte("This object should not be written.\n") 5112 5113 // A client with ReadOnly scope should be able to read bucket successfully. 5114 if _, err := bkt.Attrs(ctx); err != nil { 5115 t.Errorf("client with ScopeReadOnly was not able to read attrs: %v", err) 5116 } 5117 5118 // Should not be able to write successfully. 5119 if err := writeObject(ctx, obj, "text/plain", contents); err == nil { 5120 if err := obj.Delete(ctx); err != nil { 5121 t.Logf("obj.Delete: %v", err) 5122 } 5123 t.Error("client with ScopeReadOnly was able to write an object unexpectedly.") 5124 } 5125 5126 // Should not be able to change permissions. 5127 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}}); err == nil { 5128 t.Error("client with ScopeReadWrite was able to change unexpectedly.") 5129 } 5130 }, option.WithScopes(ScopeReadOnly)) 5131 5132 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 5133 bkt := client.Bucket(bucket) 5134 obj := bkt.Object("test-scopes") 5135 contents := []byte("This object should be written.\n") 5136 5137 // A client with ReadWrite scope should be able to read bucket successfully. 5138 if _, err := bkt.Attrs(ctx); err != nil { 5139 t.Errorf("client with ScopeReadOnly was not able to read attrs: %v", err) 5140 } 5141 5142 // Should be able to write to an object. 5143 if err := writeObject(ctx, obj, "text/plain", contents); err != nil { 5144 t.Errorf("client with ScopeReadWrite was not able to write: %v", err) 5145 } 5146 defer func() { 5147 if err := obj.Delete(ctx); err != nil { 5148 t.Logf("obj.Delete: %v", err) 5149 } 5150 }() 5151 5152 // Should not be able to change permissions. 5153 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}}); err == nil { 5154 t.Error("client with ScopeReadWrite was able to change permissions unexpectedly") 5155 } 5156 }, option.WithScopes(ScopeReadWrite)) 5157 5158 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 5159 bkt := client.Bucket(bucket) 5160 obj := bkt.Object("test-scopes") 5161 contents := []byte("This object should be written.\n") 5162 5163 // A client without any scopes should not be able to perform ops. 5164 if _, err := bkt.Attrs(ctx); err == nil { 5165 t.Errorf("client with no scopes was able to read attrs unexpectedly") 5166 } 5167 5168 if err := writeObject(ctx, obj, "text/plain", contents); err == nil { 5169 if err := obj.Delete(ctx); err != nil { 5170 t.Logf("obj.Delete: %v", err) 5171 } 5172 t.Error("client with no scopes was able to write an object unexpectedly.") 5173 } 5174 5175 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}}); err == nil { 5176 t.Error("client with no scopes was able to change permissions unexpectedly") 5177 } 5178 }, option.WithScopes("")) 5179 } 5180 5181 func TestIntegration_SignedURL_WithCreds(t *testing.T) { 5182 ctx := context.Background() 5183 5184 creds, err := findTestCredentials(ctx, "GCLOUD_TESTS_GOLANG_KEY", ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform") 5185 if err != nil { 5186 t.Fatalf("unable to find test credentials: %v", err) 5187 } 5188 5189 multiTransportTest(skipGRPC("creds capture logic must be implemented for gRPC constructor"), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 5190 // We can use any client to create the object 5191 obj := "testBucketSignedURL" 5192 contents := []byte("test") 5193 if err := writeObject(ctx, client.Bucket(bucket).Object(obj), "text/plain", contents); err != nil { 5194 t.Fatalf("writing: %v", err) 5195 } 5196 opts := SignedURLOptions{ 5197 Method: "GET", 5198 Expires: time.Now().Add(30 * time.Second), 5199 } 5200 bkt := client.Bucket(bucket) 5201 url, err := bkt.SignedURL(obj, &opts) 5202 if err != nil { 5203 t.Fatalf("unable to create signed URL: %v", err) 5204 } 5205 5206 if err := verifySignedURL(url, nil, contents); err != nil { 5207 t.Fatalf("problem with the signed URL: %v", err) 5208 } 5209 }, option.WithCredentials(creds)) 5210 } 5211 5212 func TestIntegration_SignedURL_DefaultSignBytes(t *testing.T) { 5213 ctx := context.Background() 5214 5215 // Create another client to test the sign byte function as well 5216 scopes := []string{ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform"} 5217 ts := testutil.TokenSource(ctx, scopes...) 5218 if ts == nil { 5219 t.Fatalf("Cannot get token source to create client") 5220 } 5221 5222 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) { 5223 jwt, err := testutil.JWTConfig() 5224 if err != nil { 5225 t.Fatalf("unable to find test credentials: %v", err) 5226 } 5227 5228 obj := "testBucketSignedURL" 5229 contents := []byte("test") 5230 if err := writeObject(ctx, client.Bucket(bucket).Object(obj), "text/plain", contents); err != nil { 5231 t.Fatalf("writing: %v", err) 5232 } 5233 5234 opts := SignedURLOptions{ 5235 Method: "GET", 5236 Expires: time.Now().Add(30 * time.Second), 5237 GoogleAccessID: jwt.Email, 5238 } 5239 bkt := client.Bucket(bucket) 5240 url, err := bkt.SignedURL(obj, &opts) 5241 if err != nil { 5242 t.Fatalf("unable to create signed URL: %v", err) 5243 } 5244 5245 if err := verifySignedURL(url, nil, contents); err != nil { 5246 t.Fatalf("problem with the signed URL: %v", err) 5247 } 5248 }, option.WithTokenSource(ts)) 5249 5250 } 5251 5252 func TestIntegration_PostPolicyV4_WithCreds(t *testing.T) { 5253 // By default we are authed with a token source, so don't have the context to 5254 // read some of the fields from the keyfile. 5255 // Here we explictly send the key to the client. 5256 creds, err := findTestCredentials(context.Background(), "GCLOUD_TESTS_GOLANG_KEY", ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform") 5257 if err != nil { 5258 t.Fatalf("unable to find test credentials: %v", err) 5259 } 5260 5261 ctx := skipJSONReads(skipGRPC("creds capture logic must be implemented for gRPC constructor"), "test is not testing the read behaviour") 5262 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, clientWithCredentials *Client) { 5263 h := testHelper{t} 5264 5265 statusCodeToRespond := 200 5266 5267 for _, test := range []struct { 5268 desc string 5269 opts PostPolicyV4Options 5270 client *Client 5271 }{ 5272 { 5273 desc: "signing with the private key", 5274 opts: PostPolicyV4Options{ 5275 Expires: time.Now().Add(30 * time.Minute), 5276 5277 Fields: &PolicyV4Fields{ 5278 StatusCodeOnSuccess: statusCodeToRespond, 5279 ContentType: "text/plain", 5280 ACL: "public-read", 5281 }, 5282 }, 5283 client: clientWithCredentials, 5284 }, 5285 } { 5286 t.Run(test.desc, func(t *testing.T) { 5287 objectName := uidSpace.New() 5288 object := test.client.Bucket(bucket).Object(objectName) 5289 defer h.mustDeleteObject(object) 5290 5291 pv4, err := test.client.Bucket(bucket).GenerateSignedPostPolicyV4(objectName, &test.opts) 5292 if err != nil { 5293 t.Fatal(err) 5294 } 5295 5296 if err := verifyPostPolicy(pv4, object, bytes.Repeat([]byte("a"), 25), statusCodeToRespond); err != nil { 5297 t.Fatal(err) 5298 } 5299 }) 5300 } 5301 }, option.WithCredentials(creds)) 5302 5303 } 5304 5305 func TestIntegration_PostPolicyV4_BucketDefault(t *testing.T) { 5306 ctx := skipJSONReads(context.Background(), "test is not testing the read behaviour") 5307 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, clientWithoutPrivateKey *Client) { 5308 h := testHelper{t} 5309 5310 jwt, err := testutil.JWTConfig() 5311 if err != nil { 5312 t.Fatalf("unable to find test credentials: %v", err) 5313 } 5314 5315 statusCodeToRespond := 200 5316 5317 for _, test := range []struct { 5318 desc string 5319 opts PostPolicyV4Options 5320 client *Client 5321 }{ 5322 { 5323 desc: "signing with the default sign bytes func", 5324 opts: PostPolicyV4Options{ 5325 Expires: time.Now().Add(30 * time.Minute), 5326 GoogleAccessID: jwt.Email, 5327 Fields: &PolicyV4Fields{ 5328 StatusCodeOnSuccess: statusCodeToRespond, 5329 ContentType: "text/plain", 5330 ACL: "public-read", 5331 }, 5332 }, 5333 client: clientWithoutPrivateKey, 5334 }, 5335 } { 5336 t.Run(test.desc, func(t *testing.T) { 5337 objectName := uidSpaceObjects.New() 5338 object := test.client.Bucket(bucket).Object(objectName) 5339 defer h.mustDeleteObject(object) 5340 5341 pv4, err := test.client.Bucket(bucket).GenerateSignedPostPolicyV4(object.ObjectName(), &test.opts) 5342 if err != nil { 5343 t.Fatal(err) 5344 } 5345 5346 if err := verifyPostPolicy(pv4, object, bytes.Repeat([]byte("a"), 25), statusCodeToRespond); err != nil { 5347 t.Fatal(err) 5348 } 5349 }) 5350 } 5351 }) 5352 5353 } 5354 5355 // Tests that the same SignBytes function works for both 5356 // SignRawBytes on GeneratePostPolicyV4 and SignBytes on SignedURL 5357 func TestIntegration_PostPolicyV4_SignedURL_WithSignBytes(t *testing.T) { 5358 ctx := skipJSONReads(context.Background(), "test is not testing the read behaviour") 5359 multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) { 5360 5361 h := testHelper{t} 5362 projectID := testutil.ProjID() 5363 bucketName := prefix + uidSpace.New() 5364 objectName := uidSpaceObjects.New() 5365 fileBody := bytes.Repeat([]byte("b"), 25) 5366 bucket := client.Bucket(bucketName) 5367 5368 h.mustCreate(bucket, projectID, nil) 5369 defer h.mustDeleteBucket(bucket) 5370 5371 object := bucket.Object(objectName) 5372 defer h.mustDeleteObject(object) 5373 5374 jwtConf, err := testutil.JWTConfig() 5375 if err != nil { 5376 t.Fatal(err) 5377 } 5378 if jwtConf == nil { 5379 t.Skip("JSON key file is not present") 5380 } 5381 5382 signingFunc := func(b []byte) ([]byte, error) { 5383 parsedRSAPrivKey, err := parseKey(jwtConf.PrivateKey) 5384 if err != nil { 5385 return nil, err 5386 } 5387 sum := sha256.Sum256(b) 5388 return rsa.SignPKCS1v15(cryptorand.Reader, parsedRSAPrivKey, crypto.SHA256, sum[:]) 5389 } 5390 5391 // Test Post Policy 5392 successStatusCode := 200 5393 ppv4Opts := &PostPolicyV4Options{ 5394 GoogleAccessID: jwtConf.Email, 5395 SignRawBytes: signingFunc, 5396 Expires: time.Now().Add(30 * time.Minute), 5397 Fields: &PolicyV4Fields{ 5398 StatusCodeOnSuccess: successStatusCode, 5399 ContentType: "text/plain", 5400 ACL: "public-read", 5401 }, 5402 } 5403 5404 pv4, err := GenerateSignedPostPolicyV4(bucketName, objectName, ppv4Opts) 5405 if err != nil { 5406 t.Fatal(err) 5407 } 5408 5409 if err := verifyPostPolicy(pv4, object, fileBody, successStatusCode); err != nil { 5410 t.Fatal(err) 5411 } 5412 5413 // Test Signed URL 5414 signURLOpts := &SignedURLOptions{ 5415 GoogleAccessID: jwtConf.Email, 5416 SignBytes: signingFunc, 5417 Method: "GET", 5418 Expires: time.Now().Add(30 * time.Second), 5419 } 5420 5421 url, err := bucket.SignedURL(objectName, signURLOpts) 5422 if err != nil { 5423 t.Fatalf("unable to create signed URL: %v", err) 5424 } 5425 5426 if err := verifySignedURL(url, nil, fileBody); err != nil { 5427 t.Fatal(err) 5428 } 5429 }) 5430 } 5431 5432 func TestIntegration_OCTracing(t *testing.T) { 5433 multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { 5434 te := testutil.NewTestExporter() 5435 defer te.Unregister() 5436 5437 bkt := client.Bucket(bucket) 5438 bkt.Attrs(ctx) 5439 5440 if len(te.Spans) == 0 { 5441 t.Fatalf("Expected some spans to be created, but got %d", 0) 5442 } 5443 }) 5444 } 5445 5446 // verifySignedURL gets the bytes at the provided url and verifies them against the 5447 // expectedFileBody. Make sure the SignedURLOptions set the method as "GET". 5448 func verifySignedURL(url string, headers map[string][]string, expectedFileBody []byte) error { 5449 got, err := getURL(url, headers) 5450 if err != nil { 5451 return fmt.Errorf("getURL %q: %v", url, err) 5452 } 5453 if !bytes.Equal(got, expectedFileBody) { 5454 return fmt.Errorf("got %q, want %q", got, expectedFileBody) 5455 } 5456 return nil 5457 } 5458 5459 // verifyPostPolicy uploads a file to the obj using the provided post policy and 5460 // verifies that it was uploaded correctly 5461 func verifyPostPolicy(pv4 *PostPolicyV4, obj *ObjectHandle, bytesToWrite []byte, statusCodeOnSuccess int) error { 5462 ctx := context.Background() 5463 var res *http.Response 5464 5465 // Request is sent using a vanilla net/http client, so there are no built-in 5466 // retries. We must wrap with a retry to prevent flakes. 5467 return retry(ctx, 5468 func() error { 5469 formBuf := new(bytes.Buffer) 5470 mw := multipart.NewWriter(formBuf) 5471 for fieldName, value := range pv4.Fields { 5472 if err := mw.WriteField(fieldName, value); err != nil { 5473 return fmt.Errorf("Failed to write form field: %q: %v", fieldName, err) 5474 } 5475 } 5476 5477 // Now let's perform the upload 5478 mf, err := mw.CreateFormFile("file", "myfile.txt") 5479 if err != nil { 5480 return err 5481 } 5482 if _, err := mf.Write(bytesToWrite); err != nil { 5483 return err 5484 } 5485 if err := mw.Close(); err != nil { 5486 return err 5487 } 5488 5489 // Compose the HTTP request 5490 req, err := http.NewRequest("POST", pv4.URL, formBuf) 5491 if err != nil { 5492 return fmt.Errorf("Failed to compose HTTP request: %v", err) 5493 } 5494 5495 // Ensure the Content-Type is derived from the writer 5496 req.Header.Set("Content-Type", mw.FormDataContentType()) 5497 5498 // Send request 5499 res, err = http.DefaultClient.Do(req) 5500 if err != nil { 5501 return err 5502 } 5503 return nil 5504 }, 5505 func() error { 5506 // Check response 5507 if g, w := res.StatusCode, statusCodeOnSuccess; g != w { 5508 blob, _ := httputil.DumpResponse(res, true) 5509 return fmt.Errorf("Status code in response mismatch: got %d want %d\nBody: %s", g, w, blob) 5510 } 5511 io.Copy(ioutil.Discard, res.Body) 5512 5513 // Verify that the file was properly uploaded 5514 // by reading back its attributes and content 5515 attrs, err := obj.Attrs(ctx) 5516 if err != nil { 5517 return fmt.Errorf("Failed to retrieve attributes: %v", err) 5518 } 5519 if g, w := attrs.Size, int64(len(bytesToWrite)); g != w { 5520 return fmt.Errorf("ContentLength mismatch: got %d want %d", g, w) 5521 } 5522 if g, w := attrs.MD5, md5.Sum(bytesToWrite); !bytes.Equal(g, w[:]) { 5523 return fmt.Errorf("MD5Checksum mismatch\nGot: %x\nWant: %x", g, w) 5524 } 5525 5526 // Compare the uploaded body with the expected 5527 rd, err := obj.NewReader(ctx) 5528 if err != nil { 5529 return fmt.Errorf("Failed to create a reader: %v", err) 5530 } 5531 gotBody, err := ioutil.ReadAll(rd) 5532 if err != nil { 5533 return fmt.Errorf("Failed to read the body: %v", err) 5534 } 5535 if diff := testutil.Diff(string(gotBody), string(bytesToWrite)); diff != "" { 5536 return fmt.Errorf("Body mismatch: got - want +\n%s", diff) 5537 } 5538 return nil 5539 }) 5540 } 5541 5542 func findTestCredentials(ctx context.Context, envVar string, scopes ...string) (*google.Credentials, error) { 5543 key := os.Getenv(envVar) 5544 var opts []option.ClientOption 5545 if len(scopes) > 0 { 5546 opts = append(opts, option.WithScopes(scopes...)) 5547 } 5548 if key != "" { 5549 opts = append(opts, option.WithCredentialsFile(key)) 5550 } 5551 return transport.Creds(ctx, opts...) 5552 } 5553 5554 type testHelper struct { 5555 t *testing.T 5556 } 5557 5558 func (h testHelper) mustCreate(b *BucketHandle, projID string, attrs *BucketAttrs) { 5559 h.t.Helper() 5560 if err := b.Create(context.Background(), projID, attrs); err != nil { 5561 h.t.Fatalf("bucket create: %v", err) 5562 } 5563 } 5564 5565 func (h testHelper) mustDeleteBucket(b *BucketHandle) { 5566 h.t.Helper() 5567 if err := b.Delete(context.Background()); err != nil { 5568 h.t.Fatalf("bucket delete: %v", err) 5569 } 5570 } 5571 5572 func (h testHelper) mustBucketAttrs(b *BucketHandle) *BucketAttrs { 5573 h.t.Helper() 5574 attrs, err := b.Attrs(context.Background()) 5575 if err != nil { 5576 h.t.Fatalf("bucket attrs: %v", err) 5577 } 5578 return attrs 5579 } 5580 5581 // updating a bucket is conditionally idempotent on metageneration, so we pass that in to enable retries 5582 func (h testHelper) mustUpdateBucket(b *BucketHandle, ua BucketAttrsToUpdate, metageneration int64) *BucketAttrs { 5583 h.t.Helper() 5584 attrs, err := b.If(BucketConditions{MetagenerationMatch: metageneration}).Update(context.Background(), ua) 5585 if err != nil { 5586 h.t.Fatalf("update: %v", err) 5587 } 5588 return attrs 5589 } 5590 5591 func (h testHelper) mustObjectAttrs(o *ObjectHandle) *ObjectAttrs { 5592 h.t.Helper() 5593 attrs, err := o.Attrs(context.Background()) 5594 if err != nil { 5595 h.t.Fatalf("object attrs: %v", err) 5596 } 5597 return attrs 5598 } 5599 5600 func (h testHelper) mustDeleteObject(o *ObjectHandle) { 5601 h.t.Helper() 5602 if err := o.Retryer(WithPolicy(RetryAlways)).Delete(context.Background()); err != nil { 5603 var apiErr *apierror.APIError 5604 if ok := errors.As(err, &apiErr); ok { 5605 // Object may already be deleted with retry; if so skip. 5606 if apiErr.HTTPCode() == 404 || apiErr.GRPCStatus().Code() == codes.NotFound { 5607 return 5608 } 5609 } 5610 h.t.Fatalf("delete object %s from bucket %s: %v", o.ObjectName(), o.BucketName(), err) 5611 } 5612 } 5613 5614 // updating an object is conditionally idempotent on metageneration, so we pass that in to enable retries 5615 func (h testHelper) mustUpdateObject(o *ObjectHandle, ua ObjectAttrsToUpdate, metageneration int64) *ObjectAttrs { 5616 h.t.Helper() 5617 attrs, err := o.If(Conditions{MetagenerationMatch: metageneration}).Update(context.Background(), ua) 5618 if err != nil { 5619 h.t.Fatalf("update: %v", err) 5620 } 5621 return attrs 5622 } 5623 5624 func (h testHelper) mustWrite(w *Writer, data []byte) { 5625 h.t.Helper() 5626 if _, err := w.Write(data); err != nil { 5627 w.Close() 5628 h.t.Fatalf("write: %v", err) 5629 } 5630 if err := w.Close(); err != nil { 5631 h.t.Fatalf("close write: %v", err) 5632 } 5633 } 5634 5635 func (h testHelper) mustRead(obj *ObjectHandle) []byte { 5636 h.t.Helper() 5637 data, err := readObject(context.Background(), obj) 5638 if err != nil { 5639 h.t.Fatalf("read: %v", err) 5640 } 5641 return data 5642 } 5643 5644 // deleteObjectIfExists deletes an object with a RetryAlways policy (unless another 5645 // policy is supplied in the options). It will not return an error if the object 5646 // is already deleted/doesn't exist. It will time out after 15 seconds. 5647 func deleteObjectIfExists(o *ObjectHandle, retryOpts ...RetryOption) error { 5648 ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) 5649 defer cancel() 5650 retryOpts = append([]RetryOption{WithPolicy(RetryAlways)}, retryOpts...) 5651 5652 if err := o.Retryer(retryOpts...).Delete(ctx); err != nil { 5653 var apiErr *apierror.APIError 5654 if ok := errors.As(err, &apiErr); ok { 5655 // Object may already be deleted with retry; if so, return no error. 5656 if apiErr.HTTPCode() == 404 || apiErr.GRPCStatus().Code() == codes.NotFound { 5657 return nil 5658 } 5659 } 5660 return fmt.Errorf("delete object %s from bucket %s: %v", o.ObjectName(), o.BucketName(), err) 5661 } 5662 return nil 5663 } 5664 5665 func writeContents(w *Writer, contents []byte) error { 5666 if contents != nil { 5667 if _, err := w.Write(contents); err != nil { 5668 _ = w.Close() 5669 return err 5670 } 5671 } 5672 return w.Close() 5673 } 5674 5675 func writeObject(ctx context.Context, obj *ObjectHandle, contentType string, contents []byte) error { 5676 w := newWriter(ctx, obj, contentType, false) 5677 5678 return writeContents(w, contents) 5679 } 5680 5681 func newWriter(ctx context.Context, obj *ObjectHandle, contentType string, forceEmptyContentType bool) *Writer { 5682 w := obj.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx) 5683 w.ContentType = contentType 5684 w.ForceEmptyContentType = forceEmptyContentType 5685 5686 return w 5687 } 5688 5689 func readObject(ctx context.Context, obj *ObjectHandle) ([]byte, error) { 5690 r, err := obj.NewReader(ctx) 5691 if err != nil { 5692 return nil, err 5693 } 5694 defer r.Close() 5695 return ioutil.ReadAll(r) 5696 } 5697 5698 // cleanupBuckets deletes the bucket used for testing, as well as old 5699 // testing buckets that weren't cleaned previously. 5700 func cleanupBuckets() error { 5701 if testing.Short() { 5702 return nil // Don't clean up in short mode. 5703 } 5704 ctx := context.Background() 5705 client, err := newTestClient(ctx) 5706 if err != nil { 5707 log.Fatalf("NewClient: %v", err) 5708 } 5709 if client == nil { 5710 return nil // Don't cleanup if we're not configured correctly. 5711 } 5712 defer client.Close() 5713 if err := killBucket(ctx, client, bucketName); err != nil { 5714 return err 5715 } 5716 if err := killBucket(ctx, client, grpcBucketName); err != nil { 5717 return err 5718 } 5719 5720 // Delete buckets whose name begins with our test prefix, and which were 5721 // created a while ago. (Unfortunately GCS doesn't provide last-modified 5722 // time, which would be a better way to check for staleness.) 5723 if err := deleteExpiredBuckets(ctx, client, testPrefix); err != nil { 5724 return err 5725 } 5726 return deleteExpiredBuckets(ctx, client, grpcTestPrefix) 5727 } 5728 5729 func deleteExpiredBuckets(ctx context.Context, client *Client, prefix string) error { 5730 const expireAge = 24 * time.Hour 5731 projectID := testutil.ProjID() 5732 it := client.Buckets(ctx, projectID) 5733 it.Prefix = prefix 5734 for { 5735 bktAttrs, err := it.Next() 5736 if err == iterator.Done { 5737 break 5738 } 5739 if err != nil { 5740 return err 5741 } 5742 if time.Since(bktAttrs.Created) > expireAge { 5743 log.Printf("deleting bucket %q, which is more than %s old", bktAttrs.Name, expireAge) 5744 if err := killBucket(ctx, client, bktAttrs.Name); err != nil { 5745 return err 5746 } 5747 } 5748 } 5749 return nil 5750 } 5751 5752 // killBucket deletes a bucket and all its objects. 5753 func killBucket(ctx context.Context, client *Client, bucketName string) error { 5754 bkt := client.Bucket(bucketName) 5755 // Bucket must be empty to delete. 5756 it := bkt.Objects(ctx, nil) 5757 for { 5758 objAttrs, err := it.Next() 5759 if err == iterator.Done { 5760 break 5761 } 5762 if err != nil { 5763 return err 5764 } 5765 // Objects with a hold must have the hold released. 5766 if objAttrs.EventBasedHold || objAttrs.TemporaryHold { 5767 obj := bkt.Object(objAttrs.Name) 5768 if _, err := obj.Update(ctx, ObjectAttrsToUpdate{EventBasedHold: false, TemporaryHold: false}); err != nil { 5769 return fmt.Errorf("removing hold from %q: %v", bucketName+"/"+objAttrs.Name, err) 5770 } 5771 } 5772 if err := bkt.Object(objAttrs.Name).Delete(ctx); err != nil { 5773 return fmt.Errorf("deleting %q: %v", bucketName+"/"+objAttrs.Name, err) 5774 } 5775 } 5776 // GCS is eventually consistent, so this delete may fail because the 5777 // replica still sees an object in the bucket. We log the error and expect 5778 // a later test run to delete the bucket. 5779 if err := bkt.Delete(ctx); err != nil { 5780 log.Printf("deleting %q: %v", bucketName, err) 5781 } 5782 return nil 5783 } 5784 5785 func randomContents() []byte { 5786 h := md5.New() 5787 io.WriteString(h, fmt.Sprintf("hello world%d", rng.Intn(100000))) 5788 return h.Sum(nil) 5789 } 5790 5791 type zeros struct{} 5792 5793 func (zeros) Read(p []byte) (int, error) { return len(p), nil } 5794 5795 // Make a GET request to a URL using an unauthenticated client, and return its contents. 5796 func getURL(url string, headers map[string][]string) ([]byte, error) { 5797 req, err := http.NewRequest("GET", url, nil) 5798 if err != nil { 5799 return nil, err 5800 } 5801 req.Header = headers 5802 res, err := http.DefaultClient.Do(req) 5803 if err != nil { 5804 return nil, err 5805 } 5806 defer res.Body.Close() 5807 bytes, err := ioutil.ReadAll(res.Body) 5808 if err != nil { 5809 return nil, err 5810 } 5811 if res.StatusCode != 200 { 5812 return nil, fmt.Errorf("code=%d, body=%s", res.StatusCode, string(bytes)) 5813 } 5814 return bytes, nil 5815 } 5816 5817 // Make a PUT request to a URL using an unauthenticated client, and return its contents. 5818 func putURL(url string, headers map[string][]string, payload io.Reader) ([]byte, error) { 5819 req, err := http.NewRequest("PUT", url, payload) 5820 if err != nil { 5821 return nil, err 5822 } 5823 req.Header = headers 5824 res, err := http.DefaultClient.Do(req) 5825 if err != nil { 5826 return nil, err 5827 } 5828 defer res.Body.Close() 5829 bytes, err := ioutil.ReadAll(res.Body) 5830 if err != nil { 5831 return nil, err 5832 } 5833 if res.StatusCode != 200 { 5834 return nil, fmt.Errorf("code=%d, body=%s", res.StatusCode, string(bytes)) 5835 } 5836 return bytes, nil 5837 } 5838 5839 func keyFileEmail(filename string) (string, error) { 5840 bytes, err := ioutil.ReadFile(filename) 5841 if err != nil { 5842 return "", err 5843 } 5844 var v struct { 5845 ClientEmail string `json:"client_email"` 5846 } 5847 if err := json.Unmarshal(bytes, &v); err != nil { 5848 return "", err 5849 } 5850 return v.ClientEmail, nil 5851 } 5852 5853 type comparableACL interface { 5854 equals(ACLRule) bool 5855 } 5856 5857 type testACLRule ACLRule 5858 5859 func (acl testACLRule) equals(a ACLRule) bool { 5860 return cmp.Equal(a, ACLRule(acl)) 5861 } 5862 5863 type entityRoleACL struct { 5864 entity ACLEntity 5865 role ACLRole 5866 } 5867 5868 func (er entityRoleACL) equals(a ACLRule) bool { 5869 return a.Entity == er.entity && a.Role == er.role 5870 } 5871 5872 type prefixRoleACL struct { 5873 prefix string 5874 role ACLRole 5875 } 5876 5877 func (pr prefixRoleACL) equals(a ACLRule) bool { 5878 return strings.HasPrefix(string(a.Entity), pr.prefix) && a.Role == pr.role 5879 } 5880 5881 func containsACLRule(acl []ACLRule, want comparableACL) bool { 5882 for _, acl := range acl { 5883 if want.equals(acl) { 5884 return true 5885 } 5886 } 5887 return false 5888 } 5889 5890 // retry retries a function call as well as an (optional) correctness check for up 5891 // to 60 seconds. Both call and check must run without error in order to succeed. 5892 // If the timeout is hit, the most recent error from call or check will be returned. 5893 // This function should be used to wrap calls that might cause integration test 5894 // flakes due to delays in propagation (for example, metadata updates). 5895 func retry(ctx context.Context, call func() error, check func() error) error { 5896 timeout := time.After(60 * time.Second) 5897 var err error 5898 for { 5899 select { 5900 case <-timeout: 5901 return err 5902 default: 5903 } 5904 err = call() 5905 if err == nil { 5906 if check == nil || check() == nil { 5907 return nil 5908 } 5909 err = check() 5910 } 5911 time.Sleep(200 * time.Millisecond) 5912 } 5913 } 5914 5915 func retryOnNilAndTransientErrs(err error) bool { 5916 return err == nil || ShouldRetry(err) 5917 } 5918 func retryOnTransient400and403(err error) bool { 5919 var e *googleapi.Error 5920 var ae *apierror.APIError 5921 return ShouldRetry(err) || 5922 /* http */ errors.As(err, &e) && (e.Code == 400 || e.Code == 403) || 5923 /* grpc */ errors.As(err, &ae) && (ae.GRPCStatus().Code() == codes.InvalidArgument || ae.GRPCStatus().Code() == codes.PermissionDenied) 5924 } 5925 5926 func skipGRPC(reason string) context.Context { 5927 return context.WithValue(context.Background(), skipTransportTestKey("grpc"), reason) 5928 } 5929 5930 func skipHTTP(reason string) context.Context { 5931 ctx := context.WithValue(context.Background(), skipTransportTestKey("http"), reason) 5932 return context.WithValue(ctx, skipTransportTestKey("jsonReads"), reason) 5933 } 5934 5935 func skipJSONReads(ctx context.Context, reason string) context.Context { 5936 return context.WithValue(ctx, skipTransportTestKey("jsonReads"), reason) 5937 } 5938 5939 func skipXMLReads(ctx context.Context, reason string) context.Context { 5940 return context.WithValue(ctx, skipTransportTestKey("http"), reason) 5941 } 5942 5943 // Extract the error code if it's a googleapi.Error 5944 func extractErrCode(err error) int { 5945 if err == nil { 5946 return 0 5947 } 5948 var e *googleapi.Error 5949 if errors.As(err, &e) { 5950 return e.Code 5951 } 5952 5953 return -1 5954 } 5955 5956 func setUpRequesterPaysBucket(ctx context.Context, t *testing.T, bucket, object string, addOwnerEmail string) { 5957 t.Helper() 5958 client := testConfig(ctx, t) 5959 h := testHelper{t} 5960 5961 requesterPaysBucket := client.Bucket(bucket) 5962 5963 // Create a requester-pays bucket. 5964 h.mustCreate(requesterPaysBucket, testutil.ProjID(), &BucketAttrs{RequesterPays: true}) 5965 t.Cleanup(func() { h.mustDeleteBucket(requesterPaysBucket) }) 5966 5967 // Grant ownership 5968 if err := requesterPaysBucket.ACL().Set(ctx, ACLEntity("user-"+addOwnerEmail), RoleOwner); err != nil { 5969 t.Fatalf("set ACL: %v", err) 5970 } 5971 5972 h.mustWrite(requesterPaysBucket.Object(object).NewWriter(ctx), []byte("hello")) 5973 t.Cleanup(func() { 5974 err := requesterPaysBucket.Object(object).Delete(ctx) 5975 if err != nil { 5976 // only log because object may be deleted by test 5977 t.Logf("could not delete object: %v", err) 5978 } 5979 }) 5980 }