cloud.google.com/go/storage@v1.40.0/integration_test.go (about)

     1  // Copyright 2014 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package storage
    16  
    17  import (
    18  	"bytes"
    19  	"compress/gzip"
    20  	"context"
    21  	"crypto"
    22  	"crypto/md5"
    23  	cryptorand "crypto/rand"
    24  	"crypto/rsa"
    25  	"crypto/sha256"
    26  	"encoding/base64"
    27  	"encoding/json"
    28  	"errors"
    29  	"flag"
    30  	"fmt"
    31  	"hash/crc32"
    32  	"io"
    33  	"io/ioutil"
    34  	"log"
    35  	"math"
    36  	"math/rand"
    37  	"mime/multipart"
    38  	"net/http"
    39  	"net/http/httputil"
    40  	"os"
    41  	"sort"
    42  	"strconv"
    43  	"strings"
    44  	"testing"
    45  	"time"
    46  
    47  	"cloud.google.com/go/httpreplay"
    48  	"cloud.google.com/go/iam"
    49  	"cloud.google.com/go/iam/apiv1/iampb"
    50  	"cloud.google.com/go/internal/testutil"
    51  	"cloud.google.com/go/internal/uid"
    52  	"github.com/google/go-cmp/cmp"
    53  	"github.com/google/go-cmp/cmp/cmpopts"
    54  	"github.com/googleapis/gax-go/v2/apierror"
    55  	"golang.org/x/oauth2/google"
    56  	"google.golang.org/api/googleapi"
    57  	"google.golang.org/api/iterator"
    58  	itesting "google.golang.org/api/iterator/testing"
    59  	"google.golang.org/api/option"
    60  	raw "google.golang.org/api/storage/v1"
    61  	"google.golang.org/api/transport"
    62  	"google.golang.org/grpc"
    63  	"google.golang.org/grpc/codes"
    64  	"google.golang.org/grpc/status"
    65  )
    66  
    67  type skipTransportTestKey string
    68  
    69  const (
    70  	testPrefix     = "go-integration-test"
    71  	replayFilename = "storage.replay"
    72  	// TODO(jba): move to testutil, factor out from firestore/integration_test.go.
    73  	envFirestoreProjID     = "GCLOUD_TESTS_GOLANG_FIRESTORE_PROJECT_ID"
    74  	envFirestorePrivateKey = "GCLOUD_TESTS_GOLANG_FIRESTORE_KEY"
    75  	grpcTestPrefix         = "golang-grpc-test"
    76  )
    77  
    78  var (
    79  	record = flag.Bool("record", false, "record RPCs")
    80  
    81  	uidSpace        *uid.Space
    82  	uidSpaceObjects *uid.Space
    83  	bucketName      string
    84  	grpcBucketName  string
    85  	// Use our own random number generator to isolate the sequence of random numbers from
    86  	// other packages. This makes it possible to use HTTP replay and draw the same sequence
    87  	// of numbers as during recording.
    88  	rng           *rand.Rand
    89  	newTestClient func(ctx context.Context, opts ...option.ClientOption) (*Client, error)
    90  	replaying     bool
    91  	testTime      time.Time
    92  )
    93  
    94  func TestMain(m *testing.M) {
    95  	grpc.EnableTracing = true
    96  	cleanup := initIntegrationTest()
    97  	cleanupEmulatorClients := initEmulatorClients()
    98  	exit := m.Run()
    99  	if err := cleanup(); err != nil {
   100  		// Don't fail the test if cleanup fails.
   101  		log.Printf("Post-test cleanup failed: %v", err)
   102  	}
   103  	if err := cleanupEmulatorClients(); err != nil {
   104  		// Don't fail the test if cleanup fails.
   105  		log.Printf("Post-test cleanup failed for emulator clients: %v", err)
   106  	}
   107  
   108  	os.Exit(exit)
   109  }
   110  
   111  // If integration tests will be run, create a unique bucket for them.
   112  // Also, set newTestClient to handle record/replay.
   113  // Return a cleanup function.
   114  func initIntegrationTest() func() error {
   115  	flag.Parse() // needed for testing.Short()
   116  	switch {
   117  	case testing.Short() && *record:
   118  		log.Fatal("cannot combine -short and -record")
   119  		return nil
   120  
   121  	case testing.Short() && httpreplay.Supported() && testutil.CanReplay(replayFilename) && testutil.ProjID() != "":
   122  		// go test -short with a replay file will replay the integration tests, if
   123  		// the appropriate environment variables have been set.
   124  		replaying = true
   125  		httpreplay.DebugHeaders()
   126  		replayer, err := httpreplay.NewReplayer(replayFilename)
   127  		if err != nil {
   128  			log.Fatal(err)
   129  		}
   130  		var t time.Time
   131  		if err := json.Unmarshal(replayer.Initial(), &t); err != nil {
   132  			log.Fatal(err)
   133  		}
   134  		initUIDsAndRand(t)
   135  		newTestClient = func(ctx context.Context, _ ...option.ClientOption) (*Client, error) {
   136  			hc, err := replayer.Client(ctx) // no creds needed
   137  			if err != nil {
   138  				return nil, err
   139  			}
   140  			return NewClient(ctx, option.WithHTTPClient(hc))
   141  		}
   142  		log.Printf("replaying from %s", replayFilename)
   143  		return func() error { return replayer.Close() }
   144  
   145  	case testing.Short():
   146  		// go test -short without a replay file skips the integration tests.
   147  		if testutil.CanReplay(replayFilename) && testutil.ProjID() != "" {
   148  			log.Print("replay not supported for Go versions before 1.8")
   149  		}
   150  		newTestClient = nil
   151  		return func() error { return nil }
   152  
   153  	default: // Run integration tests against a real backend.
   154  		now := time.Now().UTC()
   155  		initUIDsAndRand(now)
   156  		var cleanup func() error
   157  		if *record && httpreplay.Supported() {
   158  			// Remember the time for replay.
   159  			nowBytes, err := json.Marshal(now)
   160  			if err != nil {
   161  				log.Fatal(err)
   162  			}
   163  			recorder, err := httpreplay.NewRecorder(replayFilename, nowBytes)
   164  			if err != nil {
   165  				log.Fatalf("could not record: %v", err)
   166  			}
   167  			newTestClient = func(ctx context.Context, opts ...option.ClientOption) (*Client, error) {
   168  				hc, err := recorder.Client(ctx, opts...)
   169  				if err != nil {
   170  					return nil, err
   171  				}
   172  				return NewClient(ctx, option.WithHTTPClient(hc))
   173  			}
   174  			cleanup = func() error {
   175  				err1 := cleanupBuckets()
   176  				err2 := recorder.Close()
   177  				if err1 != nil {
   178  					return err1
   179  				}
   180  				return err2
   181  			}
   182  			log.Printf("recording to %s", replayFilename)
   183  		} else {
   184  			if *record {
   185  				log.Print("record not supported for Go versions before 1.8")
   186  			}
   187  			newTestClient = NewClient
   188  			cleanup = cleanupBuckets
   189  		}
   190  		ctx := context.Background()
   191  		client, err := newTestClient(ctx)
   192  		if err != nil {
   193  			log.Fatalf("NewClient: %v", err)
   194  		}
   195  		if client == nil {
   196  			return func() error { return nil }
   197  		}
   198  		defer client.Close()
   199  		if err := client.Bucket(bucketName).Create(ctx, testutil.ProjID(), nil); err != nil {
   200  			log.Fatalf("creating bucket %q: %v", bucketName, err)
   201  		}
   202  		if err := client.Bucket(grpcBucketName).Create(ctx, testutil.ProjID(), nil); err != nil {
   203  			log.Fatalf("creating bucket %q: %v", grpcBucketName, err)
   204  		}
   205  		return cleanup
   206  	}
   207  }
   208  
   209  func initUIDsAndRand(t time.Time) {
   210  	uidSpace = uid.NewSpace("", &uid.Options{Time: t, Short: true})
   211  	bucketName = testPrefix + uidSpace.New()
   212  	uidSpaceObjects = uid.NewSpace("obj", &uid.Options{Time: t})
   213  	grpcBucketName = grpcTestPrefix + uidSpace.New()
   214  	// Use our own random source, to avoid other parts of the program taking
   215  	// random numbers from the global source and putting record and replay
   216  	// out of sync.
   217  	rng = testutil.NewRand(t)
   218  	testTime = t
   219  }
   220  
   221  // testConfig returns the Client used to access GCS. testConfig skips
   222  // the current test if credentials are not available or when being run
   223  // in Short mode.
   224  func testConfig(ctx context.Context, t *testing.T, opts ...option.ClientOption) *Client {
   225  	if testing.Short() && !replaying {
   226  		t.Skip("Integration tests skipped in short mode")
   227  	}
   228  	client, err := newTestClient(ctx, opts...)
   229  	if err != nil {
   230  		t.Fatalf("NewClient: %v", err)
   231  	}
   232  	if client == nil {
   233  		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
   234  	}
   235  	return client
   236  }
   237  
   238  // testConfigGPRC returns a gRPC-based client to access GCS. testConfigGRPC
   239  // skips the curent test when being run in Short mode.
   240  func testConfigGRPC(ctx context.Context, t *testing.T, opts ...option.ClientOption) (gc *Client) {
   241  	if testing.Short() {
   242  		t.Skip("Integration tests skipped in short mode")
   243  	}
   244  
   245  	gc, err := NewGRPCClient(ctx, opts...)
   246  	if err != nil {
   247  		t.Fatalf("NewGRPCClient: %v", err)
   248  	}
   249  
   250  	return
   251  }
   252  
   253  // initTransportClients initializes Storage clients for each supported transport.
   254  func initTransportClients(ctx context.Context, t *testing.T, opts ...option.ClientOption) map[string]*Client {
   255  	withJSON := append(opts, WithJSONReads())
   256  	return map[string]*Client{
   257  		"http": testConfig(ctx, t, opts...),
   258  		"grpc": testConfigGRPC(ctx, t, opts...),
   259  		// TODO: remove jsonReads when support for XML reads is dropped
   260  		"jsonReads": testConfig(ctx, t, withJSON...),
   261  	}
   262  }
   263  
   264  // multiTransportTest initializes fresh clients for each transport, then runs
   265  // given testing function using each transport-specific client, supplying the
   266  // test function with the sub-test instance, the context it was given, the name
   267  // of an existing bucket to use, a bucket name to use for bucket creation, and
   268  // the client to use.
   269  func multiTransportTest(ctx context.Context, t *testing.T,
   270  	test func(*testing.T, context.Context, string, string, *Client),
   271  	opts ...option.ClientOption) {
   272  	for transport, client := range initTransportClients(ctx, t, opts...) {
   273  		t.Run(transport, func(t *testing.T) {
   274  			t.Cleanup(func() {
   275  				client.Close()
   276  			})
   277  
   278  			if reason := ctx.Value(skipTransportTestKey(transport)); reason != nil {
   279  				t.Skip("transport", fmt.Sprintf("%q", transport), "explicitly skipped:", reason)
   280  			}
   281  
   282  			bucket := bucketName
   283  			prefix := testPrefix
   284  			if transport == "grpc" {
   285  				bucket = grpcBucketName
   286  				prefix = grpcTestPrefix
   287  			}
   288  
   289  			test(t, ctx, bucket, prefix, client)
   290  		})
   291  	}
   292  }
   293  
   294  func TestIntegration_BucketCreateDelete(t *testing.T) {
   295  	ctx := skipJSONReads(context.Background(), "no reads in test")
   296  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
   297  		projectID := testutil.ProjID()
   298  
   299  		labels := map[string]string{
   300  			"l1":    "v1",
   301  			"empty": "",
   302  		}
   303  
   304  		lifecycle := Lifecycle{
   305  			Rules: []LifecycleRule{{
   306  				Action: LifecycleAction{
   307  					Type:         SetStorageClassAction,
   308  					StorageClass: "NEARLINE",
   309  				},
   310  				Condition: LifecycleCondition{
   311  					AgeInDays:             10,
   312  					Liveness:              Archived,
   313  					CreatedBefore:         time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC),
   314  					MatchesStorageClasses: []string{"STANDARD"},
   315  					NumNewerVersions:      3,
   316  				},
   317  			}, {
   318  				Action: LifecycleAction{
   319  					Type:         SetStorageClassAction,
   320  					StorageClass: "ARCHIVE",
   321  				},
   322  				Condition: LifecycleCondition{
   323  					CustomTimeBefore:      time.Date(2020, 1, 2, 0, 0, 0, 0, time.UTC),
   324  					DaysSinceCustomTime:   20,
   325  					Liveness:              Live,
   326  					MatchesStorageClasses: []string{"STANDARD"},
   327  				},
   328  			}, {
   329  				Action: LifecycleAction{
   330  					Type: DeleteAction,
   331  				},
   332  				Condition: LifecycleCondition{
   333  					DaysSinceNoncurrentTime: 30,
   334  					Liveness:                Live,
   335  					NoncurrentTimeBefore:    time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC),
   336  					MatchesStorageClasses:   []string{"NEARLINE"},
   337  					NumNewerVersions:        10,
   338  				},
   339  			}, {
   340  				Action: LifecycleAction{
   341  					Type: DeleteAction,
   342  				},
   343  				Condition: LifecycleCondition{
   344  					AgeInDays:        10,
   345  					MatchesPrefix:    []string{"testPrefix"},
   346  					MatchesSuffix:    []string{"testSuffix"},
   347  					NumNewerVersions: 3,
   348  				},
   349  			}, {
   350  				Action: LifecycleAction{
   351  					Type: DeleteAction,
   352  				},
   353  				Condition: LifecycleCondition{
   354  					AllObjects: true,
   355  				},
   356  			}},
   357  		}
   358  
   359  		// testedAttrs are the bucket attrs directly compared in this test
   360  		type testedAttrs struct {
   361  			StorageClass          string
   362  			VersioningEnabled     bool
   363  			LocationType          string
   364  			Labels                map[string]string
   365  			Location              string
   366  			Lifecycle             Lifecycle
   367  			CustomPlacementConfig *CustomPlacementConfig
   368  		}
   369  
   370  		for _, test := range []struct {
   371  			name      string
   372  			attrs     *BucketAttrs
   373  			wantAttrs testedAttrs
   374  		}{
   375  			{
   376  				name:  "no attrs",
   377  				attrs: nil,
   378  				wantAttrs: testedAttrs{
   379  					StorageClass:      "STANDARD",
   380  					VersioningEnabled: false,
   381  					LocationType:      "multi-region",
   382  					Location:          "US",
   383  				},
   384  			},
   385  			{
   386  				name: "with attrs",
   387  				attrs: &BucketAttrs{
   388  					StorageClass:      "NEARLINE",
   389  					VersioningEnabled: true,
   390  					Labels:            labels,
   391  					Lifecycle:         lifecycle,
   392  					Location:          "SOUTHAMERICA-EAST1",
   393  				},
   394  				wantAttrs: testedAttrs{
   395  					StorageClass:      "NEARLINE",
   396  					VersioningEnabled: true,
   397  					Labels:            labels,
   398  					Location:          "SOUTHAMERICA-EAST1",
   399  					LocationType:      "region",
   400  					Lifecycle:         lifecycle,
   401  				},
   402  			},
   403  			{
   404  				name: "dual-region",
   405  				attrs: &BucketAttrs{
   406  					Location: "US",
   407  					CustomPlacementConfig: &CustomPlacementConfig{
   408  						DataLocations: []string{"US-EAST1", "US-WEST1"},
   409  					},
   410  				},
   411  				wantAttrs: testedAttrs{
   412  					Location:     "US",
   413  					LocationType: "dual-region",
   414  					StorageClass: "STANDARD",
   415  					CustomPlacementConfig: &CustomPlacementConfig{
   416  						DataLocations: []string{"US-EAST1", "US-WEST1"},
   417  					},
   418  				},
   419  			},
   420  		} {
   421  			t.Run(test.name, func(t *testing.T) {
   422  				newBucketName := prefix + uidSpace.New()
   423  				b := client.Bucket(newBucketName)
   424  
   425  				if err := b.Create(ctx, projectID, test.attrs); err != nil {
   426  					t.Fatalf("bucket create: %v", err)
   427  				}
   428  
   429  				gotAttrs, err := b.Attrs(ctx)
   430  				if err != nil {
   431  					t.Fatalf("bucket attrs: %v", err)
   432  				}
   433  
   434  				// All newly created buckets should conform to the following:
   435  				if gotAttrs.MetaGeneration != 1 {
   436  					t.Errorf("metageneration: got %d, should be 1", gotAttrs.MetaGeneration)
   437  				}
   438  				if gotAttrs.ProjectNumber == 0 {
   439  					t.Errorf("got a zero ProjectNumber")
   440  				}
   441  
   442  				// Test specific wanted bucket attrs
   443  				if gotAttrs.VersioningEnabled != test.wantAttrs.VersioningEnabled {
   444  					t.Errorf("versioning enabled: got %t, want %t", gotAttrs.VersioningEnabled, test.wantAttrs.VersioningEnabled)
   445  				}
   446  				if got, want := gotAttrs.Labels, test.wantAttrs.Labels; !testutil.Equal(got, want) {
   447  					t.Errorf("labels: got %v, want %v", got, want)
   448  				}
   449  				if diff := cmp.Diff(gotAttrs.Lifecycle, test.wantAttrs.Lifecycle); diff != "" {
   450  					t.Errorf("lifecycle: diff got vs. want: %v", diff)
   451  				}
   452  				if gotAttrs.LocationType != test.wantAttrs.LocationType {
   453  					t.Errorf("location type: got %s, want %s", gotAttrs.LocationType, test.wantAttrs.LocationType)
   454  				}
   455  				if gotAttrs.StorageClass != test.wantAttrs.StorageClass {
   456  					t.Errorf("storage class: got %s, want %s", gotAttrs.StorageClass, test.wantAttrs.StorageClass)
   457  				}
   458  				if gotAttrs.Location != test.wantAttrs.Location {
   459  					t.Errorf("location: got %s, want %s", gotAttrs.Location, test.wantAttrs.Location)
   460  				}
   461  				if got, want := gotAttrs.CustomPlacementConfig, test.wantAttrs.CustomPlacementConfig; !testutil.Equal(got, want) {
   462  					t.Errorf("customPlacementConfig: \ngot\t%v\nwant\t%v", got, want)
   463  				}
   464  
   465  				// Delete the bucket and check that the deletion was succesful
   466  				if err := b.Delete(ctx); err != nil {
   467  					t.Fatalf("bucket delete: %v", err)
   468  				}
   469  				_, err = b.Attrs(ctx)
   470  				if err != ErrBucketNotExist {
   471  					t.Fatalf("expected ErrBucketNotExist, got %v", err)
   472  				}
   473  			})
   474  		}
   475  	})
   476  }
   477  
   478  func TestIntegration_BucketLifecycle(t *testing.T) {
   479  	ctx := skipJSONReads(context.Background(), "no reads in test")
   480  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
   481  		h := testHelper{t}
   482  
   483  		wantLifecycle := Lifecycle{
   484  			Rules: []LifecycleRule{
   485  				{
   486  					Action:    LifecycleAction{Type: AbortIncompleteMPUAction},
   487  					Condition: LifecycleCondition{AgeInDays: 30},
   488  				},
   489  				{
   490  					Action:    LifecycleAction{Type: DeleteAction},
   491  					Condition: LifecycleCondition{AllObjects: true},
   492  				},
   493  			},
   494  		}
   495  
   496  		bucket := client.Bucket(prefix + uidSpace.New())
   497  
   498  		// Create bucket with lifecycle rules
   499  		h.mustCreate(bucket, testutil.ProjID(), &BucketAttrs{
   500  			Lifecycle: wantLifecycle,
   501  		})
   502  		defer h.mustDeleteBucket(bucket)
   503  
   504  		attrs := h.mustBucketAttrs(bucket)
   505  		if !testutil.Equal(attrs.Lifecycle, wantLifecycle) {
   506  			t.Fatalf("got %v, want %v", attrs.Lifecycle, wantLifecycle)
   507  		}
   508  
   509  		// Remove lifecycle rules
   510  		ua := BucketAttrsToUpdate{Lifecycle: &Lifecycle{}}
   511  		attrs = h.mustUpdateBucket(bucket, ua, attrs.MetaGeneration)
   512  		if !testutil.Equal(attrs.Lifecycle, Lifecycle{}) {
   513  			t.Fatalf("got %v, want %v", attrs.Lifecycle, Lifecycle{})
   514  		}
   515  
   516  		// Update bucket with a lifecycle rule
   517  		ua = BucketAttrsToUpdate{Lifecycle: &wantLifecycle}
   518  		attrs = h.mustUpdateBucket(bucket, ua, attrs.MetaGeneration)
   519  		if !testutil.Equal(attrs.Lifecycle, wantLifecycle) {
   520  			t.Fatalf("got %v, want %v", attrs.Lifecycle, wantLifecycle)
   521  		}
   522  	})
   523  }
   524  
   525  func TestIntegration_BucketUpdate(t *testing.T) {
   526  	ctx := skipJSONReads(context.Background(), "no reads in test")
   527  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
   528  		h := testHelper{t}
   529  
   530  		b := client.Bucket(prefix + uidSpace.New())
   531  		h.mustCreate(b, testutil.ProjID(), nil)
   532  		defer h.mustDeleteBucket(b)
   533  
   534  		attrs := h.mustBucketAttrs(b)
   535  		if attrs.VersioningEnabled {
   536  			t.Fatal("bucket should not have versioning by default")
   537  		}
   538  		if len(attrs.Labels) > 0 {
   539  			t.Fatal("bucket should not have labels initially")
   540  		}
   541  
   542  		// Turn on versioning, add some labels.
   543  		ua := BucketAttrsToUpdate{VersioningEnabled: true}
   544  		ua.SetLabel("l1", "v1")
   545  		ua.SetLabel("empty", "")
   546  		attrs = h.mustUpdateBucket(b, ua, attrs.MetaGeneration)
   547  		if !attrs.VersioningEnabled {
   548  			t.Fatal("should have versioning now")
   549  		}
   550  		wantLabels := map[string]string{
   551  			"l1":    "v1",
   552  			"empty": "",
   553  		}
   554  		if !testutil.Equal(attrs.Labels, wantLabels) {
   555  			t.Fatalf("add labels: got %v, want %v", attrs.Labels, wantLabels)
   556  		}
   557  
   558  		// Turn off versioning again; add and remove some more labels.
   559  		ua = BucketAttrsToUpdate{VersioningEnabled: false}
   560  		ua.SetLabel("l1", "v2")   // update
   561  		ua.SetLabel("new", "new") // create
   562  		ua.DeleteLabel("empty")   // delete
   563  		ua.DeleteLabel("absent")  // delete non-existent
   564  		attrs = h.mustUpdateBucket(b, ua, attrs.MetaGeneration)
   565  		if attrs.VersioningEnabled {
   566  			t.Fatal("should have versioning off")
   567  		}
   568  		wantLabels = map[string]string{
   569  			"l1":  "v2",
   570  			"new": "new",
   571  		}
   572  		if !testutil.Equal(attrs.Labels, wantLabels) {
   573  			t.Fatalf("got %v, want %v", attrs.Labels, wantLabels)
   574  		}
   575  
   576  		// Configure a lifecycle
   577  		wantLifecycle := Lifecycle{
   578  			Rules: []LifecycleRule{
   579  				{
   580  					Action: LifecycleAction{Type: "Delete"},
   581  					Condition: LifecycleCondition{
   582  						AgeInDays:     30,
   583  						MatchesPrefix: []string{"testPrefix"},
   584  						MatchesSuffix: []string{"testSuffix"},
   585  					},
   586  				},
   587  			},
   588  		}
   589  		ua = BucketAttrsToUpdate{Lifecycle: &wantLifecycle}
   590  		attrs = h.mustUpdateBucket(b, ua, attrs.MetaGeneration)
   591  		if !testutil.Equal(attrs.Lifecycle, wantLifecycle) {
   592  			t.Fatalf("got %v, want %v", attrs.Lifecycle, wantLifecycle)
   593  		}
   594  		// Check that StorageClass has "STANDARD" value for unset field by default
   595  		// before passing new value.
   596  		wantStorageClass := "STANDARD"
   597  		if !testutil.Equal(attrs.StorageClass, wantStorageClass) {
   598  			t.Fatalf("got %v, want %v", attrs.StorageClass, wantStorageClass)
   599  		}
   600  		wantStorageClass = "NEARLINE"
   601  		ua = BucketAttrsToUpdate{StorageClass: wantStorageClass}
   602  		attrs = h.mustUpdateBucket(b, ua, attrs.MetaGeneration)
   603  		if !testutil.Equal(attrs.StorageClass, wantStorageClass) {
   604  			t.Fatalf("got %v, want %v", attrs.StorageClass, wantStorageClass)
   605  		}
   606  	})
   607  }
   608  
   609  func TestIntegration_BucketPolicyOnly(t *testing.T) {
   610  	ctx := skipJSONReads(context.Background(), "no reads in test")
   611  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
   612  		h := testHelper{t}
   613  
   614  		bkt := client.Bucket(prefix + uidSpace.New())
   615  		h.mustCreate(bkt, testutil.ProjID(), nil)
   616  		defer h.mustDeleteBucket(bkt)
   617  
   618  		// Insert an object with custom ACL.
   619  		o := bkt.Object("bucketPolicyOnly")
   620  		defer func() {
   621  			if err := o.Delete(ctx); err != nil {
   622  				log.Printf("failed to delete test object: %v", err)
   623  			}
   624  		}()
   625  		wc := o.NewWriter(ctx)
   626  		wc.ContentType = "text/plain"
   627  		h.mustWrite(wc, []byte("test"))
   628  		a := o.ACL()
   629  		aclEntity := ACLEntity("user-test@example.com")
   630  		err := a.Set(ctx, aclEntity, RoleReader)
   631  		if err != nil {
   632  			t.Fatalf("set ACL failed: %v", err)
   633  		}
   634  
   635  		// Enable BucketPolicyOnly.
   636  		ua := BucketAttrsToUpdate{BucketPolicyOnly: &BucketPolicyOnly{Enabled: true}}
   637  		attrs := h.mustUpdateBucket(bkt, ua, h.mustBucketAttrs(bkt).MetaGeneration)
   638  		if got, want := attrs.BucketPolicyOnly.Enabled, true; got != want {
   639  			t.Fatalf("got %v, want %v", got, want)
   640  		}
   641  		if got := attrs.BucketPolicyOnly.LockedTime; got.IsZero() {
   642  			t.Fatal("got a zero time value, want a populated value")
   643  		}
   644  
   645  		// Confirm BucketAccessControl returns error, since we cannot get legacy ACL
   646  		// for a bucket that has uniform bucket-level access.
   647  
   648  		// Metadata updates may be delayed up to 10s. Since we expect an error from
   649  		// this call, we retry on a nil error until we get the non-retryable error
   650  		// that we are expecting.
   651  		ctxWithTimeout, cancelCtx := context.WithTimeout(ctx, time.Second*10)
   652  		b := bkt.Retryer(WithErrorFunc(retryOnNilAndTransientErrs))
   653  		_, err = b.ACL().List(ctxWithTimeout)
   654  		cancelCtx()
   655  		if err == nil {
   656  			t.Errorf("ACL.List: expected bucket ACL list to fail")
   657  		}
   658  
   659  		// Confirm ObjectAccessControl returns error, for same reason as above.
   660  		ctxWithTimeout, cancelCtx = context.WithTimeout(ctx, time.Second*10)
   661  		_, err = o.Retryer(WithErrorFunc(retryOnNilAndTransientErrs)).ACL().List(ctxWithTimeout)
   662  		cancelCtx()
   663  		if err == nil {
   664  			t.Errorf("ACL.List: expected object ACL list to fail")
   665  		}
   666  
   667  		// Disable BucketPolicyOnly.
   668  		ua = BucketAttrsToUpdate{BucketPolicyOnly: &BucketPolicyOnly{Enabled: false}}
   669  		attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration)
   670  		if got, want := attrs.BucketPolicyOnly.Enabled, false; got != want {
   671  			t.Fatalf("attrs.BucketPolicyOnly.Enabled: got %v, want %v", got, want)
   672  		}
   673  
   674  		// Check that the object ACL rules are the same.
   675  
   676  		// Metadata updates may be delayed up to 10s. Before that, we can get a 400
   677  		// indicating that uniform bucket-level access is still enabled in HTTP.
   678  		// We need to retry manually as GRPC will not error but provide empty ACL.
   679  		var acl []ACLRule
   680  		err = retry(ctx, func() error {
   681  			var err error
   682  			acl, err = o.ACL().List(ctx)
   683  			if err != nil {
   684  				return fmt.Errorf("ACL.List: object ACL list failed: %v", err)
   685  			}
   686  			return nil
   687  		}, func() error {
   688  			if !containsACLRule(acl, entityRoleACL{aclEntity, RoleReader}) {
   689  				return fmt.Errorf("containsACL: expected ACL %v to include custom ACL entity %v", acl, entityRoleACL{aclEntity, RoleReader})
   690  			}
   691  			return nil
   692  		})
   693  		if err != nil {
   694  			t.Fatal(err)
   695  		}
   696  	})
   697  }
   698  
   699  func TestIntegration_UniformBucketLevelAccess(t *testing.T) {
   700  	ctx := skipJSONReads(context.Background(), "no reads in test")
   701  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
   702  		h := testHelper{t}
   703  		bkt := client.Bucket(prefix + uidSpace.New())
   704  		h.mustCreate(bkt, testutil.ProjID(), nil)
   705  		defer h.mustDeleteBucket(bkt)
   706  
   707  		// Insert an object with custom ACL.
   708  		o := bkt.Object("uniformBucketLevelAccess")
   709  		defer func() {
   710  			if err := o.Delete(ctx); err != nil {
   711  				log.Printf("failed to delete test object: %v", err)
   712  			}
   713  		}()
   714  		wc := o.NewWriter(ctx)
   715  		wc.ContentType = "text/plain"
   716  		h.mustWrite(wc, []byte("test"))
   717  		a := o.ACL()
   718  		aclEntity := ACLEntity("user-test@example.com")
   719  		err := a.Set(ctx, aclEntity, RoleReader)
   720  		if err != nil {
   721  			t.Fatalf("set ACL failed: %v", err)
   722  		}
   723  
   724  		// Enable UniformBucketLevelAccess.
   725  		ua := BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: true}}
   726  		attrs := h.mustUpdateBucket(bkt, ua, h.mustBucketAttrs(bkt).MetaGeneration)
   727  		if got, want := attrs.UniformBucketLevelAccess.Enabled, true; got != want {
   728  			t.Fatalf("got %v, want %v", got, want)
   729  		}
   730  		if got := attrs.UniformBucketLevelAccess.LockedTime; got.IsZero() {
   731  			t.Fatal("got a zero time value, want a populated value")
   732  		}
   733  
   734  		// Confirm BucketAccessControl returns error.
   735  		// We retry on nil to account for propagation delay in metadata update.
   736  		ctxWithTimeout, cancelCtx := context.WithTimeout(ctx, time.Second*10)
   737  		b := bkt.Retryer(WithErrorFunc(retryOnNilAndTransientErrs))
   738  		_, err = b.ACL().List(ctxWithTimeout)
   739  		cancelCtx()
   740  		if err == nil {
   741  			t.Errorf("ACL.List: expected bucket ACL list to fail")
   742  		}
   743  
   744  		// Confirm ObjectAccessControl returns error.
   745  		ctxWithTimeout, cancelCtx = context.WithTimeout(ctx, time.Second*10)
   746  		_, err = o.Retryer(WithErrorFunc(retryOnNilAndTransientErrs)).ACL().List(ctxWithTimeout)
   747  		cancelCtx()
   748  		if err == nil {
   749  			t.Errorf("ACL.List: expected object ACL list to fail")
   750  		}
   751  
   752  		// Disable UniformBucketLevelAccess.
   753  		ua = BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: false}}
   754  		attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration)
   755  		if got, want := attrs.UniformBucketLevelAccess.Enabled, false; got != want {
   756  			t.Fatalf("got %v, want %v", got, want)
   757  		}
   758  
   759  		// Metadata updates may be delayed up to 10s. Before that, we can get a 400
   760  		// indicating that uniform bucket-level access is still enabled in HTTP.
   761  		// We need to retry manually as GRPC will not error but provide empty ACL.
   762  		var acl []ACLRule
   763  		err = retry(ctx, func() error {
   764  			var err error
   765  			acl, err = o.ACL().List(ctx)
   766  			if err != nil {
   767  				return fmt.Errorf("ACL.List: object ACL list failed: %v", err)
   768  			}
   769  			return nil
   770  		}, func() error {
   771  			if !containsACLRule(acl, entityRoleACL{aclEntity, RoleReader}) {
   772  				return fmt.Errorf("containsACL: expected ACL %v to include custom ACL entity %v", acl, entityRoleACL{aclEntity, RoleReader})
   773  			}
   774  			return nil
   775  		})
   776  		if err != nil {
   777  			t.Fatal(err)
   778  		}
   779  	})
   780  }
   781  
   782  func TestIntegration_PublicAccessPrevention(t *testing.T) {
   783  	ctx := skipJSONReads(context.Background(), "no reads in test")
   784  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
   785  		h := testHelper{t}
   786  
   787  		// Create a bucket with PublicAccessPrevention enforced.
   788  		bkt := client.Bucket(prefix + uidSpace.New())
   789  		h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{PublicAccessPrevention: PublicAccessPreventionEnforced})
   790  		defer h.mustDeleteBucket(bkt)
   791  
   792  		// Making bucket public should fail.
   793  		policy, err := bkt.IAM().V3().Policy(ctx)
   794  		if err != nil {
   795  			t.Fatalf("fetching bucket IAM policy: %v", err)
   796  		}
   797  		policy.Bindings = append(policy.Bindings, &iampb.Binding{
   798  			Role:    "roles/storage.objectViewer",
   799  			Members: []string{iam.AllUsers},
   800  		})
   801  		if err := bkt.IAM().V3().SetPolicy(ctx, policy); err == nil {
   802  			t.Error("SetPolicy: expected adding AllUsers policy to bucket should fail")
   803  		}
   804  
   805  		// Making object public via ACL should fail.
   806  		o := bkt.Object("publicAccessPrevention")
   807  		defer func() {
   808  			if err := o.Delete(ctx); err != nil {
   809  				log.Printf("failed to delete test object: %v", err)
   810  			}
   811  		}()
   812  		wc := o.NewWriter(ctx)
   813  		wc.ContentType = "text/plain"
   814  		h.mustWrite(wc, []byte("test"))
   815  		a := o.ACL()
   816  		if err := a.Set(ctx, AllUsers, RoleReader); err == nil {
   817  			t.Error("ACL.Set: expected adding AllUsers ACL to object should fail")
   818  		}
   819  
   820  		// Update PAP setting to inherited should work and not affect UBLA setting.
   821  		attrs, err := bkt.Update(ctx, BucketAttrsToUpdate{PublicAccessPrevention: PublicAccessPreventionInherited})
   822  		if err != nil {
   823  			t.Fatalf("updating PublicAccessPrevention failed: %v", err)
   824  		}
   825  		if attrs.PublicAccessPrevention != PublicAccessPreventionInherited {
   826  			t.Errorf("updating PublicAccessPrevention: got %s, want %s", attrs.PublicAccessPrevention, PublicAccessPreventionInherited)
   827  		}
   828  		if attrs.UniformBucketLevelAccess.Enabled || attrs.BucketPolicyOnly.Enabled {
   829  			t.Error("updating PublicAccessPrevention changed UBLA setting")
   830  		}
   831  
   832  		// Now, making object public or making bucket public should succeed. Run with
   833  		// retry because ACL settings may take time to propagate.
   834  		retrier := func(err error) bool {
   835  			// Once ACL settings propagate, PAP should no longer be enforced and the call will succeed.
   836  			// In the meantime, while PAP is enforced, trying to set ACL results in:
   837  			// 	-	FailedPrecondition for gRPC
   838  			// 	-	condition not met (412) for HTTP
   839  			return ShouldRetry(err) || status.Code(err) == codes.FailedPrecondition || extractErrCode(err) == http.StatusPreconditionFailed
   840  		}
   841  
   842  		ctxWithTimeout, cancelCtx := context.WithTimeout(ctx, time.Second*10)
   843  		a = o.Retryer(WithErrorFunc(retrier), WithPolicy(RetryAlways)).ACL()
   844  		err = a.Set(ctxWithTimeout, AllUsers, RoleReader)
   845  		cancelCtx()
   846  		if err != nil {
   847  			t.Errorf("ACL.Set: making object public failed: %v", err)
   848  		}
   849  
   850  		policy, err = bkt.IAM().V3().Policy(ctx)
   851  		if err != nil {
   852  			t.Fatalf("fetching bucket IAM policy: %v", err)
   853  		}
   854  		policy.Bindings = append(policy.Bindings, &iampb.Binding{
   855  			Role:    "roles/storage.objectViewer",
   856  			Members: []string{iam.AllUsers},
   857  		})
   858  		if err := bkt.IAM().V3().SetPolicy(ctx, policy); err != nil {
   859  			t.Errorf("SetPolicy: making bucket public failed: %v", err)
   860  		}
   861  
   862  		// Updating UBLA should not affect PAP setting.
   863  		attrs, err = bkt.Update(ctx, BucketAttrsToUpdate{UniformBucketLevelAccess: &UniformBucketLevelAccess{Enabled: true}})
   864  		if err != nil {
   865  			t.Fatalf("updating UBLA failed: %v", err)
   866  		}
   867  		if !attrs.UniformBucketLevelAccess.Enabled {
   868  			t.Error("updating UBLA: got UBLA not enabled, want enabled")
   869  		}
   870  		if attrs.PublicAccessPrevention != PublicAccessPreventionInherited {
   871  			t.Errorf("updating UBLA: got %s, want %s", attrs.PublicAccessPrevention, PublicAccessPreventionInherited)
   872  		}
   873  	})
   874  }
   875  
   876  func TestIntegration_Autoclass(t *testing.T) {
   877  	ctx := skipJSONReads(context.Background(), "no reads in test")
   878  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
   879  		h := testHelper{t}
   880  
   881  		// Create a bucket with Autoclass enabled.
   882  		bkt := client.Bucket(prefix + uidSpace.New())
   883  		h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{Autoclass: &Autoclass{Enabled: true}})
   884  		defer h.mustDeleteBucket(bkt)
   885  
   886  		// Get Autoclass configuration from bucket attrs.
   887  		// Autoclass.TerminalStorageClass is defaulted to NEARLINE if not specified.
   888  		attrs, err := bkt.Attrs(ctx)
   889  		if err != nil {
   890  			t.Fatalf("get bucket attrs failed: %v", err)
   891  		}
   892  		var toggleTime time.Time
   893  		var tscUpdateTime time.Time
   894  		if attrs != nil && attrs.Autoclass != nil {
   895  			if got, want := attrs.Autoclass.Enabled, true; got != want {
   896  				t.Errorf("attr.Autoclass.Enabled = %v, want %v", got, want)
   897  			}
   898  			if toggleTime = attrs.Autoclass.ToggleTime; toggleTime.IsZero() {
   899  				t.Error("got a zero time value, want a populated value")
   900  			}
   901  			if got, want := attrs.Autoclass.TerminalStorageClass, "NEARLINE"; got != want {
   902  				t.Errorf("attr.Autoclass.TerminalStorageClass = %v, want %v", got, want)
   903  			}
   904  			if tscUpdateTime := attrs.Autoclass.TerminalStorageClassUpdateTime; tscUpdateTime.IsZero() {
   905  				t.Error("got a zero time value, want a populated value")
   906  			}
   907  		}
   908  
   909  		// Update TerminalStorageClass on the bucket.
   910  		ua := BucketAttrsToUpdate{Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"}}
   911  		attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration)
   912  		if got, want := attrs.Autoclass.Enabled, true; got != want {
   913  			t.Errorf("attr.Autoclass.Enabled = %v, want %v", got, want)
   914  		}
   915  		if got, want := attrs.Autoclass.TerminalStorageClass, "ARCHIVE"; got != want {
   916  			t.Errorf("attr.Autoclass.TerminalStorageClass = %v, want %v", got, want)
   917  		}
   918  		latestTSCUpdateTime := attrs.Autoclass.TerminalStorageClassUpdateTime
   919  		if latestTSCUpdateTime.IsZero() {
   920  			t.Error("got a zero time value, want a populated value")
   921  		}
   922  		if !latestTSCUpdateTime.After(tscUpdateTime) {
   923  			t.Error("latestTSCUpdateTime should be newer than bucket creation tscUpdateTime")
   924  		}
   925  
   926  		// Disable Autoclass on the bucket.
   927  		ua = BucketAttrsToUpdate{Autoclass: &Autoclass{Enabled: false}}
   928  		attrs = h.mustUpdateBucket(bkt, ua, attrs.MetaGeneration)
   929  		if got, want := attrs.Autoclass.Enabled, false; got != want {
   930  			t.Errorf("attr.Autoclass.Enabled = %v, want %v", got, want)
   931  		}
   932  		latestToggleTime := attrs.Autoclass.ToggleTime
   933  		if latestToggleTime.IsZero() {
   934  			t.Error("got a zero time value, want a populated value")
   935  		}
   936  		if !latestToggleTime.After(toggleTime) {
   937  			t.Error("latestToggleTime should be newer than bucket creation toggleTime")
   938  		}
   939  	})
   940  }
   941  
   942  func TestIntegration_ConditionalDelete(t *testing.T) {
   943  	ctx := skipJSONReads(context.Background(), "no reads in test")
   944  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
   945  		h := testHelper{t}
   946  
   947  		o := client.Bucket(bucket).Object("conddel")
   948  
   949  		wc := o.NewWriter(ctx)
   950  		wc.ContentType = "text/plain"
   951  		h.mustWrite(wc, []byte("foo"))
   952  
   953  		gen := wc.Attrs().Generation
   954  		metaGen := wc.Attrs().Metageneration
   955  
   956  		if err := o.Generation(gen - 1).Delete(ctx); err == nil {
   957  			t.Fatalf("Unexpected successful delete with Generation")
   958  		}
   959  		if err := o.If(Conditions{MetagenerationMatch: metaGen + 1}).Delete(ctx); err == nil {
   960  			t.Fatalf("Unexpected successful delete with IfMetaGenerationMatch")
   961  		}
   962  		if err := o.If(Conditions{MetagenerationNotMatch: metaGen}).Delete(ctx); err == nil {
   963  			t.Fatalf("Unexpected successful delete with IfMetaGenerationNotMatch")
   964  		}
   965  		if err := o.Generation(gen).Delete(ctx); err != nil {
   966  			t.Fatalf("final delete failed: %v", err)
   967  		}
   968  	})
   969  }
   970  
   971  func TestIntegration_ObjectsRangeReader(t *testing.T) {
   972  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
   973  		bkt := client.Bucket(bucket)
   974  
   975  		objName := uidSpaceObjects.New()
   976  		obj := bkt.Object(objName)
   977  		contents := []byte("Hello, world this is a range request")
   978  
   979  		w := obj.If(Conditions{DoesNotExist: true}).NewWriter(ctx)
   980  		if _, err := w.Write(contents); err != nil {
   981  			t.Errorf("Failed to write contents: %v", err)
   982  		}
   983  		if err := w.Close(); err != nil {
   984  			t.Errorf("Failed to close writer: %v", err)
   985  		}
   986  
   987  		last5s := []struct {
   988  			name   string
   989  			start  int64
   990  			length int64
   991  		}{
   992  			{name: "negative offset", start: -5, length: -1},
   993  			{name: "offset with specified length", start: int64(len(contents)) - 5, length: 5},
   994  			{name: "offset and read till end", start: int64(len(contents)) - 5, length: -1},
   995  		}
   996  
   997  		for _, last5 := range last5s {
   998  			t.Run(last5.name, func(t *testing.T) {
   999  				wantBuf := contents[len(contents)-5:]
  1000  				r, err := obj.NewRangeReader(ctx, last5.start, last5.length)
  1001  				if err != nil {
  1002  					t.Fatalf("Failed to make range read: %v", err)
  1003  				}
  1004  				defer r.Close()
  1005  
  1006  				if got, want := r.Attrs.StartOffset, int64(len(contents))-5; got != want {
  1007  					t.Errorf("StartOffset mismatch, got %d want %d", got, want)
  1008  				}
  1009  
  1010  				gotBuf := &bytes.Buffer{}
  1011  				nr, _ := io.Copy(gotBuf, r)
  1012  				if got, want := nr, int64(5); got != want {
  1013  					t.Errorf("Body length mismatch, got %d want %d", got, want)
  1014  				} else if diff := cmp.Diff(gotBuf.String(), string(wantBuf)); diff != "" {
  1015  					t.Errorf("Content read does not match - got(-),want(+):\n%s", diff)
  1016  				}
  1017  			})
  1018  		}
  1019  	})
  1020  }
  1021  
  1022  func TestIntegration_ObjectReadChunksGRPC(t *testing.T) {
  1023  	multiTransportTest(skipHTTP("gRPC implementation specific test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  1024  		h := testHelper{t}
  1025  		// Use a larger blob to test chunking logic. This is a little over 5MB.
  1026  		content := make([]byte, 5<<20)
  1027  		rand.New(rand.NewSource(0)).Read(content)
  1028  
  1029  		// Upload test data.
  1030  		obj := client.Bucket(bucket).Object(uidSpaceObjects.New())
  1031  		if err := writeObject(ctx, obj, "text/plain", content); err != nil {
  1032  			t.Fatal(err)
  1033  		}
  1034  		defer h.mustDeleteObject(obj)
  1035  
  1036  		r, err := obj.NewReader(ctx)
  1037  		if err != nil {
  1038  			t.Fatal(err)
  1039  		}
  1040  		defer r.Close()
  1041  
  1042  		if size := r.Size(); size != int64(len(content)) {
  1043  			t.Errorf("got size = %v, want %v", size, len(content))
  1044  		}
  1045  		if rem := r.Remain(); rem != int64(len(content)) {
  1046  			t.Errorf("got %v bytes remaining, want %v", rem, len(content))
  1047  		}
  1048  
  1049  		bufSize := len(content)
  1050  		buf := make([]byte, bufSize)
  1051  
  1052  		// Read in smaller chunks, offset to provoke reading across a Recv boundary.
  1053  		chunk := 4<<10 + 1234
  1054  		offset := 0
  1055  		for {
  1056  			end := math.Min(float64(offset+chunk), float64(bufSize))
  1057  			n, err := r.Read(buf[offset:int(end)])
  1058  			if err == io.EOF {
  1059  				break
  1060  			}
  1061  			if err != nil {
  1062  				t.Fatal(err)
  1063  			}
  1064  			offset += n
  1065  		}
  1066  
  1067  		if rem := r.Remain(); rem != 0 {
  1068  			t.Errorf("got %v bytes remaining, want 0", rem)
  1069  		}
  1070  		if !bytes.Equal(buf, content) {
  1071  			t.Errorf("content mismatch")
  1072  		}
  1073  	})
  1074  }
  1075  
  1076  func TestIntegration_MultiMessageWriteGRPC(t *testing.T) {
  1077  	multiTransportTest(skipHTTP("gRPC implementation specific test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  1078  		h := testHelper{t}
  1079  
  1080  		name := uidSpaceObjects.New()
  1081  		obj := client.Bucket(bucket).Object(name).Retryer(WithPolicy(RetryAlways))
  1082  		defer h.mustDeleteObject(obj)
  1083  
  1084  		// Use a larger blob to test multi-message logic. This is a little over 5MB.
  1085  		content := bytes.Repeat([]byte("a"), 5<<20)
  1086  
  1087  		crc32c := crc32.Checksum(content, crc32cTable)
  1088  		w := obj.NewWriter(ctx)
  1089  		w.ProgressFunc = func(p int64) {
  1090  			t.Logf("%s: committed %d\n", t.Name(), p)
  1091  		}
  1092  		w.SendCRC32C = true
  1093  		w.CRC32C = crc32c
  1094  		got, err := w.Write(content)
  1095  		if err != nil {
  1096  			t.Fatalf("Writer.Write: %v", err)
  1097  		}
  1098  		// Flush the buffer to finish the upload.
  1099  		if err := w.Close(); err != nil {
  1100  			t.Fatalf("Writer.Close: %v", err)
  1101  		}
  1102  
  1103  		want := len(content)
  1104  		if got != want {
  1105  			t.Errorf("While writing got: %d want %d", got, want)
  1106  		}
  1107  
  1108  		// Read back the Object for verification.
  1109  		reader, err := client.Bucket(bucket).Object(name).NewReader(ctx)
  1110  		if err != nil {
  1111  			t.Fatal(err)
  1112  		}
  1113  		defer reader.Close()
  1114  
  1115  		buf := make([]byte, want+4<<10)
  1116  		b := bytes.NewBuffer(buf)
  1117  		gotr, err := io.Copy(b, reader)
  1118  		if err != nil {
  1119  			t.Fatal(err)
  1120  		}
  1121  		if gotr != int64(want) {
  1122  			t.Errorf("While reading got: %d want %d", gotr, want)
  1123  		}
  1124  	})
  1125  }
  1126  
  1127  func TestIntegration_MultiChunkWrite(t *testing.T) {
  1128  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  1129  		h := testHelper{t}
  1130  		obj := client.Bucket(bucket).Object(uidSpaceObjects.New()).Retryer(WithPolicy(RetryAlways))
  1131  		defer h.mustDeleteObject(obj)
  1132  
  1133  		// Use a larger blob to test multi-message logic. This is a little over 5MB.
  1134  		content := bytes.Repeat([]byte("a"), 5<<20)
  1135  		crc32c := crc32.Checksum(content, crc32cTable)
  1136  
  1137  		w := obj.NewWriter(ctx)
  1138  		w.SendCRC32C = true
  1139  		w.CRC32C = crc32c
  1140  		// Use a 1 MB chunk size.
  1141  		w.ChunkSize = 1 << 20
  1142  		w.ProgressFunc = func(p int64) {
  1143  			t.Logf("%s: committed %d\n", t.Name(), p)
  1144  		}
  1145  		got, err := w.Write(content)
  1146  		if err != nil {
  1147  			t.Fatalf("Writer.Write: %v", err)
  1148  		}
  1149  		// Flush the buffer to finish the upload.
  1150  		if err := w.Close(); err != nil {
  1151  			t.Fatalf("Writer.Close: %v", err)
  1152  		}
  1153  
  1154  		want := len(content)
  1155  		if got != want {
  1156  			t.Errorf("While writing got: %d want %d", got, want)
  1157  		}
  1158  
  1159  		r, err := obj.NewReader(ctx)
  1160  		if err != nil {
  1161  			t.Fatal(err)
  1162  		}
  1163  		defer r.Close()
  1164  
  1165  		buf := make([]byte, want+4<<10)
  1166  		b := bytes.NewBuffer(buf)
  1167  		gotr, err := io.Copy(b, r)
  1168  		if err != nil {
  1169  			t.Fatal(err)
  1170  		}
  1171  		if gotr != int64(want) {
  1172  			t.Errorf("While reading got: %d want %d", gotr, want)
  1173  		}
  1174  	})
  1175  }
  1176  
  1177  func TestIntegration_ConditionalDownload(t *testing.T) {
  1178  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  1179  		h := testHelper{t}
  1180  
  1181  		o := client.Bucket(bucket).Object("condread")
  1182  		defer o.Delete(ctx)
  1183  
  1184  		wc := o.NewWriter(ctx)
  1185  		wc.ContentType = "text/plain"
  1186  		h.mustWrite(wc, []byte("foo"))
  1187  
  1188  		gen := wc.Attrs().Generation
  1189  		metaGen := wc.Attrs().Metageneration
  1190  
  1191  		if _, err := o.Generation(gen + 1).NewReader(ctx); err == nil {
  1192  			t.Fatalf("Unexpected successful download with nonexistent Generation")
  1193  		}
  1194  		if _, err := o.If(Conditions{MetagenerationMatch: metaGen + 1}).NewReader(ctx); err == nil {
  1195  			t.Fatalf("Unexpected successful download with failed preconditions IfMetaGenerationMatch")
  1196  		}
  1197  		if _, err := o.If(Conditions{GenerationMatch: gen + 1}).NewReader(ctx); err == nil {
  1198  			t.Fatalf("Unexpected successful download with failed preconditions IfGenerationMatch")
  1199  		}
  1200  		if _, err := o.If(Conditions{GenerationMatch: gen}).NewReader(ctx); err != nil {
  1201  			t.Fatalf("Download failed: %v", err)
  1202  		}
  1203  	})
  1204  }
  1205  
  1206  func TestIntegration_ObjectIteration(t *testing.T) {
  1207  	ctx := skipJSONReads(context.Background(), "no reads in test")
  1208  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  1209  		// Reset testTime, 'cause object last modification time should be within 5 min
  1210  		// from test (test iteration if -count passed) start time.
  1211  		testTime = time.Now().UTC()
  1212  		newBucketName := prefix + uidSpace.New()
  1213  		h := testHelper{t}
  1214  		bkt := client.Bucket(newBucketName).Retryer(WithPolicy(RetryAlways))
  1215  
  1216  		h.mustCreate(bkt, testutil.ProjID(), nil)
  1217  		defer func() {
  1218  			if err := killBucket(ctx, client, newBucketName); err != nil {
  1219  				log.Printf("deleting %q: %v", newBucketName, err)
  1220  			}
  1221  		}()
  1222  		const defaultType = "text/plain"
  1223  
  1224  		// Populate object names and make a map for their contents.
  1225  		objects := []string{
  1226  			"obj1",
  1227  			"obj2",
  1228  			"obj/with/slashes",
  1229  			"obj/",
  1230  		}
  1231  		contents := make(map[string][]byte)
  1232  
  1233  		// Test Writer.
  1234  		for _, obj := range objects {
  1235  			c := randomContents()
  1236  			if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil {
  1237  				t.Errorf("Write for %v failed with %v", obj, err)
  1238  			}
  1239  			contents[obj] = c
  1240  		}
  1241  
  1242  		testObjectIterator(t, bkt, objects)
  1243  		testObjectsIterateSelectedAttrs(t, bkt, objects)
  1244  		testObjectsIterateAllSelectedAttrs(t, bkt, objects)
  1245  		testObjectIteratorWithOffset(t, bkt, objects)
  1246  		testObjectsIterateWithProjection(t, bkt)
  1247  		t.Run("testObjectsIterateSelectedAttrsDelimiter", func(t *testing.T) {
  1248  			query := &Query{Prefix: "", Delimiter: "/"}
  1249  			if err := query.SetAttrSelection([]string{"Name"}); err != nil {
  1250  				t.Fatalf("selecting query attrs: %v", err)
  1251  			}
  1252  
  1253  			var gotNames []string
  1254  			var gotPrefixes []string
  1255  			it := bkt.Objects(context.Background(), query)
  1256  			for {
  1257  				attrs, err := it.Next()
  1258  				if err == iterator.Done {
  1259  					break
  1260  				}
  1261  				if err != nil {
  1262  					t.Fatalf("iterator.Next: %v", err)
  1263  				}
  1264  				if attrs.Name != "" {
  1265  					gotNames = append(gotNames, attrs.Name)
  1266  				} else if attrs.Prefix != "" {
  1267  					gotPrefixes = append(gotPrefixes, attrs.Prefix)
  1268  				}
  1269  
  1270  				if attrs.Bucket != "" {
  1271  					t.Errorf("Bucket field not selected, want empty, got = %v", attrs.Bucket)
  1272  				}
  1273  			}
  1274  
  1275  			sortedNames := []string{"obj1", "obj2"}
  1276  			if !cmp.Equal(sortedNames, gotNames) {
  1277  				t.Errorf("names = %v, want %v", gotNames, sortedNames)
  1278  			}
  1279  			sortedPrefixes := []string{"obj/"}
  1280  			if !cmp.Equal(sortedPrefixes, gotPrefixes) {
  1281  				t.Errorf("prefixes = %v, want %v", gotPrefixes, sortedPrefixes)
  1282  			}
  1283  		})
  1284  		t.Run("testObjectsIterateSelectedAttrsDelimiterIncludeTrailingDelimiter", func(t *testing.T) {
  1285  			query := &Query{Prefix: "", Delimiter: "/", IncludeTrailingDelimiter: true}
  1286  			if err := query.SetAttrSelection([]string{"Name"}); err != nil {
  1287  				t.Fatalf("selecting query attrs: %v", err)
  1288  			}
  1289  
  1290  			var gotNames []string
  1291  			var gotPrefixes []string
  1292  			it := bkt.Objects(context.Background(), query)
  1293  			for {
  1294  				attrs, err := it.Next()
  1295  				if err == iterator.Done {
  1296  					break
  1297  				}
  1298  				if err != nil {
  1299  					t.Fatalf("iterator.Next: %v", err)
  1300  				}
  1301  				if attrs.Name != "" {
  1302  					gotNames = append(gotNames, attrs.Name)
  1303  				} else if attrs.Prefix != "" {
  1304  					gotPrefixes = append(gotPrefixes, attrs.Prefix)
  1305  				}
  1306  
  1307  				if attrs.Bucket != "" {
  1308  					t.Errorf("Bucket field not selected, want empty, got = %v", attrs.Bucket)
  1309  				}
  1310  			}
  1311  
  1312  			sortedNames := []string{"obj/", "obj1", "obj2"}
  1313  			if !cmp.Equal(sortedNames, gotNames) {
  1314  				t.Errorf("names = %v, want %v", gotNames, sortedNames)
  1315  			}
  1316  			sortedPrefixes := []string{"obj/"}
  1317  			if !cmp.Equal(sortedPrefixes, gotPrefixes) {
  1318  				t.Errorf("prefixes = %v, want %v", gotPrefixes, sortedPrefixes)
  1319  			}
  1320  		})
  1321  	})
  1322  }
  1323  
  1324  func TestIntegration_ObjectIterationMatchGlob(t *testing.T) {
  1325  	multiTransportTest(skipJSONReads(context.Background(), "no reads in test"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  1326  		// Reset testTime, 'cause object last modification time should be within 5 min
  1327  		// from test (test iteration if -count passed) start time.
  1328  		testTime = time.Now().UTC()
  1329  		newBucketName := prefix + uidSpace.New()
  1330  		h := testHelper{t}
  1331  		bkt := client.Bucket(newBucketName).Retryer(WithPolicy(RetryAlways))
  1332  
  1333  		h.mustCreate(bkt, testutil.ProjID(), nil)
  1334  		defer func() {
  1335  			if err := killBucket(ctx, client, newBucketName); err != nil {
  1336  				log.Printf("deleting %q: %v", newBucketName, err)
  1337  			}
  1338  		}()
  1339  		const defaultType = "text/plain"
  1340  
  1341  		// Populate object names and make a map for their contents.
  1342  		objects := []string{
  1343  			"obj1",
  1344  			"obj2",
  1345  			"obj/with/slashes",
  1346  			"obj/",
  1347  			"other/obj1",
  1348  		}
  1349  		contents := make(map[string][]byte)
  1350  
  1351  		// Test Writer.
  1352  		for _, obj := range objects {
  1353  			c := randomContents()
  1354  			if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil {
  1355  				t.Errorf("Write for %v failed with %v", obj, err)
  1356  			}
  1357  			contents[obj] = c
  1358  		}
  1359  		query := &Query{MatchGlob: "**obj1"}
  1360  
  1361  		var gotNames []string
  1362  		it := bkt.Objects(context.Background(), query)
  1363  		for {
  1364  			attrs, err := it.Next()
  1365  			if err == iterator.Done {
  1366  				break
  1367  			}
  1368  			if err != nil {
  1369  				t.Fatalf("iterator.Next: %v", err)
  1370  			}
  1371  			if attrs.Name != "" {
  1372  				gotNames = append(gotNames, attrs.Name)
  1373  			}
  1374  		}
  1375  
  1376  		sortedNames := []string{"obj1", "other/obj1"}
  1377  		if !cmp.Equal(sortedNames, gotNames) {
  1378  			t.Errorf("names = %v, want %v", gotNames, sortedNames)
  1379  		}
  1380  	})
  1381  }
  1382  
  1383  func TestIntegration_ObjectIterationManagedFolder(t *testing.T) {
  1384  	ctx := skipGRPC("not yet implemented in gRPC")
  1385  	multiTransportTest(skipJSONReads(ctx, "no reads in test"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  1386  		newBucketName := prefix + uidSpace.New()
  1387  		h := testHelper{t}
  1388  		bkt := client.Bucket(newBucketName).Retryer(WithPolicy(RetryAlways))
  1389  
  1390  		// Create bucket with UBLA enabled as this is necessary for managed folders.
  1391  		h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{
  1392  			UniformBucketLevelAccess: UniformBucketLevelAccess{
  1393  				Enabled: true,
  1394  			},
  1395  		})
  1396  
  1397  		t.Cleanup(func() {
  1398  			if err := killBucket(ctx, client, newBucketName); err != nil {
  1399  				log.Printf("deleting %q: %v", newBucketName, err)
  1400  			}
  1401  		})
  1402  		const defaultType = "text/plain"
  1403  
  1404  		// Populate object names and make a map for their contents.
  1405  		objects := []string{
  1406  			"obj1",
  1407  			"obj2",
  1408  			"obj/with/slashes",
  1409  			"obj/",
  1410  			"other/obj1",
  1411  		}
  1412  		contents := make(map[string][]byte)
  1413  
  1414  		// Test Writer.
  1415  		for _, obj := range objects {
  1416  			c := randomContents()
  1417  			if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil {
  1418  				t.Errorf("Write for %v failed with %v", obj, err)
  1419  			}
  1420  			contents[obj] = c
  1421  		}
  1422  
  1423  		// Create a managed folder. This requires using the Apiary client as this is not available
  1424  		// in the veneer layer.
  1425  		// TODO: change to use storage control client once available.
  1426  		call := client.raw.ManagedFolders.Insert(newBucketName, &raw.ManagedFolder{Name: "mf"})
  1427  		mf, err := call.Context(ctx).Do()
  1428  		if err != nil {
  1429  			t.Fatalf("creating managed folder: %v", err)
  1430  		}
  1431  
  1432  		t.Cleanup(func() {
  1433  			// TODO: add this cleanup logic to killBucket as well once gRPC support is available.
  1434  			call := client.raw.ManagedFolders.Delete(newBucketName, mf.Name)
  1435  			call.Context(ctx).Do()
  1436  		})
  1437  
  1438  		// Test that managed folders are only included when IncludeFoldersAsPrefixes is set.
  1439  		cases := []struct {
  1440  			name  string
  1441  			query *Query
  1442  			want  []string
  1443  		}{
  1444  			{
  1445  				name:  "include folders",
  1446  				query: &Query{Delimiter: "/", IncludeFoldersAsPrefixes: true},
  1447  				want:  []string{"mf/", "obj/", "other/"},
  1448  			},
  1449  			{
  1450  				name:  "no folders",
  1451  				query: &Query{Delimiter: "/"},
  1452  				want:  []string{"obj/", "other/"},
  1453  			},
  1454  		}
  1455  
  1456  		for _, c := range cases {
  1457  			t.Run(c.name, func(t *testing.T) {
  1458  				var gotNames []string
  1459  				var gotPrefixes []string
  1460  				it := bkt.Objects(context.Background(), c.query)
  1461  				for {
  1462  					attrs, err := it.Next()
  1463  					if err == iterator.Done {
  1464  						break
  1465  					}
  1466  					if err != nil {
  1467  						t.Fatalf("iterator.Next: %v", err)
  1468  					}
  1469  					if attrs.Name != "" {
  1470  						gotNames = append(gotNames, attrs.Name)
  1471  					}
  1472  					if attrs.Prefix != "" {
  1473  						gotPrefixes = append(gotPrefixes, attrs.Prefix)
  1474  					}
  1475  				}
  1476  
  1477  				sortedNames := []string{"obj1", "obj2"}
  1478  				if !cmp.Equal(sortedNames, gotNames) {
  1479  					t.Errorf("names = %v, want %v", gotNames, sortedNames)
  1480  				}
  1481  
  1482  				if !cmp.Equal(c.want, gotPrefixes) {
  1483  					t.Errorf("prefixes = %v, want %v", gotPrefixes, c.want)
  1484  				}
  1485  			})
  1486  		}
  1487  	})
  1488  }
  1489  
  1490  func TestIntegration_ObjectUpdate(t *testing.T) {
  1491  	ctx := skipJSONReads(context.Background(), "no reads in test")
  1492  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  1493  		b := client.Bucket(bucket)
  1494  
  1495  		o := b.Object("update-obj" + uidSpaceObjects.New())
  1496  		w := o.NewWriter(ctx)
  1497  		_, err := io.Copy(w, bytes.NewReader(randomContents()))
  1498  		if err != nil {
  1499  			t.Fatalf("io.Copy: %v", err)
  1500  		}
  1501  		if err := w.Close(); err != nil {
  1502  			t.Fatalf("w.Close: %v", err)
  1503  		}
  1504  		defer func() {
  1505  			if err := o.Delete(ctx); err != nil {
  1506  				t.Errorf("o.Delete : %v", err)
  1507  			}
  1508  		}()
  1509  
  1510  		attrs, err := o.Attrs(ctx)
  1511  		if err != nil {
  1512  			t.Fatalf("o.Attrs: %v", err)
  1513  		}
  1514  
  1515  		// Test UpdateAttrs.
  1516  		metadata := map[string]string{"key": "value"}
  1517  
  1518  		updated, err := o.If(Conditions{MetagenerationMatch: attrs.Metageneration}).Update(ctx, ObjectAttrsToUpdate{
  1519  			ContentType:     "text/html",
  1520  			ContentLanguage: "en",
  1521  			Metadata:        metadata,
  1522  			ACL:             []ACLRule{{Entity: "domain-google.com", Role: RoleReader}},
  1523  		})
  1524  		if err != nil {
  1525  			t.Fatalf("o.Update: %v", err)
  1526  		}
  1527  
  1528  		if got, want := updated.ContentType, "text/html"; got != want {
  1529  			t.Errorf("updated.ContentType == %q; want %q", got, want)
  1530  		}
  1531  		if got, want := updated.ContentLanguage, "en"; got != want {
  1532  			t.Errorf("updated.ContentLanguage == %q; want %q", updated.ContentLanguage, want)
  1533  		}
  1534  		if got, want := updated.Metadata, metadata; !testutil.Equal(got, want) {
  1535  			t.Errorf("updated.Metadata == %+v; want %+v", updated.Metadata, want)
  1536  		}
  1537  		if got, want := updated.Created, attrs.Created; got != want {
  1538  			t.Errorf("updated.Created == %q; want %q", got, want)
  1539  		}
  1540  		if !updated.Created.Before(updated.Updated) {
  1541  			t.Errorf("updated.Updated should be newer than update.Created")
  1542  		}
  1543  
  1544  		// Add another metadata key
  1545  		anotherKey := map[string]string{"key2": "value2"}
  1546  		metadata["key2"] = "value2"
  1547  
  1548  		updated, err = o.Update(ctx, ObjectAttrsToUpdate{
  1549  			Metadata: anotherKey,
  1550  		})
  1551  		if err != nil {
  1552  			t.Fatalf("o.Update: %v", err)
  1553  		}
  1554  
  1555  		if got, want := updated.Metadata, metadata; !testutil.Equal(got, want) {
  1556  			t.Errorf("updated.Metadata == %+v; want %+v", updated.Metadata, want)
  1557  		}
  1558  
  1559  		// Delete ContentType and ContentLanguage and Metadata.
  1560  		updated, err = o.If(Conditions{MetagenerationMatch: updated.Metageneration}).Update(ctx, ObjectAttrsToUpdate{
  1561  			ContentType:     "",
  1562  			ContentLanguage: "",
  1563  			Metadata:        map[string]string{},
  1564  			ACL:             []ACLRule{{Entity: "domain-google.com", Role: RoleReader}},
  1565  		})
  1566  		if err != nil {
  1567  			t.Fatalf("o.Update: %v", err)
  1568  		}
  1569  
  1570  		if got, want := updated.ContentType, ""; got != want {
  1571  			t.Errorf("updated.ContentType == %q; want %q", got, want)
  1572  		}
  1573  		if got, want := updated.ContentLanguage, ""; got != want {
  1574  			t.Errorf("updated.ContentLanguage == %q; want %q", updated.ContentLanguage, want)
  1575  		}
  1576  		if updated.Metadata != nil {
  1577  			t.Errorf("updated.Metadata == %+v; want nil", updated.Metadata)
  1578  		}
  1579  		if got, want := updated.Created, attrs.Created; got != want {
  1580  			t.Errorf("updated.Created == %q; want %q", got, want)
  1581  		}
  1582  		if !updated.Created.Before(updated.Updated) {
  1583  			t.Errorf("updated.Updated should be newer than update.Created")
  1584  		}
  1585  	})
  1586  }
  1587  
  1588  func TestIntegration_ObjectChecksums(t *testing.T) {
  1589  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  1590  		b := client.Bucket(bucket)
  1591  		checksumCases := []struct {
  1592  			name     string
  1593  			contents [][]byte
  1594  			size     int64
  1595  			md5      string
  1596  			crc32c   uint32
  1597  		}{
  1598  			{
  1599  				name:     "checksum-object",
  1600  				contents: [][]byte{[]byte("hello"), []byte("world")},
  1601  				size:     10,
  1602  				md5:      "fc5e038d38a57032085441e7fe7010b0",
  1603  				crc32c:   1456190592,
  1604  			},
  1605  			{
  1606  				name:     "zero-object",
  1607  				contents: [][]byte{},
  1608  				size:     0,
  1609  				md5:      "d41d8cd98f00b204e9800998ecf8427e",
  1610  				crc32c:   0,
  1611  			},
  1612  		}
  1613  		for _, c := range checksumCases {
  1614  			wc := b.Object(c.name + uidSpaceObjects.New()).NewWriter(ctx)
  1615  			for _, data := range c.contents {
  1616  				if _, err := wc.Write(data); err != nil {
  1617  					t.Fatalf("Write(%q) failed with %q", data, err)
  1618  				}
  1619  			}
  1620  			if err := wc.Close(); err != nil {
  1621  				t.Fatalf("%q: close failed with %q", c.name, err)
  1622  			}
  1623  			obj := wc.Attrs()
  1624  			if got, want := obj.Size, c.size; got != want {
  1625  				t.Errorf("Object (%q) Size = %v; want %v", c.name, got, want)
  1626  			}
  1627  			if got, want := fmt.Sprintf("%x", obj.MD5), c.md5; got != want {
  1628  				t.Errorf("Object (%q) MD5 = %q; want %q", c.name, got, want)
  1629  			}
  1630  			if got, want := obj.CRC32C, c.crc32c; got != want {
  1631  				t.Errorf("Object (%q) CRC32C = %v; want %v", c.name, got, want)
  1632  			}
  1633  		}
  1634  	})
  1635  }
  1636  
  1637  func TestIntegration_ObjectCompose(t *testing.T) {
  1638  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  1639  		b := client.Bucket(bucket)
  1640  
  1641  		objects := []*ObjectHandle{
  1642  			b.Object("obj1" + uidSpaceObjects.New()),
  1643  			b.Object("obj2" + uidSpaceObjects.New()),
  1644  			b.Object("obj/with/slashes" + uidSpaceObjects.New()),
  1645  			b.Object("obj/" + uidSpaceObjects.New()),
  1646  		}
  1647  		var compSrcs []*ObjectHandle
  1648  		wantContents := make([]byte, 0)
  1649  
  1650  		// Write objects to compose
  1651  		for _, obj := range objects {
  1652  			c := randomContents()
  1653  			if err := writeObject(ctx, obj, "text/plain", c); err != nil {
  1654  				t.Errorf("Write for %v failed with %v", obj, err)
  1655  			}
  1656  			compSrcs = append(compSrcs, obj)
  1657  			wantContents = append(wantContents, c...)
  1658  			defer obj.Delete(ctx)
  1659  		}
  1660  
  1661  		checkCompose := func(obj *ObjectHandle, contentTypeSet *string) {
  1662  			r, err := obj.NewReader(ctx)
  1663  			if err != nil {
  1664  				t.Fatalf("new reader: %v", err)
  1665  			}
  1666  
  1667  			slurp, err := ioutil.ReadAll(r)
  1668  			if err != nil {
  1669  				t.Fatalf("ioutil.ReadAll: %v", err)
  1670  			}
  1671  			defer r.Close()
  1672  			if !bytes.Equal(slurp, wantContents) {
  1673  				t.Errorf("Composed object contents\ngot:  %q\nwant: %q", slurp, wantContents)
  1674  			}
  1675  			got := r.ContentType()
  1676  			// Accept both an empty string and octet-stream if the content type was not set;
  1677  			// HTTP will set the content type as octet-stream whilst GRPC will not set it all.
  1678  			if !(contentTypeSet == nil && (got == "" || got == "application/octet-stream")) && got != *contentTypeSet {
  1679  				t.Errorf("Composed object content-type = %q, want %q", got, *contentTypeSet)
  1680  			}
  1681  		}
  1682  
  1683  		// Compose should work even if the user sets no destination attributes.
  1684  		compDst := b.Object("composed1")
  1685  		c := compDst.ComposerFrom(compSrcs...)
  1686  		attrs, err := c.Run(ctx)
  1687  		if err != nil {
  1688  			t.Fatalf("ComposeFrom error: %v", err)
  1689  		}
  1690  		if attrs.ComponentCount != int64(len(objects)) {
  1691  			t.Errorf("mismatching ComponentCount: got %v, want %v", attrs.ComponentCount, int64(len(objects)))
  1692  		}
  1693  		checkCompose(compDst, nil)
  1694  
  1695  		// It should also work if we do.
  1696  		contentType := "text/json"
  1697  		compDst = b.Object("composed2")
  1698  		c = compDst.ComposerFrom(compSrcs...)
  1699  		c.ContentType = contentType
  1700  		attrs, err = c.Run(ctx)
  1701  		if err != nil {
  1702  			t.Fatalf("ComposeFrom error: %v", err)
  1703  		}
  1704  		if attrs.ComponentCount != int64(len(objects)) {
  1705  			t.Errorf("mismatching ComponentCount: got %v, want %v", attrs.ComponentCount, int64(len(objects)))
  1706  		}
  1707  		checkCompose(compDst, &contentType)
  1708  	})
  1709  }
  1710  
  1711  func TestIntegration_Copy(t *testing.T) {
  1712  	ctx := skipJSONReads(context.Background(), "no reads in test")
  1713  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, prefix string, client *Client) {
  1714  		h := testHelper{t}
  1715  
  1716  		bucketFrom := client.Bucket(bucket)
  1717  		bucketInSameRegion := client.Bucket(prefix + uidSpace.New())
  1718  		bucketInDifferentRegion := client.Bucket(prefix + uidSpace.New())
  1719  
  1720  		// Create new bucket
  1721  		if err := bucketInSameRegion.Create(ctx, testutil.ProjID(), nil); err != nil {
  1722  			t.Fatalf("bucket.Create: %v", err)
  1723  		}
  1724  		t.Cleanup(func() {
  1725  			h.mustDeleteBucket(bucketInSameRegion)
  1726  		})
  1727  
  1728  		// Create new bucket
  1729  		if err := bucketInDifferentRegion.Create(ctx, testutil.ProjID(), &BucketAttrs{Location: "NORTHAMERICA-NORTHEAST2"}); err != nil {
  1730  			t.Fatalf("bucket.Create: %v", err)
  1731  		}
  1732  		t.Cleanup(func() {
  1733  			h.mustDeleteBucket(bucketInDifferentRegion)
  1734  		})
  1735  
  1736  		// We use a larger object size to be able to trigger multiple rewrite calls
  1737  		minObjectSize := 2500000 // 2.5 Mb
  1738  		obj := bucketFrom.Object("copy-object-original" + uidSpaceObjects.New())
  1739  
  1740  		// Create an object to copy from
  1741  		w := obj.NewWriter(ctx)
  1742  		c := randomContents()
  1743  		for written := 0; written < minObjectSize; {
  1744  			n, err := w.Write(c)
  1745  			if err != nil {
  1746  				t.Fatalf("w.Write: %v", err)
  1747  			}
  1748  			written += n
  1749  		}
  1750  		if err := w.Close(); err != nil {
  1751  			t.Fatalf("w.Close: %v", err)
  1752  		}
  1753  		t.Cleanup(func() {
  1754  			h.mustDeleteObject(obj)
  1755  		})
  1756  
  1757  		attrs, err := obj.Attrs(ctx)
  1758  		if err != nil {
  1759  			t.Fatalf("obj.Attrs: %v", err)
  1760  		}
  1761  
  1762  		crc32c := attrs.CRC32C
  1763  
  1764  		type copierAttrs struct {
  1765  			contentEncoding string
  1766  			maxBytesPerCall int64
  1767  		}
  1768  
  1769  		for _, test := range []struct {
  1770  			desc                    string
  1771  			toObj                   string
  1772  			toBucket                *BucketHandle
  1773  			copierAttrs             *copierAttrs
  1774  			numExpectedRewriteCalls int
  1775  		}{
  1776  			{
  1777  				desc:                    "copy within bucket",
  1778  				toObj:                   "copy-within-bucket",
  1779  				toBucket:                bucketFrom,
  1780  				numExpectedRewriteCalls: 1,
  1781  			},
  1782  			{
  1783  				desc:                    "copy to new bucket",
  1784  				toObj:                   "copy-new-bucket",
  1785  				toBucket:                bucketInSameRegion,
  1786  				numExpectedRewriteCalls: 1,
  1787  			},
  1788  			{
  1789  				desc:                    "copy with attributes",
  1790  				toObj:                   "copy-with-attributes",
  1791  				toBucket:                bucketInSameRegion,
  1792  				copierAttrs:             &copierAttrs{contentEncoding: "identity"},
  1793  				numExpectedRewriteCalls: 1,
  1794  			},
  1795  			{
  1796  				// this test should trigger multiple re-write calls and may fail
  1797  				// with a rate limit error if those calls are stuck in an infinite loop
  1798  				desc:                    "copy to new region",
  1799  				toObj:                   "copy-new-region",
  1800  				toBucket:                bucketInDifferentRegion,
  1801  				copierAttrs:             &copierAttrs{maxBytesPerCall: 1048576},
  1802  				numExpectedRewriteCalls: 3,
  1803  			},
  1804  		} {
  1805  			t.Run(test.desc, func(t *testing.T) {
  1806  				copyObj := test.toBucket.Object(test.toObj)
  1807  				copier := copyObj.CopierFrom(obj)
  1808  
  1809  				if attrs := test.copierAttrs; attrs != nil {
  1810  					if attrs.contentEncoding != "" {
  1811  						copier.ContentEncoding = attrs.contentEncoding
  1812  					}
  1813  					if attrs.maxBytesPerCall != 0 {
  1814  						copier.maxBytesRewrittenPerCall = attrs.maxBytesPerCall
  1815  					}
  1816  				}
  1817  
  1818  				rewriteCallsCount := 0
  1819  				copier.ProgressFunc = func(_, _ uint64) {
  1820  					rewriteCallsCount++
  1821  				}
  1822  
  1823  				attrs, err = copier.Run(ctx)
  1824  				if err != nil {
  1825  					t.Fatalf("Copier.Run failed with %v", err)
  1826  				}
  1827  				t.Cleanup(func() {
  1828  					h.mustDeleteObject(copyObj)
  1829  				})
  1830  
  1831  				// Check copied object is in the correct bucket with the correct name
  1832  				if attrs.Bucket != test.toBucket.name || attrs.Name != test.toObj {
  1833  					t.Errorf("unexpected copy behaviour: got: %s in bucket %s, want: %s in bucket %s", attrs.Name, attrs.Bucket, attrs.Name, test.toBucket.name)
  1834  				}
  1835  
  1836  				// Check attrs
  1837  				if test.copierAttrs != nil {
  1838  					if attrs.ContentEncoding != test.copierAttrs.contentEncoding {
  1839  						t.Errorf("unexpected ContentEncoding; got: %s, want: %s", attrs.ContentEncoding, test.copierAttrs.contentEncoding)
  1840  					}
  1841  				}
  1842  
  1843  				// Check the copied contents
  1844  				if attrs.CRC32C != crc32c {
  1845  					t.Errorf("mismatching checksum: got %v, want %v", attrs.CRC32C, crc32c)
  1846  				}
  1847  
  1848  				// Check that the number of requests made is as expected
  1849  				if rewriteCallsCount != test.numExpectedRewriteCalls {
  1850  					t.Errorf("unexpected number of rewrite calls: got %v, want %v", rewriteCallsCount, test.numExpectedRewriteCalls)
  1851  				}
  1852  			})
  1853  		}
  1854  	})
  1855  }
  1856  
  1857  func TestIntegration_Encoding(t *testing.T) {
  1858  	multiTransportTest(skipGRPC("gzip transcoding not supported"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  1859  		bkt := client.Bucket(bucket)
  1860  
  1861  		// Test content encoding
  1862  		const zeroCount = 20 << 1 // TODO: should be 20 << 20
  1863  		obj := bkt.Object("gzip-test")
  1864  		w := obj.NewWriter(ctx)
  1865  		w.ContentEncoding = "gzip"
  1866  		gw := gzip.NewWriter(w)
  1867  		if _, err := io.Copy(gw, io.LimitReader(zeros{}, zeroCount)); err != nil {
  1868  			t.Fatalf("io.Copy, upload: %v", err)
  1869  		}
  1870  		if err := gw.Close(); err != nil {
  1871  			t.Errorf("gzip.Close(): %v", err)
  1872  		}
  1873  		if err := w.Close(); err != nil {
  1874  			t.Errorf("w.Close(): %v", err)
  1875  		}
  1876  		r, err := obj.NewReader(ctx)
  1877  		if err != nil {
  1878  			t.Fatalf("NewReader(gzip-test): %v", err)
  1879  		}
  1880  		n, err := io.Copy(ioutil.Discard, r)
  1881  		if err != nil {
  1882  			t.Errorf("io.Copy, download: %v", err)
  1883  		}
  1884  		if n != zeroCount {
  1885  			t.Errorf("downloaded bad data: got %d bytes, want %d", n, zeroCount)
  1886  		}
  1887  
  1888  		// Test NotFound.
  1889  		_, err = bkt.Object("obj-not-exists").NewReader(ctx)
  1890  		if err != ErrObjectNotExist {
  1891  			t.Errorf("Object should not exist, err found to be %v", err)
  1892  		}
  1893  	})
  1894  }
  1895  
  1896  func testObjectIterator(t *testing.T, bkt *BucketHandle, objects []string) {
  1897  	ctx := context.Background()
  1898  	h := testHelper{t}
  1899  	// Collect the list of items we expect: ObjectAttrs in lexical order by name.
  1900  	names := make([]string, len(objects))
  1901  	copy(names, objects)
  1902  	sort.Strings(names)
  1903  	var attrs []*ObjectAttrs
  1904  	for _, name := range names {
  1905  		attrs = append(attrs, h.mustObjectAttrs(bkt.Object(name)))
  1906  	}
  1907  	msg, ok := itesting.TestIterator(attrs,
  1908  		func() interface{} { return bkt.Objects(ctx, &Query{Prefix: "obj"}) },
  1909  		func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() })
  1910  	if !ok {
  1911  		t.Errorf("ObjectIterator.Next: %s", msg)
  1912  	}
  1913  	// TODO(jba): test query.Delimiter != ""
  1914  }
  1915  
  1916  func testObjectIteratorWithOffset(t *testing.T, bkt *BucketHandle, objects []string) {
  1917  	ctx := context.Background()
  1918  	h := testHelper{t}
  1919  	// Collect the list of items we expect: ObjectAttrs in lexical order by name.
  1920  	names := make([]string, len(objects))
  1921  	copy(names, objects)
  1922  	sort.Strings(names)
  1923  	var attrs []*ObjectAttrs
  1924  	for _, name := range names {
  1925  		attrs = append(attrs, h.mustObjectAttrs(bkt.Object(name)))
  1926  	}
  1927  	m := make(map[string][]*ObjectAttrs)
  1928  	for i, name := range names {
  1929  		// StartOffset takes the value of object names, the result must be for:
  1930  		// ― obj/with/slashes: obj/with/slashes, obj1, obj2
  1931  		// ― obj1: obj1, obj2
  1932  		// ― obj2: obj2.
  1933  		m[name] = attrs[i:]
  1934  		msg, ok := itesting.TestIterator(m[name],
  1935  			func() interface{} { return bkt.Objects(ctx, &Query{StartOffset: name}) },
  1936  			func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() })
  1937  		if !ok {
  1938  			t.Errorf("ObjectIterator.Next: %s", msg)
  1939  		}
  1940  		// EndOffset takes the value of object names, the result must be for:
  1941  		// ― obj/with/slashes: ""
  1942  		// ― obj1: obj/with/slashes
  1943  		// ― obj2: obj/with/slashes, obj1.
  1944  		m[name] = attrs[:i]
  1945  		msg, ok = itesting.TestIterator(m[name],
  1946  			func() interface{} { return bkt.Objects(ctx, &Query{EndOffset: name}) },
  1947  			func(it interface{}) (interface{}, error) { return it.(*ObjectIterator).Next() })
  1948  		if !ok {
  1949  			t.Errorf("ObjectIterator.Next: %s", msg)
  1950  		}
  1951  	}
  1952  }
  1953  
  1954  func testObjectsIterateSelectedAttrs(t *testing.T, bkt *BucketHandle, objects []string) {
  1955  	// Create a query that will only select the "Name" attr of objects, and
  1956  	// invoke object listing.
  1957  	query := &Query{Prefix: ""}
  1958  	query.SetAttrSelection([]string{"Name"})
  1959  
  1960  	var gotNames []string
  1961  	it := bkt.Objects(context.Background(), query)
  1962  	for {
  1963  		attrs, err := it.Next()
  1964  		if err == iterator.Done {
  1965  			break
  1966  		}
  1967  		if err != nil {
  1968  			t.Fatalf("iterator.Next: %v", err)
  1969  		}
  1970  		gotNames = append(gotNames, attrs.Name)
  1971  
  1972  		if len(attrs.Bucket) > 0 {
  1973  			t.Errorf("Bucket field not selected, want empty, got = %v", attrs.Bucket)
  1974  		}
  1975  	}
  1976  
  1977  	sortedNames := make([]string, len(objects))
  1978  	copy(sortedNames, objects)
  1979  	sort.Strings(sortedNames)
  1980  	sort.Strings(gotNames)
  1981  
  1982  	if !cmp.Equal(sortedNames, gotNames) {
  1983  		t.Errorf("names = %v, want %v", gotNames, sortedNames)
  1984  	}
  1985  }
  1986  
  1987  func testObjectsIterateAllSelectedAttrs(t *testing.T, bkt *BucketHandle, objects []string) {
  1988  	// Tests that all selected attributes work - query succeeds (without actually
  1989  	// verifying the returned results).
  1990  	query := &Query{
  1991  		Prefix:      "",
  1992  		StartOffset: "obj/",
  1993  		EndOffset:   "obj2",
  1994  	}
  1995  	var selectedAttrs []string
  1996  	for k := range attrToFieldMap {
  1997  		selectedAttrs = append(selectedAttrs, k)
  1998  	}
  1999  	query.SetAttrSelection(selectedAttrs)
  2000  
  2001  	count := 0
  2002  	it := bkt.Objects(context.Background(), query)
  2003  	for {
  2004  		_, err := it.Next()
  2005  		if err == iterator.Done {
  2006  			break
  2007  		}
  2008  		if err != nil {
  2009  			t.Fatalf("iterator.Next: %v", err)
  2010  		}
  2011  		count++
  2012  	}
  2013  
  2014  	if count != len(objects)-1 {
  2015  		t.Errorf("count = %v, want %v", count, len(objects)-1)
  2016  	}
  2017  }
  2018  
  2019  func testObjectsIterateWithProjection(t *testing.T, bkt *BucketHandle) {
  2020  	projections := map[Projection]bool{
  2021  		ProjectionDefault: true,
  2022  		ProjectionFull:    true,
  2023  		ProjectionNoACL:   false,
  2024  	}
  2025  
  2026  	for projection, expectACL := range projections {
  2027  		query := &Query{Projection: projection}
  2028  		it := bkt.Objects(context.Background(), query)
  2029  		attrs, err := it.Next()
  2030  		if err == iterator.Done {
  2031  			t.Fatalf("iterator: no objects")
  2032  		}
  2033  		if err != nil {
  2034  			t.Fatalf("iterator.Next: %v", err)
  2035  		}
  2036  
  2037  		if expectACL {
  2038  			if attrs.Owner == "" {
  2039  				t.Errorf("projection %q: Owner is empty, want nonempty Owner", projection)
  2040  			}
  2041  			if len(attrs.ACL) == 0 {
  2042  				t.Errorf("projection %q: ACL is empty, want at least one ACL rule", projection)
  2043  			}
  2044  		} else {
  2045  			if attrs.Owner != "" {
  2046  				t.Errorf("projection %q: got Owner = %q, want empty Owner", projection, attrs.Owner)
  2047  			}
  2048  			if len(attrs.ACL) != 0 {
  2049  				t.Errorf("projection %q: got %d ACL rules, want empty ACL", projection, len(attrs.ACL))
  2050  			}
  2051  		}
  2052  	}
  2053  }
  2054  
  2055  func TestIntegration_SignedURL(t *testing.T) {
  2056  	multiTransportTest(skipJSONReads(context.Background(), "no reads in test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  2057  		// To test SignedURL, we need a real user email and private key. Extract them
  2058  		// from the JSON key file.
  2059  		jwtConf, err := testutil.JWTConfig()
  2060  		if err != nil {
  2061  			t.Fatal(err)
  2062  		}
  2063  		if jwtConf == nil {
  2064  			t.Skip("JSON key file is not present")
  2065  		}
  2066  
  2067  		bkt := client.Bucket(bucket)
  2068  		obj := "signedURL"
  2069  		contents := []byte("This is a test of SignedURL.\n")
  2070  		md5 := "Jyxvgwm9n2MsrGTMPbMeYA==" // base64-encoded MD5 of contents
  2071  		if err := writeObject(ctx, bkt.Object(obj), "text/plain", contents); err != nil {
  2072  			t.Fatalf("writing: %v", err)
  2073  		}
  2074  		for _, test := range []struct {
  2075  			desc    string
  2076  			opts    SignedURLOptions
  2077  			headers map[string][]string
  2078  			fail    bool
  2079  		}{
  2080  			{
  2081  				desc: "basic v2",
  2082  			},
  2083  			{
  2084  				desc: "basic v4",
  2085  				opts: SignedURLOptions{Scheme: SigningSchemeV4},
  2086  			},
  2087  			{
  2088  				desc:    "MD5 sent and matches",
  2089  				opts:    SignedURLOptions{MD5: md5},
  2090  				headers: map[string][]string{"Content-MD5": {md5}},
  2091  			},
  2092  			{
  2093  				desc: "MD5 not sent",
  2094  				opts: SignedURLOptions{MD5: md5},
  2095  				fail: true,
  2096  			},
  2097  			{
  2098  				desc:    "Content-Type sent and matches",
  2099  				opts:    SignedURLOptions{ContentType: "text/plain"},
  2100  				headers: map[string][]string{"Content-Type": {"text/plain"}},
  2101  			},
  2102  			{
  2103  				desc:    "Content-Type sent but does not match",
  2104  				opts:    SignedURLOptions{ContentType: "text/plain"},
  2105  				headers: map[string][]string{"Content-Type": {"application/json"}},
  2106  				fail:    true,
  2107  			},
  2108  			{
  2109  				desc: "Canonical headers sent and match",
  2110  				opts: SignedURLOptions{Headers: []string{
  2111  					" X-Goog-Foo: Bar baz ",
  2112  					"X-Goog-Novalue", // ignored: no value
  2113  					"X-Google-Foo",   // ignored: wrong prefix
  2114  					"x-goog-meta-start-time: 2023-02-10T02:00:00Z", // with colons
  2115  				}},
  2116  				headers: map[string][]string{"X-Goog-foo": {"Bar baz  "}, "x-goog-meta-start-time": {"2023-02-10T02:00:00Z"}},
  2117  			},
  2118  			{
  2119  				desc: "Canonical headers sent and match using V4",
  2120  				opts: SignedURLOptions{Headers: []string{
  2121  					"x-goog-meta-start-time: 2023-02-10T02:", // with colons
  2122  					" X-Goog-Foo: Bar baz ",
  2123  					"X-Goog-Novalue", // ignored: no value
  2124  					"X-Google-Foo",   // ignored: wrong prefix
  2125  				},
  2126  					Scheme: SigningSchemeV4,
  2127  				},
  2128  				headers: map[string][]string{"x-goog-meta-start-time": {"2023-02-10T02:"}, "X-Goog-foo": {"Bar baz  "}},
  2129  			},
  2130  			{
  2131  				desc:    "Canonical headers sent but don't match",
  2132  				opts:    SignedURLOptions{Headers: []string{" X-Goog-Foo: Bar baz"}},
  2133  				headers: map[string][]string{"X-Goog-Foo": {"bar baz"}},
  2134  				fail:    true,
  2135  			},
  2136  			{
  2137  				desc: "Virtual hosted style with custom hostname",
  2138  				opts: SignedURLOptions{
  2139  					Style:    VirtualHostedStyle(),
  2140  					Hostname: "storage.googleapis.com:443",
  2141  				},
  2142  				fail: false,
  2143  			},
  2144  			{
  2145  				desc: "Hostname v4",
  2146  				opts: SignedURLOptions{
  2147  					Hostname: "storage.googleapis.com:443",
  2148  					Scheme:   SigningSchemeV4,
  2149  				},
  2150  				fail: false,
  2151  			},
  2152  		} {
  2153  			opts := test.opts
  2154  			opts.GoogleAccessID = jwtConf.Email
  2155  			opts.PrivateKey = jwtConf.PrivateKey
  2156  			opts.Method = "GET"
  2157  			opts.Expires = time.Now().Add(time.Hour)
  2158  
  2159  			u, err := bkt.SignedURL(obj, &opts)
  2160  			if err != nil {
  2161  				t.Errorf("%s: SignedURL: %v", test.desc, err)
  2162  				continue
  2163  			}
  2164  
  2165  			err = verifySignedURL(u, test.headers, contents)
  2166  			if err != nil && !test.fail {
  2167  				t.Errorf("%s: wanted success but got error:\n%v", test.desc, err)
  2168  			} else if err == nil && test.fail {
  2169  				t.Errorf("%s: wanted failure but test succeeded", test.desc)
  2170  			}
  2171  		}
  2172  	})
  2173  }
  2174  
  2175  func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) {
  2176  	multiTransportTest(skipJSONReads(context.Background(), "no reads in test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  2177  
  2178  		// To test SignedURL, we need a real user email and private key. Extract
  2179  		// them from the JSON key file.
  2180  		jwtConf, err := testutil.JWTConfig()
  2181  		if err != nil {
  2182  			t.Fatal(err)
  2183  		}
  2184  		if jwtConf == nil {
  2185  			t.Skip("JSON key file is not present")
  2186  		}
  2187  
  2188  		bkt := client.Bucket(bucket)
  2189  
  2190  		// TODO(deklerk): document how these were generated and their significance
  2191  		encryptionKey := "AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI="
  2192  		encryptionKeySha256 := "QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k="
  2193  		headers := map[string][]string{
  2194  			"x-goog-encryption-algorithm":  {"AES256"},
  2195  			"x-goog-encryption-key":        {encryptionKey},
  2196  			"x-goog-encryption-key-sha256": {encryptionKeySha256},
  2197  		}
  2198  		contents := []byte(`{"message":"encryption with csek works"}`)
  2199  		tests := []struct {
  2200  			desc string
  2201  			opts *SignedURLOptions
  2202  		}{
  2203  			{
  2204  				desc: "v4 URL with customer supplied encryption keys for PUT",
  2205  				opts: &SignedURLOptions{
  2206  					Method: "PUT",
  2207  					Headers: []string{
  2208  						"x-goog-encryption-algorithm:AES256",
  2209  						"x-goog-encryption-key:AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI=",
  2210  						"x-goog-encryption-key-sha256:QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k=",
  2211  					},
  2212  					Scheme: SigningSchemeV4,
  2213  				},
  2214  			},
  2215  			{
  2216  				desc: "v4 URL with customer supplied encryption keys for GET",
  2217  				opts: &SignedURLOptions{
  2218  					Method: "GET",
  2219  					Headers: []string{
  2220  						"x-goog-encryption-algorithm:AES256",
  2221  						fmt.Sprintf("x-goog-encryption-key:%s", encryptionKey),
  2222  						fmt.Sprintf("x-goog-encryption-key-sha256:%s", encryptionKeySha256),
  2223  					},
  2224  					Scheme: SigningSchemeV4,
  2225  				},
  2226  			},
  2227  		}
  2228  		defer func() {
  2229  			// Delete encrypted object.
  2230  			err := bkt.Object("csek.json").Delete(ctx)
  2231  			if err != nil {
  2232  				log.Printf("failed to deleted encrypted file: %v", err)
  2233  			}
  2234  		}()
  2235  
  2236  		for _, test := range tests {
  2237  			opts := test.opts
  2238  			opts.GoogleAccessID = jwtConf.Email
  2239  			opts.PrivateKey = jwtConf.PrivateKey
  2240  			opts.Expires = time.Now().Add(time.Hour)
  2241  
  2242  			u, err := bkt.SignedURL("csek.json", test.opts)
  2243  			if err != nil {
  2244  				t.Fatalf("%s: %v", test.desc, err)
  2245  			}
  2246  
  2247  			if test.opts.Method == "PUT" {
  2248  				if _, err := putURL(u, headers, bytes.NewReader(contents)); err != nil {
  2249  					t.Fatalf("%s: %v", test.desc, err)
  2250  				}
  2251  			}
  2252  
  2253  			if test.opts.Method == "GET" {
  2254  				if err := verifySignedURL(u, headers, contents); err != nil {
  2255  					t.Fatalf("%s: %v", test.desc, err)
  2256  				}
  2257  			}
  2258  		}
  2259  	})
  2260  }
  2261  
  2262  func TestIntegration_SignedURL_EmptyStringObjectName(t *testing.T) {
  2263  	multiTransportTest(skipJSONReads(context.Background(), "no reads in test"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  2264  
  2265  		// To test SignedURL, we need a real user email and private key. Extract them
  2266  		// from the JSON key file.
  2267  		jwtConf, err := testutil.JWTConfig()
  2268  		if err != nil {
  2269  			t.Fatal(err)
  2270  		}
  2271  		if jwtConf == nil {
  2272  			t.Skip("JSON key file is not present")
  2273  		}
  2274  
  2275  		opts := &SignedURLOptions{
  2276  			Scheme:         SigningSchemeV4,
  2277  			Method:         "GET",
  2278  			GoogleAccessID: jwtConf.Email,
  2279  			PrivateKey:     jwtConf.PrivateKey,
  2280  			Expires:        time.Now().Add(time.Hour),
  2281  		}
  2282  
  2283  		bkt := client.Bucket(bucket)
  2284  		u, err := bkt.SignedURL("", opts)
  2285  		if err != nil {
  2286  			t.Fatal(err)
  2287  		}
  2288  
  2289  		// Should be some ListBucketResult response.
  2290  		_, err = getURL(u, nil)
  2291  		if err != nil {
  2292  			t.Fatal(err)
  2293  		}
  2294  	})
  2295  
  2296  }
  2297  
  2298  func TestIntegration_BucketACL(t *testing.T) {
  2299  	ctx := skipJSONReads(context.Background(), "no reads in test")
  2300  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  2301  		h := testHelper{t}
  2302  
  2303  		bucket := prefix + uidSpace.New()
  2304  		bkt := client.Bucket(bucket)
  2305  		h.mustCreate(bkt, testutil.ProjID(), nil)
  2306  		defer h.mustDeleteBucket(bkt)
  2307  
  2308  		entity := ACLEntity("domain-google.com")
  2309  		rule := ACLRule{Entity: entity, Role: RoleReader, Domain: "google.com"}
  2310  
  2311  		if err := bkt.DefaultObjectACL().Set(ctx, entity, RoleReader); err != nil {
  2312  			t.Errorf("Can't put default ACL rule for the bucket, errored with %v", err)
  2313  		}
  2314  
  2315  		acl, err := bkt.DefaultObjectACL().List(ctx)
  2316  		if err != nil {
  2317  			t.Errorf("DefaultObjectACL.List for bucket %q: %v", bucket, err)
  2318  		}
  2319  		if !containsACLRule(acl, testACLRule(rule)) {
  2320  			t.Fatalf("default ACL rule missing; want: %#v, got rules: %+v", rule, acl)
  2321  		}
  2322  
  2323  		o := bkt.Object("acl1")
  2324  		defer h.mustDeleteObject(o)
  2325  
  2326  		// Retry to account for propagation delay in metadata update.
  2327  		err = retry(ctx, func() error {
  2328  			if err := writeObject(ctx, o, "", randomContents()); err != nil {
  2329  				return fmt.Errorf("Write for %v failed with %v", o.ObjectName(), err)
  2330  			}
  2331  			acl, err = o.ACL().List(ctx)
  2332  			return err
  2333  		}, func() error {
  2334  			if !containsACLRule(acl, testACLRule(rule)) {
  2335  				return fmt.Errorf("object ACL rule missing %+v from ACL \n%+v", rule, acl)
  2336  			}
  2337  			return nil
  2338  		})
  2339  		if err != nil {
  2340  			t.Error(err)
  2341  		}
  2342  
  2343  		if err := o.ACL().Delete(ctx, entity); err != nil {
  2344  			t.Errorf("object ACL: could not delete entity %s", entity)
  2345  		}
  2346  		// Delete the default ACL rule. We can't move this code earlier in the
  2347  		// test, because the test depends on the fact that the object ACL inherits
  2348  		// it.
  2349  		if err := bkt.DefaultObjectACL().Delete(ctx, entity); err != nil {
  2350  			t.Errorf("default ACL: could not delete entity %s", entity)
  2351  		}
  2352  
  2353  		entity2 := AllAuthenticatedUsers
  2354  		rule2 := ACLRule{Entity: entity2, Role: RoleReader}
  2355  		if err := bkt.ACL().Set(ctx, entity2, RoleReader); err != nil {
  2356  			t.Errorf("Error while putting bucket ACL rule: %v", err)
  2357  		}
  2358  
  2359  		var bACL []ACLRule
  2360  
  2361  		// Retry to account for propagation delay in metadata update.
  2362  		err = retry(ctx, func() error {
  2363  			bACL, err = bkt.ACL().List(ctx)
  2364  			return err
  2365  		}, func() error {
  2366  			if !containsACLRule(bACL, testACLRule(rule2)) {
  2367  				return fmt.Errorf("bucket ACL missing %+v", rule2)
  2368  			}
  2369  			return nil
  2370  		})
  2371  		if err != nil {
  2372  			t.Error(err)
  2373  		}
  2374  
  2375  		if err := bkt.ACL().Delete(ctx, entity2); err != nil {
  2376  			t.Errorf("Error while deleting bucket ACL rule: %v", err)
  2377  		}
  2378  	})
  2379  }
  2380  
  2381  func TestIntegration_ValidObjectNames(t *testing.T) {
  2382  	ctx := skipJSONReads(context.Background(), "no reads in test")
  2383  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  2384  		bkt := client.Bucket(bucket)
  2385  
  2386  		validNames := []string{
  2387  			"gopher",
  2388  			"Гоферови",
  2389  			"a",
  2390  			strings.Repeat("a", 1024),
  2391  		}
  2392  		for _, name := range validNames {
  2393  			if err := writeObject(ctx, bkt.Object(name), "", []byte("data")); err != nil {
  2394  				t.Errorf("Object %q write failed: %v. Want success", name, err)
  2395  				continue
  2396  			}
  2397  			defer bkt.Object(name).Delete(ctx)
  2398  		}
  2399  
  2400  		invalidNames := []string{
  2401  			"",                        // Too short.
  2402  			strings.Repeat("a", 1025), // Too long.
  2403  			"new\nlines",
  2404  			"bad\xffunicode",
  2405  		}
  2406  		for _, name := range invalidNames {
  2407  			// Invalid object names will either cause failure during Write or Close.
  2408  			if err := writeObject(ctx, bkt.Object(name), "", []byte("data")); err != nil {
  2409  				continue
  2410  			}
  2411  			defer bkt.Object(name).Delete(ctx)
  2412  			t.Errorf("%q should have failed. Didn't", name)
  2413  		}
  2414  	})
  2415  }
  2416  
  2417  func TestIntegration_WriterContentType(t *testing.T) {
  2418  	ctx := skipJSONReads(context.Background(), "no reads in test")
  2419  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  2420  		obj := client.Bucket(bucket).Object("content")
  2421  		testCases := []struct {
  2422  			content               string
  2423  			setType, wantType     string
  2424  			forceEmptyContentType bool
  2425  		}{
  2426  			{
  2427  				// Sniffed content type.
  2428  				content:  "It was the best of times, it was the worst of times.",
  2429  				wantType: "text/plain; charset=utf-8",
  2430  			},
  2431  			{
  2432  				// Sniffed content type.
  2433  				content:  "<html><head><title>My first page</title></head></html>",
  2434  				wantType: "text/html; charset=utf-8",
  2435  			},
  2436  			{
  2437  				content:  "<html><head><title>My first page</title></head></html>",
  2438  				setType:  "text/html",
  2439  				wantType: "text/html",
  2440  			},
  2441  			{
  2442  				content:  "<html><head><title>My first page</title></head></html>",
  2443  				setType:  "image/jpeg",
  2444  				wantType: "image/jpeg",
  2445  			},
  2446  			{
  2447  				// Content type sniffing disabled.
  2448  				content:               "<html><head><title>My first page</title></head></html>",
  2449  				setType:               "",
  2450  				wantType:              "",
  2451  				forceEmptyContentType: true,
  2452  			},
  2453  		}
  2454  		for i, tt := range testCases {
  2455  			writer := newWriter(ctx, obj, tt.setType, tt.forceEmptyContentType)
  2456  			if err := writeContents(writer, []byte(tt.content)); err != nil {
  2457  				t.Errorf("writing #%d: %v", i, err)
  2458  			}
  2459  			attrs, err := obj.Attrs(ctx)
  2460  			if err != nil {
  2461  				t.Errorf("obj.Attrs: %v", err)
  2462  				continue
  2463  			}
  2464  			if got := attrs.ContentType; got != tt.wantType {
  2465  				t.Errorf("Content-Type = %q; want %q\nContent: %q\nSet Content-Type: %q", got, tt.wantType, tt.content, tt.setType)
  2466  			}
  2467  		}
  2468  	})
  2469  }
  2470  
  2471  func TestIntegration_WriterChunksize(t *testing.T) {
  2472  	ctx := skipJSONReads(context.Background(), "no reads in test")
  2473  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  2474  		obj := client.Bucket(bucket).Object("writer-chunksize-test" + uidSpaceObjects.New())
  2475  		objSize := 1<<10<<10 + 1 // 1 Mib + 1 byte
  2476  		contents := bytes.Repeat([]byte("a"), objSize)
  2477  
  2478  		for _, test := range []struct {
  2479  			desc             string
  2480  			chunksize        int
  2481  			wantBytesPerCall int64
  2482  			wantCallbacks    int
  2483  		}{
  2484  			{
  2485  				desc:             "default chunksize",
  2486  				chunksize:        16 << 10 << 10,
  2487  				wantBytesPerCall: 16 << 10 << 10,
  2488  				wantCallbacks:    0,
  2489  			},
  2490  			{
  2491  				desc:             "small chunksize rounds up to 256kib",
  2492  				chunksize:        1,
  2493  				wantBytesPerCall: 256 << 10,
  2494  				wantCallbacks:    5,
  2495  			},
  2496  			{
  2497  				desc:             "chunksize of 256kib",
  2498  				chunksize:        256 << 10,
  2499  				wantBytesPerCall: 256 << 10,
  2500  				wantCallbacks:    5,
  2501  			},
  2502  			{
  2503  				desc:             "chunksize of just over 256kib rounds up",
  2504  				chunksize:        256<<10 + 1,
  2505  				wantBytesPerCall: 256 * 2 << 10,
  2506  				wantCallbacks:    3,
  2507  			},
  2508  			{
  2509  				desc:             "multiple of 256kib",
  2510  				chunksize:        256 * 3 << 10,
  2511  				wantBytesPerCall: 256 * 3 << 10,
  2512  				wantCallbacks:    2,
  2513  			},
  2514  			{
  2515  				desc:             "chunksize 0 uploads everything",
  2516  				chunksize:        0,
  2517  				wantBytesPerCall: int64(objSize),
  2518  				wantCallbacks:    0,
  2519  			},
  2520  		} {
  2521  			t.Run(test.desc, func(t *testing.T) {
  2522  				t.Cleanup(func() { obj.Delete(ctx) })
  2523  
  2524  				w := obj.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx)
  2525  				w.ChunkSize = test.chunksize
  2526  
  2527  				bytesWrittenSoFar := int64(0)
  2528  				callbacks := 0
  2529  
  2530  				w.ProgressFunc = func(i int64) {
  2531  					bytesWrittenByCall := i - bytesWrittenSoFar
  2532  
  2533  					// Error if this is not the last call and we don't write exactly wantBytesPerCall
  2534  					if i != int64(objSize) && bytesWrittenByCall != test.wantBytesPerCall {
  2535  						t.Errorf("unexpected number of bytes written by call; wanted: %d, written: %d", test.wantBytesPerCall, bytesWrittenByCall)
  2536  					}
  2537  
  2538  					bytesWrittenSoFar = i
  2539  					callbacks++
  2540  				}
  2541  
  2542  				if _, err := w.Write(contents); err != nil {
  2543  					_ = w.Close()
  2544  					t.Fatalf("writer.Write: %v", err)
  2545  				}
  2546  				if err := w.Close(); err != nil {
  2547  					t.Fatalf("writer.Close: %v", err)
  2548  				}
  2549  
  2550  				if callbacks != test.wantCallbacks {
  2551  					t.Errorf("ProgressFunc was called %d times, expected %d", callbacks, test.wantCallbacks)
  2552  				}
  2553  
  2554  				// Confirm all bytes were uploaded.
  2555  				attrs, err := obj.Attrs(ctx)
  2556  				if err != nil {
  2557  					t.Fatalf("obj.Attrs: %v", err)
  2558  				}
  2559  				if attrs.Size != int64(objSize) {
  2560  					t.Errorf("incorrect number of bytes written; got %v, want %v", attrs.Size, objSize)
  2561  				}
  2562  			})
  2563  		}
  2564  	})
  2565  }
  2566  
  2567  func TestIntegration_ZeroSizedObject(t *testing.T) {
  2568  	t.Parallel()
  2569  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  2570  		h := testHelper{t}
  2571  
  2572  		obj := client.Bucket(bucket).Object("zero")
  2573  
  2574  		// Check writing it works as expected.
  2575  		w := obj.NewWriter(ctx)
  2576  		if err := w.Close(); err != nil {
  2577  			t.Fatalf("Writer.Close: %v", err)
  2578  		}
  2579  		defer obj.Delete(ctx)
  2580  
  2581  		// Check we can read it too.
  2582  		body := h.mustRead(obj)
  2583  		if len(body) != 0 {
  2584  			t.Errorf("Body is %v, want empty []byte{}", body)
  2585  		}
  2586  	})
  2587  }
  2588  
  2589  func TestIntegration_Encryption(t *testing.T) {
  2590  	// This function tests customer-supplied encryption keys for all operations
  2591  	// involving objects. Bucket and ACL operations aren't tested because they
  2592  	// aren't affected by customer encryption. Neither is deletion.
  2593  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  2594  		h := testHelper{t}
  2595  
  2596  		obj := client.Bucket(bucket).Object("customer-encryption")
  2597  		key := []byte("my-secret-AES-256-encryption-key")
  2598  		keyHash := sha256.Sum256(key)
  2599  		keyHashB64 := base64.StdEncoding.EncodeToString(keyHash[:])
  2600  		key2 := []byte("My-Secret-AES-256-Encryption-Key")
  2601  		contents := "top secret."
  2602  
  2603  		checkMetadataCall := func(msg string, f func(o *ObjectHandle) (*ObjectAttrs, error)) {
  2604  			// Performing a metadata operation without the key should succeed.
  2605  			attrs, err := f(obj)
  2606  			if err != nil {
  2607  				t.Fatalf("%s: %v", msg, err)
  2608  			}
  2609  			// The key hash should match...
  2610  			if got, want := attrs.CustomerKeySHA256, keyHashB64; got != want {
  2611  				t.Errorf("%s: key hash: got %q, want %q", msg, got, want)
  2612  			}
  2613  			// ...but CRC and MD5 should not be present.
  2614  			if attrs.CRC32C != 0 {
  2615  				t.Errorf("%s: CRC: got %v, want 0", msg, attrs.CRC32C)
  2616  			}
  2617  			if len(attrs.MD5) > 0 {
  2618  				t.Errorf("%s: MD5: got %v, want len == 0", msg, attrs.MD5)
  2619  			}
  2620  
  2621  			// Performing a metadata operation with the key should succeed.
  2622  			attrs, err = f(obj.Key(key))
  2623  			if err != nil {
  2624  				t.Fatalf("%s: %v", msg, err)
  2625  			}
  2626  			// Check the key and content hashes.
  2627  			if got, want := attrs.CustomerKeySHA256, keyHashB64; got != want {
  2628  				t.Errorf("%s: key hash: got %q, want %q", msg, got, want)
  2629  			}
  2630  			if attrs.CRC32C == 0 {
  2631  				t.Errorf("%s: CRC: got 0, want non-zero", msg)
  2632  			}
  2633  			if len(attrs.MD5) == 0 {
  2634  				t.Errorf("%s: MD5: got len == 0, want len > 0", msg)
  2635  			}
  2636  		}
  2637  
  2638  		checkRead := func(msg string, o *ObjectHandle, k []byte, wantContents string) {
  2639  			// Reading the object without the key should fail.
  2640  			if _, err := readObject(ctx, o); err == nil {
  2641  				t.Errorf("%s: reading without key: want error, got nil", msg)
  2642  			}
  2643  			// Reading the object with the key should succeed.
  2644  			got := h.mustRead(o.Key(k))
  2645  			gotContents := string(got)
  2646  			// And the contents should match what we wrote.
  2647  			if gotContents != wantContents {
  2648  				t.Errorf("%s: contents: got %q, want %q", msg, gotContents, wantContents)
  2649  			}
  2650  		}
  2651  
  2652  		checkReadUnencrypted := func(msg string, obj *ObjectHandle, wantContents string) {
  2653  			got := h.mustRead(obj)
  2654  			gotContents := string(got)
  2655  			if gotContents != wantContents {
  2656  				t.Errorf("%s: got %q, want %q", msg, gotContents, wantContents)
  2657  			}
  2658  		}
  2659  
  2660  		// Write to obj using our own encryption key, which is a valid 32-byte
  2661  		// AES-256 key.
  2662  		h.mustWrite(obj.Key(key).NewWriter(ctx), []byte(contents))
  2663  
  2664  		checkMetadataCall("Attrs", func(o *ObjectHandle) (*ObjectAttrs, error) {
  2665  			return o.Attrs(ctx)
  2666  		})
  2667  
  2668  		checkMetadataCall("Update", func(o *ObjectHandle) (*ObjectAttrs, error) {
  2669  			return o.Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"})
  2670  		})
  2671  
  2672  		checkRead("first object", obj, key, contents)
  2673  
  2674  		// We create 2 objects here and we can interleave operations to get around
  2675  		// the rate limit for object mutation operations (create, update, and delete).
  2676  		obj2 := client.Bucket(bucket).Object("customer-encryption-2")
  2677  		obj4 := client.Bucket(bucket).Object("customer-encryption-4")
  2678  
  2679  		// Copying an object without the key should fail.
  2680  		if _, err := obj4.CopierFrom(obj).Run(ctx); err == nil {
  2681  			t.Fatal("want error, got nil")
  2682  		}
  2683  		// Copying an object with the key should succeed.
  2684  		if _, err := obj2.CopierFrom(obj.Key(key)).Run(ctx); err != nil {
  2685  			t.Fatal(err)
  2686  		}
  2687  		// The destination object is not encrypted; we can read it without a key.
  2688  		checkReadUnencrypted("copy dest", obj2, contents)
  2689  
  2690  		// Providing a key on the destination but not the source should fail,
  2691  		// since the source is encrypted.
  2692  		if _, err := obj2.Key(key2).CopierFrom(obj).Run(ctx); err == nil {
  2693  			t.Fatal("want error, got nil")
  2694  		}
  2695  
  2696  		// But copying with keys for both source and destination should succeed.
  2697  		if _, err := obj2.Key(key2).CopierFrom(obj.Key(key)).Run(ctx); err != nil {
  2698  			t.Fatal(err)
  2699  		}
  2700  		// And the destination should be encrypted, meaning we can only read it
  2701  		// with a key.
  2702  		checkRead("copy destination", obj2, key2, contents)
  2703  
  2704  		// Change obj2's key to prepare for compose, where all objects must have
  2705  		// the same key. Also illustrates key rotation: copy an object to itself
  2706  		// with a different key.
  2707  		if _, err := obj2.Key(key).CopierFrom(obj2.Key(key2)).Run(ctx); err != nil {
  2708  			t.Fatal(err)
  2709  		}
  2710  		obj3 := client.Bucket(bucket).Object("customer-encryption-3")
  2711  		// Composing without keys should fail.
  2712  		if _, err := obj3.ComposerFrom(obj, obj2).Run(ctx); err == nil {
  2713  			t.Fatal("want error, got nil")
  2714  		}
  2715  		// Keys on the source objects result in an error.
  2716  		if _, err := obj3.ComposerFrom(obj.Key(key), obj2).Run(ctx); err == nil {
  2717  			t.Fatal("want error, got nil")
  2718  		}
  2719  		// A key on the destination object both decrypts the source objects
  2720  		// and encrypts the destination.
  2721  		if _, err := obj3.Key(key).ComposerFrom(obj, obj2).Run(ctx); err != nil {
  2722  			t.Fatalf("got %v, want nil", err)
  2723  		}
  2724  		// Check that the destination in encrypted.
  2725  		checkRead("compose destination", obj3, key, contents+contents)
  2726  
  2727  		// You can't compose one or more unencrypted source objects into an
  2728  		// encrypted destination object.
  2729  		_, err := obj4.CopierFrom(obj2.Key(key)).Run(ctx) // unencrypt obj2
  2730  		if err != nil {
  2731  			t.Fatal(err)
  2732  		}
  2733  		if _, err := obj3.Key(key).ComposerFrom(obj4).Run(ctx); err == nil {
  2734  			t.Fatal("got nil, want error")
  2735  		}
  2736  	})
  2737  }
  2738  
  2739  func TestIntegration_NonexistentObjectRead(t *testing.T) {
  2740  	t.Parallel()
  2741  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  2742  		_, err := client.Bucket(bucket).Object("object-does-not-exist").NewReader(ctx)
  2743  		if !errors.Is(err, ErrObjectNotExist) {
  2744  			t.Errorf("Objects: got %v, want ErrObjectNotExist", err)
  2745  		}
  2746  	})
  2747  }
  2748  
  2749  func TestIntegration_NonexistentBucket(t *testing.T) {
  2750  	t.Parallel()
  2751  	ctx := skipJSONReads(context.Background(), "no reads in test")
  2752  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) {
  2753  		bkt := client.Bucket(prefix + uidSpace.New())
  2754  		if _, err := bkt.Attrs(ctx); err != ErrBucketNotExist {
  2755  			t.Errorf("Attrs: got %v, want ErrBucketNotExist", err)
  2756  		}
  2757  		it := bkt.Objects(ctx, nil)
  2758  		if _, err := it.Next(); err != ErrBucketNotExist {
  2759  			t.Errorf("Objects: got %v, want ErrBucketNotExist", err)
  2760  		}
  2761  	})
  2762  }
  2763  
  2764  func TestIntegration_PerObjectStorageClass(t *testing.T) {
  2765  	const (
  2766  		defaultStorageClass = "STANDARD"
  2767  		newStorageClass     = "NEARLINE"
  2768  	)
  2769  	ctx := skipJSONReads(context.Background(), "no reads in test")
  2770  
  2771  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  2772  		h := testHelper{t}
  2773  
  2774  		bkt := client.Bucket(bucket)
  2775  
  2776  		// The bucket should have the default storage class.
  2777  		battrs := h.mustBucketAttrs(bkt)
  2778  		if battrs.StorageClass != defaultStorageClass {
  2779  			t.Fatalf("bucket storage class: got %q, want %q",
  2780  				battrs.StorageClass, defaultStorageClass)
  2781  		}
  2782  		// Write an object; it should start with the bucket's storage class.
  2783  		obj := bkt.Object("posc")
  2784  		h.mustWrite(obj.NewWriter(ctx), []byte("foo"))
  2785  		oattrs, err := obj.Attrs(ctx)
  2786  		if err != nil {
  2787  			t.Fatal(err)
  2788  		}
  2789  		if oattrs.StorageClass != defaultStorageClass {
  2790  			t.Fatalf("object storage class: got %q, want %q",
  2791  				oattrs.StorageClass, defaultStorageClass)
  2792  		}
  2793  		// Now use Copy to change the storage class.
  2794  		copier := obj.CopierFrom(obj)
  2795  		copier.StorageClass = newStorageClass
  2796  		oattrs2, err := copier.Run(ctx)
  2797  		if err != nil {
  2798  			log.Fatal(err)
  2799  		}
  2800  		if oattrs2.StorageClass != newStorageClass {
  2801  			t.Fatalf("new object storage class: got %q, want %q",
  2802  				oattrs2.StorageClass, newStorageClass)
  2803  		}
  2804  
  2805  		// We can also write a new object using a non-default storage class.
  2806  		obj2 := bkt.Object("posc2")
  2807  		w := obj2.NewWriter(ctx)
  2808  		w.StorageClass = newStorageClass
  2809  		h.mustWrite(w, []byte("xxx"))
  2810  		if w.Attrs().StorageClass != newStorageClass {
  2811  			t.Fatalf("new object storage class: got %q, want %q",
  2812  				w.Attrs().StorageClass, newStorageClass)
  2813  		}
  2814  	})
  2815  }
  2816  
  2817  func TestIntegration_NoUnicodeNormalization(t *testing.T) {
  2818  	t.Parallel()
  2819  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  2820  		bkt := client.Bucket(bucket)
  2821  		h := testHelper{t}
  2822  
  2823  		for _, tst := range []struct {
  2824  			nameQuoted, content string
  2825  		}{
  2826  			{`"Caf\u00e9"`, "Normalization Form C"},
  2827  			{`"Cafe\u0301"`, "Normalization Form D"},
  2828  		} {
  2829  			name, err := strconv.Unquote(tst.nameQuoted)
  2830  			w := bkt.Object(name).NewWriter(ctx)
  2831  			h.mustWrite(w, []byte(tst.content))
  2832  			if err != nil {
  2833  				t.Fatalf("invalid name: %s: %v", tst.nameQuoted, err)
  2834  			}
  2835  			if got := string(h.mustRead(bkt.Object(name))); got != tst.content {
  2836  				t.Errorf("content of %s is %q, want %q", tst.nameQuoted, got, tst.content)
  2837  			}
  2838  		}
  2839  	})
  2840  }
  2841  
  2842  func TestIntegration_HashesOnUpload(t *testing.T) {
  2843  	// Check that the user can provide hashes on upload, and that these are checked.
  2844  	ctx := skipJSONReads(context.Background(), "no reads in test")
  2845  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  2846  		obj := client.Bucket(bucket).Object("hashesOnUpload-1")
  2847  		data := []byte("I can't wait to be verified")
  2848  
  2849  		write := func(w *Writer) error {
  2850  			if _, err := w.Write(data); err != nil {
  2851  				_ = w.Close()
  2852  				return err
  2853  			}
  2854  			return w.Close()
  2855  		}
  2856  
  2857  		crc32c := crc32.Checksum(data, crc32cTable)
  2858  		// The correct CRC should succeed.
  2859  		w := obj.NewWriter(ctx)
  2860  		w.CRC32C = crc32c
  2861  		w.SendCRC32C = true
  2862  		if err := write(w); err != nil {
  2863  			t.Error(err)
  2864  		}
  2865  
  2866  		// If we change the CRC, validation should fail.
  2867  		w = obj.NewWriter(ctx)
  2868  		w.CRC32C = crc32c + 1
  2869  		w.SendCRC32C = true
  2870  		if err := write(w); err == nil {
  2871  			t.Error("write with bad CRC32c: want error, got nil")
  2872  		}
  2873  
  2874  		// If we have the wrong CRC but forget to send it, we succeed.
  2875  		w = obj.NewWriter(ctx)
  2876  		w.CRC32C = crc32c + 1
  2877  		if err := write(w); err != nil {
  2878  			t.Error(err)
  2879  		}
  2880  
  2881  		// MD5
  2882  		md5 := md5.Sum(data)
  2883  		// The correct MD5 should succeed.
  2884  		w = obj.NewWriter(ctx)
  2885  		w.MD5 = md5[:]
  2886  		if err := write(w); err != nil {
  2887  			t.Error(err)
  2888  		}
  2889  
  2890  		// If we change the MD5, validation should fail.
  2891  		w = obj.NewWriter(ctx)
  2892  		w.MD5 = append([]byte(nil), md5[:]...)
  2893  		w.MD5[0]++
  2894  		if err := write(w); err == nil {
  2895  			t.Error("write with bad MD5: want error, got nil")
  2896  		}
  2897  	})
  2898  }
  2899  
  2900  func TestIntegration_BucketIAM(t *testing.T) {
  2901  	ctx := skipJSONReads(context.Background(), "no reads in test")
  2902  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) {
  2903  		h := testHelper{t}
  2904  		bkt := client.Bucket(prefix + uidSpace.New())
  2905  		h.mustCreate(bkt, testutil.ProjID(), nil)
  2906  		defer h.mustDeleteBucket(bkt)
  2907  		// This bucket is unique to this test run. So we don't have
  2908  		// to worry about other runs interfering with our IAM policy
  2909  		// changes.
  2910  
  2911  		member := "projectViewer:" + testutil.ProjID()
  2912  		role := iam.RoleName("roles/storage.objectViewer")
  2913  		// Get the bucket's IAM policy.
  2914  		policy, err := bkt.IAM().Policy(ctx)
  2915  		if err != nil {
  2916  			t.Fatalf("Getting policy: %v", err)
  2917  		}
  2918  		// The member should not have the role.
  2919  		if policy.HasRole(member, role) {
  2920  			t.Errorf("member %q has role %q", member, role)
  2921  		}
  2922  		// Change the policy.
  2923  		policy.Add(member, role)
  2924  		if err := bkt.IAM().SetPolicy(ctx, policy); err != nil {
  2925  			t.Fatalf("SetPolicy: %v", err)
  2926  		}
  2927  		// Confirm that the binding was added.
  2928  		policy, err = bkt.IAM().Policy(ctx)
  2929  		if err != nil {
  2930  			t.Fatalf("Getting policy: %v", err)
  2931  		}
  2932  		if !policy.HasRole(member, role) {
  2933  			t.Errorf("member %q does not have role %q", member, role)
  2934  		}
  2935  
  2936  		// Check TestPermissions.
  2937  		// This client should have all these permissions (and more).
  2938  		perms := []string{"storage.buckets.get", "storage.buckets.delete"}
  2939  		got, err := bkt.IAM().TestPermissions(ctx, perms)
  2940  		if err != nil {
  2941  			t.Fatalf("TestPermissions: %v", err)
  2942  		}
  2943  		sort.Strings(perms)
  2944  		sort.Strings(got)
  2945  		if !testutil.Equal(got, perms) {
  2946  			t.Errorf("got %v, want %v", got, perms)
  2947  		}
  2948  	})
  2949  }
  2950  
  2951  // This test tests only possibilities where the user making the request is an
  2952  // owner on the project that owns the requester pays bucket. Therefore, we don't
  2953  // need a second project for this test.
  2954  //
  2955  // There are up to three entities involved in a requester-pays call:
  2956  //
  2957  //  1. The user making the request. Here, we use the account used as credentials
  2958  //     for most of our integration tests. The following must hold for this test:
  2959  //     - this user must have resourcemanager.projects.createBillingAssignment
  2960  //     permission (Owner role) on (2) (the project, not the bucket)
  2961  //     - this user must NOT have that permission on (3b).
  2962  //  2. The project that owns the requester-pays bucket. Here, that
  2963  //     is the test project ID (see testutil.ProjID).
  2964  //  3. The project provided as the userProject parameter of the request;
  2965  //     the project to be billed. This test uses:
  2966  //     a. The project that owns the requester-pays bucket (same as (2))
  2967  //     b. Another project (the Firestore project).
  2968  func TestIntegration_RequesterPaysOwner(t *testing.T) {
  2969  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) {
  2970  		jwt, err := testutil.JWTConfig()
  2971  		if err != nil {
  2972  			t.Fatalf("testutil.JWTConfig: %v", err)
  2973  		}
  2974  		// an account that has permissions on the project that owns the bucket
  2975  		mainUserEmail := jwt.Email
  2976  
  2977  		// the project that owns the requester-pays bucket
  2978  		mainProjectID := testutil.ProjID()
  2979  
  2980  		client.SetRetry(WithPolicy(RetryAlways))
  2981  
  2982  		// Secondary project: a project that does not own the bucket.
  2983  		// The "main" user should not have permission on this.
  2984  		secondaryProject := os.Getenv(envFirestoreProjID)
  2985  		if secondaryProject == "" {
  2986  			t.Fatalf("need a second project (env var %s)", envFirestoreProjID)
  2987  		}
  2988  
  2989  		for _, test := range []struct {
  2990  			desc          string
  2991  			userProject   *string // to set on bucket, nil if it should not be set
  2992  			expectSuccess bool
  2993  		}{
  2994  			{
  2995  				desc:          "user is Owner on the project that owns the bucket",
  2996  				userProject:   nil,
  2997  				expectSuccess: true, // by the rule permitting access by owners of the containing bucket
  2998  			},
  2999  			{
  3000  				desc:          "userProject is unnecessary but allowed",
  3001  				userProject:   &mainProjectID,
  3002  				expectSuccess: true, // by the rule permitting access by owners of the containing bucket
  3003  			},
  3004  			{
  3005  				desc:          "cannot use someone else's project for billing",
  3006  				userProject:   &secondaryProject,
  3007  				expectSuccess: false, // we cannot use a project we don't have access to for billing
  3008  			},
  3009  		} {
  3010  			t.Run(test.desc, func(t *testing.T) {
  3011  				h := testHelper{t}
  3012  				ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
  3013  				defer cancel()
  3014  
  3015  				printTestCase := func() string {
  3016  					userProject := "none"
  3017  					if test.userProject != nil {
  3018  						userProject = *test.userProject
  3019  					}
  3020  					return fmt.Sprintf("user: %s\n\t\tcontaining project: %s\n\t\tUserProject: %s", mainUserEmail, mainProjectID, userProject)
  3021  				}
  3022  
  3023  				checkforErrors := func(desc string, err error) {
  3024  					if err != nil && test.expectSuccess {
  3025  						t.Errorf("%s: got unexpected error:%v\n\t\t%s", desc, err, printTestCase())
  3026  					} else if err == nil && !test.expectSuccess {
  3027  						t.Errorf("%s: got unexpected success\n\t\t%s", desc, printTestCase())
  3028  					}
  3029  				}
  3030  
  3031  				bucketName := prefix + uidSpace.New()
  3032  				requesterPaysBucket := client.Bucket(bucketName)
  3033  
  3034  				// Create a requester-pays bucket
  3035  				h.mustCreate(requesterPaysBucket, mainProjectID, &BucketAttrs{RequesterPays: true})
  3036  				t.Cleanup(func() { h.mustDeleteBucket(requesterPaysBucket) })
  3037  
  3038  				// Make sure the object exists, so we don't get confused by ErrObjectNotExist.
  3039  				// The later write we perform may fail so we always write to the object as the user
  3040  				// with permissions on the containing bucket (mainUser).
  3041  				// The storage service may perform validation in any order (perhaps in parallel),
  3042  				// so if we delete or update an object that doesn't exist and for which we lack permission,
  3043  				// we could see either of those two errors. (See Google-internal bug 78341001.)
  3044  				objectName := "acl-go-test" + uidSpaceObjects.New()
  3045  				h.mustWrite(requesterPaysBucket.Object(objectName).NewWriter(ctx), []byte("hello"))
  3046  
  3047  				// Set up the bucket to use depending on the test case
  3048  				bucket := client.Bucket(bucketName)
  3049  				if test.userProject != nil {
  3050  					bucket = bucket.UserProject(*test.userProject)
  3051  				}
  3052  
  3053  				// Get bucket attrs
  3054  				attrs, err := bucket.Attrs(ctx)
  3055  				checkforErrors("get bucket attrs", err)
  3056  				if attrs != nil {
  3057  					if got, want := attrs.RequesterPays, true; got != want {
  3058  						t.Fatalf("attr.RequesterPays = %t, want %t", got, want)
  3059  					}
  3060  				}
  3061  
  3062  				// Bucket ACL operations
  3063  				entity := ACLEntity("domain-google.com")
  3064  
  3065  				checkforErrors("bucket acl set", bucket.ACL().Set(ctx, entity, RoleReader))
  3066  				_, err = bucket.ACL().List(ctx)
  3067  				checkforErrors("bucket acl list", err)
  3068  				checkforErrors("bucket acl delete", bucket.ACL().Delete(ctx, entity))
  3069  
  3070  				// Object operations (except for delete)
  3071  				// Retry to account for propagation delay to objects in metadata update
  3072  				// (we updated the metadata to add the otherUserEmail as owner on the bucket)
  3073  				o := bucket.Object(objectName)
  3074  				ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Second*10)
  3075  				defer cancel()
  3076  				// Only retry when we expect success to avoid retrying for 10 seconds
  3077  				// when we know it will fail
  3078  				if test.expectSuccess {
  3079  					o = o.Retryer(WithErrorFunc(retryOnTransient400and403))
  3080  				}
  3081  				checkforErrors("write object", writeObject(ctxWithTimeout, o, "text/plain", []byte("hello")))
  3082  				_, err = readObject(ctx, bucket.Object(objectName))
  3083  				checkforErrors("read object", err)
  3084  				_, err = bucket.Object(objectName).Attrs(ctx)
  3085  				checkforErrors("get object attrs", err)
  3086  				_, err = bucket.Object(objectName).Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"})
  3087  				checkforErrors("update object", err)
  3088  
  3089  				// Object ACL operations
  3090  				checkforErrors("object acl set", bucket.Object(objectName).ACL().Set(ctx, entity, RoleReader))
  3091  				_, err = bucket.Object(objectName).ACL().List(ctx)
  3092  				checkforErrors("object acl list", err)
  3093  				checkforErrors("object acl list", bucket.Object(objectName).ACL().Delete(ctx, entity))
  3094  
  3095  				// Default object ACL operations
  3096  				// Once again, we interleave buckets to avoid rate limits
  3097  				checkforErrors("default object acl set", bucket.DefaultObjectACL().Set(ctx, entity, RoleReader))
  3098  				_, err = bucket.DefaultObjectACL().List(ctx)
  3099  				checkforErrors("default object acl list", err)
  3100  				checkforErrors("default object acl delete", bucket.DefaultObjectACL().Delete(ctx, entity))
  3101  
  3102  				// Copy
  3103  				_, err = bucket.Object("copy").CopierFrom(bucket.Object(objectName)).Run(ctx)
  3104  				checkforErrors("copy", err)
  3105  				// Delete "copy" object, if created
  3106  				if err == nil {
  3107  					t.Cleanup(func() {
  3108  						h.mustDeleteObject(bucket.Object("copy"))
  3109  					})
  3110  				}
  3111  
  3112  				// Compose
  3113  				_, err = bucket.Object("compose").ComposerFrom(bucket.Object(objectName), bucket.Object("copy")).Run(ctx)
  3114  				checkforErrors("compose", err)
  3115  				// Delete "compose" object, if created
  3116  				if err == nil {
  3117  					t.Cleanup(func() {
  3118  						h.mustDeleteObject(bucket.Object("compose"))
  3119  					})
  3120  				}
  3121  
  3122  				// Delete object
  3123  				if err = bucket.Object(objectName).Delete(ctx); err != nil {
  3124  					// We still want to delete object if the test errors
  3125  					h.mustDeleteObject(requesterPaysBucket.Object(objectName))
  3126  				}
  3127  				checkforErrors("delete object", err)
  3128  			})
  3129  		}
  3130  	})
  3131  }
  3132  
  3133  // This test needs a second project and user to test all possibilities. Since we
  3134  // need these things for Firestore already, we use them here.
  3135  //
  3136  // There are up to three entities involved in a requester-pays call:
  3137  //  1. The user making the request. Here, we use the account used for the
  3138  //     Firestore tests. The following must hold for this test to work:
  3139  //     - this user must NOT have resourcemanager.projects.createBillingAssignment
  3140  //     on the project that owns the bucket (2).
  3141  //     - this user must have serviceusage.services.use permission on the Firestore
  3142  //     project (3b).
  3143  //     - this user must NOT have that serviceusage.services.use permission on
  3144  //     the project that owns the bucket (3a).
  3145  //  2. The project that owns the requester-pays bucket. Here, that
  3146  //     is the test project ID (see testutil.ProjID).
  3147  //  3. The project provided as the userProject parameter of the request;
  3148  //     the project to be billed. This test uses:
  3149  //     a. The project that owns the requester-pays bucket (same as (2))
  3150  //     b. Another project (the Firestore project).
  3151  func TestIntegration_RequesterPaysNonOwner(t *testing.T) {
  3152  	if testing.Short() && !replaying {
  3153  		t.Skip("Integration tests skipped in short mode")
  3154  	}
  3155  	ctx := context.Background()
  3156  
  3157  	// Main project: the project that owns the requester-pays bucket.
  3158  	mainProject := testutil.ProjID()
  3159  
  3160  	// Secondary project: a project that does not own the bucket.
  3161  	// The "main" user does not have permission on this.
  3162  	// This project should have billing enabled.
  3163  	secondaryProject := os.Getenv(envFirestoreProjID)
  3164  	if secondaryProject == "" {
  3165  		t.Fatalf("need a second project (env var %s)", envFirestoreProjID)
  3166  	}
  3167  
  3168  	// Secondary email: an account with permissions on the secondary project,
  3169  	// but not on the main project.
  3170  	// We will grant this email permissions to the bucket created under the main
  3171  	// project, but it must provide a user project to make requests
  3172  	// against that bucket (since it's a requester-pays bucket).
  3173  	secondaryUserEmail, err := keyFileEmail(os.Getenv(envFirestorePrivateKey))
  3174  	if err != nil {
  3175  		t.Fatalf("keyFileEmail error getting second account (env var %s): %v", envFirestorePrivateKey, err)
  3176  	}
  3177  
  3178  	// Token source from secondary email to authenticate to client
  3179  	ts := testutil.TokenSourceEnv(ctx, envFirestorePrivateKey, ScopeFullControl)
  3180  	if ts == nil {
  3181  		t.Fatalf("need a second account (env var %s)", envFirestorePrivateKey)
  3182  	}
  3183  
  3184  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) {
  3185  		client.SetRetry(WithPolicy(RetryAlways))
  3186  
  3187  		for _, test := range []struct {
  3188  			desc              string
  3189  			userProject       *string // to set on bucket, nil if it should not be set
  3190  			expectSuccess     bool
  3191  			wantErrorCode     int
  3192  			wantErrorCodeGRPC codes.Code
  3193  		}{
  3194  			{
  3195  				desc:          "no UserProject",
  3196  				userProject:   nil,
  3197  				expectSuccess: false, // by the standard requester-pays rule
  3198  			},
  3199  			{
  3200  				desc:          "user is an Editor on UserProject",
  3201  				userProject:   &secondaryProject,
  3202  				expectSuccess: true, // by the standard requester-pays rule
  3203  			},
  3204  			{
  3205  				desc:              "user is not an Editor on UserProject",
  3206  				userProject:       &mainProject,
  3207  				expectSuccess:     false, // we cannot use a project we don't have access to for billing
  3208  				wantErrorCode:     403,
  3209  				wantErrorCodeGRPC: codes.PermissionDenied,
  3210  			},
  3211  		} {
  3212  			t.Run(test.desc, func(t *testing.T) {
  3213  				ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
  3214  				t.Cleanup(cancel)
  3215  
  3216  				printTestCase := func() string {
  3217  					userProject := "none"
  3218  					if test.userProject != nil {
  3219  						userProject = *test.userProject
  3220  					}
  3221  					return fmt.Sprintf("user: %s\n\t\tcontaining project: %s\n\t\tUserProject: %s", secondaryUserEmail, mainProject, userProject)
  3222  				}
  3223  
  3224  				checkforErrors := func(desc string, err error) {
  3225  					errCode := extractErrCode(err)
  3226  					if err != nil && test.expectSuccess {
  3227  						t.Errorf("%s: got unexpected error:%v\n\t\t%s", desc, err, printTestCase())
  3228  					} else if err == nil && !test.expectSuccess {
  3229  						t.Errorf("%s: got unexpected success\n\t\t%s", desc, printTestCase())
  3230  					} else if !test.expectSuccess && test.wantErrorCode != 0 {
  3231  						if (status.Code(err) != codes.OK && status.Code(err) != codes.Unknown && status.Code(err) != test.wantErrorCodeGRPC) || (errCode > 0 && errCode != test.wantErrorCode) {
  3232  							fmt.Println(status.Code(err), "   ", status.Code(err) != test.wantErrorCodeGRPC)
  3233  							t.Errorf("%s: mismatched errors; want error code: %d or grpc error: %s, got error: %v \n\t\t%s\n",
  3234  								desc, test.wantErrorCode, test.wantErrorCodeGRPC, err, printTestCase())
  3235  						}
  3236  					}
  3237  				}
  3238  
  3239  				bucketName := prefix + uidSpace.New()
  3240  				objectName := "acl-go-test" + uidSpaceObjects.New()
  3241  
  3242  				setUpRequesterPaysBucket(ctx, t, bucketName, objectName, secondaryUserEmail)
  3243  
  3244  				// Set up the bucket to use depending on the test case
  3245  				bucket := client.Bucket(bucketName)
  3246  				if test.userProject != nil {
  3247  					bucket = bucket.UserProject(*test.userProject)
  3248  				}
  3249  
  3250  				// Get bucket attrs
  3251  				attrs, err := bucket.Attrs(ctx)
  3252  				checkforErrors("get bucket attrs", err)
  3253  				if attrs != nil {
  3254  					if got, want := attrs.RequesterPays, true; got != want {
  3255  						t.Fatalf("attr.RequesterPays = %t, want %t", got, want)
  3256  					}
  3257  				}
  3258  
  3259  				// Bucket ACL operations
  3260  				entity := ACLEntity("domain-google.com")
  3261  
  3262  				checkforErrors("bucket acl set", bucket.ACL().Set(ctx, entity, RoleReader))
  3263  				_, err = bucket.ACL().List(ctx)
  3264  				checkforErrors("bucket acl list", err)
  3265  				checkforErrors("bucket acl delete", bucket.ACL().Delete(ctx, entity))
  3266  
  3267  				// Object operations (except for delete)
  3268  				// Retry to account for propagation delay to objects in metadata update
  3269  				// (we updated the metadata to add the otherUserEmail as owner on the bucket)
  3270  				o := bucket.Object(objectName)
  3271  				ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Second*15)
  3272  				defer cancel()
  3273  				// Only retry when we expect success to avoid retrying
  3274  				// when we know it will fail
  3275  				if test.expectSuccess {
  3276  					o = o.Retryer(WithErrorFunc(retryOnTransient400and403))
  3277  				}
  3278  				checkforErrors("write object", writeObject(ctxWithTimeout, o, "text/plain", []byte("hello")))
  3279  				_, err = readObject(ctx, bucket.Object(objectName))
  3280  				checkforErrors("read object", err)
  3281  				_, err = bucket.Object(objectName).Attrs(ctx)
  3282  				checkforErrors("get object attrs", err)
  3283  				_, err = bucket.Object(objectName).Update(ctx, ObjectAttrsToUpdate{ContentLanguage: "en"})
  3284  				checkforErrors("update object", err)
  3285  
  3286  				// Object ACL operations
  3287  				checkforErrors("object acl set", bucket.Object(objectName).ACL().Set(ctx, entity, RoleReader))
  3288  				_, err = bucket.Object(objectName).ACL().List(ctx)
  3289  				checkforErrors("object acl list", err)
  3290  				checkforErrors("object acl list", bucket.Object(objectName).ACL().Delete(ctx, entity))
  3291  
  3292  				// Default object ACL operations
  3293  				// Once again, we interleave buckets to avoid rate limits
  3294  				checkforErrors("default object acl set", bucket.DefaultObjectACL().Set(ctx, entity, RoleReader))
  3295  				_, err = bucket.DefaultObjectACL().List(ctx)
  3296  				checkforErrors("default object acl list", err)
  3297  				checkforErrors("default object acl delete", bucket.DefaultObjectACL().Delete(ctx, entity))
  3298  
  3299  				// Copy
  3300  				copyObj := bucket.Object("copy")
  3301  				_, err = copyObj.CopierFrom(bucket.Object(objectName)).Run(ctx)
  3302  				checkforErrors("copy", err)
  3303  				// Delete "copy" object, if created
  3304  				if err == nil {
  3305  					t.Cleanup(func() {
  3306  						if err := deleteObjectIfExists(copyObj, WithErrorFunc(retryOnTransient400and403)); err != nil {
  3307  							t.Error(err)
  3308  						}
  3309  					})
  3310  				}
  3311  
  3312  				// Compose
  3313  				composeObj := bucket.Object("compose")
  3314  				_, err = composeObj.ComposerFrom(bucket.Object(objectName), bucket.Object("copy")).Run(ctx)
  3315  				checkforErrors("compose", err)
  3316  				// Delete "compose" object, if created
  3317  				if err == nil {
  3318  					t.Cleanup(func() {
  3319  						if err := deleteObjectIfExists(composeObj, WithErrorFunc(retryOnTransient400and403)); err != nil {
  3320  							t.Error(err)
  3321  						}
  3322  					})
  3323  				}
  3324  
  3325  				// Delete object
  3326  				checkforErrors("delete object", bucket.Object(objectName).Delete(ctx))
  3327  			})
  3328  		}
  3329  	}, option.WithTokenSource(ts))
  3330  }
  3331  
  3332  func TestIntegration_Notifications(t *testing.T) {
  3333  	multiTransportTest(skipGRPC("notifications not implemented"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  3334  		bkt := client.Bucket(bucket)
  3335  
  3336  		checkNotifications := func(msg string, want map[string]*Notification) {
  3337  			got, err := bkt.Notifications(ctx)
  3338  			if err != nil {
  3339  				t.Fatal(err)
  3340  			}
  3341  			if diff := testutil.Diff(got, want); diff != "" {
  3342  				t.Errorf("%s: got=-, want=+:\n%s", msg, diff)
  3343  			}
  3344  		}
  3345  		checkNotifications("initial", map[string]*Notification{})
  3346  
  3347  		nArg := &Notification{
  3348  			TopicProjectID: testutil.ProjID(),
  3349  			TopicID:        "go-storage-notification-test",
  3350  			PayloadFormat:  NoPayload,
  3351  		}
  3352  		n, err := bkt.AddNotification(ctx, nArg)
  3353  		if err != nil {
  3354  			t.Fatal(err)
  3355  		}
  3356  		if n.ID == "" {
  3357  			t.Fatal("expected created Notification to have non-empty ID")
  3358  		}
  3359  		nArg.ID = n.ID
  3360  		if !testutil.Equal(n, nArg) {
  3361  			t.Errorf("got %+v, want %+v", n, nArg)
  3362  		}
  3363  		checkNotifications("after add", map[string]*Notification{n.ID: n})
  3364  
  3365  		if err := bkt.DeleteNotification(ctx, n.ID); err != nil {
  3366  			t.Fatal(err)
  3367  		}
  3368  		checkNotifications("after delete", map[string]*Notification{})
  3369  	})
  3370  }
  3371  
  3372  func TestIntegration_PublicBucket(t *testing.T) {
  3373  	// Confirm that an unauthenticated client can access a public bucket.
  3374  	// See https://cloud.google.com/storage/docs/public-datasets/landsat
  3375  
  3376  	multiTransportTest(skipGRPC("no public buckets for gRPC"), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  3377  		const landsatBucket = "gcp-public-data-landsat"
  3378  		const landsatPrefix = "LC08/01/001/002/LC08_L1GT_001002_20160817_20170322_01_T2/"
  3379  		const landsatObject = landsatPrefix + "LC08_L1GT_001002_20160817_20170322_01_T2_ANG.txt"
  3380  
  3381  		h := testHelper{t}
  3382  		bkt := client.Bucket(landsatBucket)
  3383  		obj := bkt.Object(landsatObject)
  3384  
  3385  		// Read a public object.
  3386  		bytes := h.mustRead(obj)
  3387  		if got, want := len(bytes), 117255; got != want {
  3388  			t.Errorf("len(bytes) = %d, want %d", got, want)
  3389  		}
  3390  
  3391  		// List objects in a public bucket.
  3392  		iter := bkt.Objects(ctx, &Query{Prefix: landsatPrefix})
  3393  		gotCount := 0
  3394  		for {
  3395  			_, err := iter.Next()
  3396  			if err == iterator.Done {
  3397  				break
  3398  			}
  3399  			if err != nil {
  3400  				t.Fatal(err)
  3401  			}
  3402  			gotCount++
  3403  		}
  3404  		if wantCount := 14; gotCount != wantCount {
  3405  			t.Errorf("object count: got %d, want %d", gotCount, wantCount)
  3406  		}
  3407  
  3408  		errCode := func(err error) int {
  3409  			var err2 *googleapi.Error
  3410  			if ok := errors.As(err, &err2); !ok {
  3411  				return -1
  3412  			}
  3413  			return err2.Code
  3414  		}
  3415  
  3416  		// Reading from or writing to a non-public bucket fails.
  3417  		c := testConfig(ctx, t)
  3418  		defer c.Close()
  3419  		nonPublicObj := client.Bucket(bucket).Object("noauth")
  3420  		// XML API calls return 403 but the JSON API returns 401. Either is
  3421  		// acceptable for reads.
  3422  		_, err := readObject(ctx, nonPublicObj)
  3423  		if got := errCode(err); got != 403 && got != 401 {
  3424  			t.Errorf("got code %d; want %v\nerror: %v", got, "401 or 403", err)
  3425  		}
  3426  		err = writeObject(ctx, nonPublicObj, "text/plain", []byte("b"))
  3427  		if got, want := errCode(err), 401; got != want {
  3428  			t.Errorf("got code %d; want %d\nerror: %v", got, want, err)
  3429  		}
  3430  	}, option.WithoutAuthentication())
  3431  }
  3432  
  3433  func TestIntegration_PublicObject(t *testing.T) {
  3434  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  3435  		publicObj := client.Bucket(bucket).Object("public-obj" + uidSpaceObjects.New())
  3436  		contents := randomContents()
  3437  
  3438  		w := publicObj.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx)
  3439  		if _, err := w.Write(contents); err != nil {
  3440  			t.Fatalf("writer.Write: %v", err)
  3441  		}
  3442  		if err := w.Close(); err != nil {
  3443  			t.Errorf("writer.Close: %v", err)
  3444  		}
  3445  
  3446  		// Set object ACL to public read.
  3447  		if err := publicObj.ACL().Set(ctx, AllUsers, RoleReader); err != nil {
  3448  			t.Fatalf("PutACLEntry failed with %v", err)
  3449  		}
  3450  
  3451  		// Create unauthenticated client.
  3452  		publicClient, err := newTestClient(ctx, option.WithoutAuthentication())
  3453  		if err != nil {
  3454  			t.Fatalf("newTestClient: %v", err)
  3455  		}
  3456  
  3457  		// Test can read public object.
  3458  		publicObjUnauthenticated := publicClient.Bucket(bucket).Object(publicObj.ObjectName())
  3459  		data, err := readObject(context.Background(), publicObjUnauthenticated)
  3460  		if err != nil {
  3461  			t.Fatalf("readObject: %v", err)
  3462  		}
  3463  
  3464  		if !bytes.Equal(data, contents) {
  3465  			t.Errorf("Public object's content: got %q, want %q", data, contents)
  3466  		}
  3467  
  3468  		// Test cannot write to read-only object without authentication.
  3469  		wc := publicObjUnauthenticated.NewWriter(ctx)
  3470  		if _, err := wc.Write([]byte("hello")); err != nil {
  3471  			t.Errorf("Write unexpectedly failed with %v", err)
  3472  		}
  3473  		if err = wc.Close(); err == nil {
  3474  			t.Error("Close expected an error, found none")
  3475  		}
  3476  	})
  3477  }
  3478  
  3479  func TestIntegration_ReadCRC(t *testing.T) {
  3480  	// Test that the checksum is handled correctly when reading files.
  3481  	// For gzipped files, see https://github.com/GoogleCloudPlatform/google-cloud-dotnet/issues/1641.
  3482  	ctx := skipJSONReads(skipGRPC("transcoding not supported"), "https://github.com/googleapis/google-cloud-go/issues/7786")
  3483  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  3484  		const (
  3485  			// This is an uncompressed file.
  3486  			// See https://cloud.google.com/storage/docs/public-datasets/landsat
  3487  			uncompressedBucket = "gcp-public-data-landsat"
  3488  			uncompressedObject = "LC08/01/001/002/LC08_L1GT_001002_20160817_20170322_01_T2/LC08_L1GT_001002_20160817_20170322_01_T2_ANG.txt"
  3489  
  3490  			gzippedObject = "gzipped-text.txt"
  3491  		)
  3492  
  3493  		h := testHelper{t}
  3494  
  3495  		// Create gzipped object.
  3496  		var buf bytes.Buffer
  3497  		zw := gzip.NewWriter(&buf)
  3498  		zw.Name = gzippedObject
  3499  		if _, err := zw.Write([]byte("gzipped object data")); err != nil {
  3500  			t.Fatalf("creating gzip: %v", err)
  3501  		}
  3502  		if err := zw.Close(); err != nil {
  3503  			t.Fatalf("closing gzip writer: %v", err)
  3504  		}
  3505  		w := client.Bucket(bucket).Object(gzippedObject).NewWriter(ctx)
  3506  		w.ContentEncoding = "gzip"
  3507  		w.ContentType = "text/plain"
  3508  		h.mustWrite(w, buf.Bytes())
  3509  
  3510  		for _, test := range []struct {
  3511  			desc           string
  3512  			obj            *ObjectHandle
  3513  			offset, length int64
  3514  			readCompressed bool // don't decompress a gzipped file
  3515  
  3516  			wantErr   bool
  3517  			wantCheck bool // Should Reader try to check the CRC?
  3518  		}{
  3519  			{
  3520  				desc:           "uncompressed, entire file",
  3521  				obj:            client.Bucket(uncompressedBucket).Object(uncompressedObject),
  3522  				offset:         0,
  3523  				length:         -1,
  3524  				readCompressed: false,
  3525  				wantCheck:      true,
  3526  			},
  3527  			{
  3528  				desc:           "uncompressed, entire file, don't decompress",
  3529  				obj:            client.Bucket(uncompressedBucket).Object(uncompressedObject),
  3530  				offset:         0,
  3531  				length:         -1,
  3532  				readCompressed: true,
  3533  				wantCheck:      true,
  3534  			},
  3535  			{
  3536  				desc:           "uncompressed, suffix",
  3537  				obj:            client.Bucket(uncompressedBucket).Object(uncompressedObject),
  3538  				offset:         1,
  3539  				length:         -1,
  3540  				readCompressed: false,
  3541  				wantCheck:      false,
  3542  			},
  3543  			{
  3544  				desc:           "uncompressed, prefix",
  3545  				obj:            client.Bucket(uncompressedBucket).Object(uncompressedObject),
  3546  				offset:         0,
  3547  				length:         18,
  3548  				readCompressed: false,
  3549  				wantCheck:      false,
  3550  			},
  3551  			{
  3552  				// When a gzipped file is unzipped on read, we can't verify the checksum
  3553  				// because it was computed against the zipped contents. We can detect
  3554  				// this case using http.Response.Uncompressed.
  3555  				desc:           "compressed, entire file, unzipped",
  3556  				obj:            client.Bucket(bucket).Object(gzippedObject),
  3557  				offset:         0,
  3558  				length:         -1,
  3559  				readCompressed: false,
  3560  				wantCheck:      false,
  3561  			},
  3562  			{
  3563  				// When we read a gzipped file uncompressed, it's like reading a regular file:
  3564  				// the served content and the CRC match.
  3565  				desc:           "compressed, entire file, read compressed",
  3566  				obj:            client.Bucket(bucket).Object(gzippedObject),
  3567  				offset:         0,
  3568  				length:         -1,
  3569  				readCompressed: true,
  3570  				wantCheck:      true,
  3571  			},
  3572  			{
  3573  				desc:           "compressed, partial, server unzips",
  3574  				obj:            client.Bucket(bucket).Object(gzippedObject),
  3575  				offset:         1,
  3576  				length:         8,
  3577  				readCompressed: false,
  3578  				wantErr:        true, // GCS can't serve part of a gzipped object
  3579  				wantCheck:      false,
  3580  			},
  3581  			{
  3582  				desc:           "compressed, partial, read compressed",
  3583  				obj:            client.Bucket(bucket).Object(gzippedObject),
  3584  				offset:         1,
  3585  				length:         8,
  3586  				readCompressed: true,
  3587  				wantCheck:      false,
  3588  			},
  3589  		} {
  3590  			obj := test.obj.ReadCompressed(test.readCompressed)
  3591  			r, err := obj.NewRangeReader(ctx, test.offset, test.length)
  3592  			if err != nil {
  3593  				if test.wantErr {
  3594  					continue
  3595  				}
  3596  				t.Fatalf("%s: %v", test.desc, err)
  3597  			}
  3598  			if got, want := r.checkCRC, test.wantCheck; got != want {
  3599  				t.Errorf("%s, checkCRC: got %t, want %t", test.desc, got, want)
  3600  			}
  3601  			_, err = ioutil.ReadAll(r)
  3602  			_ = r.Close()
  3603  			if err != nil {
  3604  				t.Fatalf("%s: %v", test.desc, err)
  3605  			}
  3606  		}
  3607  	})
  3608  
  3609  }
  3610  
  3611  func TestIntegration_CancelWrite(t *testing.T) {
  3612  	// Verify that canceling the writer's context immediately stops uploading an object
  3613  	ctx := skipJSONReads(context.Background(), "no reads in test")
  3614  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  3615  		bkt := client.Bucket(bucket)
  3616  
  3617  		cctx, cancel := context.WithCancel(ctx)
  3618  		defer cancel()
  3619  		obj := bkt.Object("cancel-write")
  3620  		w := obj.NewWriter(cctx)
  3621  		w.ChunkSize = googleapi.MinUploadChunkSize
  3622  		buf := make([]byte, w.ChunkSize)
  3623  		// Write the first chunk. This is read in its entirety before sending the request
  3624  		// (see google.golang.org/api/gensupport.PrepareUpload), so we expect it to return
  3625  		// without error.
  3626  		_, err := w.Write(buf)
  3627  		if err != nil {
  3628  			t.Fatal(err)
  3629  		}
  3630  		// Now cancel the context.
  3631  		cancel()
  3632  		// The next Write should return context.Canceled.
  3633  		_, err = w.Write(buf)
  3634  		if !errors.Is(err, context.Canceled) {
  3635  			t.Fatalf("got %v, wanted context.Canceled", err)
  3636  		}
  3637  		// The Close should too.
  3638  		err = w.Close()
  3639  		if !errors.Is(err, context.Canceled) {
  3640  			t.Fatalf("got %v, wanted context.Canceled", err)
  3641  		}
  3642  	})
  3643  }
  3644  
  3645  func TestIntegration_UpdateCORS(t *testing.T) {
  3646  	ctx := skipJSONReads(context.Background(), "no reads in test")
  3647  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  3648  		initialSettings := []CORS{
  3649  			{
  3650  				MaxAge:          time.Hour,
  3651  				Methods:         []string{"POST"},
  3652  				Origins:         []string{"some-origin.com"},
  3653  				ResponseHeaders: []string{"foo-bar"},
  3654  			},
  3655  		}
  3656  
  3657  		for _, test := range []struct {
  3658  			desc  string
  3659  			input []CORS
  3660  			want  []CORS
  3661  		}{
  3662  			{
  3663  				desc: "set new values",
  3664  				input: []CORS{
  3665  					{
  3666  						MaxAge:          time.Hour,
  3667  						Methods:         []string{"GET"},
  3668  						Origins:         []string{"*"},
  3669  						ResponseHeaders: []string{"some-header"},
  3670  					},
  3671  				},
  3672  				want: []CORS{
  3673  					{
  3674  						MaxAge:          time.Hour,
  3675  						Methods:         []string{"GET"},
  3676  						Origins:         []string{"*"},
  3677  						ResponseHeaders: []string{"some-header"},
  3678  					},
  3679  				},
  3680  			},
  3681  			{
  3682  				desc:  "set to empty to remove existing policies",
  3683  				input: []CORS{},
  3684  				want:  nil,
  3685  			},
  3686  			{
  3687  				desc:  "do not set to keep existing policies",
  3688  				input: nil,
  3689  				want: []CORS{
  3690  					{
  3691  						MaxAge:          time.Hour,
  3692  						Methods:         []string{"POST"},
  3693  						Origins:         []string{"some-origin.com"},
  3694  						ResponseHeaders: []string{"foo-bar"},
  3695  					},
  3696  				},
  3697  			},
  3698  		} {
  3699  			t.Run(test.desc, func(t *testing.T) {
  3700  				h := testHelper{t}
  3701  
  3702  				bkt := client.Bucket(prefix + uidSpace.New())
  3703  				h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{CORS: initialSettings})
  3704  				defer h.mustDeleteBucket(bkt)
  3705  				// Set VersioningEnabled so that we don't send an empty update/patch request, which is invalid for gRPC
  3706  				h.mustUpdateBucket(bkt, BucketAttrsToUpdate{CORS: test.input, VersioningEnabled: false}, h.mustBucketAttrs(bkt).MetaGeneration)
  3707  				attrs := h.mustBucketAttrs(bkt)
  3708  				if diff := testutil.Diff(attrs.CORS, test.want); diff != "" {
  3709  					t.Errorf("input: %v\ngot=-, want=+:\n%s", test.input, diff)
  3710  				}
  3711  			})
  3712  		}
  3713  	})
  3714  }
  3715  
  3716  func TestIntegration_UpdateDefaultEventBasedHold(t *testing.T) {
  3717  	ctx := skipJSONReads(context.Background(), "no reads in test")
  3718  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  3719  		h := testHelper{t}
  3720  
  3721  		bkt := client.Bucket(prefix + uidSpace.New())
  3722  		h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{})
  3723  		defer h.mustDeleteBucket(bkt)
  3724  		attrs := h.mustBucketAttrs(bkt)
  3725  		if attrs.DefaultEventBasedHold != false {
  3726  			t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, false)
  3727  		}
  3728  
  3729  		h.mustUpdateBucket(bkt, BucketAttrsToUpdate{DefaultEventBasedHold: true}, attrs.MetaGeneration)
  3730  		attrs = h.mustBucketAttrs(bkt)
  3731  		if attrs.DefaultEventBasedHold != true {
  3732  			t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, true)
  3733  		}
  3734  
  3735  		// Omitting it should leave the value unchanged.
  3736  		h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RequesterPays: true}, attrs.MetaGeneration)
  3737  		attrs = h.mustBucketAttrs(bkt)
  3738  		if attrs.DefaultEventBasedHold != true {
  3739  			t.Errorf("got=%v, want=%v", attrs.DefaultEventBasedHold, true)
  3740  		}
  3741  	})
  3742  }
  3743  
  3744  func TestIntegration_UpdateEventBasedHold(t *testing.T) {
  3745  	ctx := skipJSONReads(context.Background(), "no reads in test")
  3746  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  3747  		h := testHelper{t}
  3748  
  3749  		obj := client.Bucket(bucket).Object("some-obj")
  3750  		h.mustWrite(obj.NewWriter(ctx), randomContents())
  3751  
  3752  		defer func() {
  3753  			h.mustUpdateObject(obj, ObjectAttrsToUpdate{EventBasedHold: false}, h.mustObjectAttrs(obj).Metageneration)
  3754  			h.mustDeleteObject(obj)
  3755  		}()
  3756  
  3757  		attrs := h.mustObjectAttrs(obj)
  3758  		if attrs.EventBasedHold != false {
  3759  			t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, false)
  3760  		}
  3761  
  3762  		h.mustUpdateObject(obj, ObjectAttrsToUpdate{EventBasedHold: true}, attrs.Metageneration)
  3763  		attrs = h.mustObjectAttrs(obj)
  3764  		if attrs.EventBasedHold != true {
  3765  			t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, true)
  3766  		}
  3767  
  3768  		// Omitting it should leave the value unchanged.
  3769  		h.mustUpdateObject(obj, ObjectAttrsToUpdate{ContentType: "foo"}, attrs.Metageneration)
  3770  		attrs = h.mustObjectAttrs(obj)
  3771  		if attrs.EventBasedHold != true {
  3772  			t.Fatalf("got=%v, want=%v", attrs.EventBasedHold, true)
  3773  		}
  3774  	})
  3775  }
  3776  
  3777  func TestIntegration_UpdateTemporaryHold(t *testing.T) {
  3778  	ctx := skipJSONReads(context.Background(), "no reads in test")
  3779  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  3780  		h := testHelper{t}
  3781  
  3782  		obj := client.Bucket(bucket).Object("updatetemporaryhold-obj")
  3783  		h.mustWrite(obj.NewWriter(ctx), randomContents())
  3784  
  3785  		defer func() {
  3786  			h.mustUpdateObject(obj, ObjectAttrsToUpdate{TemporaryHold: false}, h.mustObjectAttrs(obj).Metageneration)
  3787  			h.mustDeleteObject(obj)
  3788  		}()
  3789  
  3790  		attrs := h.mustObjectAttrs(obj)
  3791  		if attrs.TemporaryHold != false {
  3792  			t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, false)
  3793  		}
  3794  
  3795  		h.mustUpdateObject(obj, ObjectAttrsToUpdate{TemporaryHold: true}, attrs.Metageneration)
  3796  		attrs = h.mustObjectAttrs(obj)
  3797  		if attrs.TemporaryHold != true {
  3798  			t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, true)
  3799  		}
  3800  
  3801  		// Omitting it should leave the value unchanged.
  3802  		h.mustUpdateObject(obj, ObjectAttrsToUpdate{ContentType: "foo"}, attrs.Metageneration)
  3803  		attrs = h.mustObjectAttrs(obj)
  3804  		if attrs.TemporaryHold != true {
  3805  			t.Fatalf("got=%v, want=%v", attrs.TemporaryHold, true)
  3806  		}
  3807  	})
  3808  }
  3809  
  3810  func TestIntegration_UpdateRetentionExpirationTime(t *testing.T) {
  3811  	ctx := skipJSONReads(context.Background(), "no reads in test")
  3812  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  3813  		h := testHelper{t}
  3814  
  3815  		bkt := client.Bucket(prefix + uidSpace.New())
  3816  		h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour}})
  3817  		obj := bkt.Object("some-obj")
  3818  		h.mustWrite(obj.NewWriter(ctx), randomContents())
  3819  
  3820  		defer func() {
  3821  			t.Helper()
  3822  			h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{RetentionPeriod: 0}}, h.mustBucketAttrs(bkt).MetaGeneration)
  3823  
  3824  			// RetentionPeriod of less than a day is explicitly called out
  3825  			// as best effort and not guaranteed, so let's log problems deleting
  3826  			// objects instead of failing.
  3827  			if err := obj.Delete(context.Background()); err != nil {
  3828  				t.Logf("object delete: %v", err)
  3829  			}
  3830  			if err := bkt.Delete(context.Background()); err != nil {
  3831  				t.Logf("bucket delete: %v", err)
  3832  			}
  3833  		}()
  3834  
  3835  		attrs := h.mustObjectAttrs(obj)
  3836  		if attrs.RetentionExpirationTime == (time.Time{}) {
  3837  			t.Fatalf("got=%v, wanted a non-zero value", attrs.RetentionExpirationTime)
  3838  		}
  3839  	})
  3840  }
  3841  
  3842  func TestIntegration_CustomTime(t *testing.T) {
  3843  	ctx := skipJSONReads(context.Background(), "no reads in test")
  3844  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  3845  		h := testHelper{t}
  3846  
  3847  		// Create object with CustomTime.
  3848  		bkt := client.Bucket(bucket)
  3849  		obj := bkt.Object("custom-time-obj")
  3850  		w := obj.NewWriter(ctx)
  3851  		ct := time.Date(2020, 8, 25, 12, 12, 12, 0, time.UTC)
  3852  		w.ObjectAttrs.CustomTime = ct
  3853  		h.mustWrite(w, randomContents())
  3854  
  3855  		// Validate that CustomTime has been set
  3856  		checkCustomTime := func(want time.Time) error {
  3857  			attrs, err := obj.Attrs(ctx)
  3858  			if err != nil {
  3859  				return fmt.Errorf("failed to get object attrs: %v", err)
  3860  			}
  3861  			if got := attrs.CustomTime; got != want {
  3862  				return fmt.Errorf("CustomTime not set correctly: got %+v, want %+v", got, ct)
  3863  			}
  3864  			return nil
  3865  		}
  3866  
  3867  		if err := checkCustomTime(ct); err != nil {
  3868  			t.Fatalf("checking CustomTime: %v", err)
  3869  		}
  3870  
  3871  		// Update CustomTime to the future should succeed.
  3872  		laterTime := ct.Add(10 * time.Hour)
  3873  		if _, err := obj.Update(ctx, ObjectAttrsToUpdate{CustomTime: laterTime}); err != nil {
  3874  			t.Fatalf("updating CustomTime: %v", err)
  3875  		}
  3876  
  3877  		// Update CustomTime to the past should give error.
  3878  		earlierTime := ct.Add(5 * time.Hour)
  3879  		if _, err := obj.Update(ctx, ObjectAttrsToUpdate{CustomTime: earlierTime}); err == nil {
  3880  			t.Fatalf("backdating CustomTime: expected error, got none")
  3881  		}
  3882  
  3883  		// Zero value for CustomTime should be ignored. Set TemporaryHold so that
  3884  		// we don't send an empty update request, which is invalid for gRPC.
  3885  		if _, err := obj.Update(ctx, ObjectAttrsToUpdate{TemporaryHold: false}); err != nil {
  3886  			t.Fatalf("empty update: %v", err)
  3887  		}
  3888  		if err := checkCustomTime(laterTime); err != nil {
  3889  			t.Fatalf("after sending zero value: %v", err)
  3890  		}
  3891  	})
  3892  }
  3893  
  3894  func TestIntegration_UpdateRetentionPolicy(t *testing.T) {
  3895  	ctx := skipJSONReads(context.Background(), "no reads in test")
  3896  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  3897  		initial := &RetentionPolicy{RetentionPeriod: time.Minute}
  3898  
  3899  		for _, test := range []struct {
  3900  			desc  string
  3901  			input *RetentionPolicy
  3902  			want  *RetentionPolicy
  3903  		}{
  3904  			{
  3905  				desc:  "update",
  3906  				input: &RetentionPolicy{RetentionPeriod: time.Hour},
  3907  				want:  &RetentionPolicy{RetentionPeriod: time.Hour},
  3908  			},
  3909  			{
  3910  				desc:  "update even with timestamp (EffectiveTime should be ignored)",
  3911  				input: &RetentionPolicy{RetentionPeriod: time.Hour, EffectiveTime: time.Now()},
  3912  				want:  &RetentionPolicy{RetentionPeriod: time.Hour},
  3913  			},
  3914  			{
  3915  				desc:  "remove",
  3916  				input: &RetentionPolicy{},
  3917  				want:  nil,
  3918  			},
  3919  			{
  3920  				desc:  "remove even with timestamp (EffectiveTime should be ignored)",
  3921  				input: &RetentionPolicy{EffectiveTime: time.Now().Add(time.Hour)},
  3922  				want:  nil,
  3923  			},
  3924  			{
  3925  				desc:  "ignore",
  3926  				input: nil,
  3927  				want:  initial,
  3928  			},
  3929  		} {
  3930  			t.Run(test.desc, func(t *testing.T) {
  3931  				h := testHelper{t}
  3932  				bkt := client.Bucket(prefix + uidSpace.New())
  3933  				h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: initial})
  3934  				defer h.mustDeleteBucket(bkt)
  3935  				// Set VersioningEnabled so that we don't send an empty update request, which is invalid for gRPC
  3936  				h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: test.input, VersioningEnabled: false}, h.mustBucketAttrs(bkt).MetaGeneration)
  3937  
  3938  				attrs := h.mustBucketAttrs(bkt)
  3939  				if attrs.RetentionPolicy != nil && attrs.RetentionPolicy.EffectiveTime.Unix() == 0 {
  3940  					// Should be set by the server and parsed by the client
  3941  					t.Fatal("EffectiveTime should be set, but it was not")
  3942  				}
  3943  				if diff := testutil.Diff(attrs.RetentionPolicy, test.want, cmpopts.IgnoreTypes(time.Time{})); diff != "" {
  3944  					t.Errorf("input: %v\ngot=-, want=+:\n%s", test.input, diff)
  3945  				}
  3946  			})
  3947  		}
  3948  	})
  3949  }
  3950  
  3951  func TestIntegration_DeleteObjectInBucketWithRetentionPolicy(t *testing.T) {
  3952  	ctx := skipJSONReads(context.Background(), "no reads in test")
  3953  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  3954  		h := testHelper{t}
  3955  
  3956  		bkt := client.Bucket(prefix + uidSpace.New())
  3957  		h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: 25 * time.Hour}})
  3958  		defer h.mustDeleteBucket(bkt)
  3959  
  3960  		o := bkt.Object("some-object")
  3961  		if err := writeObject(ctx, o, "text/plain", []byte("hello world")); err != nil {
  3962  			t.Fatal(err)
  3963  		}
  3964  
  3965  		if err := o.Delete(ctx); err == nil {
  3966  			t.Fatal("expected to err deleting an object in a bucket with retention period, but got nil")
  3967  		}
  3968  
  3969  		// Remove the retention period
  3970  		h.mustUpdateBucket(bkt, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{}}, h.mustBucketAttrs(bkt).MetaGeneration)
  3971  
  3972  		// Delete with retry, as bucket metadata changes
  3973  		// can take some time to propagate.
  3974  		retry := func(err error) bool { return err != nil }
  3975  		ctx, cancel := context.WithTimeout(ctx, time.Second*10)
  3976  		defer cancel()
  3977  
  3978  		o = o.Retryer(WithErrorFunc(retry), WithPolicy(RetryAlways))
  3979  		if err := o.Delete(ctx); err != nil {
  3980  			t.Fatalf("object delete: %v", err)
  3981  		}
  3982  	})
  3983  }
  3984  
  3985  func TestIntegration_LockBucket(t *testing.T) {
  3986  	ctx := skipJSONReads(context.Background(), "no reads in test")
  3987  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  3988  		h := testHelper{t}
  3989  
  3990  		bkt := client.Bucket(prefix + uidSpace.New())
  3991  		h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour * 25}})
  3992  		attrs := h.mustBucketAttrs(bkt)
  3993  		if attrs.RetentionPolicy.IsLocked {
  3994  			t.Fatal("Expected bucket to begin unlocked, but it was not")
  3995  		}
  3996  		err := bkt.If(BucketConditions{MetagenerationMatch: attrs.MetaGeneration}).LockRetentionPolicy(ctx)
  3997  		if err != nil {
  3998  			t.Fatal("could not lock", err)
  3999  		}
  4000  
  4001  		attrs = h.mustBucketAttrs(bkt)
  4002  		if !attrs.RetentionPolicy.IsLocked {
  4003  			t.Fatal("Expected bucket to be locked, but it was not")
  4004  		}
  4005  
  4006  		_, err = bkt.Update(ctx, BucketAttrsToUpdate{RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour}})
  4007  		if err == nil {
  4008  			t.Fatal("Expected error updating locked bucket, got nil")
  4009  		}
  4010  	})
  4011  }
  4012  
  4013  func TestIntegration_LockBucket_MetagenerationRequired(t *testing.T) {
  4014  	ctx := skipJSONReads(context.Background(), "no reads in test")
  4015  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  4016  		h := testHelper{t}
  4017  
  4018  		bkt := client.Bucket(prefix + uidSpace.New())
  4019  		h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{
  4020  			RetentionPolicy: &RetentionPolicy{RetentionPeriod: time.Hour * 25},
  4021  		})
  4022  		err := bkt.LockRetentionPolicy(ctx)
  4023  		if err == nil {
  4024  			t.Fatal("expected error locking bucket without metageneration condition, got nil")
  4025  		}
  4026  	})
  4027  }
  4028  
  4029  func TestIntegration_BucketObjectRetention(t *testing.T) {
  4030  	ctx := skipJSONReads(skipGRPC("not yet available in gRPC - b/308194853"), "no reads in test")
  4031  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  4032  		setTrue, setFalse := true, false
  4033  
  4034  		for _, test := range []struct {
  4035  			desc              string
  4036  			enable            *bool
  4037  			wantRetentionMode string
  4038  		}{
  4039  			{
  4040  				desc:              "ObjectRetentionMode is not enabled by default",
  4041  				wantRetentionMode: "",
  4042  			},
  4043  			{
  4044  				desc:              "Enable retention",
  4045  				enable:            &setTrue,
  4046  				wantRetentionMode: "Enabled",
  4047  			},
  4048  			{
  4049  				desc:              "Set object retention to false",
  4050  				enable:            &setFalse,
  4051  				wantRetentionMode: "",
  4052  			},
  4053  		} {
  4054  			t.Run(test.desc, func(t *testing.T) {
  4055  				b := client.Bucket(prefix + uidSpace.New())
  4056  				if test.enable != nil {
  4057  					b = b.SetObjectRetention(*test.enable)
  4058  				}
  4059  
  4060  				err := b.Create(ctx, testutil.ProjID(), nil)
  4061  				if err != nil {
  4062  					t.Fatalf("error creating bucket: %v", err)
  4063  				}
  4064  				t.Cleanup(func() { b.Delete(ctx) })
  4065  
  4066  				attrs, err := b.Attrs(ctx)
  4067  				if err != nil {
  4068  					t.Fatalf("b.Attrs: %v", err)
  4069  				}
  4070  				if got, want := attrs.ObjectRetentionMode, test.wantRetentionMode; got != want {
  4071  					t.Errorf("expected ObjectRetentionMode to be %q, got %q", want, got)
  4072  				}
  4073  			})
  4074  		}
  4075  	})
  4076  }
  4077  
  4078  func TestIntegration_ObjectRetention(t *testing.T) {
  4079  	ctx := skipJSONReads(skipGRPC("not yet available in gRPC - b/308194853"), "no reads in test")
  4080  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  4081  		h := testHelper{t}
  4082  
  4083  		b := client.Bucket(prefix + uidSpace.New()).SetObjectRetention(true)
  4084  
  4085  		if err := b.Create(ctx, testutil.ProjID(), nil); err != nil {
  4086  			t.Fatalf("error creating bucket: %v", err)
  4087  		}
  4088  		t.Cleanup(func() { h.mustDeleteBucket(b) })
  4089  
  4090  		retentionUnlocked := &ObjectRetention{
  4091  			Mode:        "Unlocked",
  4092  			RetainUntil: time.Now().Add(time.Minute * 20).Truncate(time.Second),
  4093  		}
  4094  		retentionUnlockedExtended := &ObjectRetention{
  4095  			Mode:        "Unlocked",
  4096  			RetainUntil: time.Now().Add(time.Hour).Truncate(time.Second),
  4097  		}
  4098  
  4099  		// Create an object with future retain until time
  4100  		o := b.Object("retention-on-create" + uidSpaceObjects.New())
  4101  		w := o.NewWriter(ctx)
  4102  		w.Retention = retentionUnlocked
  4103  		h.mustWrite(w, []byte("contents"))
  4104  		t.Cleanup(func() {
  4105  			if _, err := o.OverrideUnlockedRetention(true).Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{}}); err != nil {
  4106  				t.Fatalf("failed to remove retention from object: %v", err)
  4107  			}
  4108  			h.mustDeleteObject(o)
  4109  		})
  4110  
  4111  		if got, want := w.Attrs().Retention, retentionUnlocked; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) {
  4112  			t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want)
  4113  		}
  4114  
  4115  		// Delete object under retention returns 403
  4116  		if err := o.Delete(ctx); err == nil || extractErrCode(err) != http.StatusForbidden {
  4117  			t.Fatalf("delete should have failed with: %v, instead got:%v", http.StatusForbidden, err)
  4118  		}
  4119  
  4120  		// Extend retain until time of Unlocked object is possible
  4121  		attrs, err := o.Update(ctx, ObjectAttrsToUpdate{Retention: retentionUnlockedExtended})
  4122  		if err != nil {
  4123  			t.Fatalf("failed to add retention to object: %v", err)
  4124  		}
  4125  
  4126  		if got, want := attrs.Retention, retentionUnlockedExtended; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) {
  4127  			t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want)
  4128  		}
  4129  
  4130  		// Reduce retain until time of Unlocked object without
  4131  		// override_unlocked_retention=True returns 403
  4132  		_, err = o.Update(ctx, ObjectAttrsToUpdate{Retention: retentionUnlocked})
  4133  		if err == nil || extractErrCode(err) != http.StatusForbidden {
  4134  			t.Fatalf("o.Update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
  4135  		}
  4136  
  4137  		// Remove retention of Unlocked object without
  4138  		// override_unlocked_retention=True returns 403
  4139  		_, err = o.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{}})
  4140  		if err == nil || extractErrCode(err) != http.StatusForbidden {
  4141  			t.Fatalf("o.Update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
  4142  		}
  4143  
  4144  		// Reduce retain until time of Unlocked object with override_unlocked_retention=True
  4145  		attrs, err = o.OverrideUnlockedRetention(true).Update(ctx, ObjectAttrsToUpdate{
  4146  			Retention: retentionUnlocked,
  4147  		})
  4148  		if err != nil {
  4149  			t.Fatalf("failed to add retention to object: %v", err)
  4150  		}
  4151  
  4152  		if got, want := attrs.Retention, retentionUnlocked; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) {
  4153  			t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want)
  4154  		}
  4155  
  4156  		// Create a new object
  4157  		objectWithRetentionOnUpdate := b.Object("retention-on-update" + uidSpaceObjects.New())
  4158  		w = objectWithRetentionOnUpdate.NewWriter(ctx)
  4159  		h.mustWrite(w, []byte("contents"))
  4160  
  4161  		// Retention should not be set
  4162  		if got := w.Attrs().Retention; got != nil {
  4163  			t.Errorf("expected no ObjectRetention, got: %+v", got)
  4164  		}
  4165  
  4166  		// Update object with only one of (retain until time, retention mode) returns 400
  4167  		_, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{Mode: "Locked"}})
  4168  		if err == nil || extractErrCode(err) != http.StatusBadRequest {
  4169  			t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
  4170  		}
  4171  
  4172  		_, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{RetainUntil: time.Now().Add(time.Second)}})
  4173  		if err == nil || extractErrCode(err) != http.StatusBadRequest {
  4174  			t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
  4175  		}
  4176  
  4177  		// Update object with future retain until time
  4178  		attrs, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: retentionUnlocked})
  4179  		if err != nil {
  4180  			t.Errorf("o.Update: %v", err)
  4181  		}
  4182  
  4183  		if got, want := attrs.Retention, retentionUnlocked; got.Mode != want.Mode || !got.RetainUntil.Equal(want.RetainUntil) {
  4184  			t.Errorf("mismatching retention config, got: %+v, want:%+v", got, want)
  4185  		}
  4186  
  4187  		// Update/Patch object with retain until time in the past returns 400
  4188  		_, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{RetainUntil: time.Now().Add(-time.Second)}})
  4189  		if err == nil || extractErrCode(err) != http.StatusBadRequest {
  4190  			t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
  4191  		}
  4192  
  4193  		// Update object with only one of (retain until time, retention mode) returns 400
  4194  		_, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{Mode: "Locked"}})
  4195  		if err == nil || extractErrCode(err) != http.StatusBadRequest {
  4196  			t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
  4197  		}
  4198  
  4199  		_, err = objectWithRetentionOnUpdate.Update(ctx, ObjectAttrsToUpdate{Retention: &ObjectRetention{RetainUntil: time.Now().Add(time.Second)}})
  4200  		if err == nil || extractErrCode(err) != http.StatusBadRequest {
  4201  			t.Errorf("update should have failed with: %v, instead got:%v", http.StatusBadRequest, err)
  4202  		}
  4203  
  4204  		// Remove retention of Unlocked object with override_unlocked_retention=True
  4205  		attrs, err = objectWithRetentionOnUpdate.OverrideUnlockedRetention(true).Update(ctx, ObjectAttrsToUpdate{
  4206  			Retention: &ObjectRetention{},
  4207  		})
  4208  		if err != nil {
  4209  			t.Fatalf("failed to remove retention from object: %v", err)
  4210  		}
  4211  
  4212  		if got := attrs.Retention; got != nil {
  4213  			t.Errorf("mismatching retention config, got: %+v, wanted nil", got)
  4214  		}
  4215  
  4216  		// We should be able to delete the object as normal since retention was removed
  4217  		if err := objectWithRetentionOnUpdate.Delete(ctx); err != nil {
  4218  			t.Errorf("object.Delete:%v", err)
  4219  		}
  4220  	})
  4221  }
  4222  
  4223  func TestIntegration_KMS(t *testing.T) {
  4224  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, prefix string, client *Client) {
  4225  		h := testHelper{t}
  4226  
  4227  		keyRingName := os.Getenv("GCLOUD_TESTS_GOLANG_KEYRING")
  4228  		if keyRingName == "" {
  4229  			t.Fatal("GCLOUD_TESTS_GOLANG_KEYRING must be set. See CONTRIBUTING.md for details")
  4230  		}
  4231  		keyName1 := keyRingName + "/cryptoKeys/key1"
  4232  		keyName2 := keyRingName + "/cryptoKeys/key2"
  4233  		contents := []byte("my secret")
  4234  
  4235  		write := func(obj *ObjectHandle, setKey bool) {
  4236  			w := obj.NewWriter(ctx)
  4237  			if setKey {
  4238  				w.KMSKeyName = keyName1
  4239  			}
  4240  			h.mustWrite(w, contents)
  4241  		}
  4242  
  4243  		checkRead := func(obj *ObjectHandle) {
  4244  			got := h.mustRead(obj)
  4245  			if !bytes.Equal(got, contents) {
  4246  				t.Errorf("got %v, want %v", got, contents)
  4247  			}
  4248  			attrs := h.mustObjectAttrs(obj)
  4249  			if len(attrs.KMSKeyName) < len(keyName1) || attrs.KMSKeyName[:len(keyName1)] != keyName1 {
  4250  				t.Errorf("got %q, want %q", attrs.KMSKeyName, keyName1)
  4251  			}
  4252  		}
  4253  
  4254  		// Write an object with a key, then read it to verify its contents and the presence of the key name.
  4255  		bkt := client.Bucket(bucket)
  4256  		obj := bkt.Object("kms")
  4257  		write(obj, true)
  4258  		checkRead(obj)
  4259  		h.mustDeleteObject(obj)
  4260  
  4261  		// Encrypt an object with a CSEK, then copy it using a CMEK.
  4262  		src := bkt.Object("csek").Key(testEncryptionKey)
  4263  		if err := writeObject(ctx, src, "text/plain", contents); err != nil {
  4264  			t.Fatal(err)
  4265  		}
  4266  		dest := bkt.Object("cmek")
  4267  		c := dest.CopierFrom(src)
  4268  		c.DestinationKMSKeyName = keyName1
  4269  		if _, err := c.Run(ctx); err != nil {
  4270  			t.Fatal(err)
  4271  		}
  4272  		checkRead(dest)
  4273  		src.Delete(ctx)
  4274  		dest.Delete(ctx)
  4275  
  4276  		// Create a bucket with a default key, then write and read an object.
  4277  		bkt = client.Bucket(prefix + uidSpace.New())
  4278  		h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{
  4279  			Location:   "US",
  4280  			Encryption: &BucketEncryption{DefaultKMSKeyName: keyName1},
  4281  		})
  4282  		defer h.mustDeleteBucket(bkt)
  4283  
  4284  		attrs := h.mustBucketAttrs(bkt)
  4285  		if got, want := attrs.Encryption.DefaultKMSKeyName, keyName1; got != want {
  4286  			t.Fatalf("got %q, want %q", got, want)
  4287  		}
  4288  		obj = bkt.Object("kms")
  4289  		write(obj, false)
  4290  		checkRead(obj)
  4291  		h.mustDeleteObject(obj)
  4292  
  4293  		// Update the bucket's default key to a different name.
  4294  		// (This key doesn't have to exist.)
  4295  		attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{Encryption: &BucketEncryption{DefaultKMSKeyName: keyName2}}, attrs.MetaGeneration)
  4296  		if got, want := attrs.Encryption.DefaultKMSKeyName, keyName2; got != want {
  4297  			t.Fatalf("got %q, want %q", got, want)
  4298  		}
  4299  		attrs = h.mustBucketAttrs(bkt)
  4300  		if got, want := attrs.Encryption.DefaultKMSKeyName, keyName2; got != want {
  4301  			t.Fatalf("got %q, want %q", got, want)
  4302  		}
  4303  
  4304  		// Remove the default KMS key.
  4305  		attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{Encryption: &BucketEncryption{DefaultKMSKeyName: ""}}, attrs.MetaGeneration)
  4306  		if attrs.Encryption != nil {
  4307  			t.Fatalf("got %#v, want nil", attrs.Encryption)
  4308  		}
  4309  	})
  4310  }
  4311  
  4312  func TestIntegration_PredefinedACLs(t *testing.T) {
  4313  	projectOwners := prefixRoleACL{prefix: "project-owners", role: RoleOwner}
  4314  	userOwner := prefixRoleACL{prefix: "user", role: RoleOwner}
  4315  	authenticatedRead := entityRoleACL{entity: AllAuthenticatedUsers, role: RoleReader}
  4316  
  4317  	ctx := skipJSONReads(context.Background(), "no reads in test")
  4318  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  4319  		h := testHelper{t}
  4320  
  4321  		bkt := client.Bucket(prefix + uidSpace.New())
  4322  		h.mustCreate(bkt, testutil.ProjID(), &BucketAttrs{
  4323  			PredefinedACL:              "authenticatedRead",
  4324  			PredefinedDefaultObjectACL: "publicRead",
  4325  		})
  4326  		defer h.mustDeleteBucket(bkt)
  4327  		attrs := h.mustBucketAttrs(bkt)
  4328  
  4329  		if acl, want := attrs.ACL, projectOwners; !containsACLRule(acl, want) {
  4330  			t.Fatalf("Bucket.ACL: expected acl to contain: %+v, got acl: %+v", want, acl)
  4331  		}
  4332  		if acl, want := attrs.ACL, authenticatedRead; !containsACLRule(acl, want) {
  4333  			t.Fatalf("Bucket.ACL: expected acl to contain: %+v, got acl: %+v", want, acl)
  4334  		}
  4335  		if acl := attrs.DefaultObjectACL; !containsACLRule(acl, entityRoleACL{AllUsers, RoleReader}) {
  4336  			t.Fatalf("DefaultObjectACL: expected acl to contain: %+v, got acl: %+v", entityRoleACL{AllUsers, RoleReader}, acl)
  4337  		}
  4338  
  4339  		// Bucket update
  4340  		attrs = h.mustUpdateBucket(bkt, BucketAttrsToUpdate{
  4341  			PredefinedACL:              "private",
  4342  			PredefinedDefaultObjectACL: "authenticatedRead",
  4343  		}, attrs.MetaGeneration)
  4344  		if acl, want := attrs.ACL, projectOwners; !containsACLRule(acl, want) {
  4345  			t.Fatalf("Bucket.ACL update: expected acl to contain: %+v, got acl: %+v", want, acl)
  4346  		}
  4347  		if acl, want := attrs.DefaultObjectACL, authenticatedRead; !containsACLRule(acl, want) {
  4348  			t.Fatalf("DefaultObjectACL update: expected acl to contain: %+v, got acl: %+v", want, acl)
  4349  		}
  4350  
  4351  		// Object creation
  4352  		obj := bkt.Object("private")
  4353  		w := obj.NewWriter(ctx)
  4354  		w.PredefinedACL = "authenticatedRead"
  4355  		h.mustWrite(w, []byte("hello"))
  4356  		defer h.mustDeleteObject(obj)
  4357  		var acl []ACLRule
  4358  		err := retry(ctx, func() error {
  4359  			attrs, err := obj.Attrs(ctx)
  4360  			if err != nil {
  4361  				return fmt.Errorf("Object.Attrs: object metadata get failed: %v", err)
  4362  			}
  4363  			acl = attrs.ACL
  4364  			return nil
  4365  		}, func() error {
  4366  			if want := userOwner; !containsACLRule(acl, want) {
  4367  				return fmt.Errorf("Object.ACL: expected acl to contain: %+v, got acl: %+v", want, acl)
  4368  			}
  4369  			return nil
  4370  		})
  4371  		if err != nil {
  4372  			t.Fatal(err)
  4373  		}
  4374  		err = retry(ctx, func() error {
  4375  			attrs, err := obj.Attrs(ctx)
  4376  			if err != nil {
  4377  				return fmt.Errorf("Object.Attrs: object metadata get failed: %v", err)
  4378  			}
  4379  			acl = attrs.ACL
  4380  			return nil
  4381  		}, func() error {
  4382  			if want := authenticatedRead; !containsACLRule(acl, want) {
  4383  				return fmt.Errorf("Object.ACL: expected acl to contain: %+v, got acl: %+v", want, acl)
  4384  			}
  4385  			return nil
  4386  		})
  4387  		if err != nil {
  4388  			t.Fatal(err)
  4389  		}
  4390  
  4391  		// Object update
  4392  		oattrs := h.mustUpdateObject(obj, ObjectAttrsToUpdate{PredefinedACL: "private"}, h.mustObjectAttrs(obj).Metageneration)
  4393  		if acl, want := oattrs.ACL, userOwner; !containsACLRule(acl, want) {
  4394  			t.Fatalf("Object.ACL update: expected acl to contain: %+v, got acl: %+v", want, acl)
  4395  		}
  4396  		if got := len(oattrs.ACL); got != 1 {
  4397  			t.Errorf("got %d ACL rules, want 1", got)
  4398  		}
  4399  
  4400  		// Copy
  4401  		dst := bkt.Object("dst")
  4402  		copier := dst.CopierFrom(obj)
  4403  		copier.PredefinedACL = "publicRead"
  4404  		oattrs, err = copier.Run(ctx)
  4405  		if err != nil {
  4406  			t.Fatal(err)
  4407  		}
  4408  		defer h.mustDeleteObject(dst)
  4409  		// The copied object still retains the "private" ACL of the source object.
  4410  		if acl, want := oattrs.ACL, userOwner; !containsACLRule(acl, want) {
  4411  			t.Fatalf("copy dest: expected acl to contain: %+v, got acl: %+v", want, acl)
  4412  		}
  4413  		if !containsACLRule(oattrs.ACL, entityRoleACL{AllUsers, RoleReader}) {
  4414  			t.Fatalf("copy dest: expected acl to contain: %+v, got acl: %+v", entityRoleACL{AllUsers, RoleReader}, oattrs.ACL)
  4415  		}
  4416  
  4417  		// Compose
  4418  		comp := bkt.Object("comp")
  4419  
  4420  		composer := comp.ComposerFrom(obj, dst)
  4421  		composer.PredefinedACL = "authenticatedRead"
  4422  		oattrs, err = composer.Run(ctx)
  4423  		if err != nil {
  4424  			t.Fatal(err)
  4425  		}
  4426  		defer h.mustDeleteObject(comp)
  4427  		// The composed object still retains the "private" ACL.
  4428  		if acl, want := oattrs.ACL, userOwner; !containsACLRule(acl, want) {
  4429  			t.Fatalf("compose: expected acl to contain: %+v, got acl: %+v", want, acl)
  4430  		}
  4431  		if acl, want := oattrs.ACL, authenticatedRead; !containsACLRule(acl, want) {
  4432  			t.Fatalf("compose: expected acl to contain: %+v, got acl: %+v", want, acl)
  4433  		}
  4434  	})
  4435  }
  4436  
  4437  func TestIntegration_ServiceAccount(t *testing.T) {
  4438  	ctx := skipJSONReads(context.Background(), "no reads in test")
  4439  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, _ string, client *Client) {
  4440  		s, err := client.ServiceAccount(ctx, testutil.ProjID())
  4441  		if err != nil {
  4442  			t.Fatal(err)
  4443  		}
  4444  		want := "@gs-project-accounts.iam.gserviceaccount.com"
  4445  		if !strings.Contains(s, want) {
  4446  			t.Fatalf("got %v, want to contain %v", s, want)
  4447  		}
  4448  	})
  4449  }
  4450  
  4451  func TestIntegration_Reader(t *testing.T) {
  4452  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  4453  		b := client.Bucket(bucket)
  4454  		const defaultType = "text/plain"
  4455  
  4456  		// Populate object names and make a map for their contents.
  4457  		objects := []string{
  4458  			"obj1",
  4459  			"obj2",
  4460  			"obj/with/slashes",
  4461  			"obj/",
  4462  			"./obj",
  4463  			"!#$&'()*+,/:;=,?@,[] and spaces",
  4464  		}
  4465  		contents := make(map[string][]byte)
  4466  
  4467  		// Write objects.
  4468  		for _, obj := range objects {
  4469  			c := randomContents()
  4470  			if err := writeObject(ctx, b.Object(obj), defaultType, c); err != nil {
  4471  				t.Errorf("Write for %v failed with %v", obj, err)
  4472  			}
  4473  			contents[obj] = c
  4474  		}
  4475  		// Test Reader. Cache control and last-modified are tested separately, as
  4476  		// the JSON and XML APIs return different values for these.
  4477  		for _, obj := range objects {
  4478  			rc, err := b.Object(obj).NewReader(ctx)
  4479  			if err != nil {
  4480  				t.Errorf("Can't create a reader for %v, errored with %v", obj, err)
  4481  				continue
  4482  			}
  4483  			if !rc.checkCRC {
  4484  				t.Errorf("%v: not checking CRC", obj)
  4485  			}
  4486  
  4487  			slurp, err := ioutil.ReadAll(rc)
  4488  			if err != nil {
  4489  				t.Errorf("Can't ReadAll object %v, errored with %v", obj, err)
  4490  			}
  4491  			if got, want := slurp, contents[obj]; !bytes.Equal(got, want) {
  4492  				t.Errorf("Contents (%q) = %q; want %q", obj, got, want)
  4493  			}
  4494  			if got, want := rc.Size(), len(contents[obj]); got != int64(want) {
  4495  				t.Errorf("Size (%q) = %d; want %d", obj, got, want)
  4496  			}
  4497  			if got, want := rc.ContentType(), "text/plain"; got != want {
  4498  				t.Errorf("ContentType (%q) = %q; want %q", obj, got, want)
  4499  			}
  4500  			rc.Close()
  4501  
  4502  			// Check early close.
  4503  			buf := make([]byte, 1)
  4504  			rc, err = b.Object(obj).NewReader(ctx)
  4505  			if err != nil {
  4506  				t.Fatalf("%v: %v", obj, err)
  4507  			}
  4508  			_, err = rc.Read(buf)
  4509  			if err != nil {
  4510  				t.Fatalf("%v: %v", obj, err)
  4511  			}
  4512  			if got, want := buf, contents[obj][:1]; !bytes.Equal(got, want) {
  4513  				t.Errorf("Contents[0] (%q) = %q; want %q", obj, got, want)
  4514  			}
  4515  			if err := rc.Close(); err != nil {
  4516  				t.Errorf("%v Close: %v", obj, err)
  4517  			}
  4518  		}
  4519  
  4520  		obj := objects[0]
  4521  		objlen := int64(len(contents[obj]))
  4522  
  4523  		// Test Range Reader.
  4524  		for _, r := range []struct {
  4525  			desc                 string
  4526  			offset, length, want int64
  4527  		}{
  4528  			{"entire object", 0, objlen, objlen},
  4529  			{"first half of object", 0, objlen / 2, objlen / 2},
  4530  			{"second half of object", objlen / 2, objlen, objlen / 2},
  4531  			{"no bytes - start at beginning", 0, 0, 0},
  4532  			{"no bytes - start halfway through", objlen / 2, 0, 0},
  4533  			{"start halfway through - use negative to get rest of obj", objlen / 2, -1, objlen / 2},
  4534  			{"2 times object length", 0, objlen * 2, objlen},
  4535  			{"-2 offset", -2, -1, 2},
  4536  			{"-object length offset", -objlen, -1, objlen},
  4537  			{"-half of object length offset", -(objlen / 2), -1, objlen / 2},
  4538  		} {
  4539  			rc, err := b.Object(obj).NewRangeReader(ctx, r.offset, r.length)
  4540  			if err != nil {
  4541  				t.Errorf("%+v: Can't create a range reader for %v, errored with %v", r.desc, obj, err)
  4542  				continue
  4543  			}
  4544  			if rc.Size() != objlen {
  4545  				t.Errorf("%+v: Reader has a content-size of %d, want %d", r.desc, rc.Size(), objlen)
  4546  			}
  4547  			if rc.Remain() != r.want {
  4548  				t.Errorf("%+v: Reader's available bytes reported as %d, want %d", r.desc, rc.Remain(), r.want)
  4549  			}
  4550  			slurp, err := ioutil.ReadAll(rc)
  4551  			if err != nil {
  4552  				t.Errorf("%+v: can't ReadAll object %v, errored with %v", r, obj, err)
  4553  				continue
  4554  			}
  4555  			if len(slurp) != int(r.want) {
  4556  				t.Errorf("%+v: RangeReader (%d, %d): Read %d bytes, wanted %d bytes", r.desc, r.offset, r.length, len(slurp), r.want)
  4557  				continue
  4558  			}
  4559  
  4560  			switch {
  4561  			case r.offset < 0: // The case of reading the last N bytes.
  4562  				start := objlen + r.offset
  4563  				if got, want := slurp, contents[obj][start:]; !bytes.Equal(got, want) {
  4564  					t.Errorf("RangeReader (%d, %d) = %q; want %q", r.offset, r.length, got, want)
  4565  				}
  4566  
  4567  			default:
  4568  				if got, want := slurp, contents[obj][r.offset:r.offset+r.want]; !bytes.Equal(got, want) {
  4569  					t.Errorf("RangeReader (%d, %d) = %q; want %q", r.offset, r.length, got, want)
  4570  				}
  4571  			}
  4572  			rc.Close()
  4573  		}
  4574  
  4575  		objName := objects[0]
  4576  
  4577  		// Test NewReader googleapi.Error.
  4578  		// Since a 429 or 5xx is hard to cause, we trigger a 416 (InvalidRange).
  4579  		realLen := len(contents[objName])
  4580  		_, err := b.Object(objName).NewRangeReader(ctx, int64(realLen*2), 10)
  4581  
  4582  		var e *googleapi.Error
  4583  		if !errors.As(err, &e) {
  4584  			// Check if it is the correct GRPC error
  4585  			if !(status.Code(err) == codes.OutOfRange) {
  4586  				t.Errorf("NewRangeReader did not return a googleapi.Error nor GRPC OutOfRange error; got: %v", err)
  4587  			}
  4588  		} else {
  4589  			if e.Code != 416 {
  4590  				t.Errorf("Code = %d; want %d", e.Code, 416)
  4591  			}
  4592  			if len(e.Header) == 0 {
  4593  				t.Error("Missing googleapi.Error.Header")
  4594  			}
  4595  			if len(e.Body) == 0 {
  4596  				t.Error("Missing googleapi.Error.Body")
  4597  			}
  4598  		}
  4599  	})
  4600  }
  4601  
  4602  func TestIntegration_ReaderAttrs(t *testing.T) {
  4603  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  4604  		bkt := client.Bucket(bucket)
  4605  
  4606  		const defaultType = "text/plain"
  4607  		o := bkt.Object("reader-attrs-obj")
  4608  		c := randomContents()
  4609  		if err := writeObject(ctx, o, defaultType, c); err != nil {
  4610  			t.Errorf("Write for %v failed with %v", o.ObjectName(), err)
  4611  		}
  4612  		defer func() {
  4613  			if err := o.Delete(ctx); err != nil {
  4614  				log.Printf("failed to delete test object: %v", err)
  4615  			}
  4616  		}()
  4617  
  4618  		rc, err := o.NewReader(ctx)
  4619  		if err != nil {
  4620  			t.Fatal(err)
  4621  		}
  4622  
  4623  		attrs, err := o.Attrs(ctx)
  4624  		if err != nil {
  4625  			t.Fatal(err)
  4626  		}
  4627  
  4628  		got := rc.Attrs
  4629  		want := ReaderObjectAttrs{
  4630  			Size:            attrs.Size,
  4631  			ContentType:     attrs.ContentType,
  4632  			ContentEncoding: attrs.ContentEncoding,
  4633  			CacheControl:    got.CacheControl, // ignored, tested separately
  4634  			LastModified:    got.LastModified, // ignored, tested separately
  4635  			Generation:      attrs.Generation,
  4636  			Metageneration:  attrs.Metageneration,
  4637  		}
  4638  		if got != want {
  4639  			t.Fatalf("got\t%v,\nwanted\t%v", got, want)
  4640  		}
  4641  	})
  4642  }
  4643  
  4644  func TestIntegration_ReaderLastModified(t *testing.T) {
  4645  	ctx := skipJSONReads(context.Background(), "LastModified not populated by json response")
  4646  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  4647  		testStart := time.Now()
  4648  		b := client.Bucket(bucket)
  4649  		o := b.Object("reader-lm-obj" + uidSpaceObjects.New())
  4650  
  4651  		if err := writeObject(ctx, o, "text/plain", randomContents()); err != nil {
  4652  			t.Errorf("Write for %v failed with %v", o.ObjectName(), err)
  4653  		}
  4654  		defer func() {
  4655  			if err := o.Delete(ctx); err != nil {
  4656  				log.Printf("failed to delete test object: %v", err)
  4657  			}
  4658  		}()
  4659  
  4660  		r, err := o.NewReader(ctx)
  4661  		if err != nil {
  4662  			t.Fatal(err)
  4663  		}
  4664  
  4665  		lm := r.Attrs.LastModified
  4666  		if lm.IsZero() {
  4667  			t.Fatal("LastModified is 0, should be >0")
  4668  		}
  4669  
  4670  		// We just wrote this object, so it should have a recent last-modified time.
  4671  		// Accept a time within the start + variance of the test, to account for natural
  4672  		// variation.
  4673  		expectedVariance := time.Minute
  4674  
  4675  		if lm.After(testStart.Add(expectedVariance)) {
  4676  			t.Errorf("LastModified (%q): got %s, which is not within %v from test start (%v)", o.ObjectName(), lm, expectedVariance, testStart)
  4677  		}
  4678  	})
  4679  }
  4680  
  4681  func TestIntegration_ReaderCacheControl(t *testing.T) {
  4682  	ctx := skipJSONReads(context.Background(), "Cache control header is populated differently by the json api")
  4683  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  4684  		b := client.Bucket(bucket)
  4685  		o := b.Object("reader-cc" + uidSpaceObjects.New())
  4686  
  4687  		cacheControl := "public, max-age=60"
  4688  
  4689  		// Write object.
  4690  		w := o.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx)
  4691  		w.CacheControl = cacheControl
  4692  		if _, err := w.Write(randomContents()); err != nil {
  4693  			t.Fatalf("Write for %v failed with %v", o.ObjectName(), err)
  4694  		}
  4695  		if err := w.Close(); err != nil {
  4696  			t.Fatalf("Write close for %v failed with %v", o.ObjectName(), err)
  4697  		}
  4698  		defer func() {
  4699  			if err := o.Delete(ctx); err != nil {
  4700  				log.Printf("failed to delete test object: %v", err)
  4701  			}
  4702  		}()
  4703  
  4704  		// Check cache control on reader attrs.
  4705  		r, err := o.NewReader(ctx)
  4706  		if err != nil {
  4707  			t.Fatal(err)
  4708  		}
  4709  
  4710  		if got, want := r.Attrs.CacheControl, cacheControl; got != want {
  4711  			t.Fatalf("cache control; got: %s, want: %s", got, want)
  4712  		}
  4713  	})
  4714  }
  4715  
  4716  func TestIntegration_ReaderErrObjectNotExist(t *testing.T) {
  4717  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  4718  		o := client.Bucket(bucket).Object("non-existing")
  4719  
  4720  		_, err := o.NewReader(ctx)
  4721  		if !errors.Is(err, ErrObjectNotExist) {
  4722  			t.Fatalf("expected ErrObjectNotExist, got %v", err)
  4723  		}
  4724  	})
  4725  }
  4726  
  4727  // TestIntegration_JSONReaderConditions tests only JSON reads as some conditions
  4728  // do not work with XML.
  4729  func TestIntegration_JSONReaderConditions(t *testing.T) {
  4730  	ctx := skipXMLReads(skipGRPC("json-only test"), "json-only test")
  4731  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  4732  		b := client.Bucket(bucket)
  4733  		o := b.Object("reader-conditions" + uidSpaceObjects.New())
  4734  
  4735  		// Write object.
  4736  		w := o.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx)
  4737  		if _, err := w.Write(randomContents()); err != nil {
  4738  			t.Fatalf("Write for %v failed with %v", o.ObjectName(), err)
  4739  		}
  4740  		if err := w.Close(); err != nil {
  4741  			t.Fatalf("Write close for %v failed with %v", o.ObjectName(), err)
  4742  		}
  4743  
  4744  		t.Cleanup(func() {
  4745  			if err := o.Delete(ctx); err != nil {
  4746  				log.Printf("failed to delete test object: %v", err)
  4747  			}
  4748  		})
  4749  
  4750  		// Get current gens.
  4751  		attrs, err := o.Attrs(ctx)
  4752  		if err != nil {
  4753  			t.Fatalf("o.Attrs(%s): %v", o.ObjectName(), err)
  4754  		}
  4755  		currGen := attrs.Generation
  4756  		currMetagen := attrs.Metageneration
  4757  
  4758  		// Test each condition to make sure it is passed through correctly.
  4759  		for _, test := range []struct {
  4760  			desc        string
  4761  			conds       Conditions
  4762  			wantErrCode int
  4763  		}{
  4764  			{
  4765  				desc:        "GenerationMatch incorrect gen",
  4766  				conds:       Conditions{GenerationMatch: currGen + 2},
  4767  				wantErrCode: 412,
  4768  			},
  4769  			{
  4770  				desc:        "GenerationNotMatch current gen",
  4771  				conds:       Conditions{GenerationNotMatch: currGen},
  4772  				wantErrCode: 304,
  4773  			},
  4774  			{
  4775  				desc:        "DoesNotExist set to true",
  4776  				conds:       Conditions{DoesNotExist: true},
  4777  				wantErrCode: 412,
  4778  			},
  4779  			{
  4780  				desc:        "MetagenerationMatch incorrect gen",
  4781  				conds:       Conditions{MetagenerationMatch: currMetagen + 1},
  4782  				wantErrCode: 412,
  4783  			},
  4784  			{
  4785  				desc:        "MetagenerationNotMatch current gen",
  4786  				conds:       Conditions{MetagenerationNotMatch: currMetagen},
  4787  				wantErrCode: 304,
  4788  			},
  4789  		} {
  4790  			t.Run(test.desc, func(t *testing.T) {
  4791  				o := o.If(test.conds)
  4792  				_, err := o.NewReader(ctx)
  4793  
  4794  				got := extractErrCode(err)
  4795  				if test.wantErrCode != got {
  4796  					t.Errorf("want err code: %v, got err: %v", test.wantErrCode, err)
  4797  				}
  4798  			})
  4799  		}
  4800  	})
  4801  }
  4802  
  4803  // Test that context cancellation correctly stops a download before completion.
  4804  func TestIntegration_ReaderCancel(t *testing.T) {
  4805  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  4806  		ctx, close := context.WithDeadline(ctx, time.Now().Add(time.Second*30))
  4807  		defer close()
  4808  
  4809  		bkt := client.Bucket(bucket)
  4810  		obj := bkt.Object("reader-cancel-obj")
  4811  
  4812  		minObjectSize := 5000000 // 5 Mb
  4813  
  4814  		w := obj.NewWriter(ctx)
  4815  		c := randomContents()
  4816  		for written := 0; written < minObjectSize; {
  4817  			n, err := w.Write(c)
  4818  			if err != nil {
  4819  				t.Fatalf("w.Write: %v", err)
  4820  			}
  4821  			written += n
  4822  		}
  4823  
  4824  		if err := w.Close(); err != nil {
  4825  			t.Fatalf("writer close: %v", err)
  4826  		}
  4827  		defer func() {
  4828  			if err := obj.Delete(ctx); err != nil {
  4829  				log.Printf("failed to delete test object: %v", err)
  4830  			}
  4831  		}()
  4832  
  4833  		// Create a reader (which makes a GET request to GCS and opens the body to
  4834  		// read the object) and then cancel the context before reading.
  4835  		readerCtx, cancel := context.WithCancel(ctx)
  4836  		r, err := obj.NewReader(readerCtx)
  4837  		if err != nil {
  4838  			t.Fatalf("obj.NewReader: %v", err)
  4839  		}
  4840  		defer func() {
  4841  			if err := r.Close(); err != nil {
  4842  				log.Printf("r.Close(): %v", err)
  4843  			}
  4844  		}()
  4845  
  4846  		cancel()
  4847  
  4848  		_, err = io.Copy(io.Discard, r)
  4849  		if err == nil || !errors.Is(err, context.Canceled) && !(status.Code(err) == codes.Canceled) {
  4850  			t.Fatalf("r.Read: got error %v, want context.Canceled", err)
  4851  		}
  4852  	})
  4853  }
  4854  
  4855  // Ensures that a file stored with a:
  4856  // * Content-Encoding of "gzip"
  4857  // * Content-Type of "text/plain"
  4858  // will be properly served back.
  4859  // See:
  4860  //   - https://cloud.google.com/storage/docs/transcoding#transcoding_and_gzip
  4861  //   - https://github.com/googleapis/google-cloud-go/issues/1800
  4862  func TestIntegration_NewReaderWithContentEncodingGzip(t *testing.T) {
  4863  	multiTransportTest(skipGRPC("gzip transcoding not supported"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
  4864  		h := testHelper{t}
  4865  
  4866  		projectID := testutil.ProjID()
  4867  		bkt := client.Bucket(prefix + uidSpace.New())
  4868  		h.mustCreate(bkt, projectID, nil)
  4869  		defer h.mustDeleteBucket(bkt)
  4870  		obj := bkt.Object("decompressive-transcoding")
  4871  		original := bytes.Repeat([]byte("a"), 4<<10)
  4872  
  4873  		// Firstly upload the gzip compressed file.
  4874  		w := obj.If(Conditions{DoesNotExist: true}).NewWriter(ctx)
  4875  		// Compress and upload the content.
  4876  		gzw := gzip.NewWriter(w)
  4877  		if _, err := gzw.Write(original); err != nil {
  4878  			t.Fatalf("Failed to compress content: %v", err)
  4879  		}
  4880  		if err := gzw.Close(); err != nil {
  4881  			t.Errorf("Failed to compress content: %v", err)
  4882  		}
  4883  		if err := w.Close(); err != nil {
  4884  			t.Errorf("Failed to finish uploading the file: %v", err)
  4885  		}
  4886  
  4887  		defer h.mustDeleteObject(obj)
  4888  
  4889  		// Now update the Content-Encoding and Content-Type to enable
  4890  		// decompressive transcoding.
  4891  		updatedAttrs, err := obj.Update(ctx, ObjectAttrsToUpdate{
  4892  			ContentEncoding: "gzip",
  4893  			ContentType:     "text/plain",
  4894  		})
  4895  		if err != nil {
  4896  			t.Fatalf("Attribute update failure: %v", err)
  4897  		}
  4898  		if g, w := updatedAttrs.ContentEncoding, "gzip"; g != w {
  4899  			t.Fatalf("ContentEncoding mismtach:\nGot:  %q\nWant: %q", g, w)
  4900  		}
  4901  		if g, w := updatedAttrs.ContentType, "text/plain"; g != w {
  4902  			t.Fatalf("ContentType mismtach:\nGot:  %q\nWant: %q", g, w)
  4903  		}
  4904  
  4905  		rWhole, err := obj.NewReader(ctx)
  4906  		if err != nil {
  4907  			t.Fatalf("Failed to create wholesome reader: %v", err)
  4908  		}
  4909  		blobWhole, err := ioutil.ReadAll(rWhole)
  4910  		rWhole.Close()
  4911  		if err != nil {
  4912  			t.Fatalf("Failed to read the whole body: %v", err)
  4913  		}
  4914  		if g, w := blobWhole, original; !bytes.Equal(g, w) {
  4915  			t.Fatalf("Body mismatch\nGot:\n%s\n\nWant:\n%s", g, w)
  4916  		}
  4917  
  4918  		// Now try a range read, which should return the whole body anyways since
  4919  		// for decompressive transcoding, range requests ARE IGNORED by Cloud Storage.
  4920  		r2kBTo3kB, err := obj.NewRangeReader(ctx, 2<<10, 3<<10)
  4921  		if err != nil {
  4922  			t.Fatalf("Failed to create range reader: %v", err)
  4923  		}
  4924  		blob2kBTo3kB, err := ioutil.ReadAll(r2kBTo3kB)
  4925  		r2kBTo3kB.Close()
  4926  		if err != nil {
  4927  			t.Fatalf("Failed to read with the 2kB to 3kB range request: %v", err)
  4928  		}
  4929  		// The ENTIRE body MUST be served back regardless of the requested range.
  4930  		if g, w := blob2kBTo3kB, original; !bytes.Equal(g, w) {
  4931  			t.Fatalf("Body mismatch\nGot:\n%s\n\nWant:\n%s", g, w)
  4932  		}
  4933  	})
  4934  }
  4935  
  4936  func TestIntegration_HMACKey(t *testing.T) {
  4937  	ctx := skipJSONReads(context.Background(), "no reads in test")
  4938  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, _ string, client *Client) {
  4939  		client.SetRetry(WithPolicy(RetryAlways))
  4940  
  4941  		projectID := testutil.ProjID()
  4942  
  4943  		// Use the service account email from the user's credentials. Requires that the
  4944  		// credentials are set via a JSON credentials file.
  4945  		// Note that a service account may only have up to 5 active HMAC keys at once; if
  4946  		// we see flakes because of this, we should consider switching to using a project
  4947  		// pool.
  4948  		credentials := testutil.CredentialsEnv(ctx, "GCLOUD_TESTS_GOLANG_KEY")
  4949  		if credentials == nil {
  4950  			t.Fatal("credentials could not be determined, is GCLOUD_TESTS_GOLANG_KEY set correctly?")
  4951  		}
  4952  		if credentials.JSON == nil {
  4953  			t.Fatal("could not read the JSON key file, is GCLOUD_TESTS_GOLANG_KEY set correctly?")
  4954  		}
  4955  		conf, err := google.JWTConfigFromJSON(credentials.JSON)
  4956  		if err != nil {
  4957  			t.Fatal(err)
  4958  		}
  4959  		serviceAccountEmail := conf.Email
  4960  
  4961  		hmacKey, err := client.CreateHMACKey(ctx, projectID, serviceAccountEmail)
  4962  		if err != nil {
  4963  			t.Fatalf("Failed to create HMACKey: %v", err)
  4964  		}
  4965  		if hmacKey == nil {
  4966  			t.Fatal("Unexpectedly got back a nil HMAC key")
  4967  		}
  4968  
  4969  		if hmacKey.State != Active {
  4970  			t.Fatalf("Unexpected state %q, expected %q", hmacKey.State, Active)
  4971  		}
  4972  
  4973  		hkh := client.HMACKeyHandle(projectID, hmacKey.AccessID)
  4974  		// 1. Ensure that we CANNOT delete an ACTIVE key.
  4975  		if err := hkh.Delete(ctx); err == nil {
  4976  			t.Fatal("Unexpectedly deleted key whose state is ACTIVE: No error from Delete.")
  4977  		}
  4978  
  4979  		invalidStates := []HMACState{"", Deleted, "active", "inactive", "foo_bar"}
  4980  		for _, invalidState := range invalidStates {
  4981  			t.Run("invalid-"+string(invalidState), func(t *testing.T) {
  4982  				_, err := hkh.Update(ctx, HMACKeyAttrsToUpdate{
  4983  					State: invalidState,
  4984  				})
  4985  				if err == nil {
  4986  					t.Fatal("Unexpectedly succeeded")
  4987  				}
  4988  				invalidStateMsg := fmt.Sprintf(`storage: invalid state %q for update, must be either "ACTIVE" or "INACTIVE"`, invalidState)
  4989  				if err.Error() != invalidStateMsg {
  4990  					t.Fatalf("Mismatched error: got:  %q\nwant: %q", err, invalidStateMsg)
  4991  				}
  4992  			})
  4993  		}
  4994  
  4995  		// 2.1. Setting the State to Inactive should succeed.
  4996  		hu, err := hkh.Update(ctx, HMACKeyAttrsToUpdate{
  4997  			State: Inactive,
  4998  		})
  4999  		if err != nil {
  5000  			t.Fatalf("Unexpected Update failure: %v", err)
  5001  		}
  5002  		if got, want := hu.State, Inactive; got != want {
  5003  			t.Fatalf("Unexpected updated state %q, expected %q", got, want)
  5004  		}
  5005  
  5006  		// 2.2. Setting the State back to Active should succeed.
  5007  		hu, err = hkh.Update(ctx, HMACKeyAttrsToUpdate{
  5008  			State: Active,
  5009  		})
  5010  		if err != nil {
  5011  			t.Fatalf("Unexpected Update failure: %v", err)
  5012  		}
  5013  		if got, want := hu.State, Active; got != want {
  5014  			t.Fatalf("Unexpected updated state %q, expected %q", got, want)
  5015  		}
  5016  
  5017  		// 3. Verify that keys are listed as expected.
  5018  		iter := client.ListHMACKeys(ctx, projectID)
  5019  		count := 0
  5020  		for ; ; count++ {
  5021  			_, err := iter.Next()
  5022  			if err == iterator.Done {
  5023  				break
  5024  			}
  5025  			if err != nil {
  5026  				t.Fatalf("Failed to ListHMACKeys: %v", err)
  5027  			}
  5028  		}
  5029  		if count == 0 {
  5030  			t.Fatal("Failed to list any HMACKeys")
  5031  		}
  5032  
  5033  		// 4. Finally set it to back to Inactive and
  5034  		// then retry the deletion which should now succeed.
  5035  		_, _ = hkh.Update(ctx, HMACKeyAttrsToUpdate{
  5036  			State: Inactive,
  5037  		})
  5038  		if err := hkh.Delete(ctx); err != nil {
  5039  			t.Fatalf("Unexpected deletion failure: %v", err)
  5040  		}
  5041  
  5042  		_, err = hkh.Get(ctx)
  5043  		if err != nil && !strings.Contains(err.Error(), "404") {
  5044  			// If the deleted key has already been garbage collected, a 404 is expected.
  5045  			// Other errors should cause a failure and are not expected.
  5046  			t.Fatalf("Unexpected error: %v", err)
  5047  		}
  5048  	})
  5049  }
  5050  
  5051  func TestIntegration_PostPolicyV4(t *testing.T) {
  5052  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) {
  5053  		jwtConf, err := testutil.JWTConfig()
  5054  		if err != nil {
  5055  			t.Fatal(err)
  5056  		}
  5057  		if jwtConf == nil {
  5058  			t.Skip("JSON key file is not present")
  5059  		}
  5060  
  5061  		projectID := testutil.ProjID()
  5062  		newBucketName := prefix + uidSpace.New()
  5063  		b := client.Bucket(newBucketName)
  5064  		h := testHelper{t}
  5065  		h.mustCreate(b, projectID, nil)
  5066  		defer h.mustDeleteBucket(b)
  5067  
  5068  		statusCodeToRespond := 200
  5069  		opts := &PostPolicyV4Options{
  5070  			GoogleAccessID: jwtConf.Email,
  5071  			PrivateKey:     jwtConf.PrivateKey,
  5072  
  5073  			Expires: time.Now().Add(30 * time.Minute),
  5074  
  5075  			Fields: &PolicyV4Fields{
  5076  				StatusCodeOnSuccess: statusCodeToRespond,
  5077  				ContentType:         "text/plain",
  5078  				ACL:                 "public-read",
  5079  			},
  5080  
  5081  			// The conditions that the uploaded file will be expected to conform to.
  5082  			Conditions: []PostPolicyV4Condition{
  5083  				// Make the file a maximum of 10mB.
  5084  				ConditionContentLengthRange(0, 10<<20),
  5085  				ConditionStartsWith("$acl", "public"),
  5086  			},
  5087  		}
  5088  
  5089  		objectName := uidSpaceObjects.New()
  5090  		object := b.Object(objectName)
  5091  		defer h.mustDeleteObject(object)
  5092  
  5093  		pv4, err := b.GenerateSignedPostPolicyV4(objectName, opts)
  5094  		if err != nil {
  5095  			t.Fatal(err)
  5096  		}
  5097  
  5098  		if err := verifyPostPolicy(pv4, object, bytes.Repeat([]byte("a"), 25), statusCodeToRespond); err != nil {
  5099  			t.Fatal(err)
  5100  		}
  5101  	})
  5102  }
  5103  
  5104  // Verify that custom scopes passed in by the user are applied correctly.
  5105  func TestIntegration_Scopes(t *testing.T) {
  5106  	ctx := skipJSONReads(context.Background(), "no reads in test")
  5107  
  5108  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  5109  		bkt := client.Bucket(bucket)
  5110  		obj := bkt.Object("test-scopes")
  5111  		contents := []byte("This object should not be written.\n")
  5112  
  5113  		// A client with ReadOnly scope should be able to read bucket successfully.
  5114  		if _, err := bkt.Attrs(ctx); err != nil {
  5115  			t.Errorf("client with ScopeReadOnly was not able to read attrs: %v", err)
  5116  		}
  5117  
  5118  		// Should not be able to write successfully.
  5119  		if err := writeObject(ctx, obj, "text/plain", contents); err == nil {
  5120  			if err := obj.Delete(ctx); err != nil {
  5121  				t.Logf("obj.Delete: %v", err)
  5122  			}
  5123  			t.Error("client with ScopeReadOnly was able to write an object unexpectedly.")
  5124  		}
  5125  
  5126  		// Should not be able to change permissions.
  5127  		if _, err := obj.Update(ctx, ObjectAttrsToUpdate{ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}}); err == nil {
  5128  			t.Error("client with ScopeReadWrite was able to change unexpectedly.")
  5129  		}
  5130  	}, option.WithScopes(ScopeReadOnly))
  5131  
  5132  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  5133  		bkt := client.Bucket(bucket)
  5134  		obj := bkt.Object("test-scopes")
  5135  		contents := []byte("This object should be written.\n")
  5136  
  5137  		// A client with ReadWrite scope should be able to read bucket successfully.
  5138  		if _, err := bkt.Attrs(ctx); err != nil {
  5139  			t.Errorf("client with ScopeReadOnly was not able to read attrs: %v", err)
  5140  		}
  5141  
  5142  		// Should be able to write to an object.
  5143  		if err := writeObject(ctx, obj, "text/plain", contents); err != nil {
  5144  			t.Errorf("client with ScopeReadWrite was not able to write: %v", err)
  5145  		}
  5146  		defer func() {
  5147  			if err := obj.Delete(ctx); err != nil {
  5148  				t.Logf("obj.Delete: %v", err)
  5149  			}
  5150  		}()
  5151  
  5152  		// Should not be able to change permissions.
  5153  		if _, err := obj.Update(ctx, ObjectAttrsToUpdate{ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}}); err == nil {
  5154  			t.Error("client with ScopeReadWrite was able to change permissions unexpectedly")
  5155  		}
  5156  	}, option.WithScopes(ScopeReadWrite))
  5157  
  5158  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  5159  		bkt := client.Bucket(bucket)
  5160  		obj := bkt.Object("test-scopes")
  5161  		contents := []byte("This object should be written.\n")
  5162  
  5163  		// A client without any scopes should not be able to perform ops.
  5164  		if _, err := bkt.Attrs(ctx); err == nil {
  5165  			t.Errorf("client with no scopes was able to read attrs unexpectedly")
  5166  		}
  5167  
  5168  		if err := writeObject(ctx, obj, "text/plain", contents); err == nil {
  5169  			if err := obj.Delete(ctx); err != nil {
  5170  				t.Logf("obj.Delete: %v", err)
  5171  			}
  5172  			t.Error("client with no scopes was able to write an object unexpectedly.")
  5173  		}
  5174  
  5175  		if _, err := obj.Update(ctx, ObjectAttrsToUpdate{ACL: []ACLRule{{Entity: "domain-google.com", Role: RoleReader}}}); err == nil {
  5176  			t.Error("client with no scopes was able to change permissions unexpectedly")
  5177  		}
  5178  	}, option.WithScopes(""))
  5179  }
  5180  
  5181  func TestIntegration_SignedURL_WithCreds(t *testing.T) {
  5182  	ctx := context.Background()
  5183  
  5184  	creds, err := findTestCredentials(ctx, "GCLOUD_TESTS_GOLANG_KEY", ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform")
  5185  	if err != nil {
  5186  		t.Fatalf("unable to find test credentials: %v", err)
  5187  	}
  5188  
  5189  	multiTransportTest(skipGRPC("creds capture logic must be implemented for gRPC constructor"), t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  5190  		// We can use any client to create the object
  5191  		obj := "testBucketSignedURL"
  5192  		contents := []byte("test")
  5193  		if err := writeObject(ctx, client.Bucket(bucket).Object(obj), "text/plain", contents); err != nil {
  5194  			t.Fatalf("writing: %v", err)
  5195  		}
  5196  		opts := SignedURLOptions{
  5197  			Method:  "GET",
  5198  			Expires: time.Now().Add(30 * time.Second),
  5199  		}
  5200  		bkt := client.Bucket(bucket)
  5201  		url, err := bkt.SignedURL(obj, &opts)
  5202  		if err != nil {
  5203  			t.Fatalf("unable to create signed URL: %v", err)
  5204  		}
  5205  
  5206  		if err := verifySignedURL(url, nil, contents); err != nil {
  5207  			t.Fatalf("problem with the signed URL: %v", err)
  5208  		}
  5209  	}, option.WithCredentials(creds))
  5210  }
  5211  
  5212  func TestIntegration_SignedURL_DefaultSignBytes(t *testing.T) {
  5213  	ctx := context.Background()
  5214  
  5215  	// Create another client to test the sign byte function as well
  5216  	scopes := []string{ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform"}
  5217  	ts := testutil.TokenSource(ctx, scopes...)
  5218  	if ts == nil {
  5219  		t.Fatalf("Cannot get token source to create client")
  5220  	}
  5221  
  5222  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, client *Client) {
  5223  		jwt, err := testutil.JWTConfig()
  5224  		if err != nil {
  5225  			t.Fatalf("unable to find test credentials: %v", err)
  5226  		}
  5227  
  5228  		obj := "testBucketSignedURL"
  5229  		contents := []byte("test")
  5230  		if err := writeObject(ctx, client.Bucket(bucket).Object(obj), "text/plain", contents); err != nil {
  5231  			t.Fatalf("writing: %v", err)
  5232  		}
  5233  
  5234  		opts := SignedURLOptions{
  5235  			Method:         "GET",
  5236  			Expires:        time.Now().Add(30 * time.Second),
  5237  			GoogleAccessID: jwt.Email,
  5238  		}
  5239  		bkt := client.Bucket(bucket)
  5240  		url, err := bkt.SignedURL(obj, &opts)
  5241  		if err != nil {
  5242  			t.Fatalf("unable to create signed URL: %v", err)
  5243  		}
  5244  
  5245  		if err := verifySignedURL(url, nil, contents); err != nil {
  5246  			t.Fatalf("problem with the signed URL: %v", err)
  5247  		}
  5248  	}, option.WithTokenSource(ts))
  5249  
  5250  }
  5251  
  5252  func TestIntegration_PostPolicyV4_WithCreds(t *testing.T) {
  5253  	// By default we are authed with a token source, so don't have the context to
  5254  	// read some of the fields from the keyfile.
  5255  	// Here we explictly send the key to the client.
  5256  	creds, err := findTestCredentials(context.Background(), "GCLOUD_TESTS_GOLANG_KEY", ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform")
  5257  	if err != nil {
  5258  		t.Fatalf("unable to find test credentials: %v", err)
  5259  	}
  5260  
  5261  	ctx := skipJSONReads(skipGRPC("creds capture logic must be implemented for gRPC constructor"), "test is not testing the read behaviour")
  5262  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, clientWithCredentials *Client) {
  5263  		h := testHelper{t}
  5264  
  5265  		statusCodeToRespond := 200
  5266  
  5267  		for _, test := range []struct {
  5268  			desc   string
  5269  			opts   PostPolicyV4Options
  5270  			client *Client
  5271  		}{
  5272  			{
  5273  				desc: "signing with the private key",
  5274  				opts: PostPolicyV4Options{
  5275  					Expires: time.Now().Add(30 * time.Minute),
  5276  
  5277  					Fields: &PolicyV4Fields{
  5278  						StatusCodeOnSuccess: statusCodeToRespond,
  5279  						ContentType:         "text/plain",
  5280  						ACL:                 "public-read",
  5281  					},
  5282  				},
  5283  				client: clientWithCredentials,
  5284  			},
  5285  		} {
  5286  			t.Run(test.desc, func(t *testing.T) {
  5287  				objectName := uidSpace.New()
  5288  				object := test.client.Bucket(bucket).Object(objectName)
  5289  				defer h.mustDeleteObject(object)
  5290  
  5291  				pv4, err := test.client.Bucket(bucket).GenerateSignedPostPolicyV4(objectName, &test.opts)
  5292  				if err != nil {
  5293  					t.Fatal(err)
  5294  				}
  5295  
  5296  				if err := verifyPostPolicy(pv4, object, bytes.Repeat([]byte("a"), 25), statusCodeToRespond); err != nil {
  5297  					t.Fatal(err)
  5298  				}
  5299  			})
  5300  		}
  5301  	}, option.WithCredentials(creds))
  5302  
  5303  }
  5304  
  5305  func TestIntegration_PostPolicyV4_BucketDefault(t *testing.T) {
  5306  	ctx := skipJSONReads(context.Background(), "test is not testing the read behaviour")
  5307  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket, _ string, clientWithoutPrivateKey *Client) {
  5308  		h := testHelper{t}
  5309  
  5310  		jwt, err := testutil.JWTConfig()
  5311  		if err != nil {
  5312  			t.Fatalf("unable to find test credentials: %v", err)
  5313  		}
  5314  
  5315  		statusCodeToRespond := 200
  5316  
  5317  		for _, test := range []struct {
  5318  			desc   string
  5319  			opts   PostPolicyV4Options
  5320  			client *Client
  5321  		}{
  5322  			{
  5323  				desc: "signing with the default sign bytes func",
  5324  				opts: PostPolicyV4Options{
  5325  					Expires:        time.Now().Add(30 * time.Minute),
  5326  					GoogleAccessID: jwt.Email,
  5327  					Fields: &PolicyV4Fields{
  5328  						StatusCodeOnSuccess: statusCodeToRespond,
  5329  						ContentType:         "text/plain",
  5330  						ACL:                 "public-read",
  5331  					},
  5332  				},
  5333  				client: clientWithoutPrivateKey,
  5334  			},
  5335  		} {
  5336  			t.Run(test.desc, func(t *testing.T) {
  5337  				objectName := uidSpaceObjects.New()
  5338  				object := test.client.Bucket(bucket).Object(objectName)
  5339  				defer h.mustDeleteObject(object)
  5340  
  5341  				pv4, err := test.client.Bucket(bucket).GenerateSignedPostPolicyV4(object.ObjectName(), &test.opts)
  5342  				if err != nil {
  5343  					t.Fatal(err)
  5344  				}
  5345  
  5346  				if err := verifyPostPolicy(pv4, object, bytes.Repeat([]byte("a"), 25), statusCodeToRespond); err != nil {
  5347  					t.Fatal(err)
  5348  				}
  5349  			})
  5350  		}
  5351  	})
  5352  
  5353  }
  5354  
  5355  // Tests that the same SignBytes function works for both
  5356  // SignRawBytes on GeneratePostPolicyV4 and SignBytes on SignedURL
  5357  func TestIntegration_PostPolicyV4_SignedURL_WithSignBytes(t *testing.T) {
  5358  	ctx := skipJSONReads(context.Background(), "test is not testing the read behaviour")
  5359  	multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, _, prefix string, client *Client) {
  5360  
  5361  		h := testHelper{t}
  5362  		projectID := testutil.ProjID()
  5363  		bucketName := prefix + uidSpace.New()
  5364  		objectName := uidSpaceObjects.New()
  5365  		fileBody := bytes.Repeat([]byte("b"), 25)
  5366  		bucket := client.Bucket(bucketName)
  5367  
  5368  		h.mustCreate(bucket, projectID, nil)
  5369  		defer h.mustDeleteBucket(bucket)
  5370  
  5371  		object := bucket.Object(objectName)
  5372  		defer h.mustDeleteObject(object)
  5373  
  5374  		jwtConf, err := testutil.JWTConfig()
  5375  		if err != nil {
  5376  			t.Fatal(err)
  5377  		}
  5378  		if jwtConf == nil {
  5379  			t.Skip("JSON key file is not present")
  5380  		}
  5381  
  5382  		signingFunc := func(b []byte) ([]byte, error) {
  5383  			parsedRSAPrivKey, err := parseKey(jwtConf.PrivateKey)
  5384  			if err != nil {
  5385  				return nil, err
  5386  			}
  5387  			sum := sha256.Sum256(b)
  5388  			return rsa.SignPKCS1v15(cryptorand.Reader, parsedRSAPrivKey, crypto.SHA256, sum[:])
  5389  		}
  5390  
  5391  		// Test Post Policy
  5392  		successStatusCode := 200
  5393  		ppv4Opts := &PostPolicyV4Options{
  5394  			GoogleAccessID: jwtConf.Email,
  5395  			SignRawBytes:   signingFunc,
  5396  			Expires:        time.Now().Add(30 * time.Minute),
  5397  			Fields: &PolicyV4Fields{
  5398  				StatusCodeOnSuccess: successStatusCode,
  5399  				ContentType:         "text/plain",
  5400  				ACL:                 "public-read",
  5401  			},
  5402  		}
  5403  
  5404  		pv4, err := GenerateSignedPostPolicyV4(bucketName, objectName, ppv4Opts)
  5405  		if err != nil {
  5406  			t.Fatal(err)
  5407  		}
  5408  
  5409  		if err := verifyPostPolicy(pv4, object, fileBody, successStatusCode); err != nil {
  5410  			t.Fatal(err)
  5411  		}
  5412  
  5413  		// Test Signed URL
  5414  		signURLOpts := &SignedURLOptions{
  5415  			GoogleAccessID: jwtConf.Email,
  5416  			SignBytes:      signingFunc,
  5417  			Method:         "GET",
  5418  			Expires:        time.Now().Add(30 * time.Second),
  5419  		}
  5420  
  5421  		url, err := bucket.SignedURL(objectName, signURLOpts)
  5422  		if err != nil {
  5423  			t.Fatalf("unable to create signed URL: %v", err)
  5424  		}
  5425  
  5426  		if err := verifySignedURL(url, nil, fileBody); err != nil {
  5427  			t.Fatal(err)
  5428  		}
  5429  	})
  5430  }
  5431  
  5432  func TestIntegration_OCTracing(t *testing.T) {
  5433  	multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) {
  5434  		te := testutil.NewTestExporter()
  5435  		defer te.Unregister()
  5436  
  5437  		bkt := client.Bucket(bucket)
  5438  		bkt.Attrs(ctx)
  5439  
  5440  		if len(te.Spans) == 0 {
  5441  			t.Fatalf("Expected some spans to be created, but got %d", 0)
  5442  		}
  5443  	})
  5444  }
  5445  
  5446  // verifySignedURL gets the bytes at the provided url and verifies them against the
  5447  // expectedFileBody. Make sure the SignedURLOptions set the method as "GET".
  5448  func verifySignedURL(url string, headers map[string][]string, expectedFileBody []byte) error {
  5449  	got, err := getURL(url, headers)
  5450  	if err != nil {
  5451  		return fmt.Errorf("getURL %q: %v", url, err)
  5452  	}
  5453  	if !bytes.Equal(got, expectedFileBody) {
  5454  		return fmt.Errorf("got %q, want %q", got, expectedFileBody)
  5455  	}
  5456  	return nil
  5457  }
  5458  
  5459  // verifyPostPolicy uploads a file to the obj using the provided post policy and
  5460  // verifies that it was uploaded correctly
  5461  func verifyPostPolicy(pv4 *PostPolicyV4, obj *ObjectHandle, bytesToWrite []byte, statusCodeOnSuccess int) error {
  5462  	ctx := context.Background()
  5463  	var res *http.Response
  5464  
  5465  	// Request is sent using a vanilla net/http client, so there are no built-in
  5466  	// retries. We must wrap with a retry to prevent flakes.
  5467  	return retry(ctx,
  5468  		func() error {
  5469  			formBuf := new(bytes.Buffer)
  5470  			mw := multipart.NewWriter(formBuf)
  5471  			for fieldName, value := range pv4.Fields {
  5472  				if err := mw.WriteField(fieldName, value); err != nil {
  5473  					return fmt.Errorf("Failed to write form field: %q: %v", fieldName, err)
  5474  				}
  5475  			}
  5476  
  5477  			// Now let's perform the upload
  5478  			mf, err := mw.CreateFormFile("file", "myfile.txt")
  5479  			if err != nil {
  5480  				return err
  5481  			}
  5482  			if _, err := mf.Write(bytesToWrite); err != nil {
  5483  				return err
  5484  			}
  5485  			if err := mw.Close(); err != nil {
  5486  				return err
  5487  			}
  5488  
  5489  			// Compose the HTTP request
  5490  			req, err := http.NewRequest("POST", pv4.URL, formBuf)
  5491  			if err != nil {
  5492  				return fmt.Errorf("Failed to compose HTTP request: %v", err)
  5493  			}
  5494  
  5495  			// Ensure the Content-Type is derived from the writer
  5496  			req.Header.Set("Content-Type", mw.FormDataContentType())
  5497  
  5498  			// Send request
  5499  			res, err = http.DefaultClient.Do(req)
  5500  			if err != nil {
  5501  				return err
  5502  			}
  5503  			return nil
  5504  		},
  5505  		func() error {
  5506  			// Check response
  5507  			if g, w := res.StatusCode, statusCodeOnSuccess; g != w {
  5508  				blob, _ := httputil.DumpResponse(res, true)
  5509  				return fmt.Errorf("Status code in response mismatch: got %d want %d\nBody: %s", g, w, blob)
  5510  			}
  5511  			io.Copy(ioutil.Discard, res.Body)
  5512  
  5513  			// Verify that the file was properly uploaded
  5514  			// by reading back its attributes and content
  5515  			attrs, err := obj.Attrs(ctx)
  5516  			if err != nil {
  5517  				return fmt.Errorf("Failed to retrieve attributes: %v", err)
  5518  			}
  5519  			if g, w := attrs.Size, int64(len(bytesToWrite)); g != w {
  5520  				return fmt.Errorf("ContentLength mismatch: got %d want %d", g, w)
  5521  			}
  5522  			if g, w := attrs.MD5, md5.Sum(bytesToWrite); !bytes.Equal(g, w[:]) {
  5523  				return fmt.Errorf("MD5Checksum mismatch\nGot:  %x\nWant: %x", g, w)
  5524  			}
  5525  
  5526  			// Compare the uploaded body with the expected
  5527  			rd, err := obj.NewReader(ctx)
  5528  			if err != nil {
  5529  				return fmt.Errorf("Failed to create a reader: %v", err)
  5530  			}
  5531  			gotBody, err := ioutil.ReadAll(rd)
  5532  			if err != nil {
  5533  				return fmt.Errorf("Failed to read the body: %v", err)
  5534  			}
  5535  			if diff := testutil.Diff(string(gotBody), string(bytesToWrite)); diff != "" {
  5536  				return fmt.Errorf("Body mismatch: got - want +\n%s", diff)
  5537  			}
  5538  			return nil
  5539  		})
  5540  }
  5541  
  5542  func findTestCredentials(ctx context.Context, envVar string, scopes ...string) (*google.Credentials, error) {
  5543  	key := os.Getenv(envVar)
  5544  	var opts []option.ClientOption
  5545  	if len(scopes) > 0 {
  5546  		opts = append(opts, option.WithScopes(scopes...))
  5547  	}
  5548  	if key != "" {
  5549  		opts = append(opts, option.WithCredentialsFile(key))
  5550  	}
  5551  	return transport.Creds(ctx, opts...)
  5552  }
  5553  
  5554  type testHelper struct {
  5555  	t *testing.T
  5556  }
  5557  
  5558  func (h testHelper) mustCreate(b *BucketHandle, projID string, attrs *BucketAttrs) {
  5559  	h.t.Helper()
  5560  	if err := b.Create(context.Background(), projID, attrs); err != nil {
  5561  		h.t.Fatalf("bucket create: %v", err)
  5562  	}
  5563  }
  5564  
  5565  func (h testHelper) mustDeleteBucket(b *BucketHandle) {
  5566  	h.t.Helper()
  5567  	if err := b.Delete(context.Background()); err != nil {
  5568  		h.t.Fatalf("bucket delete: %v", err)
  5569  	}
  5570  }
  5571  
  5572  func (h testHelper) mustBucketAttrs(b *BucketHandle) *BucketAttrs {
  5573  	h.t.Helper()
  5574  	attrs, err := b.Attrs(context.Background())
  5575  	if err != nil {
  5576  		h.t.Fatalf("bucket attrs: %v", err)
  5577  	}
  5578  	return attrs
  5579  }
  5580  
  5581  // updating a bucket is conditionally idempotent on metageneration, so we pass that in to enable retries
  5582  func (h testHelper) mustUpdateBucket(b *BucketHandle, ua BucketAttrsToUpdate, metageneration int64) *BucketAttrs {
  5583  	h.t.Helper()
  5584  	attrs, err := b.If(BucketConditions{MetagenerationMatch: metageneration}).Update(context.Background(), ua)
  5585  	if err != nil {
  5586  		h.t.Fatalf("update: %v", err)
  5587  	}
  5588  	return attrs
  5589  }
  5590  
  5591  func (h testHelper) mustObjectAttrs(o *ObjectHandle) *ObjectAttrs {
  5592  	h.t.Helper()
  5593  	attrs, err := o.Attrs(context.Background())
  5594  	if err != nil {
  5595  		h.t.Fatalf("object attrs: %v", err)
  5596  	}
  5597  	return attrs
  5598  }
  5599  
  5600  func (h testHelper) mustDeleteObject(o *ObjectHandle) {
  5601  	h.t.Helper()
  5602  	if err := o.Retryer(WithPolicy(RetryAlways)).Delete(context.Background()); err != nil {
  5603  		var apiErr *apierror.APIError
  5604  		if ok := errors.As(err, &apiErr); ok {
  5605  			// Object may already be deleted with retry; if so skip.
  5606  			if apiErr.HTTPCode() == 404 || apiErr.GRPCStatus().Code() == codes.NotFound {
  5607  				return
  5608  			}
  5609  		}
  5610  		h.t.Fatalf("delete object %s from bucket %s: %v", o.ObjectName(), o.BucketName(), err)
  5611  	}
  5612  }
  5613  
  5614  // updating an object is conditionally idempotent on metageneration, so we pass that in to enable retries
  5615  func (h testHelper) mustUpdateObject(o *ObjectHandle, ua ObjectAttrsToUpdate, metageneration int64) *ObjectAttrs {
  5616  	h.t.Helper()
  5617  	attrs, err := o.If(Conditions{MetagenerationMatch: metageneration}).Update(context.Background(), ua)
  5618  	if err != nil {
  5619  		h.t.Fatalf("update: %v", err)
  5620  	}
  5621  	return attrs
  5622  }
  5623  
  5624  func (h testHelper) mustWrite(w *Writer, data []byte) {
  5625  	h.t.Helper()
  5626  	if _, err := w.Write(data); err != nil {
  5627  		w.Close()
  5628  		h.t.Fatalf("write: %v", err)
  5629  	}
  5630  	if err := w.Close(); err != nil {
  5631  		h.t.Fatalf("close write: %v", err)
  5632  	}
  5633  }
  5634  
  5635  func (h testHelper) mustRead(obj *ObjectHandle) []byte {
  5636  	h.t.Helper()
  5637  	data, err := readObject(context.Background(), obj)
  5638  	if err != nil {
  5639  		h.t.Fatalf("read: %v", err)
  5640  	}
  5641  	return data
  5642  }
  5643  
  5644  // deleteObjectIfExists deletes an object with a RetryAlways policy (unless another
  5645  // policy is supplied in the options). It will not return an error if the object
  5646  // is already deleted/doesn't exist. It will time out after 15 seconds.
  5647  func deleteObjectIfExists(o *ObjectHandle, retryOpts ...RetryOption) error {
  5648  	ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
  5649  	defer cancel()
  5650  	retryOpts = append([]RetryOption{WithPolicy(RetryAlways)}, retryOpts...)
  5651  
  5652  	if err := o.Retryer(retryOpts...).Delete(ctx); err != nil {
  5653  		var apiErr *apierror.APIError
  5654  		if ok := errors.As(err, &apiErr); ok {
  5655  			// Object may already be deleted with retry; if so, return no error.
  5656  			if apiErr.HTTPCode() == 404 || apiErr.GRPCStatus().Code() == codes.NotFound {
  5657  				return nil
  5658  			}
  5659  		}
  5660  		return fmt.Errorf("delete object %s from bucket %s: %v", o.ObjectName(), o.BucketName(), err)
  5661  	}
  5662  	return nil
  5663  }
  5664  
  5665  func writeContents(w *Writer, contents []byte) error {
  5666  	if contents != nil {
  5667  		if _, err := w.Write(contents); err != nil {
  5668  			_ = w.Close()
  5669  			return err
  5670  		}
  5671  	}
  5672  	return w.Close()
  5673  }
  5674  
  5675  func writeObject(ctx context.Context, obj *ObjectHandle, contentType string, contents []byte) error {
  5676  	w := newWriter(ctx, obj, contentType, false)
  5677  
  5678  	return writeContents(w, contents)
  5679  }
  5680  
  5681  func newWriter(ctx context.Context, obj *ObjectHandle, contentType string, forceEmptyContentType bool) *Writer {
  5682  	w := obj.Retryer(WithPolicy(RetryAlways)).NewWriter(ctx)
  5683  	w.ContentType = contentType
  5684  	w.ForceEmptyContentType = forceEmptyContentType
  5685  
  5686  	return w
  5687  }
  5688  
  5689  func readObject(ctx context.Context, obj *ObjectHandle) ([]byte, error) {
  5690  	r, err := obj.NewReader(ctx)
  5691  	if err != nil {
  5692  		return nil, err
  5693  	}
  5694  	defer r.Close()
  5695  	return ioutil.ReadAll(r)
  5696  }
  5697  
  5698  // cleanupBuckets deletes the bucket used for testing, as well as old
  5699  // testing buckets that weren't cleaned previously.
  5700  func cleanupBuckets() error {
  5701  	if testing.Short() {
  5702  		return nil // Don't clean up in short mode.
  5703  	}
  5704  	ctx := context.Background()
  5705  	client, err := newTestClient(ctx)
  5706  	if err != nil {
  5707  		log.Fatalf("NewClient: %v", err)
  5708  	}
  5709  	if client == nil {
  5710  		return nil // Don't cleanup if we're not configured correctly.
  5711  	}
  5712  	defer client.Close()
  5713  	if err := killBucket(ctx, client, bucketName); err != nil {
  5714  		return err
  5715  	}
  5716  	if err := killBucket(ctx, client, grpcBucketName); err != nil {
  5717  		return err
  5718  	}
  5719  
  5720  	// Delete buckets whose name begins with our test prefix, and which were
  5721  	// created a while ago. (Unfortunately GCS doesn't provide last-modified
  5722  	// time, which would be a better way to check for staleness.)
  5723  	if err := deleteExpiredBuckets(ctx, client, testPrefix); err != nil {
  5724  		return err
  5725  	}
  5726  	return deleteExpiredBuckets(ctx, client, grpcTestPrefix)
  5727  }
  5728  
  5729  func deleteExpiredBuckets(ctx context.Context, client *Client, prefix string) error {
  5730  	const expireAge = 24 * time.Hour
  5731  	projectID := testutil.ProjID()
  5732  	it := client.Buckets(ctx, projectID)
  5733  	it.Prefix = prefix
  5734  	for {
  5735  		bktAttrs, err := it.Next()
  5736  		if err == iterator.Done {
  5737  			break
  5738  		}
  5739  		if err != nil {
  5740  			return err
  5741  		}
  5742  		if time.Since(bktAttrs.Created) > expireAge {
  5743  			log.Printf("deleting bucket %q, which is more than %s old", bktAttrs.Name, expireAge)
  5744  			if err := killBucket(ctx, client, bktAttrs.Name); err != nil {
  5745  				return err
  5746  			}
  5747  		}
  5748  	}
  5749  	return nil
  5750  }
  5751  
  5752  // killBucket deletes a bucket and all its objects.
  5753  func killBucket(ctx context.Context, client *Client, bucketName string) error {
  5754  	bkt := client.Bucket(bucketName)
  5755  	// Bucket must be empty to delete.
  5756  	it := bkt.Objects(ctx, nil)
  5757  	for {
  5758  		objAttrs, err := it.Next()
  5759  		if err == iterator.Done {
  5760  			break
  5761  		}
  5762  		if err != nil {
  5763  			return err
  5764  		}
  5765  		// Objects with a hold must have the hold released.
  5766  		if objAttrs.EventBasedHold || objAttrs.TemporaryHold {
  5767  			obj := bkt.Object(objAttrs.Name)
  5768  			if _, err := obj.Update(ctx, ObjectAttrsToUpdate{EventBasedHold: false, TemporaryHold: false}); err != nil {
  5769  				return fmt.Errorf("removing hold from %q: %v", bucketName+"/"+objAttrs.Name, err)
  5770  			}
  5771  		}
  5772  		if err := bkt.Object(objAttrs.Name).Delete(ctx); err != nil {
  5773  			return fmt.Errorf("deleting %q: %v", bucketName+"/"+objAttrs.Name, err)
  5774  		}
  5775  	}
  5776  	// GCS is eventually consistent, so this delete may fail because the
  5777  	// replica still sees an object in the bucket. We log the error and expect
  5778  	// a later test run to delete the bucket.
  5779  	if err := bkt.Delete(ctx); err != nil {
  5780  		log.Printf("deleting %q: %v", bucketName, err)
  5781  	}
  5782  	return nil
  5783  }
  5784  
  5785  func randomContents() []byte {
  5786  	h := md5.New()
  5787  	io.WriteString(h, fmt.Sprintf("hello world%d", rng.Intn(100000)))
  5788  	return h.Sum(nil)
  5789  }
  5790  
  5791  type zeros struct{}
  5792  
  5793  func (zeros) Read(p []byte) (int, error) { return len(p), nil }
  5794  
  5795  // Make a GET request to a URL using an unauthenticated client, and return its contents.
  5796  func getURL(url string, headers map[string][]string) ([]byte, error) {
  5797  	req, err := http.NewRequest("GET", url, nil)
  5798  	if err != nil {
  5799  		return nil, err
  5800  	}
  5801  	req.Header = headers
  5802  	res, err := http.DefaultClient.Do(req)
  5803  	if err != nil {
  5804  		return nil, err
  5805  	}
  5806  	defer res.Body.Close()
  5807  	bytes, err := ioutil.ReadAll(res.Body)
  5808  	if err != nil {
  5809  		return nil, err
  5810  	}
  5811  	if res.StatusCode != 200 {
  5812  		return nil, fmt.Errorf("code=%d, body=%s", res.StatusCode, string(bytes))
  5813  	}
  5814  	return bytes, nil
  5815  }
  5816  
  5817  // Make a PUT request to a URL using an unauthenticated client, and return its contents.
  5818  func putURL(url string, headers map[string][]string, payload io.Reader) ([]byte, error) {
  5819  	req, err := http.NewRequest("PUT", url, payload)
  5820  	if err != nil {
  5821  		return nil, err
  5822  	}
  5823  	req.Header = headers
  5824  	res, err := http.DefaultClient.Do(req)
  5825  	if err != nil {
  5826  		return nil, err
  5827  	}
  5828  	defer res.Body.Close()
  5829  	bytes, err := ioutil.ReadAll(res.Body)
  5830  	if err != nil {
  5831  		return nil, err
  5832  	}
  5833  	if res.StatusCode != 200 {
  5834  		return nil, fmt.Errorf("code=%d, body=%s", res.StatusCode, string(bytes))
  5835  	}
  5836  	return bytes, nil
  5837  }
  5838  
  5839  func keyFileEmail(filename string) (string, error) {
  5840  	bytes, err := ioutil.ReadFile(filename)
  5841  	if err != nil {
  5842  		return "", err
  5843  	}
  5844  	var v struct {
  5845  		ClientEmail string `json:"client_email"`
  5846  	}
  5847  	if err := json.Unmarshal(bytes, &v); err != nil {
  5848  		return "", err
  5849  	}
  5850  	return v.ClientEmail, nil
  5851  }
  5852  
  5853  type comparableACL interface {
  5854  	equals(ACLRule) bool
  5855  }
  5856  
  5857  type testACLRule ACLRule
  5858  
  5859  func (acl testACLRule) equals(a ACLRule) bool {
  5860  	return cmp.Equal(a, ACLRule(acl))
  5861  }
  5862  
  5863  type entityRoleACL struct {
  5864  	entity ACLEntity
  5865  	role   ACLRole
  5866  }
  5867  
  5868  func (er entityRoleACL) equals(a ACLRule) bool {
  5869  	return a.Entity == er.entity && a.Role == er.role
  5870  }
  5871  
  5872  type prefixRoleACL struct {
  5873  	prefix string
  5874  	role   ACLRole
  5875  }
  5876  
  5877  func (pr prefixRoleACL) equals(a ACLRule) bool {
  5878  	return strings.HasPrefix(string(a.Entity), pr.prefix) && a.Role == pr.role
  5879  }
  5880  
  5881  func containsACLRule(acl []ACLRule, want comparableACL) bool {
  5882  	for _, acl := range acl {
  5883  		if want.equals(acl) {
  5884  			return true
  5885  		}
  5886  	}
  5887  	return false
  5888  }
  5889  
  5890  // retry retries a function call as well as an (optional) correctness check for up
  5891  // to 60 seconds. Both call and check must run without error in order to succeed.
  5892  // If the timeout is hit, the most recent error from call or check will be returned.
  5893  // This function should be used to wrap calls that might cause integration test
  5894  // flakes due to delays in propagation (for example, metadata updates).
  5895  func retry(ctx context.Context, call func() error, check func() error) error {
  5896  	timeout := time.After(60 * time.Second)
  5897  	var err error
  5898  	for {
  5899  		select {
  5900  		case <-timeout:
  5901  			return err
  5902  		default:
  5903  		}
  5904  		err = call()
  5905  		if err == nil {
  5906  			if check == nil || check() == nil {
  5907  				return nil
  5908  			}
  5909  			err = check()
  5910  		}
  5911  		time.Sleep(200 * time.Millisecond)
  5912  	}
  5913  }
  5914  
  5915  func retryOnNilAndTransientErrs(err error) bool {
  5916  	return err == nil || ShouldRetry(err)
  5917  }
  5918  func retryOnTransient400and403(err error) bool {
  5919  	var e *googleapi.Error
  5920  	var ae *apierror.APIError
  5921  	return ShouldRetry(err) ||
  5922  		/* http */ errors.As(err, &e) && (e.Code == 400 || e.Code == 403) ||
  5923  		/* grpc */ errors.As(err, &ae) && (ae.GRPCStatus().Code() == codes.InvalidArgument || ae.GRPCStatus().Code() == codes.PermissionDenied)
  5924  }
  5925  
  5926  func skipGRPC(reason string) context.Context {
  5927  	return context.WithValue(context.Background(), skipTransportTestKey("grpc"), reason)
  5928  }
  5929  
  5930  func skipHTTP(reason string) context.Context {
  5931  	ctx := context.WithValue(context.Background(), skipTransportTestKey("http"), reason)
  5932  	return context.WithValue(ctx, skipTransportTestKey("jsonReads"), reason)
  5933  }
  5934  
  5935  func skipJSONReads(ctx context.Context, reason string) context.Context {
  5936  	return context.WithValue(ctx, skipTransportTestKey("jsonReads"), reason)
  5937  }
  5938  
  5939  func skipXMLReads(ctx context.Context, reason string) context.Context {
  5940  	return context.WithValue(ctx, skipTransportTestKey("http"), reason)
  5941  }
  5942  
  5943  // Extract the error code if it's a googleapi.Error
  5944  func extractErrCode(err error) int {
  5945  	if err == nil {
  5946  		return 0
  5947  	}
  5948  	var e *googleapi.Error
  5949  	if errors.As(err, &e) {
  5950  		return e.Code
  5951  	}
  5952  
  5953  	return -1
  5954  }
  5955  
  5956  func setUpRequesterPaysBucket(ctx context.Context, t *testing.T, bucket, object string, addOwnerEmail string) {
  5957  	t.Helper()
  5958  	client := testConfig(ctx, t)
  5959  	h := testHelper{t}
  5960  
  5961  	requesterPaysBucket := client.Bucket(bucket)
  5962  
  5963  	// Create a requester-pays bucket.
  5964  	h.mustCreate(requesterPaysBucket, testutil.ProjID(), &BucketAttrs{RequesterPays: true})
  5965  	t.Cleanup(func() { h.mustDeleteBucket(requesterPaysBucket) })
  5966  
  5967  	// Grant ownership
  5968  	if err := requesterPaysBucket.ACL().Set(ctx, ACLEntity("user-"+addOwnerEmail), RoleOwner); err != nil {
  5969  		t.Fatalf("set ACL: %v", err)
  5970  	}
  5971  
  5972  	h.mustWrite(requesterPaysBucket.Object(object).NewWriter(ctx), []byte("hello"))
  5973  	t.Cleanup(func() {
  5974  		err := requesterPaysBucket.Object(object).Delete(ctx)
  5975  		if err != nil {
  5976  			// only log because object may be deleted by test
  5977  			t.Logf("could not delete object: %v", err)
  5978  		}
  5979  	})
  5980  }