github.com/thiagoyeds/go-cloud@v0.26.0/blob/blob.go (about) 1 // Copyright 2018 The Go Cloud Development Kit Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // https://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // Package blob provides an easy and portable way to interact with blobs 16 // within a storage location. Subpackages contain driver implementations of 17 // blob for supported services. 18 // 19 // See https://gocloud.dev/howto/blob/ for a detailed how-to guide. 20 // 21 // 22 // Errors 23 // 24 // The errors returned from this package can be inspected in several ways: 25 // 26 // The Code function from gocloud.dev/gcerrors will return an error code, also 27 // defined in that package, when invoked on an error. 28 // 29 // The Bucket.ErrorAs method can retrieve the driver error underlying the returned 30 // error. 31 // 32 // 33 // OpenCensus Integration 34 // 35 // OpenCensus supports tracing and metric collection for multiple languages and 36 // backend providers. See https://opencensus.io. 37 // 38 // This API collects OpenCensus traces and metrics for the following methods: 39 // - Attributes 40 // - Copy 41 // - Delete 42 // - ListPage 43 // - NewRangeReader, from creation until the call to Close. (NewReader and ReadAll 44 // are included because they call NewRangeReader.) 45 // - NewWriter, from creation until the call to Close. 46 // All trace and metric names begin with the package import path. 47 // The traces add the method name. 48 // For example, "gocloud.dev/blob/Attributes". 49 // The metrics are "completed_calls", a count of completed method calls by driver, 50 // method and status (error code); and "latency", a distribution of method latency 51 // by driver and method. 52 // For example, "gocloud.dev/blob/latency". 53 // 54 // It also collects the following metrics: 55 // - gocloud.dev/blob/bytes_read: the total number of bytes read, by driver. 56 // - gocloud.dev/blob/bytes_written: the total number of bytes written, by driver. 57 // 58 // To enable trace collection in your application, see "Configure Exporter" at 59 // https://opencensus.io/quickstart/go/tracing. 60 // To enable metric collection in your application, see "Exporting stats" at 61 // https://opencensus.io/quickstart/go/metrics. 62 package blob // import "gocloud.dev/blob" 63 64 import ( 65 "bytes" 66 "context" 67 "crypto/md5" 68 "fmt" 69 "hash" 70 "io" 71 "io/ioutil" 72 "log" 73 "mime" 74 "net/http" 75 "net/url" 76 "runtime" 77 "strings" 78 "sync" 79 "time" 80 "unicode/utf8" 81 82 "go.opencensus.io/stats" 83 "go.opencensus.io/stats/view" 84 "go.opencensus.io/tag" 85 "gocloud.dev/blob/driver" 86 "gocloud.dev/gcerrors" 87 "gocloud.dev/internal/gcerr" 88 "gocloud.dev/internal/oc" 89 "gocloud.dev/internal/openurl" 90 ) 91 92 // Reader reads bytes from a blob. 93 // It implements io.ReadCloser, and must be closed after 94 // reads are finished. 95 type Reader struct { 96 b driver.Bucket 97 r driver.Reader 98 key string 99 end func(error) // called at Close to finish trace and metric collection 100 // for metric collection; 101 statsTagMutators []tag.Mutator 102 bytesRead int 103 closed bool 104 } 105 106 // Read implements io.Reader (https://golang.org/pkg/io/#Reader). 107 func (r *Reader) Read(p []byte) (int, error) { 108 n, err := r.r.Read(p) 109 r.bytesRead += n 110 return n, wrapError(r.b, err, r.key) 111 } 112 113 // Close implements io.Closer (https://golang.org/pkg/io/#Closer). 114 func (r *Reader) Close() error { 115 r.closed = true 116 err := wrapError(r.b, r.r.Close(), r.key) 117 r.end(err) 118 // Emit only on close to avoid an allocation on each call to Read(). 119 stats.RecordWithTags( 120 context.Background(), 121 r.statsTagMutators, 122 bytesReadMeasure.M(int64(r.bytesRead))) 123 return err 124 } 125 126 // ContentType returns the MIME type of the blob. 127 func (r *Reader) ContentType() string { 128 return r.r.Attributes().ContentType 129 } 130 131 // ModTime returns the time the blob was last modified. 132 func (r *Reader) ModTime() time.Time { 133 return r.r.Attributes().ModTime 134 } 135 136 // Size returns the size of the blob content in bytes. 137 func (r *Reader) Size() int64 { 138 return r.r.Attributes().Size 139 } 140 141 // As converts i to driver-specific types. 142 // See https://gocloud.dev/concepts/as/ for background information, the "As" 143 // examples in this package for examples, and the driver package 144 // documentation for the specific types supported for that driver. 145 func (r *Reader) As(i interface{}) bool { 146 return r.r.As(i) 147 } 148 149 // WriteTo reads from r and writes to w until there's no more data or 150 // an error occurs. 151 // The return value is the number of bytes written to w. 152 // 153 // It implements the io.WriterTo interface. 154 func (r *Reader) WriteTo(w io.Writer) (int64, error) { 155 _, nw, err := readFromWriteTo(r, w) 156 return nw, err 157 } 158 159 // readFromWriteTo is a helper for ReadFrom and WriteTo. 160 // It reads data from r and writes to w, until EOF or a read/write error. 161 // It returns the number of bytes read from r and the number of bytes 162 // written to w. 163 func readFromWriteTo(r io.Reader, w io.Writer) (int64, int64, error) { 164 buf := make([]byte, 1024) 165 var totalRead, totalWritten int64 166 for { 167 numRead, rerr := r.Read(buf) 168 if numRead > 0 { 169 totalRead += int64(numRead) 170 numWritten, werr := w.Write(buf[0:numRead]) 171 totalWritten += int64(numWritten) 172 if werr != nil { 173 return totalRead, totalWritten, werr 174 } 175 } 176 if rerr == io.EOF { 177 // Done! 178 return totalRead, totalWritten, nil 179 } 180 if rerr != nil { 181 return totalRead, totalWritten, rerr 182 } 183 } 184 } 185 186 // Attributes contains attributes about a blob. 187 type Attributes struct { 188 // CacheControl specifies caching attributes that services may use 189 // when serving the blob. 190 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control 191 CacheControl string 192 // ContentDisposition specifies whether the blob content is expected to be 193 // displayed inline or as an attachment. 194 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition 195 ContentDisposition string 196 // ContentEncoding specifies the encoding used for the blob's content, if any. 197 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding 198 ContentEncoding string 199 // ContentLanguage specifies the language used in the blob's content, if any. 200 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Language 201 ContentLanguage string 202 // ContentType is the MIME type of the blob. It will not be empty. 203 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type 204 ContentType string 205 // Metadata holds key/value pairs associated with the blob. 206 // Keys are guaranteed to be in lowercase, even if the backend service 207 // has case-sensitive keys (although note that Metadata written via 208 // this package will always be lowercased). If there are duplicate 209 // case-insensitive keys (e.g., "foo" and "FOO"), only one value 210 // will be kept, and it is undefined which one. 211 Metadata map[string]string 212 // CreateTime is the time the blob was created, if available. If not available, 213 // CreateTime will be the zero time. 214 CreateTime time.Time 215 // ModTime is the time the blob was last modified. 216 ModTime time.Time 217 // Size is the size of the blob's content in bytes. 218 Size int64 219 // MD5 is an MD5 hash of the blob contents or nil if not available. 220 MD5 []byte 221 // ETag for the blob; see https://en.wikipedia.org/wiki/HTTP_ETag. 222 ETag string 223 224 asFunc func(interface{}) bool 225 } 226 227 // As converts i to driver-specific types. 228 // See https://gocloud.dev/concepts/as/ for background information, the "As" 229 // examples in this package for examples, and the driver package 230 // documentation for the specific types supported for that driver. 231 func (a *Attributes) As(i interface{}) bool { 232 if a.asFunc == nil { 233 return false 234 } 235 return a.asFunc(i) 236 } 237 238 // Writer writes bytes to a blob. 239 // 240 // It implements io.WriteCloser (https://golang.org/pkg/io/#Closer), and must be 241 // closed after all writes are done. 242 type Writer struct { 243 b driver.Bucket 244 w driver.Writer 245 key string 246 end func(error) // called at Close to finish trace and metric collection 247 cancel func() // cancels the ctx provided to NewTypedWriter if contentMD5 verification fails 248 contentMD5 []byte 249 md5hash hash.Hash 250 statsTagMutators []tag.Mutator // for metric collection 251 bytesWritten int 252 closed bool 253 254 // These fields are non-zero values only when w is nil (not yet created). 255 // 256 // A ctx is stored in the Writer since we need to pass it into NewTypedWriter 257 // when we finish detecting the content type of the blob and create the 258 // underlying driver.Writer. This step happens inside Write or Close and 259 // neither of them take a context.Context as an argument. 260 // 261 // All 3 fields are only initialized when we create the Writer without 262 // setting the w field, and are reset to zero values after w is created. 263 ctx context.Context 264 opts *driver.WriterOptions 265 buf *bytes.Buffer 266 } 267 268 // sniffLen is the byte size of Writer.buf used to detect content-type. 269 const sniffLen = 512 270 271 // Write implements the io.Writer interface (https://golang.org/pkg/io/#Writer). 272 // 273 // Writes may happen asynchronously, so the returned error can be nil 274 // even if the actual write eventually fails. The write is only guaranteed to 275 // have succeeded if Close returns no error. 276 func (w *Writer) Write(p []byte) (int, error) { 277 if len(w.contentMD5) > 0 { 278 if _, err := w.md5hash.Write(p); err != nil { 279 return 0, err 280 } 281 } 282 if w.w != nil { 283 return w.write(p) 284 } 285 286 // If w is not yet created due to no content-type being passed in, try to sniff 287 // the MIME type based on at most 512 bytes of the blob content of p. 288 289 // Detect the content-type directly if the first chunk is at least 512 bytes. 290 if w.buf.Len() == 0 && len(p) >= sniffLen { 291 return w.open(p) 292 } 293 294 // Store p in w.buf and detect the content-type when the size of content in 295 // w.buf is at least 512 bytes. 296 n, err := w.buf.Write(p) 297 if err != nil { 298 return 0, err 299 } 300 if w.buf.Len() >= sniffLen { 301 // Note that w.open will return the full length of the buffer; we don't want 302 // to return that as the length of this write since some of them were written in 303 // previous writes. Instead, we return the n from this write, above. 304 _, err := w.open(w.buf.Bytes()) 305 return n, err 306 } 307 return n, nil 308 } 309 310 // Close closes the blob writer. The write operation is not guaranteed to have succeeded until 311 // Close returns with no error. 312 // Close may return an error if the context provided to create the Writer is 313 // canceled or reaches its deadline. 314 func (w *Writer) Close() (err error) { 315 w.closed = true 316 defer func() { 317 w.end(err) 318 // Emit only on close to avoid an allocation on each call to Write(). 319 stats.RecordWithTags( 320 context.Background(), 321 w.statsTagMutators, 322 bytesWrittenMeasure.M(int64(w.bytesWritten))) 323 }() 324 if len(w.contentMD5) > 0 { 325 // Verify the MD5 hash of what was written matches the ContentMD5 provided 326 // by the user. 327 md5sum := w.md5hash.Sum(nil) 328 if !bytes.Equal(md5sum, w.contentMD5) { 329 // No match! Return an error, but first cancel the context and call the 330 // driver's Close function to ensure the write is aborted. 331 w.cancel() 332 if w.w != nil { 333 _ = w.w.Close() 334 } 335 return gcerr.Newf(gcerr.FailedPrecondition, nil, "blob: the WriterOptions.ContentMD5 you specified (%X) did not match what was written (%X)", w.contentMD5, md5sum) 336 } 337 } 338 339 defer w.cancel() 340 if w.w != nil { 341 return wrapError(w.b, w.w.Close(), w.key) 342 } 343 if _, err := w.open(w.buf.Bytes()); err != nil { 344 return err 345 } 346 return wrapError(w.b, w.w.Close(), w.key) 347 } 348 349 // open tries to detect the MIME type of p and write it to the blob. 350 // The error it returns is wrapped. 351 func (w *Writer) open(p []byte) (int, error) { 352 ct := http.DetectContentType(p) 353 var err error 354 if w.w, err = w.b.NewTypedWriter(w.ctx, w.key, ct, w.opts); err != nil { 355 return 0, wrapError(w.b, err, w.key) 356 } 357 // Set the 3 fields needed for lazy NewTypedWriter back to zero values 358 // (see the comment on Writer). 359 w.buf = nil 360 w.ctx = nil 361 w.opts = nil 362 return w.write(p) 363 } 364 365 func (w *Writer) write(p []byte) (int, error) { 366 n, err := w.w.Write(p) 367 w.bytesWritten += n 368 return n, wrapError(w.b, err, w.key) 369 } 370 371 // ReadFrom reads from r and writes to w until EOF or error. 372 // The return value is the number of bytes read from r. 373 // 374 // It implements the io.ReaderFrom interface. 375 func (w *Writer) ReadFrom(r io.Reader) (int64, error) { 376 nr, _, err := readFromWriteTo(r, w) 377 return nr, err 378 } 379 380 // ListOptions sets options for listing blobs via Bucket.List. 381 type ListOptions struct { 382 // Prefix indicates that only blobs with a key starting with this prefix 383 // should be returned. 384 Prefix string 385 // Delimiter sets the delimiter used to define a hierarchical namespace, 386 // like a filesystem with "directories". It is highly recommended that you 387 // use "" or "/" as the Delimiter. Other values should work through this API, 388 // but service UIs generally assume "/". 389 // 390 // An empty delimiter means that the bucket is treated as a single flat 391 // namespace. 392 // 393 // A non-empty delimiter means that any result with the delimiter in its key 394 // after Prefix is stripped will be returned with ListObject.IsDir = true, 395 // ListObject.Key truncated after the delimiter, and zero values for other 396 // ListObject fields. These results represent "directories". Multiple results 397 // in a "directory" are returned as a single result. 398 Delimiter string 399 400 // BeforeList is a callback that will be called before each call to the 401 // the underlying service's list functionality. 402 // asFunc converts its argument to driver-specific types. 403 // See https://gocloud.dev/concepts/as/ for background information. 404 BeforeList func(asFunc func(interface{}) bool) error 405 } 406 407 // ListIterator iterates over List results. 408 type ListIterator struct { 409 b *Bucket 410 opts *driver.ListOptions 411 page *driver.ListPage 412 nextIdx int 413 } 414 415 // Next returns a *ListObject for the next blob. It returns (nil, io.EOF) if 416 // there are no more. 417 func (i *ListIterator) Next(ctx context.Context) (*ListObject, error) { 418 if i.page != nil { 419 // We've already got a page of results. 420 if i.nextIdx < len(i.page.Objects) { 421 // Next object is in the page; return it. 422 dobj := i.page.Objects[i.nextIdx] 423 i.nextIdx++ 424 return &ListObject{ 425 Key: dobj.Key, 426 ModTime: dobj.ModTime, 427 Size: dobj.Size, 428 MD5: dobj.MD5, 429 IsDir: dobj.IsDir, 430 asFunc: dobj.AsFunc, 431 }, nil 432 } 433 if len(i.page.NextPageToken) == 0 { 434 // Done with current page, and there are no more; return io.EOF. 435 return nil, io.EOF 436 } 437 // We need to load the next page. 438 i.opts.PageToken = i.page.NextPageToken 439 } 440 i.b.mu.RLock() 441 defer i.b.mu.RUnlock() 442 if i.b.closed { 443 return nil, errClosed 444 } 445 // Loading a new page. 446 p, err := i.b.b.ListPaged(ctx, i.opts) 447 if err != nil { 448 return nil, wrapError(i.b.b, err, "") 449 } 450 i.page = p 451 i.nextIdx = 0 452 return i.Next(ctx) 453 } 454 455 // ListObject represents a single blob returned from List. 456 type ListObject struct { 457 // Key is the key for this blob. 458 Key string 459 // ModTime is the time the blob was last modified. 460 ModTime time.Time 461 // Size is the size of the blob's content in bytes. 462 Size int64 463 // MD5 is an MD5 hash of the blob contents or nil if not available. 464 MD5 []byte 465 // IsDir indicates that this result represents a "directory" in the 466 // hierarchical namespace, ending in ListOptions.Delimiter. Key can be 467 // passed as ListOptions.Prefix to list items in the "directory". 468 // Fields other than Key and IsDir will not be set if IsDir is true. 469 IsDir bool 470 471 asFunc func(interface{}) bool 472 } 473 474 // As converts i to driver-specific types. 475 // See https://gocloud.dev/concepts/as/ for background information, the "As" 476 // examples in this package for examples, and the driver package 477 // documentation for the specific types supported for that driver. 478 func (o *ListObject) As(i interface{}) bool { 479 if o.asFunc == nil { 480 return false 481 } 482 return o.asFunc(i) 483 } 484 485 // Bucket provides an easy and portable way to interact with blobs 486 // within a "bucket", including read, write, and list operations. 487 // To create a Bucket, use constructors found in driver subpackages. 488 type Bucket struct { 489 b driver.Bucket 490 tracer *oc.Tracer 491 492 // mu protects the closed variable. 493 // Read locks are kept to allow holding a read lock for long-running calls, 494 // and thereby prevent closing until a call finishes. 495 mu sync.RWMutex 496 closed bool 497 } 498 499 const pkgName = "gocloud.dev/blob" 500 501 var ( 502 latencyMeasure = oc.LatencyMeasure(pkgName) 503 bytesReadMeasure = stats.Int64(pkgName+"/bytes_read", "Total bytes read", stats.UnitBytes) 504 bytesWrittenMeasure = stats.Int64(pkgName+"/bytes_written", "Total bytes written", stats.UnitBytes) 505 506 // OpenCensusViews are predefined views for OpenCensus metrics. 507 // The views include counts and latency distributions for API method calls, 508 // and total bytes read and written. 509 // See the example at https://godoc.org/go.opencensus.io/stats/view for usage. 510 OpenCensusViews = append( 511 oc.Views(pkgName, latencyMeasure), 512 &view.View{ 513 Name: pkgName + "/bytes_read", 514 Measure: bytesReadMeasure, 515 Description: "Sum of bytes read from the service.", 516 TagKeys: []tag.Key{oc.ProviderKey}, 517 Aggregation: view.Sum(), 518 }, 519 &view.View{ 520 Name: pkgName + "/bytes_written", 521 Measure: bytesWrittenMeasure, 522 Description: "Sum of bytes written to the service.", 523 TagKeys: []tag.Key{oc.ProviderKey}, 524 Aggregation: view.Sum(), 525 }) 526 ) 527 528 // NewBucket is intended for use by drivers only. Do not use in application code. 529 var NewBucket = newBucket 530 531 // newBucket creates a new *Bucket based on a specific driver implementation. 532 // End users should use subpackages to construct a *Bucket instead of this 533 // function; see the package documentation for details. 534 func newBucket(b driver.Bucket) *Bucket { 535 return &Bucket{ 536 b: b, 537 tracer: &oc.Tracer{ 538 Package: pkgName, 539 Provider: oc.ProviderName(b), 540 LatencyMeasure: latencyMeasure, 541 }, 542 } 543 } 544 545 // As converts i to driver-specific types. 546 // See https://gocloud.dev/concepts/as/ for background information, the "As" 547 // examples in this package for examples, and the driver package 548 // documentation for the specific types supported for that driver. 549 func (b *Bucket) As(i interface{}) bool { 550 if i == nil { 551 return false 552 } 553 return b.b.As(i) 554 } 555 556 // ErrorAs converts err to driver-specific types. 557 // ErrorAs panics if i is nil or not a pointer. 558 // ErrorAs returns false if err == nil. 559 // See https://gocloud.dev/concepts/as/ for background information. 560 func (b *Bucket) ErrorAs(err error, i interface{}) bool { 561 return gcerr.ErrorAs(err, i, b.b.ErrorAs) 562 } 563 564 // ReadAll is a shortcut for creating a Reader via NewReader with nil 565 // ReaderOptions, and reading the entire blob. 566 func (b *Bucket) ReadAll(ctx context.Context, key string) (_ []byte, err error) { 567 b.mu.RLock() 568 defer b.mu.RUnlock() 569 if b.closed { 570 return nil, errClosed 571 } 572 r, err := b.NewReader(ctx, key, nil) 573 if err != nil { 574 return nil, err 575 } 576 defer r.Close() 577 return ioutil.ReadAll(r) 578 } 579 580 // List returns a ListIterator that can be used to iterate over blobs in a 581 // bucket, in lexicographical order of UTF-8 encoded keys. The underlying 582 // implementation fetches results in pages. 583 // 584 // A nil ListOptions is treated the same as the zero value. 585 // 586 // List is not guaranteed to include all recently-written blobs; 587 // some services are only eventually consistent. 588 func (b *Bucket) List(opts *ListOptions) *ListIterator { 589 if opts == nil { 590 opts = &ListOptions{} 591 } 592 dopts := &driver.ListOptions{ 593 Prefix: opts.Prefix, 594 Delimiter: opts.Delimiter, 595 BeforeList: opts.BeforeList, 596 } 597 return &ListIterator{b: b, opts: dopts} 598 } 599 600 // FirstPageToken is the pageToken to pass to ListPage to retrieve the first page of results. 601 var FirstPageToken = []byte("first page") 602 603 // ListPage returns a page of ListObject results for blobs in a bucket, in lexicographical 604 // order of UTF-8 encoded keys. 605 // 606 // To fetch the first page, pass FirstPageToken as the pageToken. For subsequent pages, pass 607 // the pageToken returned from a previous call to ListPage. 608 // It is not possible to "skip ahead" pages. 609 // 610 // Each call will return pageSize results, unless there are not enough blobs to fill the 611 // page, in which case it will return fewer results (possibly 0). 612 // 613 // If there are no more blobs available, ListPage will return an empty pageToken. Note that 614 // this may happen regardless of the number of returned results -- the last page might have 615 // 0 results (i.e., if the last item was deleted), pageSize results, or anything in between. 616 // 617 // Calling ListPage with an empty pageToken will immediately return io.EOF. When looping 618 // over pages, callers can either check for an empty pageToken, or they can make one more 619 // call and check for io.EOF. 620 // 621 // The underlying implementation fetches results in pages, but one call to ListPage may 622 // require multiple page fetches (and therefore, multiple calls to the BeforeList callback). 623 // 624 // A nil ListOptions is treated the same as the zero value. 625 // 626 // ListPage is not guaranteed to include all recently-written blobs; 627 // some services are only eventually consistent. 628 func (b *Bucket) ListPage(ctx context.Context, pageToken []byte, pageSize int, opts *ListOptions) (retval []*ListObject, nextPageToken []byte, err error) { 629 if opts == nil { 630 opts = &ListOptions{} 631 } 632 if pageSize <= 0 { 633 return nil, nil, gcerr.Newf(gcerr.InvalidArgument, nil, "blob: pageSize must be > 0") 634 } 635 636 // Nil pageToken means no more results. 637 if len(pageToken) == 0 { 638 return nil, nil, io.EOF 639 } 640 641 // FirstPageToken fetches the first page. Drivers use nil. 642 // The public API doesn't use nil for the first page because it would be too easy to 643 // keep fetching forever (since the last page return nil for the next pageToken). 644 if bytes.Equal(pageToken, FirstPageToken) { 645 pageToken = nil 646 } 647 b.mu.RLock() 648 defer b.mu.RUnlock() 649 if b.closed { 650 return nil, nil, errClosed 651 } 652 653 ctx = b.tracer.Start(ctx, "ListPage") 654 defer func() { b.tracer.End(ctx, err) }() 655 656 dopts := &driver.ListOptions{ 657 Prefix: opts.Prefix, 658 Delimiter: opts.Delimiter, 659 BeforeList: opts.BeforeList, 660 PageToken: pageToken, 661 PageSize: pageSize, 662 } 663 retval = make([]*ListObject, 0, pageSize) 664 for len(retval) < pageSize { 665 p, err := b.b.ListPaged(ctx, dopts) 666 if err != nil { 667 return nil, nil, wrapError(b.b, err, "") 668 } 669 for _, dobj := range p.Objects { 670 retval = append(retval, &ListObject{ 671 Key: dobj.Key, 672 ModTime: dobj.ModTime, 673 Size: dobj.Size, 674 MD5: dobj.MD5, 675 IsDir: dobj.IsDir, 676 asFunc: dobj.AsFunc, 677 }) 678 } 679 // ListPaged may return fewer results than pageSize. If there are more results 680 // available, signalled by non-empty p.NextPageToken, try to fetch the remainder 681 // of the page. 682 // It does not work to ask for more results than we need, because then we'd have 683 // a NextPageToken on a non-page boundary. 684 dopts.PageSize = pageSize - len(retval) 685 dopts.PageToken = p.NextPageToken 686 if len(dopts.PageToken) == 0 { 687 dopts.PageToken = nil 688 break 689 } 690 } 691 return retval, dopts.PageToken, nil 692 } 693 694 // IsAccessible returns true if the bucket is accessible, false otherwise. 695 // It is a shortcut for calling ListPage and checking if it returns an error 696 // with code gcerrors.NotFound. 697 func (b *Bucket) IsAccessible(ctx context.Context) (bool, error) { 698 _, _, err := b.ListPage(ctx, FirstPageToken, 1, nil) 699 if err == nil { 700 return true, nil 701 } 702 if gcerrors.Code(err) == gcerrors.NotFound { 703 return false, nil 704 } 705 return false, err 706 } 707 708 // Exists returns true if a blob exists at key, false if it does not exist, or 709 // an error. 710 // It is a shortcut for calling Attributes and checking if it returns an error 711 // with code gcerrors.NotFound. 712 func (b *Bucket) Exists(ctx context.Context, key string) (bool, error) { 713 _, err := b.Attributes(ctx, key) 714 if err == nil { 715 return true, nil 716 } 717 if gcerrors.Code(err) == gcerrors.NotFound { 718 return false, nil 719 } 720 return false, err 721 } 722 723 // Attributes returns attributes for the blob stored at key. 724 // 725 // If the blob does not exist, Attributes returns an error for which 726 // gcerrors.Code will return gcerrors.NotFound. 727 func (b *Bucket) Attributes(ctx context.Context, key string) (_ *Attributes, err error) { 728 if !utf8.ValidString(key) { 729 return nil, gcerr.Newf(gcerr.InvalidArgument, nil, "blob: Attributes key must be a valid UTF-8 string: %q", key) 730 } 731 732 b.mu.RLock() 733 defer b.mu.RUnlock() 734 if b.closed { 735 return nil, errClosed 736 } 737 ctx = b.tracer.Start(ctx, "Attributes") 738 defer func() { b.tracer.End(ctx, err) }() 739 740 a, err := b.b.Attributes(ctx, key) 741 if err != nil { 742 return nil, wrapError(b.b, err, key) 743 } 744 var md map[string]string 745 if len(a.Metadata) > 0 { 746 // Services are inconsistent, but at least some treat keys 747 // as case-insensitive. To make the behavior consistent, we 748 // force-lowercase them when writing and reading. 749 md = make(map[string]string, len(a.Metadata)) 750 for k, v := range a.Metadata { 751 md[strings.ToLower(k)] = v 752 } 753 } 754 return &Attributes{ 755 CacheControl: a.CacheControl, 756 ContentDisposition: a.ContentDisposition, 757 ContentEncoding: a.ContentEncoding, 758 ContentLanguage: a.ContentLanguage, 759 ContentType: a.ContentType, 760 Metadata: md, 761 CreateTime: a.CreateTime, 762 ModTime: a.ModTime, 763 Size: a.Size, 764 MD5: a.MD5, 765 ETag: a.ETag, 766 asFunc: a.AsFunc, 767 }, nil 768 } 769 770 // NewReader is a shortcut for NewRangeReader with offset=0 and length=-1. 771 func (b *Bucket) NewReader(ctx context.Context, key string, opts *ReaderOptions) (*Reader, error) { 772 return b.newRangeReader(ctx, key, 0, -1, opts) 773 } 774 775 // NewRangeReader returns a Reader to read content from the blob stored at key. 776 // It reads at most length bytes starting at offset (>= 0). 777 // If length is negative, it will read till the end of the blob. 778 // 779 // If the blob does not exist, NewRangeReader returns an error for which 780 // gcerrors.Code will return gcerrors.NotFound. Exists is a lighter-weight way 781 // to check for existence. 782 // 783 // A nil ReaderOptions is treated the same as the zero value. 784 // 785 // The caller must call Close on the returned Reader when done reading. 786 func (b *Bucket) NewRangeReader(ctx context.Context, key string, offset, length int64, opts *ReaderOptions) (_ *Reader, err error) { 787 return b.newRangeReader(ctx, key, offset, length, opts) 788 } 789 790 func (b *Bucket) newRangeReader(ctx context.Context, key string, offset, length int64, opts *ReaderOptions) (_ *Reader, err error) { 791 b.mu.RLock() 792 defer b.mu.RUnlock() 793 if b.closed { 794 return nil, errClosed 795 } 796 if offset < 0 { 797 return nil, gcerr.Newf(gcerr.InvalidArgument, nil, "blob: NewRangeReader offset must be non-negative (%d)", offset) 798 } 799 if !utf8.ValidString(key) { 800 return nil, gcerr.Newf(gcerr.InvalidArgument, nil, "blob: NewRangeReader key must be a valid UTF-8 string: %q", key) 801 } 802 if opts == nil { 803 opts = &ReaderOptions{} 804 } 805 dopts := &driver.ReaderOptions{ 806 BeforeRead: opts.BeforeRead, 807 } 808 tctx := b.tracer.Start(ctx, "NewRangeReader") 809 defer func() { 810 // If err == nil, we handed the end closure off to the returned *Writer; it 811 // will be called when the Writer is Closed. 812 if err != nil { 813 b.tracer.End(tctx, err) 814 } 815 }() 816 dr, err := b.b.NewRangeReader(ctx, key, offset, length, dopts) 817 if err != nil { 818 return nil, wrapError(b.b, err, key) 819 } 820 end := func(err error) { b.tracer.End(tctx, err) } 821 r := &Reader{ 822 b: b.b, 823 r: dr, 824 key: key, 825 end: end, 826 statsTagMutators: []tag.Mutator{tag.Upsert(oc.ProviderKey, b.tracer.Provider)}, 827 } 828 _, file, lineno, ok := runtime.Caller(2) 829 runtime.SetFinalizer(r, func(r *Reader) { 830 if !r.closed { 831 var caller string 832 if ok { 833 caller = fmt.Sprintf(" (%s:%d)", file, lineno) 834 } 835 log.Printf("A blob.Reader reading from %q was never closed%s", key, caller) 836 } 837 }) 838 return r, nil 839 } 840 841 // WriteAll is a shortcut for creating a Writer via NewWriter and writing p. 842 // 843 // If opts.ContentMD5 is not set, WriteAll will compute the MD5 of p and use it 844 // as the ContentMD5 option for the Writer it creates. 845 func (b *Bucket) WriteAll(ctx context.Context, key string, p []byte, opts *WriterOptions) (err error) { 846 realOpts := new(WriterOptions) 847 if opts != nil { 848 *realOpts = *opts 849 } 850 if len(realOpts.ContentMD5) == 0 { 851 sum := md5.Sum(p) 852 realOpts.ContentMD5 = sum[:] 853 } 854 w, err := b.NewWriter(ctx, key, realOpts) 855 if err != nil { 856 return err 857 } 858 if _, err := w.Write(p); err != nil { 859 _ = w.Close() 860 return err 861 } 862 return w.Close() 863 } 864 865 // NewWriter returns a Writer that writes to the blob stored at key. 866 // A nil WriterOptions is treated the same as the zero value. 867 // 868 // If a blob with this key already exists, it will be replaced. 869 // The blob being written is not guaranteed to be readable until Close 870 // has been called; until then, any previous blob will still be readable. 871 // Even after Close is called, newly written blobs are not guaranteed to be 872 // returned from List; some services are only eventually consistent. 873 // 874 // The returned Writer will store ctx for later use in Write and/or Close. 875 // To abort a write, cancel ctx; otherwise, it must remain open until 876 // Close is called. 877 // 878 // The caller must call Close on the returned Writer, even if the write is 879 // aborted. 880 func (b *Bucket) NewWriter(ctx context.Context, key string, opts *WriterOptions) (_ *Writer, err error) { 881 if !utf8.ValidString(key) { 882 return nil, gcerr.Newf(gcerr.InvalidArgument, nil, "blob: NewWriter key must be a valid UTF-8 string: %q", key) 883 } 884 if opts == nil { 885 opts = &WriterOptions{} 886 } 887 dopts := &driver.WriterOptions{ 888 CacheControl: opts.CacheControl, 889 ContentDisposition: opts.ContentDisposition, 890 ContentEncoding: opts.ContentEncoding, 891 ContentLanguage: opts.ContentLanguage, 892 ContentMD5: opts.ContentMD5, 893 BufferSize: opts.BufferSize, 894 BeforeWrite: opts.BeforeWrite, 895 } 896 if len(opts.Metadata) > 0 { 897 // Services are inconsistent, but at least some treat keys 898 // as case-insensitive. To make the behavior consistent, we 899 // force-lowercase them when writing and reading. 900 md := make(map[string]string, len(opts.Metadata)) 901 for k, v := range opts.Metadata { 902 if k == "" { 903 return nil, gcerr.Newf(gcerr.InvalidArgument, nil, "blob: WriterOptions.Metadata keys may not be empty strings") 904 } 905 if !utf8.ValidString(k) { 906 return nil, gcerr.Newf(gcerr.InvalidArgument, nil, "blob: WriterOptions.Metadata keys must be valid UTF-8 strings: %q", k) 907 } 908 if !utf8.ValidString(v) { 909 return nil, gcerr.Newf(gcerr.InvalidArgument, nil, "blob: WriterOptions.Metadata values must be valid UTF-8 strings: %q", v) 910 } 911 lowerK := strings.ToLower(k) 912 if _, found := md[lowerK]; found { 913 return nil, gcerr.Newf(gcerr.InvalidArgument, nil, "blob: WriterOptions.Metadata has a duplicate case-insensitive metadata key: %q", lowerK) 914 } 915 md[lowerK] = v 916 } 917 dopts.Metadata = md 918 } 919 b.mu.RLock() 920 defer b.mu.RUnlock() 921 if b.closed { 922 return nil, errClosed 923 } 924 ctx, cancel := context.WithCancel(ctx) 925 tctx := b.tracer.Start(ctx, "NewWriter") 926 end := func(err error) { b.tracer.End(tctx, err) } 927 defer func() { 928 if err != nil { 929 end(err) 930 } 931 }() 932 933 w := &Writer{ 934 b: b.b, 935 end: end, 936 cancel: cancel, 937 key: key, 938 contentMD5: opts.ContentMD5, 939 md5hash: md5.New(), 940 statsTagMutators: []tag.Mutator{tag.Upsert(oc.ProviderKey, b.tracer.Provider)}, 941 } 942 if opts.ContentType != "" { 943 t, p, err := mime.ParseMediaType(opts.ContentType) 944 if err != nil { 945 cancel() 946 return nil, err 947 } 948 ct := mime.FormatMediaType(t, p) 949 dw, err := b.b.NewTypedWriter(ctx, key, ct, dopts) 950 if err != nil { 951 cancel() 952 return nil, wrapError(b.b, err, key) 953 } 954 w.w = dw 955 } else { 956 // Save the fields needed to called NewTypedWriter later, once we've gotten 957 // sniffLen bytes; see the comment on Writer. 958 w.ctx = ctx 959 w.opts = dopts 960 w.buf = bytes.NewBuffer([]byte{}) 961 } 962 _, file, lineno, ok := runtime.Caller(1) 963 runtime.SetFinalizer(w, func(w *Writer) { 964 if !w.closed { 965 var caller string 966 if ok { 967 caller = fmt.Sprintf(" (%s:%d)", file, lineno) 968 } 969 log.Printf("A blob.Writer writing to %q was never closed%s", key, caller) 970 } 971 }) 972 return w, nil 973 } 974 975 // Copy the blob stored at srcKey to dstKey. 976 // A nil CopyOptions is treated the same as the zero value. 977 // 978 // If the source blob does not exist, Copy returns an error for which 979 // gcerrors.Code will return gcerrors.NotFound. 980 // 981 // If the destination blob already exists, it is overwritten. 982 func (b *Bucket) Copy(ctx context.Context, dstKey, srcKey string, opts *CopyOptions) (err error) { 983 if !utf8.ValidString(srcKey) { 984 return gcerr.Newf(gcerr.InvalidArgument, nil, "blob: Copy srcKey must be a valid UTF-8 string: %q", srcKey) 985 } 986 if !utf8.ValidString(dstKey) { 987 return gcerr.Newf(gcerr.InvalidArgument, nil, "blob: Copy dstKey must be a valid UTF-8 string: %q", dstKey) 988 } 989 if opts == nil { 990 opts = &CopyOptions{} 991 } 992 dopts := &driver.CopyOptions{ 993 BeforeCopy: opts.BeforeCopy, 994 } 995 b.mu.RLock() 996 defer b.mu.RUnlock() 997 if b.closed { 998 return errClosed 999 } 1000 ctx = b.tracer.Start(ctx, "Copy") 1001 defer func() { b.tracer.End(ctx, err) }() 1002 return wrapError(b.b, b.b.Copy(ctx, dstKey, srcKey, dopts), fmt.Sprintf("%s -> %s", srcKey, dstKey)) 1003 } 1004 1005 // Delete deletes the blob stored at key. 1006 // 1007 // If the blob does not exist, Delete returns an error for which 1008 // gcerrors.Code will return gcerrors.NotFound. 1009 func (b *Bucket) Delete(ctx context.Context, key string) (err error) { 1010 if !utf8.ValidString(key) { 1011 return gcerr.Newf(gcerr.InvalidArgument, nil, "blob: Delete key must be a valid UTF-8 string: %q", key) 1012 } 1013 b.mu.RLock() 1014 defer b.mu.RUnlock() 1015 if b.closed { 1016 return errClosed 1017 } 1018 ctx = b.tracer.Start(ctx, "Delete") 1019 defer func() { b.tracer.End(ctx, err) }() 1020 return wrapError(b.b, b.b.Delete(ctx, key), key) 1021 } 1022 1023 // SignedURL returns a URL that can be used to GET (default), PUT or DELETE 1024 // the blob for the duration specified in opts.Expiry. 1025 // 1026 // A nil SignedURLOptions is treated the same as the zero value. 1027 // 1028 // It is valid to call SignedURL for a key that does not exist. 1029 // 1030 // If the driver does not support this functionality, SignedURL 1031 // will return an error for which gcerrors.Code will return gcerrors.Unimplemented. 1032 func (b *Bucket) SignedURL(ctx context.Context, key string, opts *SignedURLOptions) (string, error) { 1033 if !utf8.ValidString(key) { 1034 return "", gcerr.Newf(gcerr.InvalidArgument, nil, "blob: SignedURL key must be a valid UTF-8 string: %q", key) 1035 } 1036 dopts := new(driver.SignedURLOptions) 1037 if opts == nil { 1038 opts = new(SignedURLOptions) 1039 } 1040 switch { 1041 case opts.Expiry < 0: 1042 return "", gcerr.Newf(gcerr.InvalidArgument, nil, "blob: SignedURLOptions.Expiry must be >= 0 (%v)", opts.Expiry) 1043 case opts.Expiry == 0: 1044 dopts.Expiry = DefaultSignedURLExpiry 1045 default: 1046 dopts.Expiry = opts.Expiry 1047 } 1048 switch opts.Method { 1049 case "": 1050 dopts.Method = http.MethodGet 1051 case http.MethodGet, http.MethodPut, http.MethodDelete: 1052 dopts.Method = opts.Method 1053 default: 1054 return "", fmt.Errorf("blob: unsupported SignedURLOptions.Method %q", opts.Method) 1055 } 1056 if opts.ContentType != "" && opts.Method != http.MethodPut { 1057 return "", fmt.Errorf("blob: SignedURLOptions.ContentType must be empty for signing a %s URL", opts.Method) 1058 } 1059 if opts.EnforceAbsentContentType && opts.Method != http.MethodPut { 1060 return "", fmt.Errorf("blob: SignedURLOptions.EnforceAbsentContentType must be false for signing a %s URL", opts.Method) 1061 } 1062 dopts.ContentType = opts.ContentType 1063 dopts.EnforceAbsentContentType = opts.EnforceAbsentContentType 1064 dopts.BeforeSign = opts.BeforeSign 1065 b.mu.RLock() 1066 defer b.mu.RUnlock() 1067 if b.closed { 1068 return "", errClosed 1069 } 1070 url, err := b.b.SignedURL(ctx, key, dopts) 1071 return url, wrapError(b.b, err, key) 1072 } 1073 1074 // Close releases any resources used for the bucket. 1075 func (b *Bucket) Close() error { 1076 b.mu.Lock() 1077 prev := b.closed 1078 b.closed = true 1079 b.mu.Unlock() 1080 if prev { 1081 return errClosed 1082 } 1083 return wrapError(b.b, b.b.Close(), "") 1084 } 1085 1086 // DefaultSignedURLExpiry is the default duration for SignedURLOptions.Expiry. 1087 const DefaultSignedURLExpiry = 1 * time.Hour 1088 1089 // SignedURLOptions sets options for SignedURL. 1090 type SignedURLOptions struct { 1091 // Expiry sets how long the returned URL is valid for. 1092 // Defaults to DefaultSignedURLExpiry. 1093 Expiry time.Duration 1094 1095 // Method is the HTTP method that can be used on the URL; one of "GET", "PUT", 1096 // or "DELETE". Defaults to "GET". 1097 Method string 1098 1099 // ContentType specifies the Content-Type HTTP header the user agent is 1100 // permitted to use in the PUT request. It must match exactly. See 1101 // EnforceAbsentContentType for behavior when ContentType is the empty string. 1102 // If a bucket does not implement this verification, then it returns an 1103 // Unimplemented error. 1104 // 1105 // Must be empty for non-PUT requests. 1106 ContentType string 1107 1108 // If EnforceAbsentContentType is true and ContentType is the empty string, 1109 // then PUTing to the signed URL will fail if the Content-Type header is 1110 // present. Not all buckets support this: ones that do not will return an 1111 // Unimplemented error. 1112 // 1113 // If EnforceAbsentContentType is false and ContentType is the empty string, 1114 // then PUTing without a Content-Type header will succeed, but it is 1115 // implementation-specific whether providing a Content-Type header will fail. 1116 // 1117 // Must be false for non-PUT requests. 1118 EnforceAbsentContentType bool 1119 1120 // BeforeSign is a callback that will be called before each call to the 1121 // the underlying service's sign functionality. 1122 // asFunc converts its argument to driver-specific types. 1123 // See https://gocloud.dev/concepts/as/ for background information. 1124 BeforeSign func(asFunc func(interface{}) bool) error 1125 } 1126 1127 // ReaderOptions sets options for NewReader and NewRangeReader. 1128 type ReaderOptions struct { 1129 // BeforeRead is a callback that will be called exactly once, before 1130 // any data is read (unless NewReader returns an error before then, in which 1131 // case it may not be called at all). 1132 // 1133 // asFunc converts its argument to driver-specific types. 1134 // See https://gocloud.dev/concepts/as/ for background information. 1135 BeforeRead func(asFunc func(interface{}) bool) error 1136 } 1137 1138 // WriterOptions sets options for NewWriter. 1139 type WriterOptions struct { 1140 // BufferSize changes the default size in bytes of the chunks that 1141 // Writer will upload in a single request; larger blobs will be split into 1142 // multiple requests. 1143 // 1144 // This option may be ignored by some drivers. 1145 // 1146 // If 0, the driver will choose a reasonable default. 1147 // 1148 // If the Writer is used to do many small writes concurrently, using a 1149 // smaller BufferSize may reduce memory usage. 1150 BufferSize int 1151 1152 // CacheControl specifies caching attributes that services may use 1153 // when serving the blob. 1154 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control 1155 CacheControl string 1156 1157 // ContentDisposition specifies whether the blob content is expected to be 1158 // displayed inline or as an attachment. 1159 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition 1160 ContentDisposition string 1161 1162 // ContentEncoding specifies the encoding used for the blob's content, if any. 1163 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding 1164 ContentEncoding string 1165 1166 // ContentLanguage specifies the language used in the blob's content, if any. 1167 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Language 1168 ContentLanguage string 1169 1170 // ContentType specifies the MIME type of the blob being written. If not set, 1171 // it will be inferred from the content using the algorithm described at 1172 // http://mimesniff.spec.whatwg.org/. 1173 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type 1174 ContentType string 1175 1176 // ContentMD5 is used as a message integrity check. 1177 // If len(ContentMD5) > 0, the MD5 hash of the bytes written must match 1178 // ContentMD5, or Close will return an error without completing the write. 1179 // https://tools.ietf.org/html/rfc1864 1180 ContentMD5 []byte 1181 1182 // Metadata holds key/value strings to be associated with the blob, or nil. 1183 // Keys may not be empty, and are lowercased before being written. 1184 // Duplicate case-insensitive keys (e.g., "foo" and "FOO") will result in 1185 // an error. 1186 Metadata map[string]string 1187 1188 // BeforeWrite is a callback that will be called exactly once, before 1189 // any data is written (unless NewWriter returns an error, in which case 1190 // it will not be called at all). Note that this is not necessarily during 1191 // or after the first Write call, as drivers may buffer bytes before 1192 // sending an upload request. 1193 // 1194 // asFunc converts its argument to driver-specific types. 1195 // See https://gocloud.dev/concepts/as/ for background information. 1196 BeforeWrite func(asFunc func(interface{}) bool) error 1197 } 1198 1199 // CopyOptions sets options for Copy. 1200 type CopyOptions struct { 1201 // BeforeCopy is a callback that will be called before the copy is 1202 // initiated. 1203 // 1204 // asFunc converts its argument to driver-specific types. 1205 // See https://gocloud.dev/concepts/as/ for background information. 1206 BeforeCopy func(asFunc func(interface{}) bool) error 1207 } 1208 1209 // BucketURLOpener represents types that can open buckets based on a URL. 1210 // The opener must not modify the URL argument. OpenBucketURL must be safe to 1211 // call from multiple goroutines. 1212 // 1213 // This interface is generally implemented by types in driver packages. 1214 type BucketURLOpener interface { 1215 OpenBucketURL(ctx context.Context, u *url.URL) (*Bucket, error) 1216 } 1217 1218 // URLMux is a URL opener multiplexer. It matches the scheme of the URLs 1219 // against a set of registered schemes and calls the opener that matches the 1220 // URL's scheme. 1221 // See https://gocloud.dev/concepts/urls/ for more information. 1222 // 1223 // The zero value is a multiplexer with no registered schemes. 1224 type URLMux struct { 1225 schemes openurl.SchemeMap 1226 } 1227 1228 // BucketSchemes returns a sorted slice of the registered Bucket schemes. 1229 func (mux *URLMux) BucketSchemes() []string { return mux.schemes.Schemes() } 1230 1231 // ValidBucketScheme returns true iff scheme has been registered for Buckets. 1232 func (mux *URLMux) ValidBucketScheme(scheme string) bool { return mux.schemes.ValidScheme(scheme) } 1233 1234 // RegisterBucket registers the opener with the given scheme. If an opener 1235 // already exists for the scheme, RegisterBucket panics. 1236 func (mux *URLMux) RegisterBucket(scheme string, opener BucketURLOpener) { 1237 mux.schemes.Register("blob", "Bucket", scheme, opener) 1238 } 1239 1240 // OpenBucket calls OpenBucketURL with the URL parsed from urlstr. 1241 // OpenBucket is safe to call from multiple goroutines. 1242 func (mux *URLMux) OpenBucket(ctx context.Context, urlstr string) (*Bucket, error) { 1243 opener, u, err := mux.schemes.FromString("Bucket", urlstr) 1244 if err != nil { 1245 return nil, err 1246 } 1247 return applyPrefixParam(ctx, opener.(BucketURLOpener), u) 1248 } 1249 1250 // OpenBucketURL dispatches the URL to the opener that is registered with the 1251 // URL's scheme. OpenBucketURL is safe to call from multiple goroutines. 1252 func (mux *URLMux) OpenBucketURL(ctx context.Context, u *url.URL) (*Bucket, error) { 1253 opener, err := mux.schemes.FromURL("Bucket", u) 1254 if err != nil { 1255 return nil, err 1256 } 1257 return applyPrefixParam(ctx, opener.(BucketURLOpener), u) 1258 } 1259 1260 func applyPrefixParam(ctx context.Context, opener BucketURLOpener, u *url.URL) (*Bucket, error) { 1261 prefix := u.Query().Get("prefix") 1262 if prefix != "" { 1263 // Make a copy of u with the "prefix" parameter removed. 1264 urlCopy := *u 1265 q := urlCopy.Query() 1266 q.Del("prefix") 1267 urlCopy.RawQuery = q.Encode() 1268 u = &urlCopy 1269 } 1270 bucket, err := opener.OpenBucketURL(ctx, u) 1271 if err != nil { 1272 return nil, err 1273 } 1274 if prefix != "" { 1275 bucket = PrefixedBucket(bucket, prefix) 1276 } 1277 return bucket, nil 1278 } 1279 1280 var defaultURLMux = new(URLMux) 1281 1282 // DefaultURLMux returns the URLMux used by OpenBucket. 1283 // 1284 // Driver packages can use this to register their BucketURLOpener on the mux. 1285 func DefaultURLMux() *URLMux { 1286 return defaultURLMux 1287 } 1288 1289 // OpenBucket opens the bucket identified by the URL given. 1290 // 1291 // See the URLOpener documentation in driver subpackages for 1292 // details on supported URL formats, and https://gocloud.dev/concepts/urls/ 1293 // for more information. 1294 // 1295 // In addition to driver-specific query parameters, OpenBucket supports 1296 // the following query parameters: 1297 // 1298 // - prefix: wraps the resulting Bucket using PrefixedBucket with the 1299 // given prefix. 1300 func OpenBucket(ctx context.Context, urlstr string) (*Bucket, error) { 1301 return defaultURLMux.OpenBucket(ctx, urlstr) 1302 } 1303 1304 func wrapError(b driver.Bucket, err error, key string) error { 1305 if err == nil { 1306 return nil 1307 } 1308 if gcerr.DoNotWrap(err) { 1309 return err 1310 } 1311 msg := "blob" 1312 if key != "" { 1313 msg += fmt.Sprintf(" (key %q)", key) 1314 } 1315 code := gcerrors.Code(err) 1316 if code == gcerrors.Unknown { 1317 code = b.ErrorCode(err) 1318 } 1319 return gcerr.New(code, err, 2, msg) 1320 } 1321 1322 var errClosed = gcerr.Newf(gcerr.FailedPrecondition, nil, "blob: Bucket has been closed") 1323 1324 // PrefixedBucket returns a *Bucket based on b with all keys modified to have 1325 // prefix, which will usually end with a "/" to target a subdirectory in the 1326 // bucket. 1327 // 1328 // bucket will be closed and no longer usable after this function returns. 1329 func PrefixedBucket(bucket *Bucket, prefix string) *Bucket { 1330 bucket.mu.Lock() 1331 defer bucket.mu.Unlock() 1332 bucket.closed = true 1333 return NewBucket(driver.NewPrefixedBucket(bucket.b, prefix)) 1334 }