github.com/thiagoyeds/go-cloud@v0.26.0/blob/example_test.go (about) 1 // Copyright 2018 The Go Cloud Development Kit Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // https://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package blob_test 16 17 import ( 18 "context" 19 "fmt" 20 "io" 21 "io/ioutil" 22 "log" 23 "os" 24 25 "cloud.google.com/go/storage" 26 "github.com/aws/aws-sdk-go/aws/awserr" 27 "gocloud.dev/blob" 28 "gocloud.dev/blob/fileblob" 29 _ "gocloud.dev/blob/gcsblob" 30 _ "gocloud.dev/blob/s3blob" 31 ) 32 33 func ExampleBucket_NewReader() { 34 // PRAGMA: This example is used on gocloud.dev; PRAGMA comments adjust how it is shown and can be ignored. 35 // PRAGMA: On gocloud.dev, hide lines until the next blank line. 36 ctx := context.Background() 37 var bucket *blob.Bucket 38 39 // Open the key "foo.txt" for reading with the default options. 40 r, err := bucket.NewReader(ctx, "foo.txt", nil) 41 if err != nil { 42 log.Fatal(err) 43 } 44 defer r.Close() 45 // Readers also have a limited view of the blob's metadata. 46 fmt.Println("Content-Type:", r.ContentType()) 47 fmt.Println() 48 // Copy from the reader to stdout. 49 if _, err := io.Copy(os.Stdout, r); err != nil { 50 log.Fatal(err) 51 } 52 } 53 54 func ExampleBucket_NewRangeReader() { 55 // PRAGMA: This example is used on gocloud.dev; PRAGMA comments adjust how it is shown and can be ignored. 56 // PRAGMA: On gocloud.dev, hide lines until the next blank line. 57 ctx := context.Background() 58 var bucket *blob.Bucket 59 60 // Open the key "foo.txt" for reading at offset 1024 and read up to 4096 bytes. 61 r, err := bucket.NewRangeReader(ctx, "foo.txt", 1024, 4096, nil) 62 if err != nil { 63 log.Fatal(err) 64 } 65 defer r.Close() 66 // Copy from the read range to stdout. 67 if _, err := io.Copy(os.Stdout, r); err != nil { 68 log.Fatal(err) 69 } 70 } 71 72 func ExampleBucket_NewWriter() { 73 // PRAGMA: This example is used on gocloud.dev; PRAGMA comments adjust how it is shown and can be ignored. 74 // PRAGMA: On gocloud.dev, hide lines until the next blank line. 75 ctx := context.Background() 76 var bucket *blob.Bucket 77 78 // Open the key "foo.txt" for writing with the default options. 79 w, err := bucket.NewWriter(ctx, "foo.txt", nil) 80 if err != nil { 81 log.Fatal(err) 82 } 83 _, writeErr := fmt.Fprintln(w, "Hello, World!") 84 // Always check the return value of Close when writing. 85 closeErr := w.Close() 86 if writeErr != nil { 87 log.Fatal(writeErr) 88 } 89 if closeErr != nil { 90 log.Fatal(closeErr) 91 } 92 } 93 94 func ExampleBucket_NewWriter_cancel() { 95 // PRAGMA: This example is used on gocloud.dev; PRAGMA comments adjust how it is shown and can be ignored. 96 // PRAGMA: On gocloud.dev, hide lines until the next blank line. 97 ctx := context.Background() 98 var bucket *blob.Bucket 99 100 // Create a cancelable context from the existing context. 101 writeCtx, cancelWrite := context.WithCancel(ctx) 102 defer cancelWrite() 103 104 // Open the key "foo.txt" for writing with the default options. 105 w, err := bucket.NewWriter(writeCtx, "foo.txt", nil) 106 if err != nil { 107 log.Fatal(err) 108 } 109 110 // Assume some writes happened and we encountered an error. 111 // Now we want to abort the write. 112 113 if err != nil { 114 // First cancel the context. 115 cancelWrite() 116 // You must still close the writer to avoid leaking resources. 117 w.Close() 118 } 119 } 120 121 func ExampleBucket_Delete() { 122 // PRAGMA: This example is used on gocloud.dev; PRAGMA comments adjust how it is shown and can be ignored. 123 // PRAGMA: On gocloud.dev, hide lines until the next blank line. 124 ctx := context.Background() 125 var bucket *blob.Bucket 126 127 if err := bucket.Delete(ctx, "foo.txt"); err != nil { 128 log.Fatal(err) 129 } 130 } 131 132 func Example() { 133 // Connect to a bucket when your program starts up. 134 // This example uses the file-based implementation in fileblob, and creates 135 // a temporary directory to use as the root directory. 136 dir, cleanup := newTempDir() 137 defer cleanup() 138 bucket, err := fileblob.OpenBucket(dir, nil) 139 if err != nil { 140 log.Fatal(err) 141 } 142 defer bucket.Close() 143 144 // We now have a *blob.Bucket! We can write our application using the 145 // *blob.Bucket type, and have the freedom to change the initialization code 146 // above to choose a different service-specific driver later. 147 148 // In this example, we'll write a blob and then read it. 149 ctx := context.Background() 150 if err := bucket.WriteAll(ctx, "foo.txt", []byte("Go Cloud Development Kit"), nil); err != nil { 151 log.Fatal(err) 152 } 153 b, err := bucket.ReadAll(ctx, "foo.txt") 154 if err != nil { 155 log.Fatal(err) 156 } 157 fmt.Println(string(b)) 158 159 // Output: 160 // Go Cloud Development Kit 161 } 162 163 func ExampleBucket_ErrorAs() { 164 // This example is specific to the s3blob implementation; it demonstrates 165 // access to the underlying awserr.Error type. 166 // The types exposed for ErrorAs by s3blob are documented in 167 // https://godoc.org/gocloud.dev/blob/s3blob#hdr-As 168 169 ctx := context.Background() 170 171 b, err := blob.OpenBucket(ctx, "s3://my-bucket") 172 if err != nil { 173 log.Fatal(err) 174 } 175 defer b.Close() 176 177 _, err = b.ReadAll(ctx, "nosuchfile") 178 if err != nil { 179 var awsErr awserr.Error 180 if b.ErrorAs(err, &awsErr) { 181 fmt.Println(awsErr.Code()) 182 } 183 } 184 } 185 186 func ExampleBucket_List() { 187 // Connect to a bucket when your program starts up. 188 // This example uses the file-based implementation. 189 dir, cleanup := newTempDir() 190 defer cleanup() 191 192 // Create the file-based bucket. 193 bucket, err := fileblob.OpenBucket(dir, nil) 194 if err != nil { 195 log.Fatal(err) 196 } 197 defer bucket.Close() 198 199 // Create some blob objects for listing: "foo[0..4].txt". 200 ctx := context.Background() 201 for i := 0; i < 5; i++ { 202 if err := bucket.WriteAll(ctx, fmt.Sprintf("foo%d.txt", i), []byte("Go Cloud Development Kit"), nil); err != nil { 203 log.Fatal(err) 204 } 205 } 206 207 // Iterate over them. 208 // This will list the blobs created above because fileblob is strongly 209 // consistent, but is not guaranteed to work on all services. 210 iter := bucket.List(nil) 211 for { 212 obj, err := iter.Next(ctx) 213 if err == io.EOF { 214 break 215 } 216 if err != nil { 217 log.Fatal(err) 218 } 219 fmt.Println(obj.Key) 220 } 221 222 // Output: 223 // foo0.txt 224 // foo1.txt 225 // foo2.txt 226 // foo3.txt 227 // foo4.txt 228 } 229 230 func ExampleBucket_List_withDelimiter() { 231 // Connect to a bucket when your program starts up. 232 // This example uses the file-based implementation. 233 dir, cleanup := newTempDir() 234 defer cleanup() 235 236 // Create the file-based bucket. 237 bucket, err := fileblob.OpenBucket(dir, nil) 238 if err != nil { 239 log.Fatal(err) 240 } 241 defer bucket.Close() 242 243 // Create some blob objects in a hierarchy. 244 ctx := context.Background() 245 for _, key := range []string{ 246 "dir1/subdir/a.txt", 247 "dir1/subdir/b.txt", 248 "dir2/c.txt", 249 "d.txt", 250 } { 251 if err := bucket.WriteAll(ctx, key, []byte("Go Cloud Development Kit"), nil); err != nil { 252 log.Fatal(err) 253 } 254 } 255 256 // list lists files in b starting with prefix. It uses the delimiter "/", 257 // and recurses into "directories", adding 2 spaces to indent each time. 258 // It will list the blobs created above because fileblob is strongly 259 // consistent, but is not guaranteed to work on all services. 260 var list func(context.Context, *blob.Bucket, string, string) 261 list = func(ctx context.Context, b *blob.Bucket, prefix, indent string) { 262 iter := b.List(&blob.ListOptions{ 263 Delimiter: "/", 264 Prefix: prefix, 265 }) 266 for { 267 obj, err := iter.Next(ctx) 268 if err == io.EOF { 269 break 270 } 271 if err != nil { 272 log.Fatal(err) 273 } 274 fmt.Printf("%s%s\n", indent, obj.Key) 275 if obj.IsDir { 276 list(ctx, b, obj.Key, indent+" ") 277 } 278 } 279 } 280 list(ctx, bucket, "", "") 281 282 // Output: 283 // d.txt 284 // dir1/ 285 // dir1/subdir/ 286 // dir1/subdir/a.txt 287 // dir1/subdir/b.txt 288 // dir2/ 289 // dir2/c.txt 290 } 291 292 func ExampleBucket_ListPage() { 293 // Connect to a bucket when your program starts up. 294 // This example uses the file-based implementation. 295 dir, cleanup := newTempDir() 296 defer cleanup() 297 298 // Create the file-based bucket. 299 bucket, err := fileblob.OpenBucket(dir, nil) 300 if err != nil { 301 log.Fatal(err) 302 } 303 defer bucket.Close() 304 305 // Create some blob objects for listing: "foo[0..7].txt". 306 ctx := context.Background() 307 for i := 0; i < 8; i++ { 308 if err := bucket.WriteAll(ctx, fmt.Sprintf("foo%d.txt", i), []byte("Go Cloud Development Kit"), nil); err != nil { 309 log.Fatal(err) 310 } 311 } 312 313 // Iterate over them in pages. 314 // This will list the blobs created above because fileblob is strongly 315 // consistent, but is not guaranteed to work on all services. 316 317 // The first page of 3 results. 318 objs, token, err := bucket.ListPage(ctx, blob.FirstPageToken, 3, nil) 319 if err != nil { 320 log.Fatal(err) 321 } 322 for _, obj := range objs { 323 fmt.Println(obj.Key) 324 } 325 fmt.Println("END OF PAGE 1") 326 327 // The second page of 3 results. 328 objs, token, err = bucket.ListPage(ctx, token, 3, nil) 329 if err != nil { 330 log.Fatal(err) 331 } 332 for _, obj := range objs { 333 fmt.Println(obj.Key) 334 } 335 fmt.Println("END OF PAGE 2") 336 337 // The third page with the last 2 results. 338 objs, token, err = bucket.ListPage(ctx, token, 3, nil) 339 if err != nil { 340 log.Fatal(err) 341 } 342 for _, obj := range objs { 343 fmt.Println(obj.Key) 344 } 345 fmt.Println("END OF PAGE 3") 346 347 // There are no more pages, so token is now nil. Calling ListPage again will return io.EOF. 348 if token != nil { 349 fmt.Println("Token was not nil.") 350 } 351 352 // Output: 353 // foo0.txt 354 // foo1.txt 355 // foo2.txt 356 // END OF PAGE 1 357 // foo3.txt 358 // foo4.txt 359 // foo5.txt 360 // END OF PAGE 2 361 // foo6.txt 362 // foo7.txt 363 // END OF PAGE 3 364 } 365 366 func ExampleBucket_As() { 367 // This example is specific to the gcsblob implementation; it demonstrates 368 // access to the underlying cloud.google.com/go/storage.Client type. 369 // The types exposed for As by gcsblob are documented in 370 // https://godoc.org/gocloud.dev/blob/gcsblob#hdr-As 371 372 // This URL will open the bucket "my-bucket" using default credentials. 373 ctx := context.Background() 374 b, err := blob.OpenBucket(ctx, "gs://my-bucket") 375 if err != nil { 376 log.Fatal(err) 377 } 378 defer b.Close() 379 380 // Access storage.Client fields via gcsClient here. 381 var gcsClient *storage.Client 382 if b.As(&gcsClient) { 383 email, err := gcsClient.ServiceAccount(ctx, "project-name") 384 if err != nil { 385 log.Fatal(err) 386 } 387 _ = email 388 } else { 389 log.Println("Unable to access storage.Client through Bucket.As") 390 } 391 } 392 393 func ExampleWriterOptions() { 394 // This example is specific to the gcsblob implementation; it demonstrates 395 // access to the underlying cloud.google.com/go/storage.Writer type. 396 // The types exposed for As by gcsblob are documented in 397 // https://godoc.org/gocloud.dev/blob/gcsblob#hdr-As 398 399 ctx := context.Background() 400 401 b, err := blob.OpenBucket(ctx, "gs://my-bucket") 402 if err != nil { 403 log.Fatal(err) 404 } 405 defer b.Close() 406 407 beforeWrite := func(as func(interface{}) bool) error { 408 var sw *storage.Writer 409 if as(&sw) { 410 fmt.Println(sw.ChunkSize) 411 } 412 return nil 413 } 414 415 options := blob.WriterOptions{BeforeWrite: beforeWrite} 416 if err := b.WriteAll(ctx, "newfile.txt", []byte("hello\n"), &options); err != nil { 417 log.Fatal(err) 418 } 419 } 420 421 func ExampleListObject_As() { 422 // This example is specific to the gcsblob implementation; it demonstrates 423 // access to the underlying cloud.google.com/go/storage.ObjectAttrs type. 424 // The types exposed for As by gcsblob are documented in 425 // https://godoc.org/gocloud.dev/blob/gcsblob#hdr-As 426 427 ctx := context.Background() 428 429 b, err := blob.OpenBucket(ctx, "gs://my-bucket") 430 if err != nil { 431 log.Fatal(err) 432 } 433 defer b.Close() 434 435 iter := b.List(nil) 436 for { 437 obj, err := iter.Next(ctx) 438 if err == io.EOF { 439 break 440 } 441 if err != nil { 442 log.Fatal(err) 443 } 444 // Access storage.ObjectAttrs via oa here. 445 var oa storage.ObjectAttrs 446 if obj.As(&oa) { 447 _ = oa.Owner 448 } 449 } 450 } 451 452 func ExampleListOptions() { 453 // This example is specific to the gcsblob implementation; it demonstrates 454 // access to the underlying cloud.google.com/go/storage.Query type. 455 // The types exposed for As by gcsblob are documented in 456 // https://godoc.org/gocloud.dev/blob/gcsblob#hdr-As 457 458 ctx := context.Background() 459 460 b, err := blob.OpenBucket(ctx, "gs://my-bucket") 461 if err != nil { 462 log.Fatal(err) 463 } 464 defer b.Close() 465 466 beforeList := func(as func(interface{}) bool) error { 467 // Access storage.Query via q here. 468 var q *storage.Query 469 if as(&q) { 470 _ = q.Delimiter 471 } 472 return nil 473 } 474 475 iter := b.List(&blob.ListOptions{Prefix: "", Delimiter: "/", BeforeList: beforeList}) 476 for { 477 obj, err := iter.Next(ctx) 478 if err == io.EOF { 479 break 480 } 481 if err != nil { 482 log.Fatal(err) 483 } 484 _ = obj 485 } 486 } 487 488 func ExamplePrefixedBucket() { 489 // PRAGMA: This example is used on gocloud.dev; PRAGMA comments adjust how it is shown and can be ignored. 490 // PRAGMA: On gocloud.dev, hide lines until the next blank line. 491 var bucket *blob.Bucket 492 493 // Wrap the bucket using blob.PrefixedBucket. 494 // The prefix should end with "/", so that the resulting bucket operates 495 // in a subfolder. 496 bucket = blob.PrefixedBucket(bucket, "a/subfolder/") 497 498 // The original bucket is no longer usable; it has been closed. 499 // The wrapped bucket should be closed when done. 500 defer bucket.Close() 501 502 // Bucket operations on <key> will be translated to "a/subfolder/<key>". 503 } 504 505 func ExampleReader_As() { 506 // This example is specific to the gcsblob implementation; it demonstrates 507 // access to the underlying cloud.google.com/go/storage.Reader type. 508 // The types exposed for As by gcsblob are documented in 509 // https://godoc.org/gocloud.dev/blob/gcsblob#hdr-As 510 511 ctx := context.Background() 512 513 b, err := blob.OpenBucket(ctx, "gs://my-bucket") 514 if err != nil { 515 log.Fatal(err) 516 } 517 defer b.Close() 518 519 r, err := b.NewReader(ctx, "gopher.png", nil) 520 if err != nil { 521 log.Fatal(err) 522 } 523 defer r.Close() 524 525 // Access storage.Reader via sr here. 526 var sr *storage.Reader 527 if r.As(&sr) { 528 _ = sr.Attrs 529 } 530 } 531 532 func ExampleAttributes_As() { 533 // This example is specific to the gcsblob implementation; it demonstrates 534 // access to the underlying cloud.google.com/go/storage.ObjectAttrs type. 535 // The types exposed for As by gcsblob are documented in 536 // https://godoc.org/gocloud.dev/blob/gcsblob#hdr-As 537 ctx := context.Background() 538 539 b, err := blob.OpenBucket(ctx, "gs://my-bucket") 540 if err != nil { 541 log.Fatal(err) 542 } 543 defer b.Close() 544 545 attrs, err := b.Attributes(ctx, "gopher.png") 546 if err != nil { 547 log.Fatal(err) 548 } 549 550 var oa storage.ObjectAttrs 551 if attrs.As(&oa) { 552 fmt.Println(oa.Owner) 553 } 554 } 555 556 func newTempDir() (string, func()) { 557 dir, err := ioutil.TempDir("", "go-cloud-blob-example") 558 if err != nil { 559 panic(err) 560 } 561 return dir, func() { os.RemoveAll(dir) } 562 }