github.com/oskarth/go-ethereum@v1.6.8-0.20191013093314-dac24a9d3494/swarm/api/client/client.go (about) 1 // Copyright 2017 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package client 18 19 import ( 20 "archive/tar" 21 "bytes" 22 "encoding/json" 23 "errors" 24 "fmt" 25 "io" 26 "io/ioutil" 27 "mime/multipart" 28 "net/http" 29 "net/textproto" 30 "net/url" 31 "os" 32 "path/filepath" 33 "regexp" 34 "strconv" 35 "strings" 36 37 "github.com/ethereum/go-ethereum/swarm/api" 38 "github.com/ethereum/go-ethereum/swarm/storage/feed" 39 ) 40 41 var ( 42 DefaultGateway = "http://localhost:8500" 43 DefaultClient = NewClient(DefaultGateway) 44 ) 45 46 var ( 47 ErrUnauthorized = errors.New("unauthorized") 48 ) 49 50 func NewClient(gateway string) *Client { 51 return &Client{ 52 Gateway: gateway, 53 } 54 } 55 56 // Client wraps interaction with a swarm HTTP gateway. 57 type Client struct { 58 Gateway string 59 } 60 61 // UploadRaw uploads raw data to swarm and returns the resulting hash. If toEncrypt is true it 62 // uploads encrypted data 63 func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool) (string, error) { 64 if size <= 0 { 65 return "", errors.New("data size must be greater than zero") 66 } 67 addr := "" 68 if toEncrypt { 69 addr = "encrypt" 70 } 71 req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/"+addr, r) 72 if err != nil { 73 return "", err 74 } 75 req.ContentLength = size 76 res, err := http.DefaultClient.Do(req) 77 if err != nil { 78 return "", err 79 } 80 defer res.Body.Close() 81 if res.StatusCode != http.StatusOK { 82 return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) 83 } 84 data, err := ioutil.ReadAll(res.Body) 85 if err != nil { 86 return "", err 87 } 88 return string(data), nil 89 } 90 91 // DownloadRaw downloads raw data from swarm and it returns a ReadCloser and a bool whether the 92 // content was encrypted 93 func (c *Client) DownloadRaw(hash string) (io.ReadCloser, bool, error) { 94 uri := c.Gateway + "/bzz-raw:/" + hash 95 res, err := http.DefaultClient.Get(uri) 96 if err != nil { 97 return nil, false, err 98 } 99 if res.StatusCode != http.StatusOK { 100 res.Body.Close() 101 return nil, false, fmt.Errorf("unexpected HTTP status: %s", res.Status) 102 } 103 isEncrypted := (res.Header.Get("X-Decrypted") == "true") 104 return res.Body, isEncrypted, nil 105 } 106 107 // File represents a file in a swarm manifest and is used for uploading and 108 // downloading content to and from swarm 109 type File struct { 110 io.ReadCloser 111 api.ManifestEntry 112 } 113 114 // Open opens a local file which can then be passed to client.Upload to upload 115 // it to swarm 116 func Open(path string) (*File, error) { 117 f, err := os.Open(path) 118 if err != nil { 119 return nil, err 120 } 121 stat, err := f.Stat() 122 if err != nil { 123 f.Close() 124 return nil, err 125 } 126 127 contentType, err := api.DetectContentType(f.Name(), f) 128 if err != nil { 129 return nil, err 130 } 131 132 return &File{ 133 ReadCloser: f, 134 ManifestEntry: api.ManifestEntry{ 135 ContentType: contentType, 136 Mode: int64(stat.Mode()), 137 Size: stat.Size(), 138 ModTime: stat.ModTime(), 139 }, 140 }, nil 141 } 142 143 // Upload uploads a file to swarm and either adds it to an existing manifest 144 // (if the manifest argument is non-empty) or creates a new manifest containing 145 // the file, returning the resulting manifest hash (the file will then be 146 // available at bzz:/<hash>/<path>) 147 func (c *Client) Upload(file *File, manifest string, toEncrypt bool) (string, error) { 148 if file.Size <= 0 { 149 return "", errors.New("file size must be greater than zero") 150 } 151 return c.TarUpload(manifest, &FileUploader{file}, "", toEncrypt) 152 } 153 154 // Download downloads a file with the given path from the swarm manifest with 155 // the given hash (i.e. it gets bzz:/<hash>/<path>) 156 func (c *Client) Download(hash, path string) (*File, error) { 157 uri := c.Gateway + "/bzz:/" + hash + "/" + path 158 res, err := http.DefaultClient.Get(uri) 159 if err != nil { 160 return nil, err 161 } 162 if res.StatusCode != http.StatusOK { 163 res.Body.Close() 164 return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) 165 } 166 return &File{ 167 ReadCloser: res.Body, 168 ManifestEntry: api.ManifestEntry{ 169 ContentType: res.Header.Get("Content-Type"), 170 Size: res.ContentLength, 171 }, 172 }, nil 173 } 174 175 // UploadDirectory uploads a directory tree to swarm and either adds the files 176 // to an existing manifest (if the manifest argument is non-empty) or creates a 177 // new manifest, returning the resulting manifest hash (files from the 178 // directory will then be available at bzz:/<hash>/path/to/file), with 179 // the file specified in defaultPath being uploaded to the root of the manifest 180 // (i.e. bzz:/<hash>/) 181 func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bool) (string, error) { 182 stat, err := os.Stat(dir) 183 if err != nil { 184 return "", err 185 } else if !stat.IsDir() { 186 return "", fmt.Errorf("not a directory: %s", dir) 187 } 188 if defaultPath != "" { 189 if _, err := os.Stat(filepath.Join(dir, defaultPath)); err != nil { 190 if os.IsNotExist(err) { 191 return "", fmt.Errorf("the default path %q was not found in the upload directory %q", defaultPath, dir) 192 } 193 return "", fmt.Errorf("default path: %v", err) 194 } 195 } 196 return c.TarUpload(manifest, &DirectoryUploader{dir}, defaultPath, toEncrypt) 197 } 198 199 // DownloadDirectory downloads the files contained in a swarm manifest under 200 // the given path into a local directory (existing files will be overwritten) 201 func (c *Client) DownloadDirectory(hash, path, destDir, credentials string) error { 202 stat, err := os.Stat(destDir) 203 if err != nil { 204 return err 205 } else if !stat.IsDir() { 206 return fmt.Errorf("not a directory: %s", destDir) 207 } 208 209 uri := c.Gateway + "/bzz:/" + hash + "/" + path 210 req, err := http.NewRequest("GET", uri, nil) 211 if err != nil { 212 return err 213 } 214 if credentials != "" { 215 req.SetBasicAuth("", credentials) 216 } 217 req.Header.Set("Accept", "application/x-tar") 218 res, err := http.DefaultClient.Do(req) 219 if err != nil { 220 return err 221 } 222 defer res.Body.Close() 223 switch res.StatusCode { 224 case http.StatusOK: 225 case http.StatusUnauthorized: 226 return ErrUnauthorized 227 default: 228 return fmt.Errorf("unexpected HTTP status: %s", res.Status) 229 } 230 tr := tar.NewReader(res.Body) 231 for { 232 hdr, err := tr.Next() 233 if err == io.EOF { 234 return nil 235 } else if err != nil { 236 return err 237 } 238 // ignore the default path file 239 if hdr.Name == "" { 240 continue 241 } 242 243 dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path))) 244 if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { 245 return err 246 } 247 var mode os.FileMode = 0644 248 if hdr.Mode > 0 { 249 mode = os.FileMode(hdr.Mode) 250 } 251 dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) 252 if err != nil { 253 return err 254 } 255 n, err := io.Copy(dst, tr) 256 dst.Close() 257 if err != nil { 258 return err 259 } else if n != hdr.Size { 260 return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n) 261 } 262 } 263 } 264 265 // DownloadFile downloads a single file into the destination directory 266 // if the manifest entry does not specify a file name - it will fallback 267 // to the hash of the file as a filename 268 func (c *Client) DownloadFile(hash, path, dest, credentials string) error { 269 hasDestinationFilename := false 270 if stat, err := os.Stat(dest); err == nil { 271 hasDestinationFilename = !stat.IsDir() 272 } else { 273 if os.IsNotExist(err) { 274 // does not exist - should be created 275 hasDestinationFilename = true 276 } else { 277 return fmt.Errorf("could not stat path: %v", err) 278 } 279 } 280 281 manifestList, err := c.List(hash, path, credentials) 282 if err != nil { 283 return err 284 } 285 286 switch len(manifestList.Entries) { 287 case 0: 288 return fmt.Errorf("could not find path requested at manifest address. make sure the path you've specified is correct") 289 case 1: 290 //continue 291 default: 292 return fmt.Errorf("got too many matches for this path") 293 } 294 295 uri := c.Gateway + "/bzz:/" + hash + "/" + path 296 req, err := http.NewRequest("GET", uri, nil) 297 if err != nil { 298 return err 299 } 300 if credentials != "" { 301 req.SetBasicAuth("", credentials) 302 } 303 res, err := http.DefaultClient.Do(req) 304 if err != nil { 305 return err 306 } 307 defer res.Body.Close() 308 switch res.StatusCode { 309 case http.StatusOK: 310 case http.StatusUnauthorized: 311 return ErrUnauthorized 312 default: 313 return fmt.Errorf("unexpected HTTP status: expected 200 OK, got %d", res.StatusCode) 314 } 315 filename := "" 316 if hasDestinationFilename { 317 filename = dest 318 } else { 319 // try to assert 320 re := regexp.MustCompile("[^/]+$") //everything after last slash 321 322 if results := re.FindAllString(path, -1); len(results) > 0 { 323 filename = results[len(results)-1] 324 } else { 325 if entry := manifestList.Entries[0]; entry.Path != "" && entry.Path != "/" { 326 filename = entry.Path 327 } else { 328 // assume hash as name if there's nothing from the command line 329 filename = hash 330 } 331 } 332 filename = filepath.Join(dest, filename) 333 } 334 filePath, err := filepath.Abs(filename) 335 if err != nil { 336 return err 337 } 338 339 if err := os.MkdirAll(filepath.Dir(filePath), 0777); err != nil { 340 return err 341 } 342 343 dst, err := os.Create(filename) 344 if err != nil { 345 return err 346 } 347 defer dst.Close() 348 349 _, err = io.Copy(dst, res.Body) 350 return err 351 } 352 353 // UploadManifest uploads the given manifest to swarm 354 func (c *Client) UploadManifest(m *api.Manifest, toEncrypt bool) (string, error) { 355 data, err := json.Marshal(m) 356 if err != nil { 357 return "", err 358 } 359 return c.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt) 360 } 361 362 // DownloadManifest downloads a swarm manifest 363 func (c *Client) DownloadManifest(hash string) (*api.Manifest, bool, error) { 364 res, isEncrypted, err := c.DownloadRaw(hash) 365 if err != nil { 366 return nil, isEncrypted, err 367 } 368 defer res.Close() 369 var manifest api.Manifest 370 if err := json.NewDecoder(res).Decode(&manifest); err != nil { 371 return nil, isEncrypted, err 372 } 373 return &manifest, isEncrypted, nil 374 } 375 376 // List list files in a swarm manifest which have the given prefix, grouping 377 // common prefixes using "/" as a delimiter. 378 // 379 // For example, if the manifest represents the following directory structure: 380 // 381 // file1.txt 382 // file2.txt 383 // dir1/file3.txt 384 // dir1/dir2/file4.txt 385 // 386 // Then: 387 // 388 // - a prefix of "" would return [dir1/, file1.txt, file2.txt] 389 // - a prefix of "file" would return [file1.txt, file2.txt] 390 // - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt] 391 // 392 // where entries ending with "/" are common prefixes. 393 func (c *Client) List(hash, prefix, credentials string) (*api.ManifestList, error) { 394 req, err := http.NewRequest(http.MethodGet, c.Gateway+"/bzz-list:/"+hash+"/"+prefix, nil) 395 if err != nil { 396 return nil, err 397 } 398 if credentials != "" { 399 req.SetBasicAuth("", credentials) 400 } 401 res, err := http.DefaultClient.Do(req) 402 if err != nil { 403 return nil, err 404 } 405 defer res.Body.Close() 406 switch res.StatusCode { 407 case http.StatusOK: 408 case http.StatusUnauthorized: 409 return nil, ErrUnauthorized 410 default: 411 return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) 412 } 413 var list api.ManifestList 414 if err := json.NewDecoder(res.Body).Decode(&list); err != nil { 415 return nil, err 416 } 417 return &list, nil 418 } 419 420 // Uploader uploads files to swarm using a provided UploadFn 421 type Uploader interface { 422 Upload(UploadFn) error 423 } 424 425 type UploaderFunc func(UploadFn) error 426 427 func (u UploaderFunc) Upload(upload UploadFn) error { 428 return u(upload) 429 } 430 431 // DirectoryUploader uploads all files in a directory, optionally uploading 432 // a file to the default path 433 type DirectoryUploader struct { 434 Dir string 435 } 436 437 // Upload performs the upload of the directory and default path 438 func (d *DirectoryUploader) Upload(upload UploadFn) error { 439 return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error { 440 if err != nil { 441 return err 442 } 443 if f.IsDir() { 444 return nil 445 } 446 file, err := Open(path) 447 if err != nil { 448 return err 449 } 450 relPath, err := filepath.Rel(d.Dir, path) 451 if err != nil { 452 return err 453 } 454 file.Path = filepath.ToSlash(relPath) 455 return upload(file) 456 }) 457 } 458 459 // FileUploader uploads a single file 460 type FileUploader struct { 461 File *File 462 } 463 464 // Upload performs the upload of the file 465 func (f *FileUploader) Upload(upload UploadFn) error { 466 return upload(f.File) 467 } 468 469 // UploadFn is the type of function passed to an Uploader to perform the upload 470 // of a single file (for example, a directory uploader would call a provided 471 // UploadFn for each file in the directory tree) 472 type UploadFn func(file *File) error 473 474 // TarUpload uses the given Uploader to upload files to swarm as a tar stream, 475 // returning the resulting manifest hash 476 func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, toEncrypt bool) (string, error) { 477 reqR, reqW := io.Pipe() 478 defer reqR.Close() 479 addr := hash 480 481 // If there is a hash already (a manifest), then that manifest will determine if the upload has 482 // to be encrypted or not. If there is no manifest then the toEncrypt parameter decides if 483 // there is encryption or not. 484 if hash == "" && toEncrypt { 485 // This is the built-in address for the encrypted upload endpoint 486 addr = "encrypt" 487 } 488 req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+addr, reqR) 489 if err != nil { 490 return "", err 491 } 492 req.Header.Set("Content-Type", "application/x-tar") 493 if defaultPath != "" { 494 q := req.URL.Query() 495 q.Set("defaultpath", defaultPath) 496 req.URL.RawQuery = q.Encode() 497 } 498 499 // use 'Expect: 100-continue' so we don't send the request body if 500 // the server refuses the request 501 req.Header.Set("Expect", "100-continue") 502 503 tw := tar.NewWriter(reqW) 504 505 // define an UploadFn which adds files to the tar stream 506 uploadFn := func(file *File) error { 507 hdr := &tar.Header{ 508 Name: file.Path, 509 Mode: file.Mode, 510 Size: file.Size, 511 ModTime: file.ModTime, 512 Xattrs: map[string]string{ 513 "user.swarm.content-type": file.ContentType, 514 }, 515 } 516 if err := tw.WriteHeader(hdr); err != nil { 517 return err 518 } 519 _, err = io.Copy(tw, file) 520 return err 521 } 522 523 // run the upload in a goroutine so we can send the request headers and 524 // wait for a '100 Continue' response before sending the tar stream 525 go func() { 526 err := uploader.Upload(uploadFn) 527 if err == nil { 528 err = tw.Close() 529 } 530 reqW.CloseWithError(err) 531 }() 532 533 res, err := http.DefaultClient.Do(req) 534 if err != nil { 535 return "", err 536 } 537 defer res.Body.Close() 538 if res.StatusCode != http.StatusOK { 539 return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) 540 } 541 data, err := ioutil.ReadAll(res.Body) 542 if err != nil { 543 return "", err 544 } 545 return string(data), nil 546 } 547 548 // MultipartUpload uses the given Uploader to upload files to swarm as a 549 // multipart form, returning the resulting manifest hash 550 func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) { 551 reqR, reqW := io.Pipe() 552 defer reqR.Close() 553 req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR) 554 if err != nil { 555 return "", err 556 } 557 558 // use 'Expect: 100-continue' so we don't send the request body if 559 // the server refuses the request 560 req.Header.Set("Expect", "100-continue") 561 562 mw := multipart.NewWriter(reqW) 563 req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary())) 564 565 // define an UploadFn which adds files to the multipart form 566 uploadFn := func(file *File) error { 567 hdr := make(textproto.MIMEHeader) 568 hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path)) 569 hdr.Set("Content-Type", file.ContentType) 570 hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10)) 571 w, err := mw.CreatePart(hdr) 572 if err != nil { 573 return err 574 } 575 _, err = io.Copy(w, file) 576 return err 577 } 578 579 // run the upload in a goroutine so we can send the request headers and 580 // wait for a '100 Continue' response before sending the multipart form 581 go func() { 582 err := uploader.Upload(uploadFn) 583 if err == nil { 584 err = mw.Close() 585 } 586 reqW.CloseWithError(err) 587 }() 588 589 res, err := http.DefaultClient.Do(req) 590 if err != nil { 591 return "", err 592 } 593 defer res.Body.Close() 594 if res.StatusCode != http.StatusOK { 595 return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) 596 } 597 data, err := ioutil.ReadAll(res.Body) 598 if err != nil { 599 return "", err 600 } 601 return string(data), nil 602 } 603 604 // ErrNoFeedUpdatesFound is returned when Swarm cannot find updates of the given feed 605 var ErrNoFeedUpdatesFound = errors.New("No updates found for this feed") 606 607 // CreateFeedWithManifest creates a feed manifest, initializing it with the provided 608 // data 609 // Returns the resulting feed manifest address that you can use to include in an ENS Resolver (setContent) 610 // or reference future updates (Client.UpdateFeed) 611 func (c *Client) CreateFeedWithManifest(request *feed.Request) (string, error) { 612 responseStream, err := c.updateFeed(request, true) 613 if err != nil { 614 return "", err 615 } 616 defer responseStream.Close() 617 618 body, err := ioutil.ReadAll(responseStream) 619 if err != nil { 620 return "", err 621 } 622 623 var manifestAddress string 624 if err = json.Unmarshal(body, &manifestAddress); err != nil { 625 return "", err 626 } 627 return manifestAddress, nil 628 } 629 630 // UpdateFeed allows you to set a new version of your content 631 func (c *Client) UpdateFeed(request *feed.Request) error { 632 _, err := c.updateFeed(request, false) 633 return err 634 } 635 636 func (c *Client) updateFeed(request *feed.Request, createManifest bool) (io.ReadCloser, error) { 637 URL, err := url.Parse(c.Gateway) 638 if err != nil { 639 return nil, err 640 } 641 URL.Path = "/bzz-feed:/" 642 values := URL.Query() 643 body := request.AppendValues(values) 644 if createManifest { 645 values.Set("manifest", "1") 646 } 647 URL.RawQuery = values.Encode() 648 649 req, err := http.NewRequest("POST", URL.String(), bytes.NewBuffer(body)) 650 if err != nil { 651 return nil, err 652 } 653 654 res, err := http.DefaultClient.Do(req) 655 if err != nil { 656 return nil, err 657 } 658 659 return res.Body, nil 660 } 661 662 // QueryFeed returns a byte stream with the raw content of the feed update 663 // manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver 664 // points to that address 665 func (c *Client) QueryFeed(query *feed.Query, manifestAddressOrDomain string) (io.ReadCloser, error) { 666 return c.queryFeed(query, manifestAddressOrDomain, false) 667 } 668 669 // queryFeed returns a byte stream with the raw content of the feed update 670 // manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver 671 // points to that address 672 // meta set to true will instruct the node return feed metainformation instead 673 func (c *Client) queryFeed(query *feed.Query, manifestAddressOrDomain string, meta bool) (io.ReadCloser, error) { 674 URL, err := url.Parse(c.Gateway) 675 if err != nil { 676 return nil, err 677 } 678 URL.Path = "/bzz-feed:/" + manifestAddressOrDomain 679 values := URL.Query() 680 if query != nil { 681 query.AppendValues(values) //adds query parameters 682 } 683 if meta { 684 values.Set("meta", "1") 685 } 686 URL.RawQuery = values.Encode() 687 res, err := http.Get(URL.String()) 688 if err != nil { 689 return nil, err 690 } 691 692 if res.StatusCode != http.StatusOK { 693 if res.StatusCode == http.StatusNotFound { 694 return nil, ErrNoFeedUpdatesFound 695 } 696 errorMessageBytes, err := ioutil.ReadAll(res.Body) 697 var errorMessage string 698 if err != nil { 699 errorMessage = "cannot retrieve error message: " + err.Error() 700 } else { 701 errorMessage = string(errorMessageBytes) 702 } 703 return nil, fmt.Errorf("Error retrieving feed updates: %s", errorMessage) 704 } 705 706 return res.Body, nil 707 } 708 709 // GetFeedRequest returns a structure that describes the referenced feed status 710 // manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver 711 // points to that address 712 func (c *Client) GetFeedRequest(query *feed.Query, manifestAddressOrDomain string) (*feed.Request, error) { 713 714 responseStream, err := c.queryFeed(query, manifestAddressOrDomain, true) 715 if err != nil { 716 return nil, err 717 } 718 defer responseStream.Close() 719 720 body, err := ioutil.ReadAll(responseStream) 721 if err != nil { 722 return nil, err 723 } 724 725 var metadata feed.Request 726 if err := metadata.UnmarshalJSON(body); err != nil { 727 return nil, err 728 } 729 return &metadata, nil 730 }