github.com/oskarth/go-ethereum@v1.6.8-0.20191013093314-dac24a9d3494/swarm/api/api.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package api 18 19 //go:generate mimegen --types=./../../cmd/swarm/mimegen/mime.types --package=api --out=gen_mime.go 20 //go:generate gofmt -s -w gen_mime.go 21 22 import ( 23 "archive/tar" 24 "context" 25 "crypto/ecdsa" 26 "encoding/hex" 27 "errors" 28 "fmt" 29 "io" 30 "math/big" 31 "net/http" 32 "path" 33 "strings" 34 35 "bytes" 36 "mime" 37 "path/filepath" 38 "time" 39 40 "github.com/ethereum/go-ethereum/common" 41 "github.com/ethereum/go-ethereum/contracts/ens" 42 "github.com/ethereum/go-ethereum/core/types" 43 "github.com/ethereum/go-ethereum/metrics" 44 "github.com/ethereum/go-ethereum/swarm/log" 45 "github.com/ethereum/go-ethereum/swarm/multihash" 46 "github.com/ethereum/go-ethereum/swarm/spancontext" 47 "github.com/ethereum/go-ethereum/swarm/storage" 48 "github.com/ethereum/go-ethereum/swarm/storage/feed" 49 "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" 50 51 opentracing "github.com/opentracing/opentracing-go" 52 ) 53 54 var ( 55 ErrNotFound = errors.New("not found") 56 ) 57 58 var ( 59 apiResolveCount = metrics.NewRegisteredCounter("api.resolve.count", nil) 60 apiResolveFail = metrics.NewRegisteredCounter("api.resolve.fail", nil) 61 apiPutCount = metrics.NewRegisteredCounter("api.put.count", nil) 62 apiPutFail = metrics.NewRegisteredCounter("api.put.fail", nil) 63 apiGetCount = metrics.NewRegisteredCounter("api.get.count", nil) 64 apiGetNotFound = metrics.NewRegisteredCounter("api.get.notfound", nil) 65 apiGetHTTP300 = metrics.NewRegisteredCounter("api.get.http.300", nil) 66 apiManifestUpdateCount = metrics.NewRegisteredCounter("api.manifestupdate.count", nil) 67 apiManifestUpdateFail = metrics.NewRegisteredCounter("api.manifestupdate.fail", nil) 68 apiManifestListCount = metrics.NewRegisteredCounter("api.manifestlist.count", nil) 69 apiManifestListFail = metrics.NewRegisteredCounter("api.manifestlist.fail", nil) 70 apiDeleteCount = metrics.NewRegisteredCounter("api.delete.count", nil) 71 apiDeleteFail = metrics.NewRegisteredCounter("api.delete.fail", nil) 72 apiGetTarCount = metrics.NewRegisteredCounter("api.gettar.count", nil) 73 apiGetTarFail = metrics.NewRegisteredCounter("api.gettar.fail", nil) 74 apiUploadTarCount = metrics.NewRegisteredCounter("api.uploadtar.count", nil) 75 apiUploadTarFail = metrics.NewRegisteredCounter("api.uploadtar.fail", nil) 76 apiModifyCount = metrics.NewRegisteredCounter("api.modify.count", nil) 77 apiModifyFail = metrics.NewRegisteredCounter("api.modify.fail", nil) 78 apiAddFileCount = metrics.NewRegisteredCounter("api.addfile.count", nil) 79 apiAddFileFail = metrics.NewRegisteredCounter("api.addfile.fail", nil) 80 apiRmFileCount = metrics.NewRegisteredCounter("api.removefile.count", nil) 81 apiRmFileFail = metrics.NewRegisteredCounter("api.removefile.fail", nil) 82 apiAppendFileCount = metrics.NewRegisteredCounter("api.appendfile.count", nil) 83 apiAppendFileFail = metrics.NewRegisteredCounter("api.appendfile.fail", nil) 84 apiGetInvalid = metrics.NewRegisteredCounter("api.get.invalid", nil) 85 ) 86 87 // Resolver interface resolve a domain name to a hash using ENS 88 type Resolver interface { 89 Resolve(string) (common.Hash, error) 90 } 91 92 // ResolveValidator is used to validate the contained Resolver 93 type ResolveValidator interface { 94 Resolver 95 Owner(node [32]byte) (common.Address, error) 96 HeaderByNumber(context.Context, *big.Int) (*types.Header, error) 97 } 98 99 // NoResolverError is returned by MultiResolver.Resolve if no resolver 100 // can be found for the address. 101 type NoResolverError struct { 102 TLD string 103 } 104 105 // NewNoResolverError creates a NoResolverError for the given top level domain 106 func NewNoResolverError(tld string) *NoResolverError { 107 return &NoResolverError{TLD: tld} 108 } 109 110 // Error NoResolverError implements error 111 func (e *NoResolverError) Error() string { 112 if e.TLD == "" { 113 return "no ENS resolver" 114 } 115 return fmt.Sprintf("no ENS endpoint configured to resolve .%s TLD names", e.TLD) 116 } 117 118 // MultiResolver is used to resolve URL addresses based on their TLDs. 119 // Each TLD can have multiple resolvers, and the resolution from the 120 // first one in the sequence will be returned. 121 type MultiResolver struct { 122 resolvers map[string][]ResolveValidator 123 nameHash func(string) common.Hash 124 } 125 126 // MultiResolverOption sets options for MultiResolver and is used as 127 // arguments for its constructor. 128 type MultiResolverOption func(*MultiResolver) 129 130 // MultiResolverOptionWithResolver adds a Resolver to a list of resolvers 131 // for a specific TLD. If TLD is an empty string, the resolver will be added 132 // to the list of default resolver, the ones that will be used for resolution 133 // of addresses which do not have their TLD resolver specified. 134 func MultiResolverOptionWithResolver(r ResolveValidator, tld string) MultiResolverOption { 135 return func(m *MultiResolver) { 136 m.resolvers[tld] = append(m.resolvers[tld], r) 137 } 138 } 139 140 // MultiResolverOptionWithNameHash is unused at the time of this writing 141 func MultiResolverOptionWithNameHash(nameHash func(string) common.Hash) MultiResolverOption { 142 return func(m *MultiResolver) { 143 m.nameHash = nameHash 144 } 145 } 146 147 // NewMultiResolver creates a new instance of MultiResolver. 148 func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { 149 m = &MultiResolver{ 150 resolvers: make(map[string][]ResolveValidator), 151 nameHash: ens.EnsNode, 152 } 153 for _, o := range opts { 154 o(m) 155 } 156 return m 157 } 158 159 // Resolve resolves address by choosing a Resolver by TLD. 160 // If there are more default Resolvers, or for a specific TLD, 161 // the Hash from the first one which does not return error 162 // will be returned. 163 func (m *MultiResolver) Resolve(addr string) (h common.Hash, err error) { 164 rs, err := m.getResolveValidator(addr) 165 if err != nil { 166 return h, err 167 } 168 for _, r := range rs { 169 h, err = r.Resolve(addr) 170 if err == nil { 171 return 172 } 173 } 174 return 175 } 176 177 // ValidateOwner checks the ENS to validate that the owner of the given domain is the given eth address 178 func (m *MultiResolver) ValidateOwner(name string, address common.Address) (bool, error) { 179 rs, err := m.getResolveValidator(name) 180 if err != nil { 181 return false, err 182 } 183 var addr common.Address 184 for _, r := range rs { 185 addr, err = r.Owner(m.nameHash(name)) 186 // we hide the error if it is not for the last resolver we check 187 if err == nil { 188 return addr == address, nil 189 } 190 } 191 return false, err 192 } 193 194 // HeaderByNumber uses the validator of the given domainname and retrieves the header for the given block number 195 func (m *MultiResolver) HeaderByNumber(ctx context.Context, name string, blockNr *big.Int) (*types.Header, error) { 196 rs, err := m.getResolveValidator(name) 197 if err != nil { 198 return nil, err 199 } 200 for _, r := range rs { 201 var header *types.Header 202 header, err = r.HeaderByNumber(ctx, blockNr) 203 // we hide the error if it is not for the last resolver we check 204 if err == nil { 205 return header, nil 206 } 207 } 208 return nil, err 209 } 210 211 // getResolveValidator uses the hostname to retrieve the resolver associated with the top level domain 212 func (m *MultiResolver) getResolveValidator(name string) ([]ResolveValidator, error) { 213 rs := m.resolvers[""] 214 tld := path.Ext(name) 215 if tld != "" { 216 tld = tld[1:] 217 rstld, ok := m.resolvers[tld] 218 if ok { 219 return rstld, nil 220 } 221 } 222 if len(rs) == 0 { 223 return rs, NewNoResolverError(tld) 224 } 225 return rs, nil 226 } 227 228 // SetNameHash sets the hasher function that hashes the domain into a name hash that ENS uses 229 func (m *MultiResolver) SetNameHash(nameHash func(string) common.Hash) { 230 m.nameHash = nameHash 231 } 232 233 /* 234 API implements webserver/file system related content storage and retrieval 235 on top of the FileStore 236 it is the public interface of the FileStore which is included in the ethereum stack 237 */ 238 type API struct { 239 feed *feed.Handler 240 fileStore *storage.FileStore 241 dns Resolver 242 Decryptor func(context.Context, string) DecryptFunc 243 } 244 245 // NewAPI the api constructor initialises a new API instance. 246 func NewAPI(fileStore *storage.FileStore, dns Resolver, feedHandler *feed.Handler, pk *ecdsa.PrivateKey) (self *API) { 247 self = &API{ 248 fileStore: fileStore, 249 dns: dns, 250 feed: feedHandler, 251 Decryptor: func(ctx context.Context, credentials string) DecryptFunc { 252 return self.doDecrypt(ctx, credentials, pk) 253 }, 254 } 255 return 256 } 257 258 // Retrieve FileStore reader API 259 func (a *API) Retrieve(ctx context.Context, addr storage.Address) (reader storage.LazySectionReader, isEncrypted bool) { 260 return a.fileStore.Retrieve(ctx, addr) 261 } 262 263 // Store wraps the Store API call of the embedded FileStore 264 func (a *API) Store(ctx context.Context, data io.Reader, size int64, toEncrypt bool) (addr storage.Address, wait func(ctx context.Context) error, err error) { 265 log.Debug("api.store", "size", size) 266 return a.fileStore.Store(ctx, data, size, toEncrypt) 267 } 268 269 // ErrResolve is returned when an URI cannot be resolved from ENS. 270 type ErrResolve error 271 272 // Resolve a name into a content-addressed hash 273 // where address could be an ENS name, or a content addressed hash 274 func (a *API) Resolve(ctx context.Context, address string) (storage.Address, error) { 275 // if DNS is not configured, return an error 276 if a.dns == nil { 277 if hashMatcher.MatchString(address) { 278 return common.Hex2Bytes(address), nil 279 } 280 apiResolveFail.Inc(1) 281 return nil, fmt.Errorf("no DNS to resolve name: %q", address) 282 } 283 // try and resolve the address 284 resolved, err := a.dns.Resolve(address) 285 if err != nil { 286 if hashMatcher.MatchString(address) { 287 return common.Hex2Bytes(address), nil 288 } 289 return nil, err 290 } 291 return resolved[:], nil 292 } 293 294 // Resolve resolves a URI to an Address using the MultiResolver. 295 func (a *API) ResolveURI(ctx context.Context, uri *URI, credentials string) (storage.Address, error) { 296 apiResolveCount.Inc(1) 297 log.Trace("resolving", "uri", uri.Addr) 298 299 var sp opentracing.Span 300 ctx, sp = spancontext.StartSpan( 301 ctx, 302 "api.resolve") 303 defer sp.Finish() 304 305 // if the URI is immutable, check if the address looks like a hash 306 if uri.Immutable() { 307 key := uri.Address() 308 if key == nil { 309 return nil, fmt.Errorf("immutable address not a content hash: %q", uri.Addr) 310 } 311 return key, nil 312 } 313 314 addr, err := a.Resolve(ctx, uri.Addr) 315 if err != nil { 316 return nil, err 317 } 318 319 if uri.Path == "" { 320 return addr, nil 321 } 322 walker, err := a.NewManifestWalker(ctx, addr, a.Decryptor(ctx, credentials), nil) 323 if err != nil { 324 return nil, err 325 } 326 var entry *ManifestEntry 327 walker.Walk(func(e *ManifestEntry) error { 328 // if the entry matches the path, set entry and stop 329 // the walk 330 if e.Path == uri.Path { 331 entry = e 332 // return an error to cancel the walk 333 return errors.New("found") 334 } 335 // ignore non-manifest files 336 if e.ContentType != ManifestType { 337 return nil 338 } 339 // if the manifest's path is a prefix of the 340 // requested path, recurse into it by returning 341 // nil and continuing the walk 342 if strings.HasPrefix(uri.Path, e.Path) { 343 return nil 344 } 345 return ErrSkipManifest 346 }) 347 if entry == nil { 348 return nil, errors.New("not found") 349 } 350 addr = storage.Address(common.Hex2Bytes(entry.Hash)) 351 return addr, nil 352 } 353 354 // Put provides singleton manifest creation on top of FileStore store 355 func (a *API) Put(ctx context.Context, content string, contentType string, toEncrypt bool) (k storage.Address, wait func(context.Context) error, err error) { 356 apiPutCount.Inc(1) 357 r := strings.NewReader(content) 358 key, waitContent, err := a.fileStore.Store(ctx, r, int64(len(content)), toEncrypt) 359 if err != nil { 360 apiPutFail.Inc(1) 361 return nil, nil, err 362 } 363 manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) 364 r = strings.NewReader(manifest) 365 key, waitManifest, err := a.fileStore.Store(ctx, r, int64(len(manifest)), toEncrypt) 366 if err != nil { 367 apiPutFail.Inc(1) 368 return nil, nil, err 369 } 370 return key, func(ctx context.Context) error { 371 err := waitContent(ctx) 372 if err != nil { 373 return err 374 } 375 return waitManifest(ctx) 376 }, nil 377 } 378 379 // Get uses iterative manifest retrieval and prefix matching 380 // to resolve basePath to content using FileStore retrieve 381 // it returns a section reader, mimeType, status, the key of the actual content and an error 382 func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage.Address, path string) (reader storage.LazySectionReader, mimeType string, status int, contentAddr storage.Address, err error) { 383 log.Debug("api.get", "key", manifestAddr, "path", path) 384 apiGetCount.Inc(1) 385 trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, decrypt) 386 if err != nil { 387 apiGetNotFound.Inc(1) 388 status = http.StatusNotFound 389 return nil, "", http.StatusNotFound, nil, err 390 } 391 392 log.Debug("trie getting entry", "key", manifestAddr, "path", path) 393 entry, _ := trie.getEntry(path) 394 395 if entry != nil { 396 log.Debug("trie got entry", "key", manifestAddr, "path", path, "entry.Hash", entry.Hash) 397 398 if entry.ContentType == ManifestType { 399 log.Debug("entry is manifest", "key", manifestAddr, "new key", entry.Hash) 400 adr, err := hex.DecodeString(entry.Hash) 401 if err != nil { 402 return nil, "", 0, nil, err 403 } 404 return a.Get(ctx, decrypt, adr, entry.Path) 405 } 406 407 // we need to do some extra work if this is a Swarm feed manifest 408 if entry.ContentType == FeedContentType { 409 if entry.Feed == nil { 410 return reader, mimeType, status, nil, fmt.Errorf("Cannot decode Feed in manifest") 411 } 412 _, err := a.feed.Lookup(ctx, feed.NewQueryLatest(entry.Feed, lookup.NoClue)) 413 if err != nil { 414 apiGetNotFound.Inc(1) 415 status = http.StatusNotFound 416 log.Debug(fmt.Sprintf("get feed update content error: %v", err)) 417 return reader, mimeType, status, nil, err 418 } 419 // get the data of the update 420 _, rsrcData, err := a.feed.GetContent(entry.Feed) 421 if err != nil { 422 apiGetNotFound.Inc(1) 423 status = http.StatusNotFound 424 log.Warn(fmt.Sprintf("get feed update content error: %v", err)) 425 return reader, mimeType, status, nil, err 426 } 427 428 // extract multihash 429 decodedMultihash, err := multihash.FromMultihash(rsrcData) 430 if err != nil { 431 apiGetInvalid.Inc(1) 432 status = http.StatusUnprocessableEntity 433 log.Warn("invalid multihash in feed update", "err", err) 434 return reader, mimeType, status, nil, err 435 } 436 manifestAddr = storage.Address(decodedMultihash) 437 log.Trace("feed update contains multihash", "key", manifestAddr) 438 439 // get the manifest the multihash digest points to 440 trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, NOOPDecrypt) 441 if err != nil { 442 apiGetNotFound.Inc(1) 443 status = http.StatusNotFound 444 log.Warn(fmt.Sprintf("loadManifestTrie (feed update multihash) error: %v", err)) 445 return reader, mimeType, status, nil, err 446 } 447 448 // finally, get the manifest entry 449 // it will always be the entry on path "" 450 entry, _ = trie.getEntry(path) 451 if entry == nil { 452 status = http.StatusNotFound 453 apiGetNotFound.Inc(1) 454 err = fmt.Errorf("manifest (feed update multihash) entry for '%s' not found", path) 455 log.Trace("manifest (feed update multihash) entry not found", "key", manifestAddr, "path", path) 456 return reader, mimeType, status, nil, err 457 } 458 } 459 460 // regardless of feed update manifests or normal manifests we will converge at this point 461 // get the key the manifest entry points to and serve it if it's unambiguous 462 contentAddr = common.Hex2Bytes(entry.Hash) 463 status = entry.Status 464 if status == http.StatusMultipleChoices { 465 apiGetHTTP300.Inc(1) 466 return nil, entry.ContentType, status, contentAddr, err 467 } 468 mimeType = entry.ContentType 469 log.Debug("content lookup key", "key", contentAddr, "mimetype", mimeType) 470 reader, _ = a.fileStore.Retrieve(ctx, contentAddr) 471 } else { 472 // no entry found 473 status = http.StatusNotFound 474 apiGetNotFound.Inc(1) 475 err = fmt.Errorf("manifest entry for '%s' not found", path) 476 log.Trace("manifest entry not found", "key", contentAddr, "path", path) 477 } 478 return 479 } 480 481 func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Address, error) { 482 apiDeleteCount.Inc(1) 483 uri, err := Parse("bzz:/" + addr) 484 if err != nil { 485 apiDeleteFail.Inc(1) 486 return nil, err 487 } 488 key, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) 489 490 if err != nil { 491 return nil, err 492 } 493 newKey, err := a.UpdateManifest(ctx, key, func(mw *ManifestWriter) error { 494 log.Debug(fmt.Sprintf("removing %s from manifest %s", path, key.Log())) 495 return mw.RemoveEntry(path) 496 }) 497 if err != nil { 498 apiDeleteFail.Inc(1) 499 return nil, err 500 } 501 502 return newKey, nil 503 } 504 505 // GetDirectoryTar fetches a requested directory as a tarstream 506 // it returns an io.Reader and an error. Do not forget to Close() the returned ReadCloser 507 func (a *API) GetDirectoryTar(ctx context.Context, decrypt DecryptFunc, uri *URI) (io.ReadCloser, error) { 508 apiGetTarCount.Inc(1) 509 addr, err := a.Resolve(ctx, uri.Addr) 510 if err != nil { 511 return nil, err 512 } 513 walker, err := a.NewManifestWalker(ctx, addr, decrypt, nil) 514 if err != nil { 515 apiGetTarFail.Inc(1) 516 return nil, err 517 } 518 519 piper, pipew := io.Pipe() 520 521 tw := tar.NewWriter(pipew) 522 523 go func() { 524 err := walker.Walk(func(entry *ManifestEntry) error { 525 // ignore manifests (walk will recurse into them) 526 if entry.ContentType == ManifestType { 527 return nil 528 } 529 530 // retrieve the entry's key and size 531 reader, _ := a.Retrieve(ctx, storage.Address(common.Hex2Bytes(entry.Hash))) 532 size, err := reader.Size(ctx, nil) 533 if err != nil { 534 return err 535 } 536 537 // write a tar header for the entry 538 hdr := &tar.Header{ 539 Name: entry.Path, 540 Mode: entry.Mode, 541 Size: size, 542 ModTime: entry.ModTime, 543 Xattrs: map[string]string{ 544 "user.swarm.content-type": entry.ContentType, 545 }, 546 } 547 548 if err := tw.WriteHeader(hdr); err != nil { 549 return err 550 } 551 552 // copy the file into the tar stream 553 n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size)) 554 if err != nil { 555 return err 556 } else if n != size { 557 return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n) 558 } 559 560 return nil 561 }) 562 // close tar writer before closing pipew 563 // to flush remaining data to pipew 564 // regardless of error value 565 tw.Close() 566 if err != nil { 567 apiGetTarFail.Inc(1) 568 pipew.CloseWithError(err) 569 } else { 570 pipew.Close() 571 } 572 }() 573 574 return piper, nil 575 } 576 577 // GetManifestList lists the manifest entries for the specified address and prefix 578 // and returns it as a ManifestList 579 func (a *API) GetManifestList(ctx context.Context, decryptor DecryptFunc, addr storage.Address, prefix string) (list ManifestList, err error) { 580 apiManifestListCount.Inc(1) 581 walker, err := a.NewManifestWalker(ctx, addr, decryptor, nil) 582 if err != nil { 583 apiManifestListFail.Inc(1) 584 return ManifestList{}, err 585 } 586 587 err = walker.Walk(func(entry *ManifestEntry) error { 588 // handle non-manifest files 589 if entry.ContentType != ManifestType { 590 // ignore the file if it doesn't have the specified prefix 591 if !strings.HasPrefix(entry.Path, prefix) { 592 return nil 593 } 594 595 // if the path after the prefix contains a slash, add a 596 // common prefix to the list, otherwise add the entry 597 suffix := strings.TrimPrefix(entry.Path, prefix) 598 if index := strings.Index(suffix, "/"); index > -1 { 599 list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1]) 600 return nil 601 } 602 if entry.Path == "" { 603 entry.Path = "/" 604 } 605 list.Entries = append(list.Entries, entry) 606 return nil 607 } 608 609 // if the manifest's path is a prefix of the specified prefix 610 // then just recurse into the manifest by returning nil and 611 // continuing the walk 612 if strings.HasPrefix(prefix, entry.Path) { 613 return nil 614 } 615 616 // if the manifest's path has the specified prefix, then if the 617 // path after the prefix contains a slash, add a common prefix 618 // to the list and skip the manifest, otherwise recurse into 619 // the manifest by returning nil and continuing the walk 620 if strings.HasPrefix(entry.Path, prefix) { 621 suffix := strings.TrimPrefix(entry.Path, prefix) 622 if index := strings.Index(suffix, "/"); index > -1 { 623 list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1]) 624 return ErrSkipManifest 625 } 626 return nil 627 } 628 629 // the manifest neither has the prefix or needs recursing in to 630 // so just skip it 631 return ErrSkipManifest 632 }) 633 634 if err != nil { 635 apiManifestListFail.Inc(1) 636 return ManifestList{}, err 637 } 638 639 return list, nil 640 } 641 642 func (a *API) UpdateManifest(ctx context.Context, addr storage.Address, update func(mw *ManifestWriter) error) (storage.Address, error) { 643 apiManifestUpdateCount.Inc(1) 644 mw, err := a.NewManifestWriter(ctx, addr, nil) 645 if err != nil { 646 apiManifestUpdateFail.Inc(1) 647 return nil, err 648 } 649 650 if err := update(mw); err != nil { 651 apiManifestUpdateFail.Inc(1) 652 return nil, err 653 } 654 655 addr, err = mw.Store() 656 if err != nil { 657 apiManifestUpdateFail.Inc(1) 658 return nil, err 659 } 660 log.Debug(fmt.Sprintf("generated manifest %s", addr)) 661 return addr, nil 662 } 663 664 // Modify loads manifest and checks the content hash before recalculating and storing the manifest. 665 func (a *API) Modify(ctx context.Context, addr storage.Address, path, contentHash, contentType string) (storage.Address, error) { 666 apiModifyCount.Inc(1) 667 quitC := make(chan bool) 668 trie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt) 669 if err != nil { 670 apiModifyFail.Inc(1) 671 return nil, err 672 } 673 if contentHash != "" { 674 entry := newManifestTrieEntry(&ManifestEntry{ 675 Path: path, 676 ContentType: contentType, 677 }, nil) 678 entry.Hash = contentHash 679 trie.addEntry(entry, quitC) 680 } else { 681 trie.deleteEntry(path, quitC) 682 } 683 684 if err := trie.recalcAndStore(); err != nil { 685 apiModifyFail.Inc(1) 686 return nil, err 687 } 688 return trie.ref, nil 689 } 690 691 // AddFile creates a new manifest entry, adds it to swarm, then adds a file to swarm. 692 func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content []byte, nameresolver bool) (storage.Address, string, error) { 693 apiAddFileCount.Inc(1) 694 695 uri, err := Parse("bzz:/" + mhash) 696 if err != nil { 697 apiAddFileFail.Inc(1) 698 return nil, "", err 699 } 700 mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) 701 if err != nil { 702 apiAddFileFail.Inc(1) 703 return nil, "", err 704 } 705 706 // trim the root dir we added 707 if path[:1] == "/" { 708 path = path[1:] 709 } 710 711 entry := &ManifestEntry{ 712 Path: filepath.Join(path, fname), 713 ContentType: mime.TypeByExtension(filepath.Ext(fname)), 714 Mode: 0700, 715 Size: int64(len(content)), 716 ModTime: time.Now(), 717 } 718 719 mw, err := a.NewManifestWriter(ctx, mkey, nil) 720 if err != nil { 721 apiAddFileFail.Inc(1) 722 return nil, "", err 723 } 724 725 fkey, err := mw.AddEntry(ctx, bytes.NewReader(content), entry) 726 if err != nil { 727 apiAddFileFail.Inc(1) 728 return nil, "", err 729 } 730 731 newMkey, err := mw.Store() 732 if err != nil { 733 apiAddFileFail.Inc(1) 734 return nil, "", err 735 736 } 737 738 return fkey, newMkey.String(), nil 739 } 740 741 func (a *API) UploadTar(ctx context.Context, bodyReader io.ReadCloser, manifestPath, defaultPath string, mw *ManifestWriter) (storage.Address, error) { 742 apiUploadTarCount.Inc(1) 743 var contentKey storage.Address 744 tr := tar.NewReader(bodyReader) 745 defer bodyReader.Close() 746 var defaultPathFound bool 747 for { 748 hdr, err := tr.Next() 749 if err == io.EOF { 750 break 751 } else if err != nil { 752 apiUploadTarFail.Inc(1) 753 return nil, fmt.Errorf("error reading tar stream: %s", err) 754 } 755 756 // only store regular files 757 if !hdr.FileInfo().Mode().IsRegular() { 758 continue 759 } 760 761 // add the entry under the path from the request 762 manifestPath := path.Join(manifestPath, hdr.Name) 763 contentType := hdr.Xattrs["user.swarm.content-type"] 764 if contentType == "" { 765 contentType = mime.TypeByExtension(filepath.Ext(hdr.Name)) 766 } 767 //DetectContentType("") 768 entry := &ManifestEntry{ 769 Path: manifestPath, 770 ContentType: contentType, 771 Mode: hdr.Mode, 772 Size: hdr.Size, 773 ModTime: hdr.ModTime, 774 } 775 contentKey, err = mw.AddEntry(ctx, tr, entry) 776 if err != nil { 777 apiUploadTarFail.Inc(1) 778 return nil, fmt.Errorf("error adding manifest entry from tar stream: %s", err) 779 } 780 if hdr.Name == defaultPath { 781 contentType := hdr.Xattrs["user.swarm.content-type"] 782 if contentType == "" { 783 contentType = mime.TypeByExtension(filepath.Ext(hdr.Name)) 784 } 785 786 entry := &ManifestEntry{ 787 Hash: contentKey.Hex(), 788 Path: "", // default entry 789 ContentType: contentType, 790 Mode: hdr.Mode, 791 Size: hdr.Size, 792 ModTime: hdr.ModTime, 793 } 794 contentKey, err = mw.AddEntry(ctx, nil, entry) 795 if err != nil { 796 apiUploadTarFail.Inc(1) 797 return nil, fmt.Errorf("error adding default manifest entry from tar stream: %s", err) 798 } 799 defaultPathFound = true 800 } 801 } 802 if defaultPath != "" && !defaultPathFound { 803 return contentKey, fmt.Errorf("default path %q not found", defaultPath) 804 } 805 return contentKey, nil 806 } 807 808 // RemoveFile removes a file entry in a manifest. 809 func (a *API) RemoveFile(ctx context.Context, mhash string, path string, fname string, nameresolver bool) (string, error) { 810 apiRmFileCount.Inc(1) 811 812 uri, err := Parse("bzz:/" + mhash) 813 if err != nil { 814 apiRmFileFail.Inc(1) 815 return "", err 816 } 817 mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) 818 if err != nil { 819 apiRmFileFail.Inc(1) 820 return "", err 821 } 822 823 // trim the root dir we added 824 if path[:1] == "/" { 825 path = path[1:] 826 } 827 828 mw, err := a.NewManifestWriter(ctx, mkey, nil) 829 if err != nil { 830 apiRmFileFail.Inc(1) 831 return "", err 832 } 833 834 err = mw.RemoveEntry(filepath.Join(path, fname)) 835 if err != nil { 836 apiRmFileFail.Inc(1) 837 return "", err 838 } 839 840 newMkey, err := mw.Store() 841 if err != nil { 842 apiRmFileFail.Inc(1) 843 return "", err 844 845 } 846 847 return newMkey.String(), nil 848 } 849 850 // AppendFile removes old manifest, appends file entry to new manifest and adds it to Swarm. 851 func (a *API) AppendFile(ctx context.Context, mhash, path, fname string, existingSize int64, content []byte, oldAddr storage.Address, offset int64, addSize int64, nameresolver bool) (storage.Address, string, error) { 852 apiAppendFileCount.Inc(1) 853 854 buffSize := offset + addSize 855 if buffSize < existingSize { 856 buffSize = existingSize 857 } 858 859 buf := make([]byte, buffSize) 860 861 oldReader, _ := a.Retrieve(ctx, oldAddr) 862 io.ReadAtLeast(oldReader, buf, int(offset)) 863 864 newReader := bytes.NewReader(content) 865 io.ReadAtLeast(newReader, buf[offset:], int(addSize)) 866 867 if buffSize < existingSize { 868 io.ReadAtLeast(oldReader, buf[addSize:], int(buffSize)) 869 } 870 871 combinedReader := bytes.NewReader(buf) 872 totalSize := int64(len(buf)) 873 874 // TODO(jmozah): to append using pyramid chunker when it is ready 875 //oldReader := a.Retrieve(oldKey) 876 //newReader := bytes.NewReader(content) 877 //combinedReader := io.MultiReader(oldReader, newReader) 878 879 uri, err := Parse("bzz:/" + mhash) 880 if err != nil { 881 apiAppendFileFail.Inc(1) 882 return nil, "", err 883 } 884 mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) 885 if err != nil { 886 apiAppendFileFail.Inc(1) 887 return nil, "", err 888 } 889 890 // trim the root dir we added 891 if path[:1] == "/" { 892 path = path[1:] 893 } 894 895 mw, err := a.NewManifestWriter(ctx, mkey, nil) 896 if err != nil { 897 apiAppendFileFail.Inc(1) 898 return nil, "", err 899 } 900 901 err = mw.RemoveEntry(filepath.Join(path, fname)) 902 if err != nil { 903 apiAppendFileFail.Inc(1) 904 return nil, "", err 905 } 906 907 entry := &ManifestEntry{ 908 Path: filepath.Join(path, fname), 909 ContentType: mime.TypeByExtension(filepath.Ext(fname)), 910 Mode: 0700, 911 Size: totalSize, 912 ModTime: time.Now(), 913 } 914 915 fkey, err := mw.AddEntry(ctx, io.Reader(combinedReader), entry) 916 if err != nil { 917 apiAppendFileFail.Inc(1) 918 return nil, "", err 919 } 920 921 newMkey, err := mw.Store() 922 if err != nil { 923 apiAppendFileFail.Inc(1) 924 return nil, "", err 925 926 } 927 928 return fkey, newMkey.String(), nil 929 } 930 931 // BuildDirectoryTree used by swarmfs_unix 932 func (a *API) BuildDirectoryTree(ctx context.Context, mhash string, nameresolver bool) (addr storage.Address, manifestEntryMap map[string]*manifestTrieEntry, err error) { 933 934 uri, err := Parse("bzz:/" + mhash) 935 if err != nil { 936 return nil, nil, err 937 } 938 addr, err = a.Resolve(ctx, uri.Addr) 939 if err != nil { 940 return nil, nil, err 941 } 942 943 quitC := make(chan bool) 944 rootTrie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt) 945 if err != nil { 946 return nil, nil, fmt.Errorf("can't load manifest %v: %v", addr.String(), err) 947 } 948 949 manifestEntryMap = map[string]*manifestTrieEntry{} 950 err = rootTrie.listWithPrefix(uri.Path, quitC, func(entry *manifestTrieEntry, suffix string) { 951 manifestEntryMap[suffix] = entry 952 }) 953 954 if err != nil { 955 return nil, nil, fmt.Errorf("list with prefix failed %v: %v", addr.String(), err) 956 } 957 return addr, manifestEntryMap, nil 958 } 959 960 // FeedsLookup finds Swarm feeds updates at specific points in time, or the latest update 961 func (a *API) FeedsLookup(ctx context.Context, query *feed.Query) ([]byte, error) { 962 _, err := a.feed.Lookup(ctx, query) 963 if err != nil { 964 return nil, err 965 } 966 var data []byte 967 _, data, err = a.feed.GetContent(&query.Feed) 968 if err != nil { 969 return nil, err 970 } 971 return data, nil 972 } 973 974 // FeedsNewRequest creates a Request object to update a specific feed 975 func (a *API) FeedsNewRequest(ctx context.Context, feed *feed.Feed) (*feed.Request, error) { 976 return a.feed.NewRequest(ctx, feed) 977 } 978 979 // FeedsUpdate publishes a new update on the given feed 980 func (a *API) FeedsUpdate(ctx context.Context, request *feed.Request) (storage.Address, error) { 981 return a.feed.Update(ctx, request) 982 } 983 984 // FeedsHashSize returned the size of the digest produced by Swarm feeds' hashing function 985 func (a *API) FeedsHashSize() int { 986 return a.feed.HashSize 987 } 988 989 // ErrCannotLoadFeedManifest is returned when looking up a feeds manifest fails 990 var ErrCannotLoadFeedManifest = errors.New("Cannot load feed manifest") 991 992 // ErrNotAFeedManifest is returned when the address provided returned something other than a valid manifest 993 var ErrNotAFeedManifest = errors.New("Not a feed manifest") 994 995 // ResolveFeedManifest retrieves the Swarm feed manifest for the given address, and returns the referenced Feed. 996 func (a *API) ResolveFeedManifest(ctx context.Context, addr storage.Address) (*feed.Feed, error) { 997 trie, err := loadManifest(ctx, a.fileStore, addr, nil, NOOPDecrypt) 998 if err != nil { 999 return nil, ErrCannotLoadFeedManifest 1000 } 1001 1002 entry, _ := trie.getEntry("") 1003 if entry.ContentType != FeedContentType { 1004 return nil, ErrNotAFeedManifest 1005 } 1006 1007 return entry.Feed, nil 1008 } 1009 1010 // ErrCannotResolveFeedURI is returned when the ENS resolver is not able to translate a name to a Swarm feed 1011 var ErrCannotResolveFeedURI = errors.New("Cannot resolve Feed URI") 1012 1013 // ErrCannotResolveFeed is returned when values provided are not enough or invalid to recreate a 1014 // feed out of them. 1015 var ErrCannotResolveFeed = errors.New("Cannot resolve Feed") 1016 1017 // ResolveFeed attempts to extract feed information out of the manifest, if provided 1018 // If not, it attempts to extract the feed out of a set of key-value pairs 1019 func (a *API) ResolveFeed(ctx context.Context, uri *URI, values feed.Values) (*feed.Feed, error) { 1020 var fd *feed.Feed 1021 var err error 1022 if uri.Addr != "" { 1023 // resolve the content key. 1024 manifestAddr := uri.Address() 1025 if manifestAddr == nil { 1026 manifestAddr, err = a.Resolve(ctx, uri.Addr) 1027 if err != nil { 1028 return nil, ErrCannotResolveFeedURI 1029 } 1030 } 1031 1032 // get the Swarm feed from the manifest 1033 fd, err = a.ResolveFeedManifest(ctx, manifestAddr) 1034 if err != nil { 1035 return nil, err 1036 } 1037 log.Debug("handle.get.feed: resolved", "manifestkey", manifestAddr, "feed", fd.Hex()) 1038 } else { 1039 var f feed.Feed 1040 if err := f.FromValues(values); err != nil { 1041 return nil, ErrCannotResolveFeed 1042 1043 } 1044 fd = &f 1045 } 1046 return fd, nil 1047 } 1048 1049 // MimeOctetStream default value of http Content-Type header 1050 const MimeOctetStream = "application/octet-stream" 1051 1052 // DetectContentType by file file extension, or fallback to content sniff 1053 func DetectContentType(fileName string, f io.ReadSeeker) (string, error) { 1054 ctype := mime.TypeByExtension(filepath.Ext(fileName)) 1055 if ctype != "" { 1056 return ctype, nil 1057 } 1058 1059 // save/rollback to get content probe from begin of file 1060 currentPosition, err := f.Seek(0, io.SeekCurrent) 1061 if err != nil { 1062 return MimeOctetStream, fmt.Errorf("seeker can't seek, %s", err) 1063 } 1064 1065 // read a chunk to decide between utf-8 text and binary 1066 var buf [512]byte 1067 n, _ := f.Read(buf[:]) 1068 ctype = http.DetectContentType(buf[:n]) 1069 1070 _, err = f.Seek(currentPosition, io.SeekStart) // rewind to output whole file 1071 if err != nil { 1072 return MimeOctetStream, fmt.Errorf("seeker can't seek, %s", err) 1073 } 1074 1075 return ctype, nil 1076 }