github.com/gobitfly/go-ethereum@v1.8.12/swarm/api/http/server.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 /* 18 A simple http server interface to Swarm 19 */ 20 package http 21 22 import ( 23 "archive/tar" 24 "bufio" 25 "bytes" 26 "encoding/json" 27 "errors" 28 "fmt" 29 "io" 30 "io/ioutil" 31 "mime" 32 "mime/multipart" 33 "net/http" 34 "os" 35 "path" 36 "regexp" 37 "strconv" 38 "strings" 39 "time" 40 41 "github.com/ethereum/go-ethereum/common" 42 "github.com/ethereum/go-ethereum/common/hexutil" 43 "github.com/ethereum/go-ethereum/metrics" 44 "github.com/ethereum/go-ethereum/swarm/api" 45 "github.com/ethereum/go-ethereum/swarm/log" 46 "github.com/ethereum/go-ethereum/swarm/storage" 47 "github.com/ethereum/go-ethereum/swarm/storage/mru" 48 "github.com/pborman/uuid" 49 "github.com/rs/cors" 50 ) 51 52 type resourceResponse struct { 53 Manifest storage.Address `json:"manifest"` 54 Resource string `json:"resource"` 55 Update storage.Address `json:"update"` 56 } 57 58 var ( 59 postRawCount = metrics.NewRegisteredCounter("api.http.post.raw.count", nil) 60 postRawFail = metrics.NewRegisteredCounter("api.http.post.raw.fail", nil) 61 postFilesCount = metrics.NewRegisteredCounter("api.http.post.files.count", nil) 62 postFilesFail = metrics.NewRegisteredCounter("api.http.post.files.fail", nil) 63 deleteCount = metrics.NewRegisteredCounter("api.http.delete.count", nil) 64 deleteFail = metrics.NewRegisteredCounter("api.http.delete.fail", nil) 65 getCount = metrics.NewRegisteredCounter("api.http.get.count", nil) 66 getFail = metrics.NewRegisteredCounter("api.http.get.fail", nil) 67 getFileCount = metrics.NewRegisteredCounter("api.http.get.file.count", nil) 68 getFileNotFound = metrics.NewRegisteredCounter("api.http.get.file.notfound", nil) 69 getFileFail = metrics.NewRegisteredCounter("api.http.get.file.fail", nil) 70 getFilesCount = metrics.NewRegisteredCounter("api.http.get.files.count", nil) 71 getFilesFail = metrics.NewRegisteredCounter("api.http.get.files.fail", nil) 72 getListCount = metrics.NewRegisteredCounter("api.http.get.list.count", nil) 73 getListFail = metrics.NewRegisteredCounter("api.http.get.list.fail", nil) 74 ) 75 76 // ServerConfig is the basic configuration needed for the HTTP server and also 77 // includes CORS settings. 78 type ServerConfig struct { 79 Addr string 80 CorsString string 81 } 82 83 // browser API for registering bzz url scheme handlers: 84 // https://developer.mozilla.org/en/docs/Web-based_protocol_handlers 85 // electron (chromium) api for registering bzz url scheme handlers: 86 // https://github.com/atom/electron/blob/master/docs/api/protocol.md 87 88 // starts up http server 89 func StartHTTPServer(api *api.API, config *ServerConfig) { 90 var allowedOrigins []string 91 for _, domain := range strings.Split(config.CorsString, ",") { 92 allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain)) 93 } 94 c := cors.New(cors.Options{ 95 AllowedOrigins: allowedOrigins, 96 AllowedMethods: []string{"POST", "GET", "DELETE", "PATCH", "PUT"}, 97 MaxAge: 600, 98 AllowedHeaders: []string{"*"}, 99 }) 100 hdlr := c.Handler(NewServer(api)) 101 102 go http.ListenAndServe(config.Addr, hdlr) 103 } 104 105 func NewServer(api *api.API) *Server { 106 return &Server{api} 107 } 108 109 type Server struct { 110 api *api.API 111 } 112 113 // Request wraps http.Request and also includes the parsed bzz URI 114 type Request struct { 115 http.Request 116 117 uri *api.URI 118 ruid string // request unique id 119 } 120 121 // HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request 122 // body in swarm and returns the resulting storage address as a text/plain response 123 func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { 124 log.Debug("handle.post.raw", "ruid", r.ruid) 125 126 postRawCount.Inc(1) 127 128 toEncrypt := false 129 if r.uri.Addr == "encrypt" { 130 toEncrypt = true 131 } 132 133 if r.uri.Path != "" { 134 postRawFail.Inc(1) 135 Respond(w, r, "raw POST request cannot contain a path", http.StatusBadRequest) 136 return 137 } 138 139 if r.uri.Addr != "" && r.uri.Addr != "encrypt" { 140 postRawFail.Inc(1) 141 Respond(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest) 142 return 143 } 144 145 if r.Header.Get("Content-Length") == "" { 146 postRawFail.Inc(1) 147 Respond(w, r, "missing Content-Length header in request", http.StatusBadRequest) 148 return 149 } 150 addr, _, err := s.api.Store(r.Body, r.ContentLength, toEncrypt) 151 if err != nil { 152 postRawFail.Inc(1) 153 Respond(w, r, err.Error(), http.StatusInternalServerError) 154 return 155 } 156 157 log.Debug("stored content", "ruid", r.ruid, "key", addr) 158 159 w.Header().Set("Content-Type", "text/plain") 160 w.WriteHeader(http.StatusOK) 161 fmt.Fprint(w, addr) 162 } 163 164 // HandlePostFiles handles a POST request to 165 // bzz:/<hash>/<path> which contains either a single file or multiple files 166 // (either a tar archive or multipart form), adds those files either to an 167 // existing manifest or to a new manifest under <path> and returns the 168 // resulting manifest hash as a text/plain response 169 func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { 170 log.Debug("handle.post.files", "ruid", r.ruid) 171 172 postFilesCount.Inc(1) 173 contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) 174 if err != nil { 175 postFilesFail.Inc(1) 176 Respond(w, r, err.Error(), http.StatusBadRequest) 177 return 178 } 179 180 toEncrypt := false 181 if r.uri.Addr == "encrypt" { 182 toEncrypt = true 183 } 184 185 var addr storage.Address 186 if r.uri.Addr != "" && r.uri.Addr != "encrypt" { 187 addr, err = s.api.Resolve(r.uri) 188 if err != nil { 189 postFilesFail.Inc(1) 190 Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError) 191 return 192 } 193 log.Debug("resolved key", "ruid", r.ruid, "key", addr) 194 } else { 195 addr, err = s.api.NewManifest(toEncrypt) 196 if err != nil { 197 postFilesFail.Inc(1) 198 Respond(w, r, err.Error(), http.StatusInternalServerError) 199 return 200 } 201 log.Debug("new manifest", "ruid", r.ruid, "key", addr) 202 } 203 204 newAddr, err := s.updateManifest(addr, func(mw *api.ManifestWriter) error { 205 switch contentType { 206 207 case "application/x-tar": 208 return s.handleTarUpload(r, mw) 209 210 case "multipart/form-data": 211 return s.handleMultipartUpload(r, params["boundary"], mw) 212 213 default: 214 return s.handleDirectUpload(r, mw) 215 } 216 }) 217 if err != nil { 218 postFilesFail.Inc(1) 219 Respond(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError) 220 return 221 } 222 223 log.Debug("stored content", "ruid", r.ruid, "key", newAddr) 224 225 w.Header().Set("Content-Type", "text/plain") 226 w.WriteHeader(http.StatusOK) 227 fmt.Fprint(w, newAddr) 228 } 229 230 func (s *Server) handleTarUpload(req *Request, mw *api.ManifestWriter) error { 231 log.Debug("handle.tar.upload", "ruid", req.ruid) 232 tr := tar.NewReader(req.Body) 233 for { 234 hdr, err := tr.Next() 235 if err == io.EOF { 236 return nil 237 } else if err != nil { 238 return fmt.Errorf("error reading tar stream: %s", err) 239 } 240 241 // only store regular files 242 if !hdr.FileInfo().Mode().IsRegular() { 243 continue 244 } 245 246 // add the entry under the path from the request 247 path := path.Join(req.uri.Path, hdr.Name) 248 entry := &api.ManifestEntry{ 249 Path: path, 250 ContentType: hdr.Xattrs["user.swarm.content-type"], 251 Mode: hdr.Mode, 252 Size: hdr.Size, 253 ModTime: hdr.ModTime, 254 } 255 log.Debug("adding path to new manifest", "ruid", req.ruid, "bytes", entry.Size, "path", entry.Path) 256 contentKey, err := mw.AddEntry(tr, entry) 257 if err != nil { 258 return fmt.Errorf("error adding manifest entry from tar stream: %s", err) 259 } 260 log.Debug("stored content", "ruid", req.ruid, "key", contentKey) 261 } 262 } 263 264 func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error { 265 log.Debug("handle.multipart.upload", "ruid", req.ruid) 266 mr := multipart.NewReader(req.Body, boundary) 267 for { 268 part, err := mr.NextPart() 269 if err == io.EOF { 270 return nil 271 } else if err != nil { 272 return fmt.Errorf("error reading multipart form: %s", err) 273 } 274 275 var size int64 276 var reader io.Reader = part 277 if contentLength := part.Header.Get("Content-Length"); contentLength != "" { 278 size, err = strconv.ParseInt(contentLength, 10, 64) 279 if err != nil { 280 return fmt.Errorf("error parsing multipart content length: %s", err) 281 } 282 reader = part 283 } else { 284 // copy the part to a tmp file to get its size 285 tmp, err := ioutil.TempFile("", "swarm-multipart") 286 if err != nil { 287 return err 288 } 289 defer os.Remove(tmp.Name()) 290 defer tmp.Close() 291 size, err = io.Copy(tmp, part) 292 if err != nil { 293 return fmt.Errorf("error copying multipart content: %s", err) 294 } 295 if _, err := tmp.Seek(0, io.SeekStart); err != nil { 296 return fmt.Errorf("error copying multipart content: %s", err) 297 } 298 reader = tmp 299 } 300 301 // add the entry under the path from the request 302 name := part.FileName() 303 if name == "" { 304 name = part.FormName() 305 } 306 path := path.Join(req.uri.Path, name) 307 entry := &api.ManifestEntry{ 308 Path: path, 309 ContentType: part.Header.Get("Content-Type"), 310 Size: size, 311 ModTime: time.Now(), 312 } 313 log.Debug("adding path to new manifest", "ruid", req.ruid, "bytes", entry.Size, "path", entry.Path) 314 contentKey, err := mw.AddEntry(reader, entry) 315 if err != nil { 316 return fmt.Errorf("error adding manifest entry from multipart form: %s", err) 317 } 318 log.Debug("stored content", "ruid", req.ruid, "key", contentKey) 319 } 320 } 321 322 func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error { 323 log.Debug("handle.direct.upload", "ruid", req.ruid) 324 key, err := mw.AddEntry(req.Body, &api.ManifestEntry{ 325 Path: req.uri.Path, 326 ContentType: req.Header.Get("Content-Type"), 327 Mode: 0644, 328 Size: req.ContentLength, 329 ModTime: time.Now(), 330 }) 331 if err != nil { 332 return err 333 } 334 log.Debug("stored content", "ruid", req.ruid, "key", key) 335 return nil 336 } 337 338 // HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes 339 // <path> from <manifest> and returns the resulting manifest hash as a 340 // text/plain response 341 func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { 342 log.Debug("handle.delete", "ruid", r.ruid) 343 344 deleteCount.Inc(1) 345 key, err := s.api.Resolve(r.uri) 346 if err != nil { 347 deleteFail.Inc(1) 348 Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError) 349 return 350 } 351 352 newKey, err := s.updateManifest(key, func(mw *api.ManifestWriter) error { 353 log.Debug(fmt.Sprintf("removing %s from manifest %s", r.uri.Path, key.Log()), "ruid", r.ruid) 354 return mw.RemoveEntry(r.uri.Path) 355 }) 356 if err != nil { 357 deleteFail.Inc(1) 358 Respond(w, r, fmt.Sprintf("cannot update manifest: %s", err), http.StatusInternalServerError) 359 return 360 } 361 362 w.Header().Set("Content-Type", "text/plain") 363 w.WriteHeader(http.StatusOK) 364 fmt.Fprint(w, newKey) 365 } 366 367 // Parses a resource update post url to corresponding action 368 // possible combinations: 369 // / add multihash update to existing hash 370 // /raw add raw update to existing hash 371 // /# create new resource with first update as mulitihash 372 // /raw/# create new resource with first update raw 373 func resourcePostMode(path string) (isRaw bool, frequency uint64, err error) { 374 re, err := regexp.Compile("^(raw)?/?([0-9]+)?$") 375 if err != nil { 376 return isRaw, frequency, err 377 } 378 m := re.FindAllStringSubmatch(path, 2) 379 var freqstr = "0" 380 if len(m) > 0 { 381 if m[0][1] != "" { 382 isRaw = true 383 } 384 if m[0][2] != "" { 385 freqstr = m[0][2] 386 } 387 } else if len(path) > 0 { 388 return isRaw, frequency, fmt.Errorf("invalid path") 389 } 390 frequency, err = strconv.ParseUint(freqstr, 10, 64) 391 return isRaw, frequency, err 392 } 393 394 // Handles creation of new mutable resources and adding updates to existing mutable resources 395 // There are two types of updates available, "raw" and "multihash." 396 // If the latter is used, a subsequent bzz:// GET call to the manifest of the resource will return 397 // the page that the multihash is pointing to, as if it held a normal swarm content manifest 398 // 399 // The resource name will be verbatim what is passed as the address part of the url. 400 // For example, if a POST is made to /bzz-resource:/foo.eth/raw/13 a new resource with frequency 13 401 // and name "foo.eth" will be created 402 func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { 403 log.Debug("handle.post.resource", "ruid", r.ruid) 404 var err error 405 var addr storage.Address 406 var name string 407 var outdata []byte 408 isRaw, frequency, err := resourcePostMode(r.uri.Path) 409 if err != nil { 410 Respond(w, r, err.Error(), http.StatusBadRequest) 411 return 412 } 413 414 // new mutable resource creation will always have a frequency field larger than 0 415 if frequency > 0 { 416 417 name = r.uri.Addr 418 419 // the key is the content addressed root chunk holding mutable resource metadata information 420 addr, err = s.api.ResourceCreate(r.Context(), name, frequency) 421 if err != nil { 422 code, err2 := s.translateResourceError(w, r, "resource creation fail", err) 423 424 Respond(w, r, err2.Error(), code) 425 return 426 } 427 428 // we create a manifest so we can retrieve the resource with bzz:// later 429 // this manifest has a special "resource type" manifest, and its hash is the key of the mutable resource 430 // root chunk 431 m, err := s.api.NewResourceManifest(addr.Hex()) 432 if err != nil { 433 Respond(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError) 434 return 435 } 436 437 // the key to the manifest will be passed back to the client 438 // the client can access the root chunk key directly through its Hash member 439 // the manifest key should be set as content in the resolver of the ENS name 440 // \TODO update manifest key automatically in ENS 441 outdata, err = json.Marshal(m) 442 if err != nil { 443 Respond(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError) 444 return 445 } 446 } else { 447 // to update the resource through http we need to retrieve the key for the mutable resource root chunk 448 // that means that we retrieve the manifest and inspect its Hash member. 449 manifestAddr := r.uri.Address() 450 if manifestAddr == nil { 451 manifestAddr, err = s.api.Resolve(r.uri) 452 if err != nil { 453 getFail.Inc(1) 454 Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) 455 return 456 } 457 } else { 458 w.Header().Set("Cache-Control", "max-age=2147483648") 459 } 460 461 // get the root chunk key from the manifest 462 addr, err = s.api.ResolveResourceManifest(manifestAddr) 463 if err != nil { 464 getFail.Inc(1) 465 Respond(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", r.uri.Addr, err), http.StatusNotFound) 466 return 467 } 468 469 log.Debug("handle.post.resource: resolved", "ruid", r.ruid, "manifestkey", manifestAddr, "rootchunkkey", addr) 470 471 name, _, err = s.api.ResourceLookup(r.Context(), addr, 0, 0, &mru.LookupParams{}) 472 if err != nil { 473 Respond(w, r, err.Error(), http.StatusNotFound) 474 return 475 } 476 } 477 478 // Creation and update must send data aswell. This data constitutes the update data itself. 479 data, err := ioutil.ReadAll(r.Body) 480 if err != nil { 481 Respond(w, r, err.Error(), http.StatusInternalServerError) 482 return 483 } 484 485 // Multihash will be passed as hex-encoded data, so we need to parse this to bytes 486 if isRaw { 487 _, _, _, err = s.api.ResourceUpdate(r.Context(), name, data) 488 if err != nil { 489 Respond(w, r, err.Error(), http.StatusBadRequest) 490 return 491 } 492 } else { 493 bytesdata, err := hexutil.Decode(string(data)) 494 if err != nil { 495 Respond(w, r, err.Error(), http.StatusBadRequest) 496 return 497 } 498 _, _, _, err = s.api.ResourceUpdateMultihash(r.Context(), name, bytesdata) 499 if err != nil { 500 Respond(w, r, err.Error(), http.StatusBadRequest) 501 return 502 } 503 } 504 505 // If we have data to return, write this now 506 // \TODO there should always be data to return here 507 if len(outdata) > 0 { 508 w.Header().Add("Content-type", "text/plain") 509 w.WriteHeader(http.StatusOK) 510 fmt.Fprint(w, string(outdata)) 511 return 512 } 513 w.WriteHeader(http.StatusOK) 514 } 515 516 // Retrieve mutable resource updates: 517 // bzz-resource://<id> - get latest update 518 // bzz-resource://<id>/<n> - get latest update on period n 519 // bzz-resource://<id>/<n>/<m> - get update version m of period n 520 // <id> = ens name or hash 521 func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) { 522 s.handleGetResource(w, r) 523 } 524 525 // TODO: Enable pass maxPeriod parameter 526 func (s *Server) handleGetResource(w http.ResponseWriter, r *Request) { 527 log.Debug("handle.get.resource", "ruid", r.ruid) 528 var err error 529 530 // resolve the content key. 531 manifestAddr := r.uri.Address() 532 if manifestAddr == nil { 533 manifestAddr, err = s.api.Resolve(r.uri) 534 if err != nil { 535 getFail.Inc(1) 536 Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) 537 return 538 } 539 } else { 540 w.Header().Set("Cache-Control", "max-age=2147483648") 541 } 542 543 // get the root chunk key from the manifest 544 key, err := s.api.ResolveResourceManifest(manifestAddr) 545 if err != nil { 546 getFail.Inc(1) 547 Respond(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", r.uri.Addr, err), http.StatusNotFound) 548 return 549 } 550 551 log.Debug("handle.get.resource: resolved", "ruid", r.ruid, "manifestkey", manifestAddr, "rootchunk key", key) 552 553 // determine if the query specifies period and version 554 var params []string 555 if len(r.uri.Path) > 0 { 556 params = strings.Split(r.uri.Path, "/") 557 } 558 var name string 559 var period uint64 560 var version uint64 561 var data []byte 562 now := time.Now() 563 564 switch len(params) { 565 case 0: // latest only 566 name, data, err = s.api.ResourceLookup(r.Context(), key, 0, 0, nil) 567 case 2: // specific period and version 568 version, err = strconv.ParseUint(params[1], 10, 32) 569 if err != nil { 570 break 571 } 572 period, err = strconv.ParseUint(params[0], 10, 32) 573 if err != nil { 574 break 575 } 576 name, data, err = s.api.ResourceLookup(r.Context(), key, uint32(period), uint32(version), nil) 577 case 1: // last version of specific period 578 period, err = strconv.ParseUint(params[0], 10, 32) 579 if err != nil { 580 break 581 } 582 name, data, err = s.api.ResourceLookup(r.Context(), key, uint32(period), uint32(version), nil) 583 default: // bogus 584 err = mru.NewError(storage.ErrInvalidValue, "invalid mutable resource request") 585 } 586 587 // any error from the switch statement will end up here 588 if err != nil { 589 code, err2 := s.translateResourceError(w, r, "mutable resource lookup fail", err) 590 Respond(w, r, err2.Error(), code) 591 return 592 } 593 594 // All ok, serve the retrieved update 595 log.Debug("Found update", "name", name, "ruid", r.ruid) 596 w.Header().Set("Content-Type", "application/octet-stream") 597 http.ServeContent(w, &r.Request, "", now, bytes.NewReader(data)) 598 } 599 600 func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supErr string, err error) (int, error) { 601 code := 0 602 defaultErr := fmt.Errorf("%s: %v", supErr, err) 603 rsrcErr, ok := err.(*mru.Error) 604 if !ok && rsrcErr != nil { 605 code = rsrcErr.Code() 606 } 607 switch code { 608 case storage.ErrInvalidValue: 609 return http.StatusBadRequest, defaultErr 610 case storage.ErrNotFound, storage.ErrNotSynced, storage.ErrNothingToReturn, storage.ErrInit: 611 return http.StatusNotFound, defaultErr 612 case storage.ErrUnauthorized, storage.ErrInvalidSignature: 613 return http.StatusUnauthorized, defaultErr 614 case storage.ErrDataOverflow: 615 return http.StatusRequestEntityTooLarge, defaultErr 616 } 617 618 return http.StatusInternalServerError, defaultErr 619 } 620 621 // HandleGet handles a GET request to 622 // - bzz-raw://<key> and responds with the raw content stored at the 623 // given storage key 624 // - bzz-hash://<key> and responds with the hash of the content stored 625 // at the given storage key as a text/plain response 626 func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { 627 log.Debug("handle.get", "ruid", r.ruid, "uri", r.uri) 628 getCount.Inc(1) 629 var err error 630 addr := r.uri.Address() 631 if addr == nil { 632 addr, err = s.api.Resolve(r.uri) 633 if err != nil { 634 getFail.Inc(1) 635 Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) 636 return 637 } 638 } else { 639 w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable. 640 } 641 642 log.Debug("handle.get: resolved", "ruid", r.ruid, "key", addr) 643 644 // if path is set, interpret <key> as a manifest and return the 645 // raw entry at the given path 646 if r.uri.Path != "" { 647 walker, err := s.api.NewManifestWalker(addr, nil) 648 if err != nil { 649 getFail.Inc(1) 650 Respond(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest) 651 return 652 } 653 var entry *api.ManifestEntry 654 walker.Walk(func(e *api.ManifestEntry) error { 655 // if the entry matches the path, set entry and stop 656 // the walk 657 if e.Path == r.uri.Path { 658 entry = e 659 // return an error to cancel the walk 660 return errors.New("found") 661 } 662 663 // ignore non-manifest files 664 if e.ContentType != api.ManifestType { 665 return nil 666 } 667 668 // if the manifest's path is a prefix of the 669 // requested path, recurse into it by returning 670 // nil and continuing the walk 671 if strings.HasPrefix(r.uri.Path, e.Path) { 672 return nil 673 } 674 675 return api.ErrSkipManifest 676 }) 677 if entry == nil { 678 getFail.Inc(1) 679 Respond(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound) 680 return 681 } 682 addr = storage.Address(common.Hex2Bytes(entry.Hash)) 683 } 684 etag := common.Bytes2Hex(addr) 685 noneMatchEtag := r.Header.Get("If-None-Match") 686 w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to manifest key or raw entry key. 687 if noneMatchEtag != "" { 688 if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), addr) { 689 Respond(w, r, "Not Modified", http.StatusNotModified) 690 return 691 } 692 } 693 694 // check the root chunk exists by retrieving the file's size 695 reader, isEncrypted := s.api.Retrieve(addr) 696 if _, err := reader.Size(nil); err != nil { 697 getFail.Inc(1) 698 Respond(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound) 699 return 700 } 701 702 w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted)) 703 704 switch { 705 case r.uri.Raw(): 706 // allow the request to overwrite the content type using a query 707 // parameter 708 contentType := "application/octet-stream" 709 if typ := r.URL.Query().Get("content_type"); typ != "" { 710 contentType = typ 711 } 712 w.Header().Set("Content-Type", contentType) 713 http.ServeContent(w, &r.Request, "", time.Now(), reader) 714 case r.uri.Hash(): 715 w.Header().Set("Content-Type", "text/plain") 716 w.WriteHeader(http.StatusOK) 717 fmt.Fprint(w, addr) 718 } 719 } 720 721 // HandleGetFiles handles a GET request to bzz:/<manifest> with an Accept 722 // header of "application/x-tar" and returns a tar stream of all files 723 // contained in the manifest 724 func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { 725 log.Debug("handle.get.files", "ruid", r.ruid, "uri", r.uri) 726 getFilesCount.Inc(1) 727 if r.uri.Path != "" { 728 getFilesFail.Inc(1) 729 Respond(w, r, "files request cannot contain a path", http.StatusBadRequest) 730 return 731 } 732 733 addr, err := s.api.Resolve(r.uri) 734 if err != nil { 735 getFilesFail.Inc(1) 736 Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) 737 return 738 } 739 log.Debug("handle.get.files: resolved", "ruid", r.ruid, "key", addr) 740 741 walker, err := s.api.NewManifestWalker(addr, nil) 742 if err != nil { 743 getFilesFail.Inc(1) 744 Respond(w, r, err.Error(), http.StatusInternalServerError) 745 return 746 } 747 748 tw := tar.NewWriter(w) 749 defer tw.Close() 750 w.Header().Set("Content-Type", "application/x-tar") 751 w.WriteHeader(http.StatusOK) 752 753 err = walker.Walk(func(entry *api.ManifestEntry) error { 754 // ignore manifests (walk will recurse into them) 755 if entry.ContentType == api.ManifestType { 756 return nil 757 } 758 759 // retrieve the entry's key and size 760 reader, isEncrypted := s.api.Retrieve(storage.Address(common.Hex2Bytes(entry.Hash))) 761 size, err := reader.Size(nil) 762 if err != nil { 763 return err 764 } 765 w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted)) 766 767 // write a tar header for the entry 768 hdr := &tar.Header{ 769 Name: entry.Path, 770 Mode: entry.Mode, 771 Size: size, 772 ModTime: entry.ModTime, 773 Xattrs: map[string]string{ 774 "user.swarm.content-type": entry.ContentType, 775 }, 776 } 777 if err := tw.WriteHeader(hdr); err != nil { 778 return err 779 } 780 781 // copy the file into the tar stream 782 n, err := io.Copy(tw, io.LimitReader(reader, hdr.Size)) 783 if err != nil { 784 return err 785 } else if n != size { 786 return fmt.Errorf("error writing %s: expected %d bytes but sent %d", entry.Path, size, n) 787 } 788 789 return nil 790 }) 791 if err != nil { 792 getFilesFail.Inc(1) 793 log.Error(fmt.Sprintf("error generating tar stream: %s", err)) 794 } 795 } 796 797 // HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns 798 // a list of all files contained in <manifest> under <path> grouped into 799 // common prefixes using "/" as a delimiter 800 func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { 801 log.Debug("handle.get.list", "ruid", r.ruid, "uri", r.uri) 802 getListCount.Inc(1) 803 // ensure the root path has a trailing slash so that relative URLs work 804 if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { 805 http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently) 806 return 807 } 808 809 addr, err := s.api.Resolve(r.uri) 810 if err != nil { 811 getListFail.Inc(1) 812 Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) 813 return 814 } 815 log.Debug("handle.get.list: resolved", "ruid", r.ruid, "key", addr) 816 817 list, err := s.getManifestList(addr, r.uri.Path) 818 819 if err != nil { 820 getListFail.Inc(1) 821 Respond(w, r, err.Error(), http.StatusInternalServerError) 822 return 823 } 824 825 // if the client wants HTML (e.g. a browser) then render the list as a 826 // HTML index with relative URLs 827 if strings.Contains(r.Header.Get("Accept"), "text/html") { 828 w.Header().Set("Content-Type", "text/html") 829 err := htmlListTemplate.Execute(w, &htmlListData{ 830 URI: &api.URI{ 831 Scheme: "bzz", 832 Addr: r.uri.Addr, 833 Path: r.uri.Path, 834 }, 835 List: &list, 836 }) 837 if err != nil { 838 getListFail.Inc(1) 839 log.Error(fmt.Sprintf("error rendering list HTML: %s", err)) 840 } 841 return 842 } 843 844 w.Header().Set("Content-Type", "application/json") 845 json.NewEncoder(w).Encode(&list) 846 } 847 848 func (s *Server) getManifestList(addr storage.Address, prefix string) (list api.ManifestList, err error) { 849 walker, err := s.api.NewManifestWalker(addr, nil) 850 if err != nil { 851 return 852 } 853 854 err = walker.Walk(func(entry *api.ManifestEntry) error { 855 // handle non-manifest files 856 if entry.ContentType != api.ManifestType { 857 // ignore the file if it doesn't have the specified prefix 858 if !strings.HasPrefix(entry.Path, prefix) { 859 return nil 860 } 861 862 // if the path after the prefix contains a slash, add a 863 // common prefix to the list, otherwise add the entry 864 suffix := strings.TrimPrefix(entry.Path, prefix) 865 if index := strings.Index(suffix, "/"); index > -1 { 866 list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1]) 867 return nil 868 } 869 if entry.Path == "" { 870 entry.Path = "/" 871 } 872 list.Entries = append(list.Entries, entry) 873 return nil 874 } 875 876 // if the manifest's path is a prefix of the specified prefix 877 // then just recurse into the manifest by returning nil and 878 // continuing the walk 879 if strings.HasPrefix(prefix, entry.Path) { 880 return nil 881 } 882 883 // if the manifest's path has the specified prefix, then if the 884 // path after the prefix contains a slash, add a common prefix 885 // to the list and skip the manifest, otherwise recurse into 886 // the manifest by returning nil and continuing the walk 887 if strings.HasPrefix(entry.Path, prefix) { 888 suffix := strings.TrimPrefix(entry.Path, prefix) 889 if index := strings.Index(suffix, "/"); index > -1 { 890 list.CommonPrefixes = append(list.CommonPrefixes, prefix+suffix[:index+1]) 891 return api.ErrSkipManifest 892 } 893 return nil 894 } 895 896 // the manifest neither has the prefix or needs recursing in to 897 // so just skip it 898 return api.ErrSkipManifest 899 }) 900 901 return list, nil 902 } 903 904 // HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds 905 // with the content of the file at <path> from the given <manifest> 906 func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { 907 log.Debug("handle.get.file", "ruid", r.ruid) 908 getFileCount.Inc(1) 909 // ensure the root path has a trailing slash so that relative URLs work 910 if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { 911 http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently) 912 return 913 } 914 var err error 915 manifestAddr := r.uri.Address() 916 917 if manifestAddr == nil { 918 manifestAddr, err = s.api.Resolve(r.uri) 919 if err != nil { 920 getFileFail.Inc(1) 921 Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound) 922 return 923 } 924 } else { 925 w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable. 926 } 927 928 log.Debug("handle.get.file: resolved", "ruid", r.ruid, "key", manifestAddr) 929 930 reader, contentType, status, contentKey, err := s.api.Get(manifestAddr, r.uri.Path) 931 932 etag := common.Bytes2Hex(contentKey) 933 noneMatchEtag := r.Header.Get("If-None-Match") 934 w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to actual content key. 935 if noneMatchEtag != "" { 936 if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), contentKey) { 937 Respond(w, r, "Not Modified", http.StatusNotModified) 938 return 939 } 940 } 941 942 if err != nil { 943 switch status { 944 case http.StatusNotFound: 945 getFileNotFound.Inc(1) 946 Respond(w, r, err.Error(), http.StatusNotFound) 947 default: 948 getFileFail.Inc(1) 949 Respond(w, r, err.Error(), http.StatusInternalServerError) 950 } 951 return 952 } 953 954 //the request results in ambiguous files 955 //e.g. /read with readme.md and readinglist.txt available in manifest 956 if status == http.StatusMultipleChoices { 957 list, err := s.getManifestList(manifestAddr, r.uri.Path) 958 959 if err != nil { 960 getFileFail.Inc(1) 961 Respond(w, r, err.Error(), http.StatusInternalServerError) 962 return 963 } 964 965 log.Debug(fmt.Sprintf("Multiple choices! --> %v", list), "ruid", r.ruid) 966 //show a nice page links to available entries 967 ShowMultipleChoices(w, r, list) 968 return 969 } 970 971 // check the root chunk exists by retrieving the file's size 972 if _, err := reader.Size(nil); err != nil { 973 getFileNotFound.Inc(1) 974 Respond(w, r, fmt.Sprintf("file not found %s: %s", r.uri, err), http.StatusNotFound) 975 return 976 } 977 978 w.Header().Set("Content-Type", contentType) 979 http.ServeContent(w, &r.Request, "", time.Now(), newBufferedReadSeeker(reader, getFileBufferSize)) 980 } 981 982 // The size of buffer used for bufio.Reader on LazyChunkReader passed to 983 // http.ServeContent in HandleGetFile. 984 // Warning: This value influences the number of chunk requests and chunker join goroutines 985 // per file request. 986 // Recommended value is 4 times the io.Copy default buffer value which is 32kB. 987 const getFileBufferSize = 4 * 32 * 1024 988 989 // bufferedReadSeeker wraps bufio.Reader to expose Seek method 990 // from the provied io.ReadSeeker in newBufferedReadSeeker. 991 type bufferedReadSeeker struct { 992 r io.Reader 993 s io.Seeker 994 } 995 996 // newBufferedReadSeeker creates a new instance of bufferedReadSeeker, 997 // out of io.ReadSeeker. Argument `size` is the size of the read buffer. 998 func newBufferedReadSeeker(readSeeker io.ReadSeeker, size int) bufferedReadSeeker { 999 return bufferedReadSeeker{ 1000 r: bufio.NewReaderSize(readSeeker, size), 1001 s: readSeeker, 1002 } 1003 } 1004 1005 func (b bufferedReadSeeker) Read(p []byte) (n int, err error) { 1006 return b.r.Read(p) 1007 } 1008 1009 func (b bufferedReadSeeker) Seek(offset int64, whence int) (int64, error) { 1010 return b.s.Seek(offset, whence) 1011 } 1012 1013 func (s *Server) ServeHTTP(rw http.ResponseWriter, r *http.Request) { 1014 defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(time.Now()) 1015 req := &Request{Request: *r, ruid: uuid.New()[:8]} 1016 metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1) 1017 log.Info("serving request", "ruid", req.ruid, "method", r.Method, "url", r.RequestURI) 1018 1019 // wrapping the ResponseWriter, so that we get the response code set by http.ServeContent 1020 w := newLoggingResponseWriter(rw) 1021 1022 if r.RequestURI == "/" && strings.Contains(r.Header.Get("Accept"), "text/html") { 1023 1024 err := landingPageTemplate.Execute(w, nil) 1025 if err != nil { 1026 log.Error(fmt.Sprintf("error rendering landing page: %s", err)) 1027 } 1028 return 1029 } 1030 1031 if r.URL.Path == "/robots.txt" { 1032 w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat)) 1033 fmt.Fprintf(w, "User-agent: *\nDisallow: /") 1034 return 1035 } 1036 1037 if r.RequestURI == "/" && strings.Contains(r.Header.Get("Accept"), "application/json") { 1038 w.Header().Set("Content-Type", "application/json") 1039 w.WriteHeader(http.StatusOK) 1040 json.NewEncoder(w).Encode("Welcome to Swarm!") 1041 return 1042 } 1043 1044 uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) 1045 if err != nil { 1046 Respond(w, req, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest) 1047 return 1048 } 1049 1050 req.uri = uri 1051 1052 log.Debug("parsed request path", "ruid", req.ruid, "method", req.Method, "uri.Addr", req.uri.Addr, "uri.Path", req.uri.Path, "uri.Scheme", req.uri.Scheme) 1053 1054 switch r.Method { 1055 case "POST": 1056 if uri.Raw() { 1057 log.Debug("handlePostRaw") 1058 s.HandlePostRaw(w, req) 1059 } else if uri.Resource() { 1060 log.Debug("handlePostResource") 1061 s.HandlePostResource(w, req) 1062 } else if uri.Immutable() || uri.List() || uri.Hash() { 1063 log.Debug("POST not allowed on immutable, list or hash") 1064 Respond(w, req, fmt.Sprintf("POST method on scheme %s not allowed", uri.Scheme), http.StatusMethodNotAllowed) 1065 } else { 1066 log.Debug("handlePostFiles") 1067 s.HandlePostFiles(w, req) 1068 } 1069 1070 case "PUT": 1071 Respond(w, req, fmt.Sprintf("PUT method to %s not allowed", uri), http.StatusBadRequest) 1072 return 1073 1074 case "DELETE": 1075 if uri.Raw() { 1076 Respond(w, req, fmt.Sprintf("DELETE method to %s not allowed", uri), http.StatusBadRequest) 1077 return 1078 } 1079 s.HandleDelete(w, req) 1080 1081 case "GET": 1082 1083 if uri.Resource() { 1084 s.HandleGetResource(w, req) 1085 return 1086 } 1087 1088 if uri.Raw() || uri.Hash() { 1089 s.HandleGet(w, req) 1090 return 1091 } 1092 1093 if uri.List() { 1094 s.HandleGetList(w, req) 1095 return 1096 } 1097 1098 if r.Header.Get("Accept") == "application/x-tar" { 1099 s.HandleGetFiles(w, req) 1100 return 1101 } 1102 1103 s.HandleGetFile(w, req) 1104 1105 default: 1106 Respond(w, req, fmt.Sprintf("%s method is not supported", r.Method), http.StatusMethodNotAllowed) 1107 } 1108 1109 log.Info("served response", "ruid", req.ruid, "code", w.statusCode) 1110 } 1111 1112 func (s *Server) updateManifest(addr storage.Address, update func(mw *api.ManifestWriter) error) (storage.Address, error) { 1113 mw, err := s.api.NewManifestWriter(addr, nil) 1114 if err != nil { 1115 return nil, err 1116 } 1117 1118 if err := update(mw); err != nil { 1119 return nil, err 1120 } 1121 1122 addr, err = mw.Store() 1123 if err != nil { 1124 return nil, err 1125 } 1126 log.Debug(fmt.Sprintf("generated manifest %s", addr)) 1127 return addr, nil 1128 } 1129 1130 type loggingResponseWriter struct { 1131 http.ResponseWriter 1132 statusCode int 1133 } 1134 1135 func newLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter { 1136 return &loggingResponseWriter{w, http.StatusOK} 1137 } 1138 1139 func (lrw *loggingResponseWriter) WriteHeader(code int) { 1140 lrw.statusCode = code 1141 lrw.ResponseWriter.WriteHeader(code) 1142 }