github.com/wallyworld/juju@v0.0.0-20161013125918-6cf1bc9d917a/apiserver/charms.go (about) 1 // Copyright 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package apiserver 5 6 import ( 7 "archive/zip" 8 "bytes" 9 "crypto/sha256" 10 "encoding/hex" 11 "io" 12 "io/ioutil" 13 "mime" 14 "net/http" 15 "os" 16 "path" 17 "path/filepath" 18 "sort" 19 "strconv" 20 "strings" 21 22 "github.com/juju/errors" 23 ziputil "github.com/juju/utils/zip" 24 "gopkg.in/juju/charm.v6-unstable" 25 26 "github.com/juju/juju/apiserver/application" 27 "github.com/juju/juju/apiserver/common" 28 "github.com/juju/juju/apiserver/params" 29 "github.com/juju/juju/state" 30 "github.com/juju/juju/state/storage" 31 ) 32 33 type FailableHandlerFunc func(http.ResponseWriter, *http.Request) error 34 35 // CharmsHTTPHandler creates is a http.Handler which serves POST 36 // requests to a PostHandler and GET requests to a GetHandler. 37 // 38 // TODO(katco): This is the beginning of inverting the dependencies in 39 // this callstack by splitting out the serving mechanism from the 40 // modules that are processing the requests. The next step is to 41 // publically expose construction of a suitable PostHandler and 42 // GetHandler whose goals should be clearly called out in their names, 43 // (e.g. charmPersitAPI for POSTs). 44 // 45 // To accomplish this, we'll have to make the httpContext type public 46 // so that we can pass it into these public functions. 47 // 48 // After we do this, we can then test the individual funcs/structs 49 // without standing up an entire HTTP server. I.e. actual unit 50 // tests. If you're in this area and can, please chissle away at this 51 // problem and update this TODO as needed! Many thanks, hacker! 52 type CharmsHTTPHandler struct { 53 PostHandler FailableHandlerFunc 54 GetHandler FailableHandlerFunc 55 } 56 57 func (h *CharmsHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 58 var err error 59 switch r.Method { 60 case "POST": 61 err = errors.Annotate(h.PostHandler(w, r), "cannot upload charm") 62 case "GET": 63 err = errors.Annotate(h.GetHandler(w, r), "cannot retrieve charm") 64 default: 65 err = emitUnsupportedMethodErr(r.Method) 66 } 67 68 if err != nil { 69 if err := sendJSONError(w, r, errors.Trace(err)); err != nil { 70 logger.Errorf("%v", errors.Annotate(err, "cannot return error to user")) 71 } 72 } 73 } 74 75 // charmsHandler handles charm upload through HTTPS in the API server. 76 type charmsHandler struct { 77 ctxt httpContext 78 dataDir string 79 } 80 81 // bundleContentSenderFunc functions are responsible for sending a 82 // response related to a charm bundle. 83 type bundleContentSenderFunc func(w http.ResponseWriter, r *http.Request, bundle *charm.CharmArchive) error 84 85 func (h *charmsHandler) ServePost(w http.ResponseWriter, r *http.Request) error { 86 if r.Method != "POST" { 87 return errors.Trace(emitUnsupportedMethodErr(r.Method)) 88 } 89 90 st, _, err := h.ctxt.stateForRequestAuthenticatedUser(r) 91 if err != nil { 92 return errors.Trace(err) 93 } 94 // Add a charm to the store provider. 95 charmURL, err := h.processPost(r, st) 96 if err != nil { 97 return errors.NewBadRequest(err, "") 98 } 99 return errors.Trace(sendStatusAndJSON(w, http.StatusOK, ¶ms.CharmsResponse{CharmURL: charmURL.String()})) 100 } 101 102 func (h *charmsHandler) ServeGet(w http.ResponseWriter, r *http.Request) error { 103 if r.Method != "GET" { 104 return errors.Trace(emitUnsupportedMethodErr(r.Method)) 105 } 106 107 st, _, err := h.ctxt.stateForRequestAuthenticated(r) 108 if err != nil { 109 return errors.Trace(err) 110 } 111 // Retrieve or list charm files. 112 // Requires "url" (charm URL) and an optional "file" (the path to the 113 // charm file) to be included in the query. Optionally also receives an 114 // "icon" query for returning the charm icon or a default one in case the 115 // charm has no icon. 116 charmArchivePath, fileArg, serveIcon, err := h.processGet(r, st) 117 if err != nil { 118 // An error occurred retrieving the charm bundle. 119 if errors.IsNotFound(err) { 120 return errors.Trace(err) 121 } 122 123 return errors.NewBadRequest(err, "") 124 } 125 defer os.Remove(charmArchivePath) 126 127 var sender bundleContentSenderFunc 128 switch fileArg { 129 case "": 130 // The client requested the list of charm files. 131 sender = h.manifestSender 132 case "*": 133 // The client requested the archive. 134 sender = h.archiveSender 135 default: 136 // The client requested a specific file. 137 sender = h.archiveEntrySender(fileArg, serveIcon) 138 } 139 140 return errors.Trace(sendBundleContent(w, r, charmArchivePath, sender)) 141 } 142 143 // manifestSender sends a JSON-encoded response to the client including the 144 // list of files contained in the charm bundle. 145 func (h *charmsHandler) manifestSender(w http.ResponseWriter, r *http.Request, bundle *charm.CharmArchive) error { 146 manifest, err := bundle.Manifest() 147 if err != nil { 148 return errors.Annotatef(err, "unable to read manifest in %q", bundle.Path) 149 } 150 return errors.Trace(sendStatusAndJSON(w, http.StatusOK, ¶ms.CharmsResponse{ 151 Files: manifest.SortedValues(), 152 })) 153 } 154 155 // archiveEntrySender returns a bundleContentSenderFunc which is responsible 156 // for sending the contents of filePath included in the given charm bundle. If 157 // filePath does not identify a file or a symlink, a 403 forbidden error is 158 // returned. If serveIcon is true, then the charm icon.svg file is sent, or a 159 // default icon if that file is not included in the charm. 160 func (h *charmsHandler) archiveEntrySender(filePath string, serveIcon bool) bundleContentSenderFunc { 161 return func(w http.ResponseWriter, r *http.Request, bundle *charm.CharmArchive) error { 162 // TODO(fwereade) 2014-01-27 bug #1285685 163 // This doesn't handle symlinks helpfully, and should be talking in 164 // terms of bundles rather than zip readers; but this demands thought 165 // and design and is not amenable to a quick fix. 166 zipReader, err := zip.OpenReader(bundle.Path) 167 if err != nil { 168 return errors.Annotatef(err, "unable to read charm") 169 } 170 defer zipReader.Close() 171 for _, file := range zipReader.File { 172 if path.Clean(file.Name) != filePath { 173 continue 174 } 175 fileInfo := file.FileInfo() 176 if fileInfo.IsDir() { 177 return ¶ms.Error{ 178 Message: "directory listing not allowed", 179 Code: params.CodeForbidden, 180 } 181 } 182 contents, err := file.Open() 183 if err != nil { 184 return errors.Annotatef(err, "unable to read file %q", filePath) 185 } 186 defer contents.Close() 187 ctype := mime.TypeByExtension(filepath.Ext(filePath)) 188 if ctype != "" { 189 // Older mime.types may map .js to x-javascript. 190 // Map it to javascript for consistency. 191 if ctype == params.ContentTypeXJS { 192 ctype = params.ContentTypeJS 193 } 194 w.Header().Set("Content-Type", ctype) 195 } 196 w.Header().Set("Content-Length", strconv.FormatInt(fileInfo.Size(), 10)) 197 w.WriteHeader(http.StatusOK) 198 io.Copy(w, contents) 199 return nil 200 } 201 if serveIcon { 202 // An icon was requested but none was found in the archive so 203 // return the default icon instead. 204 w.Header().Set("Content-Type", "image/svg+xml") 205 w.WriteHeader(http.StatusOK) 206 io.Copy(w, strings.NewReader(defaultIcon)) 207 return nil 208 } 209 return errors.NotFoundf("charm file") 210 } 211 } 212 213 // archiveSender is a bundleContentSenderFunc which is responsible for sending 214 // the contents of the given charm bundle. 215 func (h *charmsHandler) archiveSender(w http.ResponseWriter, r *http.Request, bundle *charm.CharmArchive) error { 216 // Note that http.ServeFile's error responses are not our standard JSON 217 // responses (they are the usual textual error messages as produced 218 // by http.Error), but there's not a great deal we can do about that, 219 // except accept non-JSON error responses in the client, because 220 // http.ServeFile does not provide a way of customizing its 221 // error responses. 222 http.ServeFile(w, r, bundle.Path) 223 return nil 224 } 225 226 // processPost handles a charm upload POST request after authentication. 227 func (h *charmsHandler) processPost(r *http.Request, st *state.State) (*charm.URL, error) { 228 query := r.URL.Query() 229 schema := query.Get("schema") 230 if schema == "" { 231 schema = "local" 232 } 233 234 series := query.Get("series") 235 if series != "" { 236 if err := charm.ValidateSeries(series); err != nil { 237 return nil, errors.NewBadRequest(err, "") 238 } 239 } 240 241 // Make sure the content type is zip. 242 contentType := r.Header.Get("Content-Type") 243 if contentType != "application/zip" { 244 return nil, errors.BadRequestf("expected Content-Type: application/zip, got: %v", contentType) 245 } 246 247 charmFileName, err := writeCharmToTempFile(r.Body) 248 if err != nil { 249 return nil, errors.Trace(err) 250 } 251 defer os.Remove(charmFileName) 252 253 err = h.processUploadedArchive(charmFileName) 254 if err != nil { 255 return nil, err 256 } 257 archive, err := charm.ReadCharmArchive(charmFileName) 258 if err != nil { 259 return nil, errors.BadRequestf("invalid charm archive: %v", err) 260 } 261 262 name := archive.Meta().Name 263 if err := charm.ValidateName(name); err != nil { 264 return nil, errors.NewBadRequest(err, "") 265 } 266 267 // We got it, now let's reserve a charm URL for it in state. 268 curl := &charm.URL{ 269 Schema: schema, 270 Name: archive.Meta().Name, 271 Revision: archive.Revision(), 272 Series: series, 273 } 274 if schema == "local" { 275 curl, err = st.PrepareLocalCharmUpload(curl) 276 if err != nil { 277 return nil, errors.Trace(err) 278 } 279 } else { 280 // "cs:" charms may only be uploaded into models which are 281 // being imported during model migrations. There's currently 282 // no other time where it makes sense to accept charm store 283 // charms through this endpoint. 284 if isImporting, err := modelIsImporting(st); err != nil { 285 return nil, errors.Trace(err) 286 } else if !isImporting { 287 return nil, errors.New("cs charms may only be uploaded during model migration import") 288 } 289 290 // If a revision argument is provided, it takes precedence 291 // over the revision in the charm archive. This is required to 292 // handle the revision differences between unpublished and 293 // published charms in the charm store. 294 revisionStr := query.Get("revision") 295 if revisionStr != "" { 296 curl.Revision, err = strconv.Atoi(revisionStr) 297 if err != nil { 298 return nil, errors.NewBadRequest(errors.NewNotValid(err, "revision"), "") 299 } 300 } 301 if _, err := st.PrepareStoreCharmUpload(curl); err != nil { 302 return nil, errors.Trace(err) 303 } 304 } 305 306 // Now we need to repackage it with the reserved URL, upload it to 307 // provider storage and update the state. 308 err = h.repackageAndUploadCharm(st, archive, curl) 309 if err != nil { 310 return nil, errors.Trace(err) 311 } 312 return curl, nil 313 } 314 315 // processUploadedArchive opens the given charm archive from path, 316 // inspects it to see if it has all files at the root of the archive 317 // or it has subdirs. It repackages the archive so it has all the 318 // files at the root dir, if necessary, replacing the original archive 319 // at path. 320 func (h *charmsHandler) processUploadedArchive(path string) error { 321 // Open the archive as a zip. 322 f, err := os.OpenFile(path, os.O_RDWR, 0644) 323 if err != nil { 324 return err 325 } 326 defer f.Close() 327 fi, err := f.Stat() 328 if err != nil { 329 return err 330 } 331 zipr, err := zip.NewReader(f, fi.Size()) 332 if err != nil { 333 return errors.Annotate(err, "cannot open charm archive") 334 } 335 336 // Find out the root dir prefix from the archive. 337 rootDir, err := h.findArchiveRootDir(zipr) 338 if err != nil { 339 return errors.Annotate(err, "cannot read charm archive") 340 } 341 if rootDir == "." { 342 // Normal charm, just use charm.ReadCharmArchive). 343 return nil 344 } 345 346 // There is one or more subdirs, so we need extract it to a temp 347 // dir and then read it as a charm dir. 348 tempDir, err := ioutil.TempDir("", "charm-extract") 349 if err != nil { 350 return errors.Annotate(err, "cannot create temp directory") 351 } 352 defer os.RemoveAll(tempDir) 353 if err := ziputil.Extract(zipr, tempDir, rootDir); err != nil { 354 return errors.Annotate(err, "cannot extract charm archive") 355 } 356 dir, err := charm.ReadCharmDir(tempDir) 357 if err != nil { 358 return errors.Annotate(err, "cannot read extracted archive") 359 } 360 361 // Now repackage the dir as a bundle at the original path. 362 if err := f.Truncate(0); err != nil { 363 return err 364 } 365 if err := dir.ArchiveTo(f); err != nil { 366 return err 367 } 368 return nil 369 } 370 371 // findArchiveRootDir scans a zip archive and returns the rootDir of 372 // the archive, the one containing metadata.yaml, config.yaml and 373 // revision files, or an error if the archive appears invalid. 374 func (h *charmsHandler) findArchiveRootDir(zipr *zip.Reader) (string, error) { 375 paths, err := ziputil.Find(zipr, "metadata.yaml") 376 if err != nil { 377 return "", err 378 } 379 switch len(paths) { 380 case 0: 381 return "", errors.Errorf("invalid charm archive: missing metadata.yaml") 382 case 1: 383 default: 384 sort.Sort(byDepth(paths)) 385 if depth(paths[0]) == depth(paths[1]) { 386 return "", errors.Errorf("invalid charm archive: ambiguous root directory") 387 } 388 } 389 return filepath.Dir(paths[0]), nil 390 } 391 392 func depth(path string) int { 393 return strings.Count(path, "/") 394 } 395 396 type byDepth []string 397 398 func (d byDepth) Len() int { return len(d) } 399 func (d byDepth) Swap(i, j int) { d[i], d[j] = d[j], d[i] } 400 func (d byDepth) Less(i, j int) bool { return depth(d[i]) < depth(d[j]) } 401 402 // repackageAndUploadCharm expands the given charm archive to a 403 // temporary directoy, repackages it with the given curl's revision, 404 // then uploads it to storage, and finally updates the state. 405 func (h *charmsHandler) repackageAndUploadCharm(st *state.State, archive *charm.CharmArchive, curl *charm.URL) error { 406 // Create a temp dir to contain the extracted charm dir. 407 tempDir, err := ioutil.TempDir("", "charm-download") 408 if err != nil { 409 return errors.Annotate(err, "cannot create temp directory") 410 } 411 defer os.RemoveAll(tempDir) 412 extractPath := filepath.Join(tempDir, "extracted") 413 414 // Expand and repack it with the revision specified by curl. 415 archive.SetRevision(curl.Revision) 416 if err := archive.ExpandTo(extractPath); err != nil { 417 return errors.Annotate(err, "cannot extract uploaded charm") 418 } 419 charmDir, err := charm.ReadCharmDir(extractPath) 420 if err != nil { 421 return errors.Annotate(err, "cannot read extracted charm") 422 } 423 424 // Bundle the charm and calculate its sha256 hash at the same time. 425 var repackagedArchive bytes.Buffer 426 hash := sha256.New() 427 err = charmDir.ArchiveTo(io.MultiWriter(hash, &repackagedArchive)) 428 if err != nil { 429 return errors.Annotate(err, "cannot repackage uploaded charm") 430 } 431 bundleSHA256 := hex.EncodeToString(hash.Sum(nil)) 432 433 info := application.CharmArchive{ 434 ID: curl, 435 Charm: archive, 436 Data: &repackagedArchive, 437 Size: int64(repackagedArchive.Len()), 438 SHA256: bundleSHA256, 439 } 440 // Store the charm archive in environment storage. 441 return application.StoreCharmArchive(st, info) 442 } 443 444 // processGet handles a charm file GET request after authentication. 445 // It returns the bundle path, the requested file path (if any), whether the 446 // default charm icon has been requested and an error. 447 func (h *charmsHandler) processGet(r *http.Request, st *state.State) ( 448 archivePath string, 449 fileArg string, 450 serveIcon bool, 451 err error, 452 ) { 453 errRet := func(err error) (string, string, bool, error) { 454 return "", "", false, err 455 } 456 457 query := r.URL.Query() 458 459 // Retrieve and validate query parameters. 460 curlString := query.Get("url") 461 if curlString == "" { 462 return errRet(errors.Errorf("expected url=CharmURL query argument")) 463 } 464 curl, err := charm.ParseURL(curlString) 465 if err != nil { 466 return errRet(errors.Trace(err)) 467 } 468 fileArg = query.Get("file") 469 if fileArg != "" { 470 fileArg = path.Clean(fileArg) 471 } else if query.Get("icon") == "1" { 472 serveIcon = true 473 fileArg = "icon.svg" 474 } 475 476 // Ensure the working directory exists. 477 tmpDir := filepath.Join(h.dataDir, "charm-get-tmp") 478 if err = os.MkdirAll(tmpDir, 0755); err != nil { 479 return errRet(errors.Annotate(err, "cannot create charms tmp directory")) 480 } 481 482 // Use the storage to retrieve and save the charm archive. 483 storage := storage.NewStorage(st.ModelUUID(), st.MongoSession()) 484 ch, err := st.Charm(curl) 485 if err != nil { 486 return errRet(errors.Annotate(err, "cannot get charm from state")) 487 } 488 489 reader, _, err := storage.Get(ch.StoragePath()) 490 if err != nil { 491 return errRet(errors.Annotate(err, "cannot get charm from model storage")) 492 } 493 defer reader.Close() 494 495 charmFile, err := ioutil.TempFile(tmpDir, "charm") 496 if err != nil { 497 return errRet(errors.Annotate(err, "cannot create charm archive file")) 498 } 499 if _, err = io.Copy(charmFile, reader); err != nil { 500 cleanupFile(charmFile) 501 return errRet(errors.Annotate(err, "error processing charm archive download")) 502 } 503 504 charmFile.Close() 505 return charmFile.Name(), fileArg, serveIcon, nil 506 } 507 508 // sendJSONError sends a JSON-encoded error response. Note the 509 // difference from the error response sent by the sendError function - 510 // the error is encoded in the Error field as a string, not an Error 511 // object. 512 func sendJSONError(w http.ResponseWriter, req *http.Request, err error) error { 513 logger.Errorf("returning error from %s %s: %s", req.Method, req.URL, errors.Details(err)) 514 perr, status := common.ServerErrorAndStatus(err) 515 return errors.Trace(sendStatusAndJSON(w, status, ¶ms.CharmsResponse{ 516 Error: perr.Message, 517 ErrorCode: perr.Code, 518 ErrorInfo: perr.Info, 519 })) 520 } 521 522 // sendBundleContent uses the given bundleContentSenderFunc to send a 523 // response related to the charm archive located in the given 524 // archivePath. 525 func sendBundleContent( 526 w http.ResponseWriter, 527 r *http.Request, 528 archivePath string, 529 sender bundleContentSenderFunc, 530 ) error { 531 bundle, err := charm.ReadCharmArchive(archivePath) 532 if err != nil { 533 return errors.Annotatef(err, "unable to read archive in %q", archivePath) 534 } 535 // The bundleContentSenderFunc will set up and send an appropriate response. 536 if err := sender(w, r, bundle); err != nil { 537 return errors.Trace(err) 538 } 539 return nil 540 } 541 542 // On windows we cannot remove a file until it has been closed 543 // If this poses an active problem somewhere else it will be refactored in 544 // utils and used everywhere. 545 func cleanupFile(file *os.File) { 546 // Errors are ignored because it is ok for this to be called when 547 // the file is already closed or has been moved. 548 file.Close() 549 os.Remove(file.Name()) 550 } 551 552 func writeCharmToTempFile(r io.Reader) (string, error) { 553 tempFile, err := ioutil.TempFile("", "charm") 554 if err != nil { 555 return "", errors.Annotate(err, "creating temp file") 556 } 557 defer tempFile.Close() 558 if _, err := io.Copy(tempFile, r); err != nil { 559 return "", errors.Annotate(err, "processing upload") 560 } 561 return tempFile.Name(), nil 562 } 563 564 func modelIsImporting(st *state.State) (bool, error) { 565 model, err := st.Model() 566 if err != nil { 567 return false, errors.Trace(err) 568 } 569 return model.MigrationMode() == state.MigrationModeImporting, nil 570 } 571 572 func emitUnsupportedMethodErr(method string) error { 573 return errors.MethodNotAllowedf("unsupported method: %q", method) 574 }