github.com/timstclair/heapster@v0.20.0-alpha1/Godeps/_workspace/src/google.golang.org/appengine/blobstore/blobstore.go (about) 1 // Copyright 2011 Google Inc. All rights reserved. 2 // Use of this source code is governed by the Apache 2.0 3 // license that can be found in the LICENSE file. 4 5 // Package blobstore provides a client for App Engine's persistent blob 6 // storage service. 7 package blobstore 8 9 import ( 10 "bufio" 11 "encoding/base64" 12 "fmt" 13 "io" 14 "io/ioutil" 15 "mime" 16 "mime/multipart" 17 "net/http" 18 "net/textproto" 19 "net/url" 20 "strconv" 21 "strings" 22 "time" 23 24 "github.com/golang/protobuf/proto" 25 "golang.org/x/net/context" 26 27 "google.golang.org/appengine" 28 "google.golang.org/appengine/datastore" 29 "google.golang.org/appengine/internal" 30 31 basepb "google.golang.org/appengine/internal/base" 32 blobpb "google.golang.org/appengine/internal/blobstore" 33 ) 34 35 const ( 36 blobInfoKind = "__BlobInfo__" 37 blobFileIndexKind = "__BlobFileIndex__" 38 zeroKey = appengine.BlobKey("") 39 ) 40 41 // BlobInfo is the blob metadata that is stored in the datastore. 42 // Filename may be empty. 43 type BlobInfo struct { 44 BlobKey appengine.BlobKey 45 ContentType string `datastore:"content_type"` 46 CreationTime time.Time `datastore:"creation"` 47 Filename string `datastore:"filename"` 48 Size int64 `datastore:"size"` 49 MD5 string `datastore:"md5_hash"` 50 51 // ObjectName is the Google Cloud Storage name for this blob. 52 ObjectName string `datastore:"gs_object_name"` 53 } 54 55 // isErrFieldMismatch returns whether err is a datastore.ErrFieldMismatch. 56 // 57 // The blobstore stores blob metadata in the datastore. When loading that 58 // metadata, it may contain fields that we don't care about. datastore.Get will 59 // return datastore.ErrFieldMismatch in that case, so we ignore that specific 60 // error. 61 func isErrFieldMismatch(err error) bool { 62 _, ok := err.(*datastore.ErrFieldMismatch) 63 return ok 64 } 65 66 // Stat returns the BlobInfo for a provided blobKey. If no blob was found for 67 // that key, Stat returns datastore.ErrNoSuchEntity. 68 func Stat(c context.Context, blobKey appengine.BlobKey) (*BlobInfo, error) { 69 c, _ = appengine.Namespace(c, "") // Blobstore is always in the empty string namespace 70 dskey := datastore.NewKey(c, blobInfoKind, string(blobKey), 0, nil) 71 bi := &BlobInfo{ 72 BlobKey: blobKey, 73 } 74 if err := datastore.Get(c, dskey, bi); err != nil && !isErrFieldMismatch(err) { 75 return nil, err 76 } 77 return bi, nil 78 } 79 80 // Send sets the headers on response to instruct App Engine to send a blob as 81 // the response body. This is more efficient than reading and writing it out 82 // manually and isn't subject to normal response size limits. 83 func Send(response http.ResponseWriter, blobKey appengine.BlobKey) { 84 hdr := response.Header() 85 hdr.Set("X-AppEngine-BlobKey", string(blobKey)) 86 87 if hdr.Get("Content-Type") == "" { 88 // This value is known to dev_appserver to mean automatic. 89 // In production this is remapped to the empty value which 90 // means automatic. 91 hdr.Set("Content-Type", "application/vnd.google.appengine.auto") 92 } 93 } 94 95 // UploadURL creates an upload URL for the form that the user will 96 // fill out, passing the application path to load when the POST of the 97 // form is completed. These URLs expire and should not be reused. The 98 // opts parameter may be nil. 99 func UploadURL(c context.Context, successPath string, opts *UploadURLOptions) (*url.URL, error) { 100 req := &blobpb.CreateUploadURLRequest{ 101 SuccessPath: proto.String(successPath), 102 } 103 if opts != nil { 104 if n := opts.MaxUploadBytes; n != 0 { 105 req.MaxUploadSizeBytes = &n 106 } 107 if n := opts.MaxUploadBytesPerBlob; n != 0 { 108 req.MaxUploadSizePerBlobBytes = &n 109 } 110 if s := opts.StorageBucket; s != "" { 111 req.GsBucketName = &s 112 } 113 } 114 res := &blobpb.CreateUploadURLResponse{} 115 if err := internal.Call(c, "blobstore", "CreateUploadURL", req, res); err != nil { 116 return nil, err 117 } 118 return url.Parse(*res.Url) 119 } 120 121 // UploadURLOptions are the options to create an upload URL. 122 type UploadURLOptions struct { 123 MaxUploadBytes int64 // optional 124 MaxUploadBytesPerBlob int64 // optional 125 126 // StorageBucket specifies the Google Cloud Storage bucket in which 127 // to store the blob. 128 // This is required if you use Cloud Storage instead of Blobstore. 129 // Your application must have permission to write to the bucket. 130 // You may optionally specify a bucket name and path in the format 131 // "bucket_name/path", in which case the included path will be the 132 // prefix of the uploaded object's name. 133 StorageBucket string 134 } 135 136 // Delete deletes a blob. 137 func Delete(c context.Context, blobKey appengine.BlobKey) error { 138 return DeleteMulti(c, []appengine.BlobKey{blobKey}) 139 } 140 141 // DeleteMulti deletes multiple blobs. 142 func DeleteMulti(c context.Context, blobKey []appengine.BlobKey) error { 143 s := make([]string, len(blobKey)) 144 for i, b := range blobKey { 145 s[i] = string(b) 146 } 147 req := &blobpb.DeleteBlobRequest{ 148 BlobKey: s, 149 } 150 res := &basepb.VoidProto{} 151 if err := internal.Call(c, "blobstore", "DeleteBlob", req, res); err != nil { 152 return err 153 } 154 return nil 155 } 156 157 func errorf(format string, args ...interface{}) error { 158 return fmt.Errorf("blobstore: "+format, args...) 159 } 160 161 // ParseUpload parses the synthetic POST request that your app gets from 162 // App Engine after a user's successful upload of blobs. Given the request, 163 // ParseUpload returns a map of the blobs received (keyed by HTML form 164 // element name) and other non-blob POST parameters. 165 func ParseUpload(req *http.Request) (blobs map[string][]*BlobInfo, other url.Values, err error) { 166 _, params, err := mime.ParseMediaType(req.Header.Get("Content-Type")) 167 if err != nil { 168 return nil, nil, err 169 } 170 boundary := params["boundary"] 171 if boundary == "" { 172 return nil, nil, errorf("did not find MIME multipart boundary") 173 } 174 175 blobs = make(map[string][]*BlobInfo) 176 other = make(url.Values) 177 178 mreader := multipart.NewReader(io.MultiReader(req.Body, strings.NewReader("\r\n\r\n")), boundary) 179 for { 180 part, perr := mreader.NextPart() 181 if perr == io.EOF { 182 break 183 } 184 if perr != nil { 185 return nil, nil, errorf("error reading next mime part with boundary %q (len=%d): %v", 186 boundary, len(boundary), perr) 187 } 188 189 bi := &BlobInfo{} 190 ctype, params, err := mime.ParseMediaType(part.Header.Get("Content-Disposition")) 191 if err != nil { 192 return nil, nil, err 193 } 194 bi.Filename = params["filename"] 195 formKey := params["name"] 196 197 ctype, params, err = mime.ParseMediaType(part.Header.Get("Content-Type")) 198 if err != nil { 199 return nil, nil, err 200 } 201 bi.BlobKey = appengine.BlobKey(params["blob-key"]) 202 if ctype != "message/external-body" || bi.BlobKey == "" { 203 if formKey != "" { 204 slurp, serr := ioutil.ReadAll(part) 205 if serr != nil { 206 return nil, nil, errorf("error reading %q MIME part", formKey) 207 } 208 other[formKey] = append(other[formKey], string(slurp)) 209 } 210 continue 211 } 212 213 // App Engine sends a MIME header as the body of each MIME part. 214 tp := textproto.NewReader(bufio.NewReader(part)) 215 header, mimeerr := tp.ReadMIMEHeader() 216 if mimeerr != nil { 217 return nil, nil, mimeerr 218 } 219 bi.Size, err = strconv.ParseInt(header.Get("Content-Length"), 10, 64) 220 if err != nil { 221 return nil, nil, err 222 } 223 bi.ContentType = header.Get("Content-Type") 224 225 // Parse the time from the MIME header like: 226 // X-AppEngine-Upload-Creation: 2011-03-15 21:38:34.712136 227 createDate := header.Get("X-AppEngine-Upload-Creation") 228 if createDate == "" { 229 return nil, nil, errorf("expected to find an X-AppEngine-Upload-Creation header") 230 } 231 bi.CreationTime, err = time.Parse("2006-01-02 15:04:05.000000", createDate) 232 if err != nil { 233 return nil, nil, errorf("error parsing X-AppEngine-Upload-Creation: %s", err) 234 } 235 236 if hdr := header.Get("Content-MD5"); hdr != "" { 237 md5, err := base64.URLEncoding.DecodeString(hdr) 238 if err != nil { 239 return nil, nil, errorf("bad Content-MD5 %q: %v", hdr, err) 240 } 241 bi.MD5 = string(md5) 242 } 243 244 // If the GCS object name was provided, record it. 245 bi.ObjectName = header.Get("X-AppEngine-Cloud-Storage-Object") 246 247 blobs[formKey] = append(blobs[formKey], bi) 248 } 249 return 250 } 251 252 // Reader is a blob reader. 253 type Reader interface { 254 io.Reader 255 io.ReaderAt 256 io.Seeker 257 } 258 259 // NewReader returns a reader for a blob. It always succeeds; if the blob does 260 // not exist then an error will be reported upon first read. 261 func NewReader(c context.Context, blobKey appengine.BlobKey) Reader { 262 return openBlob(c, blobKey) 263 } 264 265 // BlobKeyForFile returns a BlobKey for a Google Storage file. 266 // The filename should be of the form "/gs/bucket_name/object_name". 267 func BlobKeyForFile(c context.Context, filename string) (appengine.BlobKey, error) { 268 req := &blobpb.CreateEncodedGoogleStorageKeyRequest{ 269 Filename: &filename, 270 } 271 res := &blobpb.CreateEncodedGoogleStorageKeyResponse{} 272 if err := internal.Call(c, "blobstore", "CreateEncodedGoogleStorageKey", req, res); err != nil { 273 return "", err 274 } 275 return appengine.BlobKey(*res.BlobKey), nil 276 }