github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/blobserver/handlers/upload.go (about) 1 /* 2 Copyright 2011 Google Inc. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package handlers 18 19 import ( 20 "bytes" 21 "crypto/sha1" 22 "errors" 23 "fmt" 24 "io" 25 "log" 26 "mime" 27 "net/http" 28 "path" 29 "strings" 30 "time" 31 32 "camlistore.org/pkg/blob" 33 "camlistore.org/pkg/blobserver" 34 "camlistore.org/pkg/blobserver/protocol" 35 "camlistore.org/pkg/httputil" 36 "camlistore.org/pkg/jsonsign/signhandler" 37 "camlistore.org/pkg/readerutil" 38 "camlistore.org/pkg/schema" 39 ) 40 41 // CreateBatchUploadHandler returns the handler that receives multi-part form uploads 42 // to upload many blobs at once. See doc/protocol/blob-upload-protocol.txt. 43 func CreateBatchUploadHandler(storage blobserver.BlobReceiveConfiger) http.Handler { 44 return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { 45 handleMultiPartUpload(rw, req, storage) 46 }) 47 } 48 49 // CreatePutUploadHandler returns the handler that receives a single 50 // blob at the blob's final URL, via the PUT method. See 51 // doc/protocol/blob-upload-protocol.txt. 52 func CreatePutUploadHandler(storage blobserver.BlobReceiver) http.Handler { 53 return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { 54 if req.Method != "PUT" { 55 log.Printf("Inconfigured upload handler.") 56 httputil.BadRequestError(rw, "Inconfigured handler.") 57 return 58 } 59 // For non-chunked uploads, we catch it here. For chunked uploads, it's caught 60 // by blobserver.Receive's LimitReader. 61 if req.ContentLength > blobserver.MaxBlobSize { 62 httputil.BadRequestError(rw, "blob too big") 63 return 64 } 65 blobrefStr := path.Base(req.URL.Path) 66 br, ok := blob.Parse(blobrefStr) 67 if !ok { 68 log.Printf("Invalid PUT request to %q", req.URL.Path) 69 httputil.BadRequestError(rw, "Bad path") 70 return 71 } 72 if !br.IsSupported() { 73 httputil.BadRequestError(rw, "unsupported object hash function") 74 return 75 } 76 _, err := blobserver.Receive(storage, br, req.Body) 77 if err == blobserver.ErrCorruptBlob { 78 httputil.BadRequestError(rw, "data doesn't match declared digest") 79 return 80 } 81 if err != nil { 82 httputil.ServeError(rw, req, err) 83 return 84 } 85 rw.WriteHeader(http.StatusNoContent) 86 }) 87 } 88 89 // vivify verifies that all the chunks for the file described by fileblob are on the blobserver. 90 // It makes a planned permanode, signs it, and uploads it. It finally makes a camliContent claim 91 // on that permanode for fileblob, signs it, and uploads it to the blobserver. 92 func vivify(blobReceiver blobserver.BlobReceiveConfiger, fileblob blob.SizedRef) error { 93 sf, ok := blobReceiver.(blob.Fetcher) 94 if !ok { 95 return fmt.Errorf("BlobReceiver is not a Fetcher") 96 } 97 fr, err := schema.NewFileReader(sf, fileblob.Ref) 98 if err != nil { 99 return fmt.Errorf("Filereader error for blobref %v: %v", fileblob.Ref.String(), err) 100 } 101 defer fr.Close() 102 103 h := sha1.New() 104 n, err := io.Copy(h, fr) 105 if err != nil { 106 return fmt.Errorf("Could not read all file of blobref %v: %v", fileblob.Ref.String(), err) 107 } 108 if n != fr.Size() { 109 return fmt.Errorf("Could not read all file of blobref %v. Wanted %v, got %v", fileblob.Ref.String(), fr.Size(), n) 110 } 111 112 config := blobReceiver.Config() 113 if config == nil { 114 return errors.New("blobReceiver has no config") 115 } 116 hf := config.HandlerFinder 117 if hf == nil { 118 return errors.New("blobReceiver config has no HandlerFinder") 119 } 120 JSONSignRoot, sh, err := hf.FindHandlerByType("jsonsign") 121 if err != nil || sh == nil { 122 return errors.New("jsonsign handler not found") 123 } 124 sigHelper, ok := sh.(*signhandler.Handler) 125 if !ok { 126 return errors.New("handler is not a JSON signhandler") 127 } 128 discoMap := sigHelper.DiscoveryMap(JSONSignRoot) 129 publicKeyBlobRef, ok := discoMap["publicKeyBlobRef"].(string) 130 if !ok { 131 return fmt.Errorf("Discovery: json decoding error: %v", err) 132 } 133 134 // The file schema must have a modtime to vivify, as the modtime is used for all three of: 135 // 1) the permanode's signature 136 // 2) the camliContent attribute claim's "claimDate" 137 // 3) the signature time of 2) 138 claimDate, err := time.Parse(time.RFC3339, fr.FileSchema().UnixMtime) 139 if err != nil { 140 return fmt.Errorf("While parsing modtime for file %v: %v", fr.FileSchema().FileName, err) 141 } 142 143 permanodeBB := schema.NewHashPlannedPermanode(h) 144 permanodeBB.SetSigner(blob.MustParse(publicKeyBlobRef)) 145 permanodeBB.SetClaimDate(claimDate) 146 permanodeSigned, err := sigHelper.Sign(permanodeBB) 147 if err != nil { 148 return fmt.Errorf("Signing permanode %v: %v", permanodeSigned, err) 149 } 150 permanodeRef := blob.SHA1FromString(permanodeSigned) 151 _, err = blobserver.ReceiveNoHash(blobReceiver, permanodeRef, strings.NewReader(permanodeSigned)) 152 if err != nil { 153 return fmt.Errorf("While uploading signed permanode %v, %v: %v", permanodeRef, permanodeSigned, err) 154 } 155 156 contentClaimBB := schema.NewSetAttributeClaim(permanodeRef, "camliContent", fileblob.Ref.String()) 157 contentClaimBB.SetSigner(blob.MustParse(publicKeyBlobRef)) 158 contentClaimBB.SetClaimDate(claimDate) 159 contentClaimSigned, err := sigHelper.Sign(contentClaimBB) 160 if err != nil { 161 return fmt.Errorf("Signing camliContent claim: %v", err) 162 } 163 contentClaimRef := blob.SHA1FromString(contentClaimSigned) 164 _, err = blobserver.ReceiveNoHash(blobReceiver, contentClaimRef, strings.NewReader(contentClaimSigned)) 165 if err != nil { 166 return fmt.Errorf("While uploading signed camliContent claim %v, %v: %v", contentClaimRef, contentClaimSigned, err) 167 } 168 return nil 169 } 170 171 func handleMultiPartUpload(rw http.ResponseWriter, req *http.Request, blobReceiver blobserver.BlobReceiveConfiger) { 172 res := new(protocol.UploadResponse) 173 174 if !(req.Method == "POST" && strings.Contains(req.URL.Path, "/camli/upload")) { 175 log.Printf("Inconfigured handler upload handler") 176 httputil.BadRequestError(rw, "Inconfigured handler.") 177 return 178 } 179 180 receivedBlobs := make([]blob.SizedRef, 0, 10) 181 182 multipart, err := req.MultipartReader() 183 if multipart == nil { 184 httputil.BadRequestError(rw, fmt.Sprintf( 185 "Expected multipart/form-data POST request; %v", err)) 186 return 187 } 188 189 var errBuf bytes.Buffer 190 addError := func(s string) { 191 log.Printf("Client error: %s", s) 192 if errBuf.Len() > 0 { 193 errBuf.WriteByte('\n') 194 } 195 errBuf.WriteString(s) 196 } 197 198 for { 199 mimePart, err := multipart.NextPart() 200 if err == io.EOF { 201 break 202 } 203 if err != nil { 204 addError(fmt.Sprintf("Error reading multipart section: %v", err)) 205 break 206 } 207 208 contentDisposition, params, err := mime.ParseMediaType(mimePart.Header.Get("Content-Disposition")) 209 if err != nil { 210 addError("invalid Content-Disposition") 211 break 212 } 213 214 if contentDisposition != "form-data" { 215 addError(fmt.Sprintf("Expected Content-Disposition of \"form-data\"; got %q", contentDisposition)) 216 break 217 } 218 219 formName := params["name"] 220 ref, ok := blob.Parse(formName) 221 if !ok { 222 addError(fmt.Sprintf("Ignoring form key %q", formName)) 223 continue 224 } 225 226 var tooBig int64 = blobserver.MaxBlobSize + 1 227 var readBytes int64 228 blobGot, err := blobserver.Receive(blobReceiver, ref, &readerutil.CountingReader{ 229 io.LimitReader(mimePart, tooBig), 230 &readBytes, 231 }) 232 if readBytes == tooBig { 233 err = fmt.Errorf("blob over the limit of %d bytes", blobserver.MaxBlobSize) 234 } 235 if err != nil { 236 addError(fmt.Sprintf("Error receiving blob %v: %v\n", ref, err)) 237 break 238 } 239 log.Printf("Received blob %v\n", blobGot) 240 receivedBlobs = append(receivedBlobs, blobGot) 241 } 242 243 for _, got := range receivedBlobs { 244 res.Received = append(res.Received, &protocol.RefAndSize{ 245 Ref: got.Ref, 246 Size: uint32(got.Size), 247 }) 248 } 249 250 if req.Header.Get("X-Camlistore-Vivify") == "1" { 251 for _, got := range receivedBlobs { 252 err := vivify(blobReceiver, got) 253 if err != nil { 254 addError(fmt.Sprintf("Error vivifying blob %v: %v\n", got.Ref.String(), err)) 255 } else { 256 rw.Header().Add("X-Camlistore-Vivified", got.Ref.String()) 257 } 258 } 259 } 260 261 res.ErrorText = errBuf.String() 262 263 httputil.ReturnJSON(rw, res) 264 }