github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/blobserver/handlers/upload.go (about)

     1  /*
     2  Copyright 2011 Google Inc.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package handlers
    18  
    19  import (
    20  	"bytes"
    21  	"crypto/sha1"
    22  	"errors"
    23  	"fmt"
    24  	"io"
    25  	"log"
    26  	"mime"
    27  	"net/http"
    28  	"path"
    29  	"strings"
    30  	"time"
    31  
    32  	"camlistore.org/pkg/blob"
    33  	"camlistore.org/pkg/blobserver"
    34  	"camlistore.org/pkg/blobserver/protocol"
    35  	"camlistore.org/pkg/httputil"
    36  	"camlistore.org/pkg/jsonsign/signhandler"
    37  	"camlistore.org/pkg/readerutil"
    38  	"camlistore.org/pkg/schema"
    39  )
    40  
    41  // CreateBatchUploadHandler returns the handler that receives multi-part form uploads
    42  // to upload many blobs at once. See doc/protocol/blob-upload-protocol.txt.
    43  func CreateBatchUploadHandler(storage blobserver.BlobReceiveConfiger) http.Handler {
    44  	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
    45  		handleMultiPartUpload(rw, req, storage)
    46  	})
    47  }
    48  
    49  // CreatePutUploadHandler returns the handler that receives a single
    50  // blob at the blob's final URL, via the PUT method.  See
    51  // doc/protocol/blob-upload-protocol.txt.
    52  func CreatePutUploadHandler(storage blobserver.BlobReceiver) http.Handler {
    53  	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
    54  		if req.Method != "PUT" {
    55  			log.Printf("Inconfigured upload handler.")
    56  			httputil.BadRequestError(rw, "Inconfigured handler.")
    57  			return
    58  		}
    59  		// For non-chunked uploads, we catch it here. For chunked uploads, it's caught
    60  		// by blobserver.Receive's LimitReader.
    61  		if req.ContentLength > blobserver.MaxBlobSize {
    62  			httputil.BadRequestError(rw, "blob too big")
    63  			return
    64  		}
    65  		blobrefStr := path.Base(req.URL.Path)
    66  		br, ok := blob.Parse(blobrefStr)
    67  		if !ok {
    68  			log.Printf("Invalid PUT request to %q", req.URL.Path)
    69  			httputil.BadRequestError(rw, "Bad path")
    70  			return
    71  		}
    72  		if !br.IsSupported() {
    73  			httputil.BadRequestError(rw, "unsupported object hash function")
    74  			return
    75  		}
    76  		_, err := blobserver.Receive(storage, br, req.Body)
    77  		if err == blobserver.ErrCorruptBlob {
    78  			httputil.BadRequestError(rw, "data doesn't match declared digest")
    79  			return
    80  		}
    81  		if err != nil {
    82  			httputil.ServeError(rw, req, err)
    83  			return
    84  		}
    85  		rw.WriteHeader(http.StatusNoContent)
    86  	})
    87  }
    88  
    89  // vivify verifies that all the chunks for the file described by fileblob are on the blobserver.
    90  // It makes a planned permanode, signs it, and uploads it. It finally makes a camliContent claim
    91  // on that permanode for fileblob, signs it, and uploads it to the blobserver.
    92  func vivify(blobReceiver blobserver.BlobReceiveConfiger, fileblob blob.SizedRef) error {
    93  	sf, ok := blobReceiver.(blob.Fetcher)
    94  	if !ok {
    95  		return fmt.Errorf("BlobReceiver is not a Fetcher")
    96  	}
    97  	fr, err := schema.NewFileReader(sf, fileblob.Ref)
    98  	if err != nil {
    99  		return fmt.Errorf("Filereader error for blobref %v: %v", fileblob.Ref.String(), err)
   100  	}
   101  	defer fr.Close()
   102  
   103  	h := sha1.New()
   104  	n, err := io.Copy(h, fr)
   105  	if err != nil {
   106  		return fmt.Errorf("Could not read all file of blobref %v: %v", fileblob.Ref.String(), err)
   107  	}
   108  	if n != fr.Size() {
   109  		return fmt.Errorf("Could not read all file of blobref %v. Wanted %v, got %v", fileblob.Ref.String(), fr.Size(), n)
   110  	}
   111  
   112  	config := blobReceiver.Config()
   113  	if config == nil {
   114  		return errors.New("blobReceiver has no config")
   115  	}
   116  	hf := config.HandlerFinder
   117  	if hf == nil {
   118  		return errors.New("blobReceiver config has no HandlerFinder")
   119  	}
   120  	JSONSignRoot, sh, err := hf.FindHandlerByType("jsonsign")
   121  	if err != nil || sh == nil {
   122  		return errors.New("jsonsign handler not found")
   123  	}
   124  	sigHelper, ok := sh.(*signhandler.Handler)
   125  	if !ok {
   126  		return errors.New("handler is not a JSON signhandler")
   127  	}
   128  	discoMap := sigHelper.DiscoveryMap(JSONSignRoot)
   129  	publicKeyBlobRef, ok := discoMap["publicKeyBlobRef"].(string)
   130  	if !ok {
   131  		return fmt.Errorf("Discovery: json decoding error: %v", err)
   132  	}
   133  
   134  	// The file schema must have a modtime to vivify, as the modtime is used for all three of:
   135  	// 1) the permanode's signature
   136  	// 2) the camliContent attribute claim's "claimDate"
   137  	// 3) the signature time of 2)
   138  	claimDate, err := time.Parse(time.RFC3339, fr.FileSchema().UnixMtime)
   139  	if err != nil {
   140  		return fmt.Errorf("While parsing modtime for file %v: %v", fr.FileSchema().FileName, err)
   141  	}
   142  
   143  	permanodeBB := schema.NewHashPlannedPermanode(h)
   144  	permanodeBB.SetSigner(blob.MustParse(publicKeyBlobRef))
   145  	permanodeBB.SetClaimDate(claimDate)
   146  	permanodeSigned, err := sigHelper.Sign(permanodeBB)
   147  	if err != nil {
   148  		return fmt.Errorf("Signing permanode %v: %v", permanodeSigned, err)
   149  	}
   150  	permanodeRef := blob.SHA1FromString(permanodeSigned)
   151  	_, err = blobserver.ReceiveNoHash(blobReceiver, permanodeRef, strings.NewReader(permanodeSigned))
   152  	if err != nil {
   153  		return fmt.Errorf("While uploading signed permanode %v, %v: %v", permanodeRef, permanodeSigned, err)
   154  	}
   155  
   156  	contentClaimBB := schema.NewSetAttributeClaim(permanodeRef, "camliContent", fileblob.Ref.String())
   157  	contentClaimBB.SetSigner(blob.MustParse(publicKeyBlobRef))
   158  	contentClaimBB.SetClaimDate(claimDate)
   159  	contentClaimSigned, err := sigHelper.Sign(contentClaimBB)
   160  	if err != nil {
   161  		return fmt.Errorf("Signing camliContent claim: %v", err)
   162  	}
   163  	contentClaimRef := blob.SHA1FromString(contentClaimSigned)
   164  	_, err = blobserver.ReceiveNoHash(blobReceiver, contentClaimRef, strings.NewReader(contentClaimSigned))
   165  	if err != nil {
   166  		return fmt.Errorf("While uploading signed camliContent claim %v, %v: %v", contentClaimRef, contentClaimSigned, err)
   167  	}
   168  	return nil
   169  }
   170  
   171  func handleMultiPartUpload(rw http.ResponseWriter, req *http.Request, blobReceiver blobserver.BlobReceiveConfiger) {
   172  	res := new(protocol.UploadResponse)
   173  
   174  	if !(req.Method == "POST" && strings.Contains(req.URL.Path, "/camli/upload")) {
   175  		log.Printf("Inconfigured handler upload handler")
   176  		httputil.BadRequestError(rw, "Inconfigured handler.")
   177  		return
   178  	}
   179  
   180  	receivedBlobs := make([]blob.SizedRef, 0, 10)
   181  
   182  	multipart, err := req.MultipartReader()
   183  	if multipart == nil {
   184  		httputil.BadRequestError(rw, fmt.Sprintf(
   185  			"Expected multipart/form-data POST request; %v", err))
   186  		return
   187  	}
   188  
   189  	var errBuf bytes.Buffer
   190  	addError := func(s string) {
   191  		log.Printf("Client error: %s", s)
   192  		if errBuf.Len() > 0 {
   193  			errBuf.WriteByte('\n')
   194  		}
   195  		errBuf.WriteString(s)
   196  	}
   197  
   198  	for {
   199  		mimePart, err := multipart.NextPart()
   200  		if err == io.EOF {
   201  			break
   202  		}
   203  		if err != nil {
   204  			addError(fmt.Sprintf("Error reading multipart section: %v", err))
   205  			break
   206  		}
   207  
   208  		contentDisposition, params, err := mime.ParseMediaType(mimePart.Header.Get("Content-Disposition"))
   209  		if err != nil {
   210  			addError("invalid Content-Disposition")
   211  			break
   212  		}
   213  
   214  		if contentDisposition != "form-data" {
   215  			addError(fmt.Sprintf("Expected Content-Disposition of \"form-data\"; got %q", contentDisposition))
   216  			break
   217  		}
   218  
   219  		formName := params["name"]
   220  		ref, ok := blob.Parse(formName)
   221  		if !ok {
   222  			addError(fmt.Sprintf("Ignoring form key %q", formName))
   223  			continue
   224  		}
   225  
   226  		var tooBig int64 = blobserver.MaxBlobSize + 1
   227  		var readBytes int64
   228  		blobGot, err := blobserver.Receive(blobReceiver, ref, &readerutil.CountingReader{
   229  			io.LimitReader(mimePart, tooBig),
   230  			&readBytes,
   231  		})
   232  		if readBytes == tooBig {
   233  			err = fmt.Errorf("blob over the limit of %d bytes", blobserver.MaxBlobSize)
   234  		}
   235  		if err != nil {
   236  			addError(fmt.Sprintf("Error receiving blob %v: %v\n", ref, err))
   237  			break
   238  		}
   239  		log.Printf("Received blob %v\n", blobGot)
   240  		receivedBlobs = append(receivedBlobs, blobGot)
   241  	}
   242  
   243  	for _, got := range receivedBlobs {
   244  		res.Received = append(res.Received, &protocol.RefAndSize{
   245  			Ref:  got.Ref,
   246  			Size: uint32(got.Size),
   247  		})
   248  	}
   249  
   250  	if req.Header.Get("X-Camlistore-Vivify") == "1" {
   251  		for _, got := range receivedBlobs {
   252  			err := vivify(blobReceiver, got)
   253  			if err != nil {
   254  				addError(fmt.Sprintf("Error vivifying blob %v: %v\n", got.Ref.String(), err))
   255  			} else {
   256  				rw.Header().Add("X-Camlistore-Vivified", got.Ref.String())
   257  			}
   258  		}
   259  	}
   260  
   261  	res.ErrorText = errBuf.String()
   262  
   263  	httputil.ReturnJSON(rw, res)
   264  }