github.com/olivere/camlistore@v0.0.0-20140121221811-1b7ac2da0199/pkg/blobserver/handlers/enumerate.go (about)

     1  /*
     2  Copyright 2011 Google Inc.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package handlers
    18  
    19  import (
    20  	"fmt"
    21  	"log"
    22  	"net/http"
    23  	"os"
    24  	"strconv"
    25  	"time"
    26  
    27  	"camlistore.org/pkg/blob"
    28  	"camlistore.org/pkg/blobserver"
    29  	"camlistore.org/pkg/context"
    30  )
    31  
    32  const defaultMaxEnumerate = 10000
    33  const defaultEnumerateSize = 100
    34  
    35  type blobInfo struct {
    36  	blob.Ref
    37  	os.FileInfo
    38  	error
    39  }
    40  
    41  func CreateEnumerateHandler(storage blobserver.BlobEnumerator) http.Handler {
    42  	return http.HandlerFunc(func(conn http.ResponseWriter, req *http.Request) {
    43  		handleEnumerateBlobs(conn, req, storage)
    44  	})
    45  }
    46  
    47  const errMsgMaxWaitSecWithAfter = "Can't use 'maxwaitsec' with 'after'.\n"
    48  
    49  func handleEnumerateBlobs(conn http.ResponseWriter, req *http.Request, storage blobserver.BlobEnumerator) {
    50  	// Potential input parameters
    51  	formValueLimit := req.FormValue("limit")
    52  	formValueMaxWaitSec := req.FormValue("maxwaitsec")
    53  	formValueAfter := req.FormValue("after")
    54  
    55  	maxEnumerate := defaultMaxEnumerate
    56  	if config, ok := storage.(blobserver.MaxEnumerateConfig); ok {
    57  		maxEnumerate = config.MaxEnumerate() - 1 // Since we'll add one below.
    58  	}
    59  
    60  	limit := defaultEnumerateSize
    61  	if formValueLimit != "" {
    62  		n, err := strconv.ParseUint(formValueLimit, 10, 32)
    63  		if err != nil || n > uint64(maxEnumerate) {
    64  			limit = maxEnumerate
    65  		} else {
    66  			limit = int(n)
    67  		}
    68  	}
    69  
    70  	waitSeconds := 0
    71  	if formValueMaxWaitSec != "" {
    72  		waitSeconds, _ = strconv.Atoi(formValueMaxWaitSec)
    73  		if waitSeconds != 0 && formValueAfter != "" {
    74  			conn.WriteHeader(http.StatusBadRequest)
    75  			fmt.Fprintf(conn, errMsgMaxWaitSecWithAfter)
    76  			return
    77  		}
    78  		switch {
    79  		case waitSeconds < 0:
    80  			waitSeconds = 0
    81  		case waitSeconds > 30:
    82  			// TODO: don't hard-code 30.  push this up into a blobserver interface
    83  			// for getting the configuration of the server (ultimately a flag in
    84  			// in the binary)
    85  			waitSeconds = 30
    86  		}
    87  	}
    88  
    89  	conn.Header().Set("Content-Type", "text/javascript; charset=utf-8")
    90  	fmt.Fprintf(conn, "{\n  \"blobs\": [\n")
    91  
    92  	loop := true
    93  	needsComma := false
    94  	deadline := time.Now().Add(time.Duration(waitSeconds) * time.Second)
    95  	after := ""
    96  	for loop && (waitSeconds == 0 || time.Now().After(deadline)) {
    97  		if waitSeconds == 0 {
    98  			loop = false
    99  		}
   100  
   101  		blobch := make(chan blob.SizedRef, 100)
   102  		resultch := make(chan error, 1)
   103  		go func() {
   104  			resultch <- storage.EnumerateBlobs(context.TODO(), blobch, formValueAfter, limit+1)
   105  		}()
   106  
   107  		endsReached := 0
   108  		gotBlobs := 0
   109  		for endsReached < 2 {
   110  			select {
   111  			case sb, ok := <-blobch:
   112  				if !ok {
   113  					endsReached++
   114  					if gotBlobs <= limit {
   115  						after = ""
   116  					}
   117  					continue
   118  				}
   119  				gotBlobs++
   120  				loop = false
   121  				if gotBlobs > limit {
   122  					// We requested one more from storage than the user asked for.
   123  					// Now we know to return a "continueAfter" response key.
   124  					// But we don't return this blob.
   125  					continue
   126  				}
   127  				blobName := sb.Ref.String()
   128  				if needsComma {
   129  					fmt.Fprintf(conn, ",\n")
   130  				}
   131  				fmt.Fprintf(conn, "    {\"blobRef\": \"%s\", \"size\": %d}",
   132  					blobName, sb.Size)
   133  				after = blobName
   134  				needsComma = true
   135  			case err := <-resultch:
   136  				if err != nil {
   137  					log.Printf("Error during enumerate: %v", err)
   138  					fmt.Fprintf(conn, "{{{ SERVER ERROR }}}")
   139  					return
   140  				}
   141  				endsReached++
   142  			}
   143  		}
   144  
   145  		if loop {
   146  			blobserver.WaitForBlob(storage, deadline, nil)
   147  		}
   148  	}
   149  	fmt.Fprintf(conn, "\n  ]")
   150  	if after != "" {
   151  		fmt.Fprintf(conn, ",\n  \"continueAfter\": \"%s\"", after)
   152  	}
   153  	const longPollSupported = true
   154  	if longPollSupported {
   155  		fmt.Fprintf(conn, ",\n  \"canLongPoll\": true")
   156  	}
   157  	fmt.Fprintf(conn, "\n}\n")
   158  }