github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/client/enumerate.go (about) 1 /* 2 Copyright 2011 Google Inc. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package client 18 19 import ( 20 "errors" 21 "fmt" 22 "log" 23 "math" 24 "net/url" 25 "time" 26 27 "camlistore.org/pkg/blob" 28 "camlistore.org/pkg/context" 29 ) 30 31 // EnumerateOpts are the options to Client.EnumerateBlobsOpts. 32 type EnumerateOpts struct { 33 After string // last blobref seen; start with ones greater than this 34 MaxWait time.Duration // how long to poll for (second granularity), waiting for any blob, or 0 for no limit 35 Limit int // if non-zero, the max blobs to return 36 } 37 38 // SimpleEnumerateBlobs sends all blobs to the provided channel. 39 // The channel will be closed, regardless of whether an error is returned. 40 func (c *Client) SimpleEnumerateBlobs(ctx *context.Context, ch chan<- blob.SizedRef) error { 41 return c.EnumerateBlobsOpts(ctx, ch, EnumerateOpts{}) 42 } 43 44 func (c *Client) EnumerateBlobs(ctx *context.Context, dest chan<- blob.SizedRef, after string, limit int) error { 45 if c.sto != nil { 46 return c.sto.EnumerateBlobs(ctx, dest, after, limit) 47 } 48 if limit == 0 { 49 log.Printf("Warning: Client.EnumerateBlobs called with a limit of zero") 50 close(dest) 51 return nil 52 } 53 return c.EnumerateBlobsOpts(ctx, dest, EnumerateOpts{ 54 After: after, 55 Limit: limit, 56 }) 57 } 58 59 const enumerateBatchSize = 1000 60 61 // EnumerateBlobsOpts sends blobs to the provided channel, as directed by opts. 62 // The channel will be closed, regardless of whether an error is returned. 63 func (c *Client) EnumerateBlobsOpts(ctx *context.Context, ch chan<- blob.SizedRef, opts EnumerateOpts) error { 64 defer close(ch) 65 if opts.After != "" && opts.MaxWait != 0 { 66 return errors.New("client error: it's invalid to use enumerate After and MaxWaitSec together") 67 } 68 pfx, err := c.prefix() 69 if err != nil { 70 return err 71 } 72 73 error := func(msg string, e error) error { 74 err := fmt.Errorf("client enumerate error: %s: %v", msg, e) 75 c.log.Print(err.Error()) 76 return err 77 } 78 79 nSent := 0 80 keepGoing := true 81 after := opts.After 82 for keepGoing { 83 waitSec := 0 84 if after == "" { 85 if opts.MaxWait > 0 { 86 waitSec = int(opts.MaxWait.Seconds()) 87 if waitSec == 0 { 88 waitSec = 1 89 } 90 } 91 } 92 url_ := fmt.Sprintf("%s/camli/enumerate-blobs?after=%s&limit=%d&maxwaitsec=%d", 93 pfx, url.QueryEscape(after), enumerateBatchSize, waitSec) 94 req := c.newRequest("GET", url_) 95 resp, err := c.httpClient.Do(req) 96 if err != nil { 97 return error("http request", err) 98 } 99 100 json, err := c.responseJSONMap("enumerate-blobs", resp) 101 if err != nil { 102 return error("stat json parse error", err) 103 } 104 105 blobs, ok := getJSONMapArray(json, "blobs") 106 if !ok { 107 return error("response JSON didn't contain 'blobs' array", nil) 108 } 109 for _, v := range blobs { 110 itemJSON, ok := v.(map[string]interface{}) 111 if !ok { 112 return error("item in 'blobs' was malformed", nil) 113 } 114 blobrefStr, ok := getJSONMapString(itemJSON, "blobRef") 115 if !ok { 116 return error("item in 'blobs' was missing string 'blobRef'", nil) 117 } 118 size, ok := getJSONMapUint32(itemJSON, "size") 119 if !ok { 120 return error("item in 'blobs' was missing numeric 'size'", nil) 121 } 122 br, ok := blob.Parse(blobrefStr) 123 if !ok { 124 return error("item in 'blobs' had invalid blobref.", nil) 125 } 126 select { 127 case ch <- blob.SizedRef{Ref: br, Size: uint32(size)}: 128 case <-ctx.Done(): 129 return context.ErrCanceled 130 } 131 nSent++ 132 if opts.Limit == nSent { 133 // nSent can't be zero at this point, so opts.Limit being 0 134 // is okay. 135 return nil 136 } 137 } 138 139 after, keepGoing = getJSONMapString(json, "continueAfter") 140 } 141 return nil 142 } 143 144 func getJSONMapString(m map[string]interface{}, key string) (string, bool) { 145 if v, ok := m[key]; ok { 146 if s, ok := v.(string); ok { 147 return s, true 148 } 149 } 150 return "", false 151 } 152 153 func getJSONMapInt64(m map[string]interface{}, key string) (int64, bool) { 154 if v, ok := m[key]; ok { 155 if n, ok := v.(float64); ok { 156 return int64(n), true 157 } 158 } 159 return 0, false 160 } 161 162 func getJSONMapUint32(m map[string]interface{}, key string) (uint32, bool) { 163 u, ok := getJSONMapInt64(m, key) 164 if !ok { 165 return 0, false 166 } 167 if u < 0 || u > math.MaxUint32 { 168 return 0, false 169 } 170 return uint32(u), true 171 } 172 173 func getJSONMapArray(m map[string]interface{}, key string) ([]interface{}, bool) { 174 if v, ok := m[key]; ok { 175 if a, ok := v.([]interface{}); ok { 176 return a, true 177 } 178 } 179 return nil, false 180 }