github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/schema/filewriter.go (about) 1 /* 2 Copyright 2011 Google Inc. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package schema 18 19 import ( 20 "bufio" 21 "bytes" 22 "fmt" 23 "io" 24 "os" 25 "strings" 26 27 "camlistore.org/pkg/blob" 28 "camlistore.org/pkg/blobserver" 29 "camlistore.org/pkg/rollsum" 30 "camlistore.org/pkg/syncutil" 31 ) 32 33 const ( 34 // maxBlobSize is the largest blob we ever make when cutting up 35 // a file. 36 maxBlobSize = 1 << 20 37 38 // firstChunkSize is the ideal size of the first chunk of a 39 // file. It's kept smaller for the file(1) command, which 40 // likes to read 96 kB on Linux and 256 kB on OS X. Related 41 // are tools which extract the EXIF metadata from JPEGs, 42 // ID3 from mp3s, etc. Nautilus, OS X Finder, etc. 43 // The first chunk may be larger than this if cutting the file 44 // here would create a small subsequent chunk (e.g. a file one 45 // byte larger than firstChunkSize) 46 firstChunkSize = 256 << 10 47 48 // bufioReaderSize is an explicit size for our bufio.Reader, 49 // so we don't rely on NewReader's implicit size. 50 // We care about the buffer size because it affects how far 51 // in advance we can detect EOF from an io.Reader that doesn't 52 // know its size. Detecting an EOF bufioReaderSize bytes early 53 // means we can plan for the final chunk. 54 bufioReaderSize = 32 << 10 55 56 // tooSmallThreshold is the threshold at which rolling checksum 57 // boundaries are ignored if the current chunk being built is 58 // smaller than this. 59 tooSmallThreshold = 64 << 10 60 ) 61 62 // WriteFileFromReader creates and uploads a "file" JSON schema 63 // composed of chunks of r, also uploading the chunks. The returned 64 // BlobRef is of the JSON file schema blob. 65 // The filename is optional. 66 func WriteFileFromReader(bs blobserver.StatReceiver, filename string, r io.Reader) (blob.Ref, error) { 67 if strings.Contains(filename, "/") { 68 return blob.Ref{}, fmt.Errorf("schema.WriteFileFromReader: filename %q shouldn't contain a slash", filename) 69 } 70 m := NewFileMap(filename) 71 return WriteFileMap(bs, m, r) 72 } 73 74 // WriteFileMap uploads chunks of r to bs while populating file and 75 // finally uploading file's Blob. The returned blobref is of file's 76 // JSON blob. 77 func WriteFileMap(bs blobserver.StatReceiver, file *Builder, r io.Reader) (blob.Ref, error) { 78 return writeFileMapRolling(bs, file, r) 79 } 80 81 // This is the simple 1MB chunk version. The rolling checksum version is below. 82 func writeFileMapOld(bs blobserver.StatReceiver, file *Builder, r io.Reader) (blob.Ref, error) { 83 parts, size := []BytesPart{}, int64(0) 84 85 var buf bytes.Buffer 86 for { 87 buf.Reset() 88 n, err := io.Copy(&buf, io.LimitReader(r, maxBlobSize)) 89 if err != nil { 90 return blob.Ref{}, err 91 } 92 if n == 0 { 93 break 94 } 95 96 hash := blob.NewHash() 97 io.Copy(hash, bytes.NewReader(buf.Bytes())) 98 br := blob.RefFromHash(hash) 99 hasBlob, err := serverHasBlob(bs, br) 100 if err != nil { 101 return blob.Ref{}, err 102 } 103 if !hasBlob { 104 sb, err := bs.ReceiveBlob(br, &buf) 105 if err != nil { 106 return blob.Ref{}, err 107 } 108 if want := (blob.SizedRef{br, uint32(n)}); sb != want { 109 return blob.Ref{}, fmt.Errorf("schema/filewriter: wrote %s, expect", sb, want) 110 } 111 } 112 113 size += n 114 parts = append(parts, BytesPart{ 115 BlobRef: br, 116 Size: uint64(n), 117 Offset: 0, // into BlobRef to read from (not of dest) 118 }) 119 } 120 121 err := file.PopulateParts(size, parts) 122 if err != nil { 123 return blob.Ref{}, err 124 } 125 126 json := file.Blob().JSON() 127 if err != nil { 128 return blob.Ref{}, err 129 } 130 br := blob.SHA1FromString(json) 131 sb, err := bs.ReceiveBlob(br, strings.NewReader(json)) 132 if err != nil { 133 return blob.Ref{}, err 134 } 135 if expect := (blob.SizedRef{br, uint32(len(json))}); expect != sb { 136 return blob.Ref{}, fmt.Errorf("schema/filewriter: wrote %s bytes, got %s ack'd", expect, sb) 137 } 138 139 return br, nil 140 } 141 142 func serverHasBlob(bs blobserver.BlobStatter, br blob.Ref) (have bool, err error) { 143 _, err = blobserver.StatBlob(bs, br) 144 if err == nil { 145 have = true 146 } else if err == os.ErrNotExist { 147 err = nil 148 } 149 return 150 } 151 152 type span struct { 153 from, to int64 154 bits int 155 br blob.Ref 156 children []span 157 } 158 159 func (s *span) isSingleBlob() bool { 160 return len(s.children) == 0 161 } 162 163 func (s *span) size() int64 { 164 size := s.to - s.from 165 for _, cs := range s.children { 166 size += cs.size() 167 } 168 return size 169 } 170 171 // noteEOFReader keeps track of when it's seen EOF, but otherwise 172 // delegates entirely to r. 173 type noteEOFReader struct { 174 r io.Reader 175 sawEOF bool 176 } 177 178 func (r *noteEOFReader) Read(p []byte) (n int, err error) { 179 n, err = r.r.Read(p) 180 if err == io.EOF { 181 r.sawEOF = true 182 } 183 return 184 } 185 186 func uploadString(bs blobserver.StatReceiver, br blob.Ref, s string) (blob.Ref, error) { 187 if !br.Valid() { 188 panic("invalid blobref") 189 } 190 hasIt, err := serverHasBlob(bs, br) 191 if err != nil { 192 return blob.Ref{}, err 193 } 194 if hasIt { 195 return br, nil 196 } 197 _, err = blobserver.ReceiveNoHash(bs, br, strings.NewReader(s)) 198 if err != nil { 199 return blob.Ref{}, err 200 } 201 return br, nil 202 } 203 204 // uploadBytes populates bb (a builder of either type "bytes" or 205 // "file", which is a superset of "bytes"), sets it to the provided 206 // size, and populates with provided spans. The bytes or file schema 207 // blob is uploaded and its blobref is returned. 208 func uploadBytes(bs blobserver.StatReceiver, bb *Builder, size int64, s []span) *uploadBytesFuture { 209 future := newUploadBytesFuture() 210 parts := []BytesPart{} 211 addBytesParts(bs, &parts, s, future) 212 213 if err := bb.PopulateParts(size, parts); err != nil { 214 future.errc <- err 215 return future 216 } 217 218 // Hack until camlistore.org/issue/102 is fixed. If we happen to upload 219 // the "file" schema before any of its parts arrive, then the indexer 220 // can get confused. So wait on the parts before, and then upload 221 // the "file" blob afterwards. 222 if bb.Type() == "file" { 223 future.errc <- nil 224 _, err := future.Get() // may not be nil, if children parts failed 225 future = newUploadBytesFuture() 226 if err != nil { 227 future.errc <- err 228 return future 229 } 230 } 231 232 json := bb.Blob().JSON() 233 br := blob.SHA1FromString(json) 234 future.br = br 235 go func() { 236 _, err := uploadString(bs, br, json) 237 future.errc <- err 238 }() 239 return future 240 } 241 242 func newUploadBytesFuture() *uploadBytesFuture { 243 return &uploadBytesFuture{ 244 errc: make(chan error, 1), 245 } 246 } 247 248 // An uploadBytesFuture is an eager result of a still-in-progress uploadBytes call. 249 // Call Get to wait and get its final result. 250 type uploadBytesFuture struct { 251 br blob.Ref 252 errc chan error 253 children []*uploadBytesFuture 254 } 255 256 // BlobRef returns the optimistic blobref of this uploadBytes call without blocking. 257 func (f *uploadBytesFuture) BlobRef() blob.Ref { 258 return f.br 259 } 260 261 // Get blocks for all children and returns any final error. 262 func (f *uploadBytesFuture) Get() (blob.Ref, error) { 263 for _, f := range f.children { 264 if _, err := f.Get(); err != nil { 265 return blob.Ref{}, err 266 } 267 } 268 return f.br, <-f.errc 269 } 270 271 // addBytesParts uploads the provided spans to bs, appending elements to *dst. 272 func addBytesParts(bs blobserver.StatReceiver, dst *[]BytesPart, spans []span, parent *uploadBytesFuture) { 273 for _, sp := range spans { 274 if len(sp.children) == 1 && sp.children[0].isSingleBlob() { 275 // Remove an occasional useless indirection of 276 // what would become a bytes schema blob 277 // pointing to a single blobref. Just promote 278 // the blobref child instead. 279 child := sp.children[0] 280 *dst = append(*dst, BytesPart{ 281 BlobRef: child.br, 282 Size: uint64(child.size()), 283 }) 284 sp.children = nil 285 } 286 if len(sp.children) > 0 { 287 childrenSize := int64(0) 288 for _, cs := range sp.children { 289 childrenSize += cs.size() 290 } 291 future := uploadBytes(bs, newBytes(), childrenSize, sp.children) 292 parent.children = append(parent.children, future) 293 *dst = append(*dst, BytesPart{ 294 BytesRef: future.BlobRef(), 295 Size: uint64(childrenSize), 296 }) 297 } 298 if sp.from == sp.to { 299 panic("Shouldn't happen. " + fmt.Sprintf("weird span with same from & to: %#v", sp)) 300 } 301 *dst = append(*dst, BytesPart{ 302 BlobRef: sp.br, 303 Size: uint64(sp.to - sp.from), 304 }) 305 } 306 } 307 308 // writeFileMap uploads chunks of r to bs while populating fileMap and 309 // finally uploading fileMap. The returned blobref is of fileMap's 310 // JSON blob. It uses rolling checksum for the chunks sizes. 311 func writeFileMapRolling(bs blobserver.StatReceiver, file *Builder, r io.Reader) (blob.Ref, error) { 312 n, spans, err := writeFileChunks(bs, file, r) 313 if err != nil { 314 return blob.Ref{}, err 315 } 316 // The top-level content parts 317 return uploadBytes(bs, file, n, spans).Get() 318 } 319 320 // WriteFileChunks uploads chunks of r to bs while populating file. 321 // It does not upload file. 322 func WriteFileChunks(bs blobserver.StatReceiver, file *Builder, r io.Reader) error { 323 size, spans, err := writeFileChunks(bs, file, r) 324 if err != nil { 325 return err 326 } 327 parts := []BytesPart{} 328 future := newUploadBytesFuture() 329 addBytesParts(bs, &parts, spans, future) 330 future.errc <- nil // Get will still block on addBytesParts' children 331 if _, err := future.Get(); err != nil { 332 return err 333 } 334 return file.PopulateParts(size, parts) 335 } 336 337 func writeFileChunks(bs blobserver.StatReceiver, file *Builder, r io.Reader) (n int64, spans []span, outerr error) { 338 src := ¬eEOFReader{r: r} 339 bufr := bufio.NewReaderSize(src, bufioReaderSize) 340 spans = []span{} // the tree of spans, cut on interesting rollsum boundaries 341 rs := rollsum.New() 342 var last int64 343 var buf bytes.Buffer 344 blobSize := 0 // of the next blob being built, should be same as buf.Len() 345 346 const chunksInFlight = 32 // at ~64 KB chunks, this is ~2MB memory per file 347 gatec := syncutil.NewGate(chunksInFlight) 348 firsterrc := make(chan error, 1) 349 350 // uploadLastSpan runs in the same goroutine as the loop below and is responsible for 351 // starting uploading the contents of the buf. It returns false if there's been 352 // an error and the loop below should be stopped. 353 uploadLastSpan := func() bool { 354 chunk := buf.String() 355 buf.Reset() 356 br := blob.SHA1FromString(chunk) 357 spans[len(spans)-1].br = br 358 select { 359 case outerr = <-firsterrc: 360 return false 361 default: 362 // No error seen so far, continue. 363 } 364 gatec.Start() 365 go func() { 366 defer gatec.Done() 367 if _, err := uploadString(bs, br, chunk); err != nil { 368 select { 369 case firsterrc <- err: 370 default: 371 } 372 } 373 }() 374 return true 375 } 376 377 for { 378 c, err := bufr.ReadByte() 379 if err == io.EOF { 380 if n != last { 381 spans = append(spans, span{from: last, to: n}) 382 if !uploadLastSpan() { 383 return 384 } 385 } 386 break 387 } 388 if err != nil { 389 return 0, nil, err 390 } 391 392 buf.WriteByte(c) 393 n++ 394 blobSize++ 395 rs.Roll(c) 396 397 var bits int 398 onRollSplit := rs.OnSplit() 399 switch { 400 case blobSize == maxBlobSize: 401 bits = 20 // arbitrary node weight; 1<<20 == 1MB 402 case src.sawEOF: 403 // Don't split. End is coming soon enough. 404 continue 405 case onRollSplit && n > firstChunkSize && blobSize > tooSmallThreshold: 406 bits = rs.Bits() 407 case n == firstChunkSize: 408 bits = 18 // 1 << 18 == 256KB 409 default: 410 // Don't split. 411 continue 412 } 413 blobSize = 0 414 415 // Take any spans from the end of the spans slice that 416 // have a smaller 'bits' score and make them children 417 // of this node. 418 var children []span 419 childrenFrom := len(spans) 420 for childrenFrom > 0 && spans[childrenFrom-1].bits < bits { 421 childrenFrom-- 422 } 423 if nCopy := len(spans) - childrenFrom; nCopy > 0 { 424 children = make([]span, nCopy) 425 copy(children, spans[childrenFrom:]) 426 spans = spans[:childrenFrom] 427 } 428 429 spans = append(spans, span{from: last, to: n, bits: bits, children: children}) 430 last = n 431 if !uploadLastSpan() { 432 return 433 } 434 } 435 436 // Loop was already hit earlier. 437 if outerr != nil { 438 return 0, nil, outerr 439 } 440 441 // Wait for all uploads to finish, one way or another, and then 442 // see if any generated errors. 443 // Once this loop is done, we own all the tokens in gatec, so nobody 444 // else can have one outstanding. 445 for i := 0; i < chunksInFlight; i++ { 446 gatec.Start() 447 } 448 select { 449 case err := <-firsterrc: 450 return 0, nil, err 451 default: 452 } 453 454 return n, spans, nil 455 456 }