github.com/janelia-flyem/dvid@v1.0.0/datatype/imageblk/read.go (about) 1 package imageblk 2 3 import ( 4 "encoding/binary" 5 "fmt" 6 "log" 7 "sync" 8 9 "github.com/janelia-flyem/dvid/datastore" 10 "github.com/janelia-flyem/dvid/datatype/roi" 11 "github.com/janelia-flyem/dvid/dvid" 12 "github.com/janelia-flyem/dvid/server" 13 "github.com/janelia-flyem/dvid/storage" 14 ) 15 16 // ROI encapsulates a request-specific ROI check with a given scaling for voxels outside the ROI. 17 type ROI struct { 18 Iter *roi.Iterator 19 attenuation uint8 20 } 21 22 // ComputeTransform determines the block coordinate and beginning + ending voxel points 23 // for the data corresponding to the given Block. 24 func (v *Voxels) ComputeTransform(block *storage.TKeyValue, blockSize dvid.Point) (blockBeg, dataBeg, dataEnd dvid.Point, err error) { 25 var ptIndex *dvid.IndexZYX 26 ptIndex, err = DecodeTKey(block.K) 27 if err != nil { 28 return 29 } 30 31 // Get the bounding voxel coordinates for this block. 32 minBlockVoxel := ptIndex.MinPoint(blockSize) 33 maxBlockVoxel := ptIndex.MaxPoint(blockSize) 34 35 // Compute the boundary voxel coordinates for the ExtData and adjust 36 // to our block bounds. 37 minDataVoxel := v.StartPoint() 38 maxDataVoxel := v.EndPoint() 39 begVolCoord, _ := minDataVoxel.Max(minBlockVoxel) 40 endVolCoord, _ := maxDataVoxel.Min(maxBlockVoxel) 41 42 // Adjust the DVID volume voxel coordinates for the data so that (0,0,0) 43 // is where we expect this slice/subvolume's data to begin. 44 dataBeg = begVolCoord.Sub(v.StartPoint()) 45 dataEnd = endVolCoord.Sub(v.StartPoint()) 46 47 // Compute block coord matching dataBeg 48 blockBeg = begVolCoord.Sub(minBlockVoxel) 49 50 return 51 } 52 53 // ReadBlock reads the possibly intersecting block data into the receiver Voxels. 54 func (v *Voxels) ReadBlock(block *storage.TKeyValue, blockSize dvid.Point, attenuation uint8) error { 55 if attenuation != 0 { 56 return v.readScaledBlock(block, blockSize, attenuation) 57 } 58 return v.readBlock(block, blockSize) 59 } 60 61 // readScaledBlock reads the possibly intersecting block data into the receiver Voxels and applies an attenuation 62 // to the values. 63 func (v *Voxels) readScaledBlock(block *storage.TKeyValue, blockSize dvid.Point, attenuation uint8) error { 64 if blockSize.NumDims() > 3 { 65 return fmt.Errorf("DVID voxel blocks currently only supports up to 3d, not 4+ dimensions") 66 } 67 blockBeg, dataBeg, dataEnd, err := v.ComputeTransform(block, blockSize) 68 if err != nil { 69 return err 70 } 71 data := v.Data() 72 bytesPerVoxel := int64(v.Values().BytesPerElement()) 73 if bytesPerVoxel != 1 { 74 return fmt.Errorf("Can only scale non-ROI blocks with 1 byte voxels") 75 } 76 77 // Compute the strides (in bytes) 78 bX := int64(blockSize.Value(0)) * bytesPerVoxel 79 bY := int64(blockSize.Value(1)) * bX 80 dX := int64(v.Stride()) 81 82 // Get the block beginning coordinates. 83 blockBegX := int64(blockBeg.Value(0)) 84 blockBegY := int64(blockBeg.Value(1)) 85 blockBegZ := int64(blockBeg.Value(2)) 86 87 // Do the transfers depending on shape of the external voxels. 88 switch { 89 case v.DataShape().Equals(dvid.XY): 90 blockI := blockBegZ*bY + blockBegY*bX + blockBegX*bytesPerVoxel 91 dataI := int64(dataBeg.Value(1))*dX + int64(dataBeg.Value(0))*bytesPerVoxel 92 for y := dataBeg.Value(1); y <= dataEnd.Value(1); y++ { 93 for x := int64(dataBeg.Value(0)); x <= int64(dataEnd.Value(0)); x++ { 94 data[dataI+x] = (block.V[blockI+x] >> attenuation) 95 } 96 blockI += bX 97 dataI += dX 98 } 99 100 case v.DataShape().Equals(dvid.XZ): 101 blockI := blockBegZ*bY + blockBegY*bX + blockBegX*bytesPerVoxel 102 dataI := int64(dataBeg.Value(2))*dX + int64(dataBeg.Value(0))*bytesPerVoxel 103 for y := dataBeg.Value(2); y <= dataEnd.Value(2); y++ { 104 for x := int64(dataBeg.Value(0)); x <= int64(dataEnd.Value(0)); x++ { 105 data[dataI+x] = (block.V[blockI+x] >> attenuation) 106 } 107 blockI += bY 108 dataI += dX 109 } 110 111 case v.DataShape().Equals(dvid.YZ): 112 bz := blockBegZ 113 for y := int64(dataBeg.Value(2)); y <= int64(dataEnd.Value(2)); y++ { 114 blockI := blockBegZ*bY + blockBegY*bX + blockBegX*bytesPerVoxel 115 dataI := y*dX + int64(dataBeg.Value(1))*bytesPerVoxel 116 for x := dataBeg.Value(1); x <= dataEnd.Value(1); x++ { 117 data[dataI] = (block.V[blockI] >> attenuation) 118 blockI += bX 119 dataI += bytesPerVoxel 120 } 121 bz++ 122 } 123 124 case v.DataShape().ShapeDimensions() == 2: 125 // TODO: General code for handling 2d ExtData in n-d space. 126 return fmt.Errorf("DVID currently does not support 2d in n-d space.") 127 128 case v.DataShape().Equals(dvid.Vol3d): 129 blockOffset := blockBegX * bytesPerVoxel 130 dX := int64(v.Size().Value(0)) * bytesPerVoxel 131 dY := int64(v.Size().Value(1)) * dX 132 dataOffset := int64(dataBeg.Value(0)) * bytesPerVoxel 133 blockZ := blockBegZ 134 135 for dataZ := dataBeg.Value(2); dataZ <= dataEnd.Value(2); dataZ++ { 136 blockY := blockBegY 137 for dataY := dataBeg.Value(1); dataY <= dataEnd.Value(1); dataY++ { 138 blockI := blockZ*bY + blockY*bX + blockOffset 139 dataI := int64(dataZ)*dY + int64(dataY)*dX + dataOffset 140 for x := int64(dataBeg.Value(0)); x <= int64(dataEnd.Value(0)); x++ { 141 data[dataI+x] = (block.V[blockI+x] >> attenuation) 142 } 143 blockY++ 144 } 145 blockZ++ 146 } 147 148 default: 149 return fmt.Errorf("Cannot readScaledBlock() unsupported voxels data shape %s", v.DataShape()) 150 } 151 return nil 152 } 153 154 // readBlock reads the possibly intersecting block data into the receiver Voxels. 155 func (v *Voxels) readBlock(block *storage.TKeyValue, blockSize dvid.Point) error { 156 if blockSize.NumDims() > 3 { 157 return fmt.Errorf("DVID voxel blocks currently only supports up to 3d, not 4+ dimensions") 158 } 159 blockBeg, dataBeg, dataEnd, err := v.ComputeTransform(block, blockSize) 160 if err != nil { 161 return err 162 } 163 data := v.Data() 164 bytesPerVoxel := int64(v.Values().BytesPerElement()) 165 166 // Compute the strides (in bytes) 167 bX := int64(blockSize.Value(0)) * bytesPerVoxel 168 bY := int64(blockSize.Value(1)) * bX 169 dX := int64(v.Stride()) 170 171 blockBegX := int64(blockBeg.Value(0)) 172 blockBegY := int64(blockBeg.Value(1)) 173 blockBegZ := int64(blockBeg.Value(2)) 174 175 // Do the transfers depending on shape of the external voxels. 176 switch { 177 case v.DataShape().Equals(dvid.XY): 178 dataI := int64(dataBeg.Value(1))*dX + int64(dataBeg.Value(0))*bytesPerVoxel 179 blockI := blockBegZ*bY + blockBegY*bX + blockBegX*bytesPerVoxel 180 bytes := int64(dataEnd.Value(0)-dataBeg.Value(0)+1) * bytesPerVoxel 181 for y := dataBeg.Value(1); y <= dataEnd.Value(1); y++ { 182 copy(data[dataI:dataI+bytes], block.V[blockI:blockI+bytes]) 183 blockI += bX 184 dataI += dX 185 } 186 187 case v.DataShape().Equals(dvid.XZ): 188 dataI := int64(dataBeg.Value(2))*dX + int64(dataBeg.Value(0))*bytesPerVoxel 189 blockI := blockBegZ*bY + blockBegY*bX + blockBegX*bytesPerVoxel 190 bytes := int64(dataEnd.Value(0)-dataBeg.Value(0)+1) * bytesPerVoxel 191 for y := dataBeg.Value(2); y <= dataEnd.Value(2); y++ { 192 copy(data[dataI:dataI+bytes], block.V[blockI:blockI+bytes]) 193 blockI += bY 194 dataI += dX 195 } 196 197 case v.DataShape().Equals(dvid.YZ): 198 bz := blockBegZ 199 for y := int64(dataBeg.Value(2)); y <= int64(dataEnd.Value(2)); y++ { 200 dataI := y*dX + int64(dataBeg.Value(1))*bytesPerVoxel 201 blockI := bz*bY + blockBegY*bX + blockBegX*bytesPerVoxel 202 for x := dataBeg.Value(1); x <= dataEnd.Value(1); x++ { 203 copy(data[dataI:dataI+bytesPerVoxel], block.V[blockI:blockI+bytesPerVoxel]) 204 blockI += bX 205 dataI += bytesPerVoxel 206 } 207 bz++ 208 } 209 210 case v.DataShape().ShapeDimensions() == 2: 211 // TODO: General code for handling 2d ExtData in n-d space. 212 return fmt.Errorf("DVID currently does not support 2d in n-d space.") 213 214 case v.DataShape().Equals(dvid.Vol3d): 215 blockOffset := blockBegX * bytesPerVoxel 216 dX := int64(v.Size().Value(0)) * bytesPerVoxel 217 dY := int64(v.Size().Value(1)) * dX 218 dataOffset := int64(dataBeg.Value(0)) * bytesPerVoxel 219 bytes := int64(dataEnd.Value(0)-dataBeg.Value(0)+1) * bytesPerVoxel 220 blockZ := blockBegZ 221 222 for dataZ := int64(dataBeg.Value(2)); dataZ <= int64(dataEnd.Value(2)); dataZ++ { 223 blockY := blockBegY 224 for dataY := int64(dataBeg.Value(1)); dataY <= int64(dataEnd.Value(1)); dataY++ { 225 blockI := blockZ*bY + blockY*bX + blockOffset 226 dataI := dataZ*dY + dataY*dX + dataOffset 227 copy(data[dataI:dataI+bytes], block.V[blockI:blockI+bytes]) 228 blockY++ 229 } 230 blockZ++ 231 } 232 233 default: 234 return fmt.Errorf("Cannot readBlock() unsupported voxels data shape %s", v.DataShape()) 235 } 236 return nil 237 } 238 239 // Handler conversion of little to big endian for voxels larger than 1 byte. 240 func (v *Voxels) littleToBigEndian(data []uint8) (bigendian []uint8, err error) { 241 bytesPerVoxel := v.Values().BytesPerElement() 242 if bytesPerVoxel == 1 { 243 return data, nil 244 } 245 bigendian = make([]uint8, len(data)) 246 switch bytesPerVoxel { 247 case 2: 248 for beg := 0; beg < len(data)-1; beg += 2 { 249 bigendian[beg], bigendian[beg+1] = data[beg+1], data[beg] 250 } 251 case 4: 252 for beg := 0; beg < len(data)-3; beg += 4 { 253 value := binary.LittleEndian.Uint32(data[beg : beg+4]) 254 binary.BigEndian.PutUint32(bigendian[beg:beg+4], value) 255 } 256 case 8: 257 for beg := 0; beg < len(data)-7; beg += 8 { 258 value := binary.LittleEndian.Uint64(data[beg : beg+8]) 259 binary.BigEndian.PutUint64(bigendian[beg:beg+8], value) 260 } 261 } 262 return 263 } 264 265 // GetROI returns an imageblk.ROI that can iterate over the provided Voxels 266 // unless roiname is empty, which prompts a nil ROI returned. 267 func GetROI(v dvid.VersionID, roiname dvid.InstanceName, bnd dvid.Bounder) (*ROI, error) { 268 if roiname != "" { 269 r := new(ROI) 270 var err error 271 r.Iter, err = roi.NewIterator(roiname, v, bnd) 272 if err != nil { 273 return nil, err 274 } 275 return r, nil 276 } 277 return nil, nil 278 } 279 280 func (d *Data) AllocateBlock() []byte { 281 numElements := d.BlockSize().Prod() 282 bytesPerElement := int64(d.Values.BytesPerElement()) 283 return make([]byte, numElements*bytesPerElement) 284 } 285 286 // BackgroundBlock returns a block buffer that has been preinitialized to the background value. 287 func (d *Data) BackgroundBlock() []byte { 288 numElements := d.BlockSize().Prod() 289 bytesPerElement := int64(d.Values.BytesPerElement()) 290 blockData := make([]byte, numElements*bytesPerElement) 291 if d.Background != 0 && bytesPerElement == 1 { 292 background := byte(d.Background) 293 for i := range blockData { 294 blockData[i] = background 295 } 296 } 297 return blockData 298 } 299 300 // GetImage retrieves a 2d image from a version node given a geometry of voxels. 301 func (d *Data) GetImage(v dvid.VersionID, vox *Voxels, roiname dvid.InstanceName) (*dvid.Image, error) { 302 if err := d.GetVoxels(v, vox, roiname); err != nil { 303 return nil, err 304 } 305 return vox.GetImage2d() 306 } 307 308 // GetVolume retrieves a n-d volume from a version node given a geometry of voxels. 309 func (d *Data) GetVolume(v dvid.VersionID, vox *Voxels, roiname dvid.InstanceName) ([]byte, error) { 310 if err := d.GetVoxels(v, vox, roiname); err != nil { 311 return nil, err 312 } 313 return vox.Data(), nil 314 } 315 316 type getOperation struct { 317 voxels *Voxels 318 blocksInROI map[string]bool 319 attenuation uint8 320 } 321 322 // GetVoxels copies voxels from the storage engine to Voxels, a requested subvolume or 2d image. 323 func (d *Data) GetVoxels(v dvid.VersionID, vox *Voxels, roiname dvid.InstanceName) error { 324 r, err := GetROI(v, roiname, vox) 325 if err != nil { 326 return err 327 } 328 329 timedLog := dvid.NewTimeLog() 330 defer timedLog.Infof("GetVoxels %s", vox) 331 332 store, err := datastore.GetOrderedKeyValueDB(d) 333 if err != nil { 334 return fmt.Errorf("Data type imageblk had error initializing store: %v\n", err) 335 } 336 337 // Only do one request at a time, although each request can start many goroutines. 338 if vox.NumVoxels() > 256*256*256 { 339 server.LargeMutationMutex.Lock() 340 defer server.LargeMutationMutex.Unlock() 341 } 342 343 ctx := datastore.NewVersionedCtx(d, v) 344 345 wg := new(sync.WaitGroup) 346 347 okv := store.(storage.BufferableOps) 348 // extract buffer interface 349 req, hasbuffer := okv.(storage.KeyValueRequester) 350 if hasbuffer { 351 okv = req.NewBuffer(ctx) 352 } 353 354 // // get store for data -- we will either have gridStore or okv/req/hasbuffer 355 // var gridStore storage.GridStoreGetter 356 // var okvDB storage.OrderedKeyValueDB 357 // // TODO -- Add gridStore support to GetVoxels 358 // gridStore, okvDB, _, err = d.gridStoreGetter() 359 // if err != nil { 360 // return fmt.Errorf("data type imageblk had error initializing store: %v", err) 361 // } 362 // ctx := datastore.NewVersionedCtx(d, v) 363 // var okv storage.BufferableOps 364 // var req storage.KeyValueRequester 365 // var hasbuffer bool 366 // if gridStore == nil { 367 // okv = okvDB.(storage.BufferableOps) 368 // // extract buffer interface 369 // req, hasbuffer = okv.(storage.KeyValueRequester) 370 // if hasbuffer { 371 // okv = req.NewBuffer(ctx) 372 // } 373 // } 374 375 for it, err := vox.NewIndexIterator(d.BlockSize()); err == nil && it.Valid(); it.NextSpan() { 376 indexBeg, indexEnd, err := it.IndexSpan() 377 if err != nil { 378 return err 379 } 380 begTKey := NewTKey(indexBeg) 381 endTKey := NewTKey(indexEnd) 382 383 // Get set of blocks in ROI if ROI provided 384 var chunkOp *storage.ChunkOp 385 if r != nil && r.Iter != nil { 386 ptBeg := indexBeg.Duplicate().(dvid.ChunkIndexer) 387 ptEnd := indexEnd.Duplicate().(dvid.ChunkIndexer) 388 begX := ptBeg.Value(0) 389 endX := ptEnd.Value(0) 390 391 blocksInROI := make(map[string]bool, (endX - begX + 1)) 392 c := dvid.ChunkPoint3d{begX, ptBeg.Value(1), ptBeg.Value(2)} 393 for x := begX; x <= endX; x++ { 394 c[0] = x 395 curIndex := dvid.IndexZYX(c) 396 if r.Iter.InsideFast(curIndex) { 397 indexString := string(curIndex.Bytes()) 398 blocksInROI[indexString] = true 399 } 400 } 401 chunkOp = &storage.ChunkOp{&getOperation{vox, blocksInROI, r.attenuation}, wg} 402 } else { 403 chunkOp = &storage.ChunkOp{&getOperation{vox, nil, 0}, wg} 404 } 405 406 if !hasbuffer { 407 // Send the entire range of key-value pairs to chunk processor 408 err = okv.ProcessRange(ctx, begTKey, endTKey, chunkOp, storage.ChunkFunc(d.ReadChunk)) 409 if err != nil { 410 return fmt.Errorf("Unable to GET data %s: %v", ctx, err) 411 } 412 } else { 413 // Extract block list 414 tkeys := make([]storage.TKey, 0) 415 416 ptBeg := indexBeg.Duplicate().(dvid.ChunkIndexer) 417 ptEnd := indexEnd.Duplicate().(dvid.ChunkIndexer) 418 begX := ptBeg.Value(0) 419 endX := ptEnd.Value(0) 420 421 c := dvid.ChunkPoint3d{begX, ptBeg.Value(1), ptBeg.Value(2)} 422 for x := begX; x <= endX; x++ { 423 c[0] = x 424 curIndex := dvid.IndexZYX(c) 425 currTKey := NewTKey(&curIndex) 426 tkeys = append(tkeys, currTKey) 427 428 } 429 430 err = okv.(storage.RequestBuffer).ProcessList(ctx, tkeys, chunkOp, storage.ChunkFunc(d.ReadChunk)) 431 if err != nil { 432 return fmt.Errorf("Unable to GET data %s: %v", ctx, err) 433 } 434 } 435 } 436 437 if hasbuffer { 438 // submit the entire buffer to the DB 439 err = okv.(storage.RequestBuffer).Flush() 440 441 if err != nil { 442 return fmt.Errorf("Unable to GET data %s: %v", ctx, err) 443 444 } 445 } 446 447 if err != nil { 448 return err 449 } 450 wg.Wait() 451 return nil 452 } 453 454 // GetBlocks returns a slice of bytes corresponding to all the blocks along a span in X 455 func (d *Data) GetBlocks(v dvid.VersionID, start dvid.ChunkPoint3d, span int32) ([]byte, error) { 456 timedLog := dvid.NewTimeLog() 457 defer timedLog.Infof("GetBlocks start at %s, span %d", start, span) 458 459 // get store for data 460 var gridStore storage.GridStoreGetter 461 var okvDB storage.OrderedKeyValueDB 462 var err error 463 gridStore, okvDB, _, err = d.gridStoreGetter() 464 if err != nil { 465 return nil, fmt.Errorf("data type imageblk had error initializing store: %v", err) 466 } 467 468 // Allocate one uncompressed-sized slice with background values. 469 blockBytes := int32(d.BlockSize().Prod()) * d.Values.BytesPerElement() 470 numBytes := blockBytes * span 471 472 buf := make([]byte, numBytes, numBytes) 473 if d.Background != 0 { 474 for i := range buf { 475 buf[i] = byte(d.Background) 476 } 477 } 478 479 if gridStore != nil { 480 blockCoord := start 481 for i := int32(0); i < span; i++ { 482 value, err := gridStore.GridGet(d.ScaleLevel, blockCoord) 483 if err != nil { 484 return nil, datastore.ErrBranchUnlockedNode 485 } 486 // TODO -- This append of serialization data wouldn't be necessary if 487 // compression was broken out. 488 data, err := dvid.SerializePrecompressedData(value, d.Compression(), dvid.NoChecksum) 489 if err != nil { 490 return nil, fmt.Errorf("can't serialize precompressed data: %v", err) 491 } 492 uncompress := true 493 block, _, err := dvid.DeserializeData(data, uncompress) 494 if err != nil { 495 return nil, fmt.Errorf("Unable to deserialize block %s: %v", blockCoord, err) 496 } 497 if int32(len(block)) != blockBytes { 498 return nil, fmt.Errorf("Deserialized block size (%d) != expected (%d)", len(block), blockBytes) 499 } 500 pos := i * blockBytes 501 copy(buf[pos:pos+blockBytes], block) 502 blockCoord[0]++ 503 } 504 return buf, nil 505 } 506 507 indexBeg := dvid.IndexZYX(start) 508 sx, sy, sz := indexBeg.Unpack() 509 510 end := start 511 end[0] += int32(span - 1) 512 indexEnd := dvid.IndexZYX(end) 513 keyBeg := NewTKey(&indexBeg) 514 keyEnd := NewTKey(&indexEnd) 515 516 // Write the blocks that we can get concurrently on this byte slice. 517 ctx := datastore.NewVersionedCtx(d, v) 518 519 var numBlocks int 520 var wg sync.WaitGroup 521 err = okvDB.ProcessRange(ctx, keyBeg, keyEnd, &storage.ChunkOp{}, func(c *storage.Chunk) error { 522 if c == nil || c.TKeyValue == nil { 523 return nil 524 } 525 kv := c.TKeyValue 526 if kv.V == nil { 527 return nil 528 } 529 530 // Determine which block this is. 531 indexZYX, err := DecodeTKey(kv.K) 532 if err != nil { 533 return err 534 } 535 x, y, z := indexZYX.Unpack() 536 if z != sz || y != sy || x < sx || x >= sx+int32(span) { 537 return fmt.Errorf("Received key-value for %s, not supposed to be within span range %s, length %d", indexZYX, start, span) 538 } 539 n := x - sx 540 i := n * blockBytes 541 j := i + blockBytes 542 543 // Spawn goroutine to transfer data 544 numBlocks++ 545 wg.Add(1) 546 go xferBlock(buf[i:j], c, &wg) 547 return nil 548 }) 549 if err != nil { 550 return nil, err 551 } 552 wg.Wait() 553 return buf, nil 554 } 555 556 func xferBlock(buf []byte, chunk *storage.Chunk, wg *sync.WaitGroup) { 557 defer wg.Done() 558 559 kv := chunk.TKeyValue 560 uncompress := true 561 block, _, err := dvid.DeserializeData(kv.V, uncompress) 562 if err != nil { 563 dvid.Errorf("Unable to deserialize block (%v): %v", kv.K, err) 564 return 565 } 566 if len(block) != len(buf) { 567 dvid.Errorf("Deserialized block length (%d) != allocated block length (%d)", len(block), len(buf)) 568 return 569 } 570 copy(buf, block) 571 } 572 573 // LoadOldBlocks loads blocks with old data if they exist. 574 func (d *Data) LoadOldBlocks(v dvid.VersionID, vox *Voxels, blocks storage.TKeyValues) error { 575 store, err := datastore.GetOrderedKeyValueDB(d) 576 if err != nil { 577 return fmt.Errorf("Data type imageblk had error initializing store: %v\n", err) 578 } 579 ctx := datastore.NewVersionedCtx(d, v) 580 581 // Create a map of old blocks indexed by the index 582 oldBlocks := map[dvid.IZYXString]([]byte){} 583 584 // Iterate through index space for this data using ZYX ordering. 585 blockSize := d.BlockSize() 586 blockNum := 0 587 for it, err := vox.NewIndexIterator(blockSize); err == nil && it.Valid(); it.NextSpan() { 588 indexBeg, indexEnd, err := it.IndexSpan() 589 if err != nil { 590 return err 591 } 592 begTKey := NewTKey(indexBeg) 593 endTKey := NewTKey(indexEnd) 594 595 // Get previous data. 596 keyvalues, err := store.GetRange(ctx, begTKey, endTKey) 597 if err != nil { 598 return err 599 } 600 for _, kv := range keyvalues { 601 indexZYX, err := DecodeTKey(kv.K) 602 if err != nil { 603 return err 604 } 605 block, _, err := dvid.DeserializeData(kv.V, true) 606 if err != nil { 607 return fmt.Errorf("Unable to deserialize block, %s: %v", ctx, err) 608 } 609 oldBlocks[indexZYX.ToIZYXString()] = block 610 } 611 612 // Load previous data into blocks 613 ptBeg := indexBeg.Duplicate().(dvid.ChunkIndexer) 614 ptEnd := indexEnd.Duplicate().(dvid.ChunkIndexer) 615 begX := ptBeg.Value(0) 616 endX := ptEnd.Value(0) 617 c := dvid.ChunkPoint3d{begX, ptBeg.Value(1), ptBeg.Value(2)} 618 for x := begX; x <= endX; x++ { 619 c[0] = x 620 curIndex := dvid.IndexZYX(c) 621 curTKey := NewTKey(&curIndex) 622 blocks[blockNum].K = curTKey 623 block, ok := oldBlocks[curIndex.ToIZYXString()] 624 if ok { 625 copy(blocks[blockNum].V, block) 626 } 627 blockNum++ 628 } 629 } 630 return nil 631 } 632 633 // GetBlock returns a block with data from this instance's preferred storage. 634 func (d *Data) GetBlock(v dvid.VersionID, k storage.TKey) ([]byte, error) { 635 store, err := datastore.GetOrderedKeyValueDB(d) 636 if err != nil { 637 return nil, fmt.Errorf("Data type imageblk had error initializing store: %v\n", err) 638 } 639 640 ctx := datastore.NewVersionedCtx(d, v) 641 serialization, err := store.Get(ctx, k) 642 if err != nil { 643 return nil, err 644 } 645 data, _, err := dvid.DeserializeData(serialization, true) 646 if err != nil { 647 return nil, fmt.Errorf("Unable to deserialize block, %s: %v", ctx, err) 648 } 649 return data, err 650 } 651 652 // ReadChunk reads a chunk of data as part of a mapped operation. 653 // Only some multiple of the # of CPU cores can be used for chunk handling before 654 // it waits for chunk processing to abate via the buffered server.HandlerToken channel. 655 func (d *Data) ReadChunk(chunk *storage.Chunk) error { 656 server.CheckChunkThrottling() 657 go d.readChunk(chunk) 658 return nil 659 } 660 661 func (d *Data) readChunk(chunk *storage.Chunk) { 662 defer func() { 663 // After processing a chunk, return the token. 664 server.HandlerToken <- 1 665 666 // Notify the requestor that this chunk is done. 667 if chunk.Wg != nil { 668 chunk.Wg.Done() 669 } 670 }() 671 672 op, ok := chunk.Op.(*getOperation) 673 if !ok { 674 log.Fatalf("Illegal operation passed to readChunk() for data %s\n", d.DataName()) 675 } 676 677 // Make sure our received chunk is valid. 678 if chunk == nil { 679 dvid.Errorf("Received nil chunk in readChunk. Ignoring chunk.\n") 680 return 681 } 682 if chunk.K == nil { 683 dvid.Errorf("Received nil chunk key in readChunk. Ignoring chunk.\n") 684 return 685 } 686 687 // If there's an ROI, if outside ROI, use blank buffer or allow scaling via attenuation. 688 var zeroOut bool 689 var attenuation uint8 690 indexZYX, err := DecodeTKey(chunk.K) 691 if err != nil { 692 dvid.Errorf("Error processing voxel block: %v\n", err) 693 return 694 } 695 if op.blocksInROI != nil { 696 indexString := string(indexZYX.Bytes()) 697 _, insideROI := op.blocksInROI[indexString] 698 if !insideROI { 699 if op.attenuation == 0 { 700 zeroOut = true 701 } 702 attenuation = op.attenuation 703 } 704 } 705 706 // Initialize the block buffer using the chunk of data. For voxels, this chunk of 707 // data needs to be uncompressed and deserialized. 708 var blockData []byte 709 if zeroOut || chunk.V == nil { 710 blockData = d.BackgroundBlock() 711 } else { 712 blockData, _, err = dvid.DeserializeData(chunk.V, true) 713 if err != nil { 714 dvid.Errorf("Unable to deserialize block in '%s': %v\n", d.DataName(), err) 715 return 716 } 717 } 718 719 // Perform the operation. 720 block := &storage.TKeyValue{chunk.K, blockData} 721 if err = op.voxels.ReadBlock(block, d.BlockSize(), attenuation); err != nil { 722 dvid.Errorf("Unable to ReadFromBlock() in %q: %v\n", d.DataName(), err) 723 return 724 } 725 }