github.com/janelia-flyem/dvid@v1.0.0/datatype/labelvol/labelvol.go (about) 1 /* 2 Package labelvol supports label-specific sparse volumes. It can be synced 3 with labelblk and is a different view of 64-bit label data. 4 */ 5 package labelvol 6 7 import ( 8 "bytes" 9 "compress/gzip" 10 "encoding/binary" 11 "encoding/gob" 12 "encoding/json" 13 "fmt" 14 "io" 15 "io/ioutil" 16 "net/http" 17 "os" 18 "path/filepath" 19 "strconv" 20 "strings" 21 "sync" 22 "time" 23 24 humanize "github.com/dustin/go-humanize" 25 26 "github.com/janelia-flyem/dvid/datastore" 27 "github.com/janelia-flyem/dvid/datatype/common/labels" 28 "github.com/janelia-flyem/dvid/datatype/labelblk" 29 "github.com/janelia-flyem/dvid/dvid" 30 "github.com/janelia-flyem/dvid/server" 31 "github.com/janelia-flyem/dvid/storage" 32 33 lz4 "github.com/janelia-flyem/go/golz4-updated" 34 ) 35 36 const ( 37 Version = "0.1" 38 RepoURL = "github.com/janelia-flyem/dvid/datatype/labelvol" 39 TypeName = "labelvol" 40 ) 41 42 const helpMessage = ` 43 API for label sparse volume data type (github.com/janelia-flyem/dvid/datatype/labelvol) 44 ======================================================================================= 45 46 Note: UUIDs referenced below are strings that may either be a unique prefix of a 47 hexadecimal UUID string (e.g., 3FA22) or a branch leaf specification that adds 48 a colon (":") followed by the case-dependent branch name. In the case of a 49 branch leaf specification, the unique UUID prefix just identifies the repo of 50 the branch, and the UUID referenced is really the leaf of the branch name. 51 For example, if we have a DAG with root A -> B -> C where C is the current 52 HEAD or leaf of the "master" (default) branch, then asking for "B:master" is 53 the same as asking for "C". If we add another version so A -> B -> C -> D, then 54 references to "B:master" now return the data from "D". 55 56 ----- 57 58 Denormalizations like sparse volumes are *not* performed for the "0" label, which is 59 considered a special label useful for designating background. This allows users to define 60 sparse labeled structures in a large volume without requiring processing of entire volume. 61 62 63 Command-line: 64 65 $ dvid repo <UUID> new labelvol <data name> <settings...> 66 67 Adds newly named data of the 'type name' to repo with specified UUID. 68 69 Example: 70 71 $ dvid repo 3f8c new labelvol sparsevols 72 73 Arguments: 74 75 UUID Hexadecimal string with enough characters to uniquely identify a version node. 76 data name Name of data to create, e.g., "sparsevols" 77 settings Configuration settings in "key=value" format separated by spaces. 78 79 Configuration Settings (case-insensitive keys) 80 81 BlockSize Size in pixels (default: %s) 82 VoxelSize Resolution of voxels (default: 8.0, 8.0, 8.0) 83 VoxelUnits Resolution units (default: "nanometers") 84 85 $ dvid node <UUID> <data name> dump <server-accessible directory> 86 87 Dumps files, one per 512^3 voxel subvolume, for all block RLEs for each label within the subvolume. 88 File names will be formatted as subvols-X_Y_Z.dat, where (X,Y,Z) is the subvolume index; for example, 89 the file subvols-0_1_2.dat has all label block RLEs for the subvolume with smallest voxel coordinate 90 at (0, 512, 1024). The encoding has the following format where integers are little endian and the 91 order of data is exactly as specified below: 92 93 uint64 Label ID 94 uint32 # Spans 95 Repeating unit of: 96 int32 Coordinate of run start (dimension 0) 97 int32 Coordinate of run start (dimension 1) 98 int32 Coordinate of run start (dimension 2) 99 int32 Length of run 100 ... 101 102 This is similar to the voxel RLEs returned by the sparsevol endpoint, except the initial 8 byte header 103 is replaced with a label identifier. Spans are always in X direction. 104 105 106 ------------------ 107 108 HTTP API (Level 2 REST): 109 110 GET <api URL>/node/<UUID>/<data name>/help 111 112 Returns data-specific help message. 113 114 115 GET <api URL>/node/<UUID>/<data name>/info 116 POST <api URL>/node/<UUID>/<data name>/info 117 118 Retrieves or puts DVID-specific data properties for these voxels. 119 120 Example: 121 122 GET <api URL>/node/3f8c/bodies/info 123 124 Returns JSON with configuration settings that include location in DVID space and 125 min/max block indices. 126 127 Arguments: 128 129 UUID Hexadecimal string with enough characters to uniquely identify a version node. 130 data name Name of labelvol data. 131 132 133 POST <api URL>/node/<UUID>/<data name>/sync?<options> 134 135 Establishes labelblk data instances with which the annotations are synced. Expects JSON to be POSTed 136 with the following format: 137 138 { "sync": "labels" } 139 140 To delete syncs, pass an empty string of names with query string "replace=true": 141 142 { "sync": "" } 143 144 The "sync" property should be followed by a comma-delimited list of data instances that MUST 145 already exist. Currently, syncs should be created before any annotations are pushed to 146 the server. If annotations already exist, these are currently not synced. 147 148 The labelvol data type only accepts syncs to labelblk data instances. 149 150 GET Query-string Options: 151 152 replace Set to "true" if you want passed syncs to replace and not be appended to current syncs. 153 Default operation is false. 154 155 156 GET <api URL>/node/<UUID>/<data name>/sparsevol/<label>?<options> 157 158 Returns a sparse volume with voxels of the given label in encoded RLE format. The returned 159 data can be optionally compressed using the "compression" option below. 160 161 The encoding has the following format where integers are little endian and the order 162 of data is exactly as specified below: 163 164 byte Payload descriptor: 165 Bit 0 (LSB) - 8-bit grayscale 166 Bit 1 - 16-bit grayscale 167 Bit 2 - 16-bit normal 168 If set to all 0, there is no payload and it's a binary sparse volume. 169 uint8 Number of dimensions 170 uint8 Dimension of run (typically 0 = X) 171 byte Reserved (to be used later) 172 uint32 # Voxels [TODO. 0 for now] 173 uint32 # Spans 174 Repeating unit of: 175 int32 Coordinate of run start (dimension 0) 176 int32 Coordinate of run start (dimension 1) 177 int32 Coordinate of run start (dimension 2) 178 int32 Length of run 179 bytes Optional payload dependent on first byte descriptor 180 ... 181 182 GET Query-string Options: 183 184 minx Spans must be equal to or larger than this minimum x voxel coordinate. 185 maxx Spans must be equal to or smaller than this maximum x voxel coordinate. 186 miny Spans must be equal to or larger than this minimum y voxel coordinate. 187 maxy Spans must be equal to or smaller than this maximum y voxel coordinate. 188 minz Spans must be equal to or larger than this minimum z voxel coordinate. 189 maxz Spans must be equal to or smaller than this maximum z voxel coordinate. 190 exact "false" if RLEs can extend a bit outside voxel bounds within border blocks. 191 This will give slightly faster responses. 192 193 compression Allows retrieval of data in "lz4" and "gzip" 194 compressed format. 195 196 197 HEAD <api URL>/node/<UUID>/<data name>/sparsevol/<label>?<options> 198 199 Returns: 200 200 (OK) if a sparse volume of the given label exists within any optional bounds. 201 204 (No Content) if there is no sparse volume for the given label within any optional bounds. 202 404 (Not found) if the label has no sparse volume. 203 204 Note that for speed, the optional bounds are always expanded to the block-aligned containing 205 subvolume, i.e., it's as if exact=false for the corresponding GET. 206 207 GET Query-string Options: 208 209 minx Spans must be equal to or larger than this minimum x voxel coordinate. 210 maxx Spans must be equal to or smaller than this maximum x voxel coordinate. 211 miny Spans must be equal to or larger than this minimum y voxel coordinate. 212 maxy Spans must be equal to or smaller than this maximum y voxel coordinate. 213 minz Spans must be equal to or larger than this minimum z voxel coordinate. 214 maxz Spans must be equal to or smaller than this maximum z voxel coordinate. 215 216 217 GET <api URL>/node/<UUID>/<data name>/sparsevol-by-point/<coord> 218 219 Returns a sparse volume with voxels that pass through a given voxel. 220 The encoding is described in the "sparsevol" request above. 221 222 Arguments: 223 224 UUID Hexadecimal string with enough characters to uniquely identify a version node. 225 data name Name of mapping data. 226 coord Coordinate of voxel with underscore as separator, e.g., 10_20_30 227 228 229 GET <api URL>/node/<UUID>/<data name>/sparsevol-coarse/<label> 230 231 Returns a sparse volume with blocks of the given label in encoded RLE format. 232 The encoding has the following format where integers are little endian and the order 233 of data is exactly as specified below: 234 235 byte Set to 0 236 uint8 Number of dimensions 237 uint8 Dimension of run (typically 0 = X) 238 byte Reserved (to be used later) 239 uint32 # Blocks [TODO. 0 for now] 240 uint32 # Spans 241 Repeating unit of: 242 int32 Block coordinate of run start (dimension 0) 243 int32 Block coordinate of run start (dimension 1) 244 int32 Block coordinate of run start (dimension 2) 245 ... 246 int32 Length of run 247 248 Note that the above format is the RLE encoding of sparsevol, where voxel coordinates 249 have been replaced by block coordinates. 250 251 252 GET <api URL>/node/<UUID>/<data name>/maxlabel 253 254 GET returns the maximum label for the version of data in JSON form: 255 256 { "maxlabel": <label #> } 257 258 259 GET <api URL>/node/<UUID>/<data name>/nextlabel 260 POST <api URL>/node/<UUID>/<data name>/nextlabel 261 262 GET returns the next label for the version of data in JSON form: 263 264 { "nextlabel": <label #> } 265 266 POST allows the client to request some # of labels that will be reserved. 267 This is used if the client wants to introduce new labels. 268 269 The request: 270 271 { "needed": <# of labels> } 272 273 Response: 274 275 { "start": <starting label #>, "end": <ending label #> } 276 277 278 POST <api URL>/node/<UUID>/<data name>/merge 279 280 Merges labels. Requires JSON in request body using the following format: 281 282 [toLabel1, fromLabel1, fromLabel2, fromLabel3, ...] 283 284 The first element of the JSON array specifies the label to be used as the merge result. 285 Note that it's computationally more efficient to group a number of merges into the 286 same toLabel as a single merge request instead of multiple merge requests. 287 288 289 POST <api URL>/node/<UUID>/<data name>/split/<label>[?splitlabel=X] 290 291 Splits a portion of a label's voxels into a new label or, if "splitlabel" is specified 292 as an optional query string, the given split label. Returns the following JSON: 293 294 { "label": <new label> } 295 296 This request requires a binary sparse volume in the POSTed body with the following 297 encoded RLE format, which is compatible with the format returned by a GET on the 298 "sparsevol" endpoint described above: 299 300 All integers are in little-endian format. 301 302 byte Payload descriptor: 303 Set to 0 to indicate it's a binary sparse volume. 304 uint8 Number of dimensions 305 uint8 Dimension of run (typically 0 = X) 306 byte Reserved (to be used later) 307 uint32 # Voxels [TODO. 0 for now] 308 uint32 # Spans 309 Repeating unit of: 310 int32 Coordinate of run start (dimension 0) 311 int32 Coordinate of run start (dimension 1) 312 int32 Coordinate of run start (dimension 2) 313 ... 314 int32 Length of run 315 316 NOTE 1: The POSTed split sparse volume must be a subset of the given label's voxels. You cannot 317 give an arbitrary sparse volume that may span multiple labels. 318 319 NOTE 2: If a split label is specified, it is the client's responsibility to make sure the given 320 label will not create conflict with labels in other versions. It should primarily be used in 321 chain operations like "split-coarse" followed by "split" using voxels, where the new label 322 created by the split coarse is used as the split label for the smaller, higher-res "split". 323 324 POST <api URL>/node/<UUID>/<data name>/split-coarse/<label>[?splitlabel=X] 325 326 Splits a portion of a label's blocks into a new label or, if "splitlabel" is specified 327 as an optional query string, the given split label. Returns the following JSON: 328 329 { "label": <new label> } 330 331 This request requires a binary sparse volume in the POSTed body with the following 332 encoded RLE format, which is similar to the "split" request format but uses block 333 instead of voxel coordinates: 334 335 All integers are in little-endian format. 336 337 byte Payload descriptor: 338 Set to 0 to indicate it's a binary sparse volume. 339 uint8 Number of dimensions 340 uint8 Dimension of run (typically 0 = X) 341 byte Reserved (to be used later) 342 uint32 # Blocks [TODO. 0 for now] 343 uint32 # Spans 344 Repeating unit of: 345 int32 Coordinate of run start (dimension 0) 346 int32 Coordinate of run start (dimension 1) 347 int32 Coordinate of run start (dimension 2) 348 ... 349 int32 Length of run 350 351 The Notes for "split" endpoint above are applicable to this "split-coarse" endpoint. 352 353 POST <api URL>/node/<UUID>/<data name>/resync/<label> 354 355 Regenerates the sparse volume for the given label from its synced labelblk. 356 This is used to repair databases that have been inadvertantly shutdown or 357 crashed while undergoing merges/splits. 358 359 This request requires a binary sparse volume in the POSTed body with the following 360 encoded RLE format, which is similar to the "split" request format but uses block 361 instead of voxel coordinates: 362 363 All integers are in little-endian format. 364 365 byte Payload descriptor: 366 Set to 0 to indicate it's a binary sparse volume. 367 uint8 Number of dimensions 368 uint8 Dimension of run (typically 0 = X) 369 byte Reserved (to be used later) 370 uint32 # Blocks [TODO. 0 for now] 371 uint32 # Spans 372 Repeating unit of: 373 int32 Coordinate of run start (dimension 0) 374 int32 Coordinate of run start (dimension 1) 375 int32 Coordinate of run start (dimension 2) 376 ... 377 int32 Length of run 378 379 DELETE <api URL>/node/<UUID>/<data name>/area/<label>/<size>/<offset> 380 381 NOTE: Does not honor syncs and is intended purely for low-level mods. 382 Deletes all of a label's sparse volume within a given 3d volume from a 3d offset. 383 384 Example: 385 386 DELETE <api URL>/node/3f8c/bodies/area/23/512_512_512/0_0_128 387 388 The above example deletes all sparsevol associated with label 23 in subvolume 389 that is 512 x 512 x 512 starting at offset (0,0,128). 390 ` 391 392 var ( 393 dtype *Type 394 395 DefaultBlockSize int32 = labelblk.DefaultBlockSize 396 DefaultRes float32 = labelblk.DefaultRes 397 DefaultUnits = labelblk.DefaultUnits 398 ) 399 400 func init() { 401 dtype = new(Type) 402 dtype.Type = datastore.Type{ 403 Name: TypeName, 404 URL: RepoURL, 405 Version: Version, 406 Requirements: &storage.Requirements{ 407 Batcher: true, 408 }, 409 } 410 411 // See doc for package on why channels are segregated instead of interleaved. 412 // Data types must be registered with the datastore to be used. 413 datastore.Register(dtype) 414 415 // Need to register types that will be used to fulfill interfaces. 416 gob.Register(&Type{}) 417 gob.Register(&Data{}) 418 } 419 420 // NewData returns a pointer to labelvol data. 421 func NewData(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (*Data, error) { 422 // Initialize the Data for this data type 423 basedata, err := datastore.NewDataService(dtype, uuid, id, name, c) 424 if err != nil { 425 return nil, err 426 } 427 props := new(Properties) 428 props.setDefault() 429 if err := props.setByConfig(c); err != nil { 430 return nil, err 431 } 432 data := &Data{ 433 Data: basedata, 434 Properties: *props, 435 } 436 data.Properties.MaxLabel = make(map[dvid.VersionID]uint64) 437 return data, nil 438 } 439 440 // --- Labelvol Datatype ----- 441 442 type Type struct { 443 datastore.Type 444 } 445 446 // --- TypeService interface --- 447 448 func (dtype *Type) NewDataService(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (datastore.DataService, error) { 449 return NewData(uuid, id, name, c) 450 } 451 452 func (dtype *Type) Help() string { 453 return helpMessage 454 } 455 456 // Properties are additional properties for data beyond those in standard datastore.Data. 457 type Properties struct { 458 Resolution dvid.Resolution 459 460 // Block size for this repo 461 BlockSize dvid.Point3d 462 463 mlMu sync.RWMutex // For atomic access of MaxLabel and MaxRepoLabel 464 465 // The maximum label id found in each version of this instance. 466 // Can be unset if no new label was added at that version, in which case 467 // you must traverse DAG to find max label of parent. 468 MaxLabel map[dvid.VersionID]uint64 469 470 // The maximum label for this instance in the entire repo. This allows us to do 471 // conflict-free merges without any relabeling. Note that relabeling (rebasing) 472 // is required if we move data between repos, e.g., when pushing remote nodes, 473 // since we have no control over which labels were created remotely and there 474 // could be conflicts between the local and remote repos. When mutations only 475 // occur within a single repo, however, this atomic label allows us to prevent 476 // conflict across all versions within this repo. 477 MaxRepoLabel uint64 478 } 479 480 // CopyPropertiesFrom copies the data instance-specific properties from a given 481 // data instance into the receiver's properties. Fulfills the datastore.PropertyCopier interface. 482 func (d *Data) CopyPropertiesFrom(src datastore.DataService, fs storage.FilterSpec) error { 483 d2, ok := src.(*Data) 484 if !ok { 485 return fmt.Errorf("unable to copy properties from non-labelvol data %q", src.DataName()) 486 } 487 d.Properties.copyImmutable(&(d2.Properties)) 488 489 // TODO -- Handle mutable data that could be potentially altered by filter. 490 d.MaxLabel = make(map[dvid.VersionID]uint64, len(d2.MaxLabel)) 491 for k, v := range d2.MaxLabel { 492 d.MaxLabel[k] = v 493 } 494 d.MaxRepoLabel = d2.MaxRepoLabel 495 496 return nil 497 } 498 499 func (p *Properties) copyImmutable(p2 *Properties) { 500 p.BlockSize = p2.BlockSize 501 502 p.Resolution.VoxelSize = make(dvid.NdFloat32, 3) 503 copy(p.Resolution.VoxelSize, p2.Resolution.VoxelSize) 504 p.Resolution.VoxelUnits = make(dvid.NdString, 3) 505 copy(p.Resolution.VoxelUnits, p2.Resolution.VoxelUnits) 506 } 507 508 func (p Properties) MarshalJSON() ([]byte, error) { 509 maxLabels := make(map[string]uint64) 510 for v, max := range p.MaxLabel { 511 uuid, err := datastore.UUIDFromVersion(v) 512 if err != nil { 513 return nil, err 514 } 515 maxLabels[string(uuid)] = max 516 } 517 return json.Marshal(struct { 518 dvid.Resolution 519 BlockSize dvid.Point3d 520 MaxLabel map[string]uint64 521 }{ 522 p.Resolution, 523 p.BlockSize, 524 maxLabels, 525 }) 526 } 527 528 func (p *Properties) setDefault() { 529 for d := 0; d < 3; d++ { 530 p.BlockSize[d] = DefaultBlockSize 531 } 532 p.Resolution.VoxelSize = make(dvid.NdFloat32, 3) 533 for d := 0; d < 3; d++ { 534 p.Resolution.VoxelSize[d] = DefaultRes 535 } 536 p.Resolution.VoxelUnits = make(dvid.NdString, 3) 537 for d := 0; d < 3; d++ { 538 p.Resolution.VoxelUnits[d] = DefaultUnits 539 } 540 } 541 542 func (p *Properties) setByConfig(config dvid.Config) error { 543 s, found, err := config.GetString("BlockSize") 544 if err != nil { 545 return err 546 } 547 if found { 548 p.BlockSize, err = dvid.StringToPoint3d(s, ",") 549 if err != nil { 550 return err 551 } 552 } 553 s, found, err = config.GetString("VoxelSize") 554 if err != nil { 555 return err 556 } 557 if found { 558 dvid.Infof("Changing resolution of voxels to %s\n", s) 559 p.Resolution.VoxelSize, err = dvid.StringToNdFloat32(s, ",") 560 if err != nil { 561 return err 562 } 563 } 564 s, found, err = config.GetString("VoxelUnits") 565 if err != nil { 566 return err 567 } 568 if found { 569 p.Resolution.VoxelUnits, err = dvid.StringToNdString(s, ",") 570 if err != nil { 571 return err 572 } 573 } 574 return nil 575 } 576 577 // Data instance of labelvol, label sparse volumes. 578 type Data struct { 579 *datastore.Data 580 Properties 581 582 // Keep track of sync operations that could be updating the data. 583 // TODO: Think about making this per label since sync status is pessimistic, assuming 584 // all labels are being updated. 585 datastore.Updater 586 587 // channels for mutations and downres caching. 588 syncCh chan datastore.SyncMessage 589 syncDone chan *sync.WaitGroup 590 } 591 592 // RemapVersions modifies internal data instance properties that rely 593 // on server-specific version ids. This is necessary after DVID-to-DVID 594 // transmission. 595 func (d *Data) RemapVersions(vmap dvid.VersionMap) error { 596 maxLabels := make(map[dvid.VersionID]uint64, len(d.Properties.MaxLabel)) 597 for oldv, label := range d.Properties.MaxLabel { 598 newv, found := vmap[oldv] 599 if !found { 600 dvid.Infof("No version %d in labelvol %q... discarding max label", oldv, d.DataName()) 601 continue 602 } 603 maxLabels[newv] = label 604 } 605 d.Properties.MaxLabel = maxLabels 606 return nil 607 } 608 609 // GetSyncedLabelblk returns the synced labelblk data instance or returns 610 // an error if there is no synced labelblk. 611 func (d *Data) GetSyncedLabelblk() (*labelblk.Data, error) { 612 // Go through all synced names, and checking if there's a valid source. 613 for dataUUID := range d.SyncedData() { 614 source, err := labelblk.GetByDataUUID(dataUUID) 615 if err == nil { 616 return source, nil 617 } 618 } 619 return nil, fmt.Errorf("no labelblk data is syncing with %s", d.DataName()) 620 } 621 622 // GetByDataUUID returns a pointer to labelvol data given a data UUID. 623 func GetByDataUUID(dataUUID dvid.UUID) (*Data, error) { 624 source, err := datastore.GetDataByDataUUID(dataUUID) 625 if err != nil { 626 return nil, err 627 } 628 data, ok := source.(*Data) 629 if !ok { 630 return nil, fmt.Errorf("Instance '%s' is not a labelvol datatype!", source.DataName()) 631 } 632 return data, nil 633 } 634 635 // GetByUUIDName returns a pointer to labelvol data given a version (UUID) and data name. 636 func GetByUUIDName(uuid dvid.UUID, name dvid.InstanceName) (*Data, error) { 637 source, err := datastore.GetDataByUUIDName(uuid, name) 638 if err != nil { 639 return nil, err 640 } 641 data, ok := source.(*Data) 642 if !ok { 643 return nil, fmt.Errorf("Instance '%s' is not a labelvol datatype!", name) 644 } 645 return data, nil 646 } 647 648 // GetByVersionName returns a pointer to labelblk data given a UUID and data name. 649 func GetByVersionName(v dvid.VersionID, name dvid.InstanceName) (*Data, error) { 650 source, err := datastore.GetDataByVersionName(v, name) 651 if err != nil { 652 return nil, err 653 } 654 data, ok := source.(*Data) 655 if !ok { 656 return nil, fmt.Errorf("Instance '%s' is not a labelvol datatype!", name) 657 } 658 return data, nil 659 } 660 661 // --- datastore.InstanceMutator interface ----- 662 663 // LoadMutable loads mutable properties of label volumes like the maximum labels 664 // for each version. Note that we load these max labels from key-value pairs 665 // rather than data instance properties persistence, because in the case of a crash, 666 // the actually stored repo data structure may be out-of-date compared to the guaranteed 667 // up-to-date key-value pairs for max labels. 668 func (d *Data) LoadMutable(root dvid.VersionID, storedVersion, expectedVersion uint64) (bool, error) { 669 ctx := storage.NewDataContext(d, 0) 670 store, err := datastore.GetOrderedKeyValueDB(d) 671 if err != nil { 672 return false, fmt.Errorf("Data type labelvol had error initializing store: %v\n", err) 673 } 674 675 wg := new(sync.WaitGroup) 676 wg.Add(1) 677 ch := make(chan *storage.KeyValue) 678 679 // Start appropriate migration function if any. 680 var saveRequired bool 681 682 switch storedVersion { 683 case 0: 684 // Need to update all max labels and set repo-level max label. 685 saveRequired = true 686 dvid.Infof("Migrating old version of labelvol %q to new version\n", d.DataName()) 687 go d.migrateMaxLabels(root, wg, ch) 688 default: 689 // Load in each version max label without migration. 690 go d.loadMaxLabels(wg, ch) 691 } 692 693 // Send the max label data per version 694 minKey, err := ctx.MinVersionKey(maxLabelTKey) 695 if err != nil { 696 return false, err 697 } 698 maxKey, err := ctx.MaxVersionKey(maxLabelTKey) 699 if err != nil { 700 return false, err 701 } 702 keysOnly := false 703 if err = store.RawRangeQuery(minKey, maxKey, keysOnly, ch, nil); err != nil { 704 return false, err 705 } 706 wg.Wait() 707 708 dvid.Infof("Loaded max label values for labelvol %q with repo-wide max %d\n", d.DataName(), d.MaxRepoLabel) 709 return saveRequired, nil 710 } 711 712 func (d *Data) migrateMaxLabels(root dvid.VersionID, wg *sync.WaitGroup, ch chan *storage.KeyValue) { 713 ctx := storage.NewDataContext(d, 0) 714 store, err := datastore.GetOrderedKeyValueDB(d) 715 if err != nil { 716 dvid.Errorf("Can't initialize store for labelvol %q: %v\n", d.DataName(), err) 717 } 718 719 var maxRepoLabel uint64 720 d.MaxLabel = make(map[dvid.VersionID]uint64) 721 for { 722 kv := <-ch 723 if kv == nil { 724 break 725 } 726 v, err := ctx.VersionFromKey(kv.K) 727 if err != nil { 728 dvid.Errorf("Can't decode key when loading mutable data for %s", d.DataName()) 729 continue 730 } 731 if len(kv.V) != 8 { 732 dvid.Errorf("Got bad value. Expected 64-bit label, got %v", kv.V) 733 continue 734 } 735 label := binary.LittleEndian.Uint64(kv.V) 736 d.MaxLabel[v] = label 737 if label > maxRepoLabel { 738 maxRepoLabel = label 739 } 740 } 741 742 // Adjust the MaxLabel data to make sure we correct for any case of child max < parent max. 743 d.adjustMaxLabels(store, root) 744 745 // Set the repo-wide max label. 746 d.MaxRepoLabel = maxRepoLabel 747 748 buf := make([]byte, 8) 749 binary.LittleEndian.PutUint64(buf, maxRepoLabel) 750 store.Put(ctx, maxRepoLabelTKey, buf) 751 752 wg.Done() 753 return 754 } 755 756 func (d *Data) adjustMaxLabels(store storage.KeyValueSetter, root dvid.VersionID) error { 757 buf := make([]byte, 8) 758 759 parentMax, ok := d.MaxLabel[root] 760 if !ok { 761 return fmt.Errorf("can't adjust version id %d since none exists in metadata", root) 762 } 763 childIDs, err := datastore.GetChildrenByVersion(root) 764 if err != nil { 765 return err 766 } 767 for _, childID := range childIDs { 768 var save bool 769 childMax, ok := d.MaxLabel[childID] 770 if !ok { 771 // set to parent max 772 d.MaxLabel[childID] = parentMax 773 save = true 774 } else if childMax < parentMax { 775 d.MaxLabel[childID] = parentMax + childMax + 1 776 save = true 777 } 778 779 // save the key-value 780 if save { 781 binary.LittleEndian.PutUint64(buf, d.MaxLabel[childID]) 782 ctx := datastore.NewVersionedCtx(d, childID) 783 store.Put(ctx, maxLabelTKey, buf) 784 } 785 786 // recurse for depth-first 787 if err := d.adjustMaxLabels(store, childID); err != nil { 788 return err 789 } 790 } 791 return nil 792 } 793 794 func (d *Data) loadMaxLabels(wg *sync.WaitGroup, ch chan *storage.KeyValue) { 795 ctx := storage.NewDataContext(d, 0) 796 var repoMax uint64 797 d.MaxLabel = make(map[dvid.VersionID]uint64) 798 for { 799 kv := <-ch 800 if kv == nil { 801 break 802 } 803 v, err := ctx.VersionFromKey(kv.K) 804 if err != nil { 805 dvid.Errorf("Can't decode key when loading mutable data for %s", d.DataName()) 806 continue 807 } 808 if len(kv.V) != 8 { 809 dvid.Errorf("Got bad value. Expected 64-bit label, got %v", kv.V) 810 continue 811 } 812 label := binary.LittleEndian.Uint64(kv.V) 813 d.MaxLabel[v] = label 814 if label > repoMax { 815 repoMax = label 816 } 817 } 818 819 // Load in the repo-wide max label. 820 store, err := datastore.GetOrderedKeyValueDB(d) 821 if err != nil { 822 dvid.Errorf("Data type labelvol had error initializing store: %v\n", err) 823 return 824 } 825 data, err := store.Get(ctx, maxRepoLabelTKey) 826 if err != nil { 827 dvid.Errorf("Error getting repo-wide max label: %v\n", err) 828 return 829 } 830 if data == nil || len(data) != 8 { 831 dvid.Errorf("Could not load repo-wide max label for instance %q. Only got %d bytes, not 64-bit label.\n", d.DataName(), len(data)) 832 dvid.Errorf("Using max label across versions: %d\n", repoMax) 833 d.MaxRepoLabel = repoMax 834 } else { 835 d.MaxRepoLabel = binary.LittleEndian.Uint64(data) 836 if d.MaxRepoLabel < repoMax { 837 dvid.Errorf("Saved repo-wide max for instance %q was %d, changed to largest version max %d\n", d.DataName(), d.MaxRepoLabel, repoMax) 838 d.MaxRepoLabel = repoMax 839 } 840 } 841 wg.Done() 842 } 843 844 // --- datastore.DataService interface --------- 845 846 func (d *Data) Help() string { 847 return helpMessage 848 } 849 850 func (d *Data) MarshalJSON() ([]byte, error) { 851 return json.Marshal(struct { 852 Base *datastore.Data 853 Extended Properties 854 }{ 855 d.Data, 856 d.Properties, 857 }) 858 } 859 860 func (d *Data) GobDecode(b []byte) error { 861 buf := bytes.NewBuffer(b) 862 dec := gob.NewDecoder(buf) 863 if err := dec.Decode(&(d.Data)); err != nil { 864 return err 865 } 866 if err := dec.Decode(&(d.Properties)); err != nil { 867 return err 868 } 869 return nil 870 } 871 872 func (d *Data) GobEncode() ([]byte, error) { 873 var buf bytes.Buffer 874 enc := gob.NewEncoder(&buf) 875 if err := enc.Encode(d.Data); err != nil { 876 return nil, err 877 } 878 if err := enc.Encode(d.Properties); err != nil { 879 return nil, err 880 } 881 return buf.Bytes(), nil 882 } 883 884 // DoRPC acts as a switchboard for RPC commands. 885 func (d *Data) DoRPC(req datastore.Request, reply *datastore.Response) error { 886 switch req.TypeCommand() { 887 case "dump": 888 if len(req.Command) < 5 { 889 return fmt.Errorf("Poorly formatted dump command. See command-line help.") 890 } 891 // Parse the request 892 var uuidStr, dataName, cmdStr, dirStr string 893 req.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &dirStr) 894 895 uuid, versionID, err := datastore.MatchingUUID(uuidStr) 896 if err != nil { 897 return err 898 } 899 go func() { 900 d.DumpSubvols(uuid, versionID, dirStr) 901 }() 902 reply.Text = fmt.Sprintf("Asynchronously dumping sparse volumes in directory: %s\n", dirStr) 903 904 default: 905 return fmt.Errorf("Unknown command. Data type '%s' [%s] does not support '%s' command.", 906 d.DataName(), d.TypeName(), req.TypeCommand()) 907 } 908 return nil 909 } 910 911 // ServeHTTP handles all incoming HTTP requests for this data. 912 func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) (activity map[string]interface{}) { 913 timedLog := dvid.NewTimeLog() 914 versionID := ctx.VersionID() 915 916 // Get the action (GET, POST) 917 action := strings.ToLower(r.Method) 918 919 // Break URL request into arguments 920 url := r.URL.Path[len(server.WebAPIPath):] 921 parts := strings.Split(url, "/") 922 if len(parts[len(parts)-1]) == 0 { 923 parts = parts[:len(parts)-1] 924 } 925 926 // Handle POST on data -> setting of configuration 927 if len(parts) == 3 && action == "put" { 928 config, err := server.DecodeJSON(r) 929 if err != nil { 930 server.BadRequest(w, r, err) 931 return 932 } 933 if err := d.ModifyConfig(config); err != nil { 934 server.BadRequest(w, r, err) 935 return 936 } 937 if err := datastore.SaveDataByUUID(uuid, d); err != nil { 938 server.BadRequest(w, r, err) 939 return 940 } 941 fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config) 942 return 943 } 944 945 if len(parts) < 4 { 946 server.BadRequest(w, r, "Incomplete API request") 947 return 948 } 949 950 // Process help and info. 951 switch parts[3] { 952 case "help": 953 w.Header().Set("Content-Type", "text/plain") 954 fmt.Fprintln(w, dtype.Help()) 955 956 case "info": 957 jsonBytes, err := d.MarshalJSON() 958 if err != nil { 959 server.BadRequest(w, r, err) 960 return 961 } 962 w.Header().Set("Content-Type", "application/json") 963 fmt.Fprintf(w, string(jsonBytes)) 964 965 case "sync": 966 if action != "post" { 967 server.BadRequest(w, r, "Only POST allowed to sync endpoint") 968 return 969 } 970 replace := r.URL.Query().Get("replace") == "true" 971 if err := datastore.SetSyncByJSON(d, uuid, replace, r.Body); err != nil { 972 server.BadRequest(w, r, err) 973 return 974 } 975 976 case "sparsevol": 977 // GET <api URL>/node/<UUID>/<data name>/sparsevol/<label> 978 // POST <api URL>/node/<UUID>/<data name>/sparsevol/<label> 979 // HEAD <api URL>/node/<UUID>/<data name>/sparsevol/<label> 980 if len(parts) < 5 { 981 server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'sparsevol' command") 982 return 983 } 984 label, err := strconv.ParseUint(parts[4], 10, 64) 985 if err != nil { 986 server.BadRequest(w, r, err) 987 return 988 } 989 if label == 0 { 990 server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n") 991 return 992 } 993 queryStrings := r.URL.Query() 994 var b dvid.Bounds 995 b.Voxel, err = dvid.OptionalBoundsFromQueryString(r) 996 if err != nil { 997 server.BadRequest(w, r, "Error parsing bounds from query string: %v\n", err) 998 return 999 } 1000 b.Block = b.Voxel.Divide(d.BlockSize) 1001 b.Exact = true 1002 if queryStrings.Get("exact") == "false" { 1003 b.Exact = false 1004 } 1005 1006 compression := queryStrings.Get("compression") 1007 1008 switch action { 1009 case "get": 1010 data, err := d.GetSparseVol(ctx, label, b) 1011 if err != nil { 1012 server.BadRequest(w, r, err) 1013 return 1014 } 1015 fmt.Printf("Sparsevol on %d returned %d bytes\n", label, len(data)) 1016 if data == nil { 1017 w.WriteHeader(http.StatusNotFound) 1018 return 1019 } 1020 w.Header().Set("Content-type", "application/octet-stream") 1021 switch compression { 1022 case "": 1023 _, err = w.Write(data) 1024 if err != nil { 1025 server.BadRequest(w, r, err) 1026 return 1027 } 1028 case "lz4": 1029 compressed := make([]byte, lz4.CompressBound(data)) 1030 var n, outSize int 1031 if outSize, err = lz4.Compress(data, compressed); err != nil { 1032 server.BadRequest(w, r, err) 1033 return 1034 } 1035 compressed = compressed[:outSize] 1036 if n, err = w.Write(compressed); err != nil { 1037 server.BadRequest(w, r, err) 1038 return 1039 } 1040 if n != outSize { 1041 errmsg := fmt.Sprintf("Only able to write %d of %d lz4 compressed bytes\n", n, outSize) 1042 dvid.Errorf(errmsg) 1043 server.BadRequest(w, r, errmsg) 1044 return 1045 } 1046 case "gzip": 1047 gw := gzip.NewWriter(w) 1048 if _, err = gw.Write(data); err != nil { 1049 server.BadRequest(w, r, err) 1050 return 1051 } 1052 if err = gw.Close(); err != nil { 1053 server.BadRequest(w, r, err) 1054 return 1055 } 1056 default: 1057 server.BadRequest(w, r, "unknown compression type %q", compression) 1058 return 1059 } 1060 1061 case "head": 1062 w.Header().Set("Content-type", "text/html") 1063 found, err := d.FoundSparseVol(ctx, label, b) 1064 if err != nil { 1065 server.BadRequest(w, r, err) 1066 return 1067 } 1068 if found { 1069 w.WriteHeader(http.StatusOK) 1070 } else { 1071 w.WriteHeader(http.StatusNoContent) 1072 } 1073 return 1074 1075 case "post": 1076 server.BadRequest(w, r, "POST of sparsevol not currently implemented\n") 1077 return 1078 // if err := d.PutSparseVol(versionID, label, r.Body); err != nil { 1079 // server.BadRequest(w, r, err) 1080 // return 1081 // } 1082 default: 1083 server.BadRequest(w, r, "Unable to handle HTTP action %s on sparsevol endpoint", action) 1084 return 1085 } 1086 timedLog.Infof("HTTP %s: sparsevol on label %d (%s)", r.Method, label, r.URL) 1087 1088 case "sparsevol-by-point": 1089 // GET <api URL>/node/<UUID>/<data name>/sparsevol-by-point/<coord> 1090 if len(parts) < 5 { 1091 server.BadRequest(w, r, "ERROR: DVID requires coord to follow 'sparsevol-by-point' command") 1092 return 1093 } 1094 coord, err := dvid.StringToPoint(parts[4], "_") 1095 if err != nil { 1096 server.BadRequest(w, r, err) 1097 return 1098 } 1099 source, err := d.GetSyncedLabelblk() 1100 if err != nil { 1101 server.BadRequest(w, r, err) 1102 return 1103 } 1104 label, err := source.GetLabelAtPoint(ctx.VersionID(), coord) 1105 if err != nil { 1106 server.BadRequest(w, r, err) 1107 return 1108 } 1109 if label == 0 { 1110 server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n") 1111 return 1112 } 1113 data, err := d.GetSparseVol(ctx, label, dvid.Bounds{}) 1114 if err != nil { 1115 server.BadRequest(w, r, err) 1116 return 1117 } 1118 if data == nil { 1119 w.WriteHeader(http.StatusNotFound) 1120 return 1121 } 1122 w.Header().Set("Content-type", "application/octet-stream") 1123 _, err = w.Write(data) 1124 if err != nil { 1125 server.BadRequest(w, r, err) 1126 return 1127 } 1128 timedLog.Infof("HTTP %s: sparsevol-by-point at %s (%s)", r.Method, coord, r.URL) 1129 1130 case "sparsevol-coarse": 1131 // GET <api URL>/node/<UUID>/<data name>/sparsevol-coarse/<label> 1132 if len(parts) < 5 { 1133 server.BadRequest(w, r, "DVID requires label ID to follow 'sparsevol-coarse' command") 1134 return 1135 } 1136 label, err := strconv.ParseUint(parts[4], 10, 64) 1137 if err != nil { 1138 server.BadRequest(w, r, err) 1139 return 1140 } 1141 if label == 0 { 1142 server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n") 1143 return 1144 } 1145 data, err := d.GetSparseCoarseVol(ctx, label) 1146 if err != nil { 1147 server.BadRequest(w, r, err) 1148 return 1149 } 1150 if data == nil { 1151 w.WriteHeader(http.StatusNotFound) 1152 return 1153 } 1154 w.Header().Set("Content-type", "application/octet-stream") 1155 _, err = w.Write(data) 1156 if err != nil { 1157 server.BadRequest(w, r, err) 1158 return 1159 } 1160 timedLog.Infof("HTTP %s: sparsevol-coarse on label %d (%s)", r.Method, label, r.URL) 1161 1162 case "area": 1163 // DELETE <api URL>/node/<UUID>/<data name>/area/<label>/<size>/<offset> 1164 if len(parts) < 6 { 1165 server.BadRequest(w, r, "DVID requires label ID, size, and offset to follow 'area' endpoint") 1166 return 1167 } 1168 label, err := strconv.ParseUint(parts[4], 10, 64) 1169 if err != nil { 1170 server.BadRequest(w, r, err) 1171 return 1172 } 1173 if label == 0 { 1174 server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n") 1175 return 1176 } 1177 sizeStr, offsetStr := parts[5], parts[6] 1178 1179 subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_") 1180 if err != nil { 1181 server.BadRequest(w, r, err) 1182 return 1183 } 1184 if subvol.StartPoint().NumDims() != 3 || subvol.Size().NumDims() != 3 { 1185 server.BadRequest(w, r, "must specify 3D subvolumes", subvol.StartPoint(), subvol.EndPoint()) 1186 return 1187 } 1188 1189 // Make sure subvolume gets align with blocks 1190 if !dvid.BlockAligned(subvol, d.BlockSize) { 1191 server.BadRequest(w, r, "cannot delete sparsevol in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint()) 1192 return 1193 } 1194 1195 if action != "delete" { 1196 server.BadRequest(w, r, "DVID does not accept the %s action on the 'blocks' endpoint", action) 1197 return 1198 } 1199 1200 if err := d.DeleteArea(ctx, label, subvol); err != nil { 1201 server.BadRequest(w, r, err) 1202 return 1203 } 1204 timedLog.Infof("HTTP %s: area %s (%s)", r.Method, subvol, r.URL) 1205 1206 case "maxlabel": 1207 // GET <api URL>/node/<UUID>/<data name>/maxlabel 1208 w.Header().Set("Content-Type", "application/json") 1209 switch action { 1210 case "get": 1211 maxlabel, ok := d.MaxLabel[versionID] 1212 if !ok { 1213 server.BadRequest(w, r, "No maximum label found for %s version %d\n", d.DataName(), versionID) 1214 return 1215 } 1216 fmt.Fprintf(w, "{%q: %d}", "maxlabel", maxlabel) 1217 default: 1218 server.BadRequest(w, r, "Unknown action %q requested: %s\n", action, r.URL) 1219 return 1220 } 1221 timedLog.Infof("HTTP maxlabel request (%s)", r.URL) 1222 1223 case "nextlabel": 1224 // GET <api URL>/node/<UUID>/<data name>/nextlabel 1225 // POST <api URL>/node/<UUID>/<data name>/nextlabel 1226 w.Header().Set("Content-Type", "application/json") 1227 switch action { 1228 case "get": 1229 fmt.Fprintf(w, "{%q: %d}", "nextlabel", d.MaxRepoLabel+1) 1230 case "post": 1231 server.BadRequest(w, r, "POST on maxlabel is not supported yet.\n") 1232 return 1233 default: 1234 server.BadRequest(w, r, "Unknown action %q requested: %s\n", action, r.URL) 1235 return 1236 } 1237 timedLog.Infof("HTTP maxlabel request (%s)", r.URL) 1238 1239 case "split": 1240 // POST <api URL>/node/<UUID>/<data name>/split/<label>[?splitlabel=X] 1241 if action != "post" { 1242 server.BadRequest(w, r, "Split requests must be POST actions.") 1243 return 1244 } 1245 if len(parts) < 5 { 1246 server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'split' command") 1247 return 1248 } 1249 fromLabel, err := strconv.ParseUint(parts[4], 10, 64) 1250 if err != nil { 1251 server.BadRequest(w, r, err) 1252 return 1253 } 1254 if fromLabel == 0 { 1255 server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n") 1256 return 1257 } 1258 var splitLabel uint64 1259 queryStrings := r.URL.Query() 1260 splitStr := queryStrings.Get("splitlabel") 1261 if splitStr != "" { 1262 splitLabel, err = strconv.ParseUint(splitStr, 10, 64) 1263 if err != nil { 1264 server.BadRequest(w, r, "Bad parameter for 'splitlabel' query string (%q). Must be uint64.\n", splitStr) 1265 } 1266 } 1267 toLabel, err := d.SplitLabels(ctx.VersionID(), fromLabel, splitLabel, r.Body) 1268 if err != nil { 1269 server.BadRequest(w, r, fmt.Sprintf("split: %v", err)) 1270 return 1271 } 1272 w.Header().Set("Content-Type", "application/json") 1273 fmt.Fprintf(w, "{%q: %d}", "label", toLabel) 1274 timedLog.Infof("HTTP split request (%s)", r.URL) 1275 1276 case "split-coarse": 1277 // POST <api URL>/node/<UUID>/<data name>/split-coarse/<label>[?splitlabel=X] 1278 if action != "post" { 1279 server.BadRequest(w, r, "Split-coarse requests must be POST actions.") 1280 return 1281 } 1282 if len(parts) < 5 { 1283 server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'split' command") 1284 return 1285 } 1286 fromLabel, err := strconv.ParseUint(parts[4], 10, 64) 1287 if err != nil { 1288 server.BadRequest(w, r, err) 1289 return 1290 } 1291 if fromLabel == 0 { 1292 server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n") 1293 return 1294 } 1295 var splitLabel uint64 1296 queryStrings := r.URL.Query() 1297 splitStr := queryStrings.Get("splitlabel") 1298 if splitStr != "" { 1299 splitLabel, err = strconv.ParseUint(splitStr, 10, 64) 1300 if err != nil { 1301 server.BadRequest(w, r, "Bad parameter for 'splitlabel' query string (%q). Must be uint64.\n", splitStr) 1302 } 1303 } 1304 toLabel, err := d.SplitCoarseLabels(ctx.VersionID(), fromLabel, splitLabel, r.Body) 1305 if err != nil { 1306 server.BadRequest(w, r, fmt.Sprintf("split-coarse: %v", err)) 1307 return 1308 } 1309 w.Header().Set("Content-Type", "application/json") 1310 fmt.Fprintf(w, "{%q: %d}", "label", toLabel) 1311 timedLog.Infof("HTTP split-coarse request (%s)", r.URL) 1312 1313 case "merge": 1314 // POST <api URL>/node/<UUID>/<data name>/merge 1315 if action != "post" { 1316 server.BadRequest(w, r, "Merge requests must be POST actions.") 1317 return 1318 } 1319 data, err := ioutil.ReadAll(r.Body) 1320 if err != nil { 1321 server.BadRequest(w, r, "Bad POSTed data for merge. Should be JSON.") 1322 return 1323 } 1324 var tuple labels.MergeTuple 1325 if err := json.Unmarshal(data, &tuple); err != nil { 1326 server.BadRequest(w, r, fmt.Sprintf("Bad merge op JSON: %v", err)) 1327 return 1328 } 1329 mergeOp, err := tuple.Op() 1330 if err != nil { 1331 server.BadRequest(w, r, err) 1332 return 1333 } 1334 mergeOp.MutID = d.NewMutationID() 1335 if err := d.MergeLabels(ctx.VersionID(), mergeOp); err != nil { 1336 server.BadRequest(w, r, fmt.Sprintf("Error on merge: %v", err)) 1337 return 1338 } 1339 timedLog.Infof("HTTP merge request (%s)", r.URL) 1340 1341 case "resync": 1342 // POST <api URL>/node/<UUID>/<data name>/resync/<label> 1343 if action != "post" { 1344 server.BadRequest(w, r, "resync requests must be POST actions.") 1345 return 1346 } 1347 if len(parts) < 5 { 1348 server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'resync' command") 1349 return 1350 } 1351 relabel, err := strconv.ParseUint(parts[4], 10, 64) 1352 if err != nil { 1353 server.BadRequest(w, r, err) 1354 return 1355 } 1356 if relabel == 0 { 1357 server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n") 1358 return 1359 } 1360 d.StartUpdate() 1361 go func() { 1362 if err := d.resyncLabel(ctx, relabel, r.Body); err != nil { 1363 dvid.Errorf("Error on resync of data %q, label %d: %v\n", d.DataName(), relabel, err) 1364 } 1365 d.StopUpdate() 1366 }() 1367 timedLog.Infof("HTTP resync %d started (%s)", relabel, r.URL) 1368 1369 default: 1370 server.BadAPIRequest(w, r, d) 1371 } 1372 return 1373 } 1374 1375 // Given a stored label, make sure our max label tracking is updated. 1376 func (d *Data) casMaxLabel(batch storage.Batch, v dvid.VersionID, label uint64) { 1377 d.mlMu.Lock() 1378 defer d.mlMu.Unlock() 1379 1380 save := false 1381 maxLabel, found := d.MaxLabel[v] 1382 if !found { 1383 dvid.Infof("Bad max label of version %d -- none found!\n", v) 1384 maxLabel = 1 1385 } 1386 if maxLabel < label { 1387 maxLabel = label 1388 save = true 1389 } 1390 if save { 1391 buf := make([]byte, 8) 1392 binary.LittleEndian.PutUint64(buf, maxLabel) 1393 batch.Put(maxLabelTKey, buf) 1394 d.MaxLabel[v] = maxLabel 1395 1396 if d.MaxRepoLabel < maxLabel { 1397 d.MaxRepoLabel = maxLabel 1398 ctx := storage.NewDataContext(d, 0) 1399 store, err := datastore.GetOrderedKeyValueDB(d) 1400 if err != nil { 1401 dvid.Errorf("Data type labelvol had error initializing store: %v\n", err) 1402 } else { 1403 store.Put(ctx, maxRepoLabelTKey, buf) 1404 } 1405 } 1406 } 1407 if err := batch.Commit(); err != nil { 1408 dvid.Errorf("batch put: %v\n", err) 1409 return 1410 } 1411 } 1412 1413 // NewLabel returns a new label for the given version. 1414 func (d *Data) NewLabel(v dvid.VersionID) (uint64, error) { 1415 d.mlMu.Lock() 1416 defer d.mlMu.Unlock() 1417 1418 // Make sure we aren't trying to increment a label on a locked node. 1419 locked, err := datastore.LockedVersion(v) 1420 if err != nil { 1421 return 0, err 1422 } 1423 if locked { 1424 return 0, fmt.Errorf("can't ask for new label in a locked version id %d", v) 1425 } 1426 1427 // Increment and store. 1428 d.MaxRepoLabel++ 1429 d.MaxLabel[v] = d.MaxRepoLabel 1430 1431 store, err := datastore.GetOrderedKeyValueDB(d) 1432 if err != nil { 1433 return 0, err 1434 } 1435 buf := make([]byte, 8) 1436 binary.LittleEndian.PutUint64(buf, d.MaxRepoLabel) 1437 ctx := datastore.NewVersionedCtx(d, v) 1438 if err := store.Put(ctx, maxLabelTKey, buf); err != nil { 1439 return 0, err 1440 } 1441 1442 ctx2 := storage.NewDataContext(d, 0) 1443 if err := store.Put(ctx2, maxRepoLabelTKey, buf); err != nil { 1444 return 0, err 1445 } 1446 1447 return d.MaxRepoLabel, nil 1448 } 1449 1450 // Returns RLEs for a given label where the key of the returned map is the block index 1451 // in string format. 1452 func (d *Data) GetLabelRLEs(v dvid.VersionID, label uint64) (dvid.BlockRLEs, error) { 1453 store, err := datastore.GetOrderedKeyValueDB(d) 1454 if err != nil { 1455 return nil, fmt.Errorf("Data type labelvol had error initializing store: %v\n", err) 1456 } 1457 1458 // Get the start/end indices for this body's KeyLabelSpatialMap (b + s) keys. 1459 begIndex := NewTKey(label, dvid.MinIndexZYX.ToIZYXString()) 1460 endIndex := NewTKey(label, dvid.MaxIndexZYX.ToIZYXString()) 1461 1462 // Process all the b+s keys and their values, which contain RLE runs for that label. 1463 labelRLEs := dvid.BlockRLEs{} 1464 var f storage.ChunkFunc = func(chunk *storage.Chunk) error { 1465 // Get the block index where the fromLabel is present 1466 _, blockStr, err := DecodeTKey(chunk.K) 1467 if err != nil { 1468 return fmt.Errorf("Can't recover block index with chunk key %v: %v\n", chunk.K, err) 1469 } 1470 1471 var blockRLEs dvid.RLEs 1472 if err := blockRLEs.UnmarshalBinary(chunk.V); err != nil { 1473 return fmt.Errorf("Unable to unmarshal RLE for label in block %v", chunk.K) 1474 } 1475 labelRLEs[blockStr] = blockRLEs 1476 return nil 1477 } 1478 ctx := datastore.NewVersionedCtx(d, v) 1479 err = store.ProcessRange(ctx, begIndex, endIndex, &storage.ChunkOp{}, f) 1480 if err != nil { 1481 return nil, err 1482 } 1483 return labelRLEs, nil 1484 } 1485 1486 type sparseOp struct { 1487 versionID dvid.VersionID 1488 encoding []byte 1489 numBlocks uint32 1490 numRuns uint32 1491 //numVoxels int32 1492 } 1493 1494 // Alter serialized RLEs by the bounds. 1495 func boundRLEs(b []byte, bounds *dvid.OptionalBounds) ([]byte, error) { 1496 var oldRLEs dvid.RLEs 1497 err := oldRLEs.UnmarshalBinary(b) 1498 if err != nil { 1499 return nil, err 1500 } 1501 newRLEs := oldRLEs.FitToBounds(bounds) 1502 return newRLEs.MarshalBinary() 1503 } 1504 1505 // FoundSparseVol returns true if a sparse volume is found for the given label 1506 // within the given bounds. 1507 func (d *Data) FoundSparseVol(ctx *datastore.VersionedCtx, label uint64, bounds dvid.Bounds) (bool, error) { 1508 store, err := datastore.GetOrderedKeyValueDB(d) 1509 if err != nil { 1510 return false, fmt.Errorf("Data type labelvol had error initializing store: %v\n", err) 1511 } 1512 1513 // Get the start/end indices for this body's KeyLabelSpatialMap (b + s) keys. 1514 minZYX := dvid.MinIndexZYX 1515 maxZYX := dvid.MaxIndexZYX 1516 if minZ, ok := bounds.Block.MinZ(); ok { 1517 minZYX[2] = minZ 1518 } 1519 if maxZ, ok := bounds.Block.MaxZ(); ok { 1520 maxZYX[2] = maxZ 1521 } 1522 1523 // Scan through all keys coming from label blocks to see if we have any hits. 1524 var constituents labels.Set 1525 iv := d.getMergeIV(ctx.VersionID()) 1526 mapping := labels.LabelMap(iv) 1527 1528 if mapping == nil { 1529 constituents = labels.Set{label: struct{}{}} 1530 } else { 1531 // Check if this label has been merged. 1532 if _, found := mapping.Get(label); found { 1533 return false, nil 1534 } 1535 1536 // If not, see if labels have been merged into it. 1537 constituents = mapping.ConstituentLabels(label) 1538 } 1539 1540 shortCircuitErr := fmt.Errorf("Found data, aborting.") 1541 var f storage.ChunkFunc = func(chunk *storage.Chunk) error { 1542 // Make sure this block is within the optinonal bounding. 1543 if bounds.Block.BoundedX() || bounds.Block.BoundedY() { 1544 _, blockStr, err := DecodeTKey(chunk.K) 1545 if err != nil { 1546 return fmt.Errorf("Error decoding sparse volume key (%v): %v\n", chunk.K, err) 1547 } 1548 indexZYX, err := blockStr.IndexZYX() 1549 if err != nil { 1550 return fmt.Errorf("Error decoding block coordinate (%v) for sparse volume: %v\n", blockStr, err) 1551 } 1552 blockX, blockY, _ := indexZYX.Unpack() 1553 if bounds.Block.OutsideX(blockX) || bounds.Block.OutsideY(blockY) { 1554 return nil 1555 } 1556 } 1557 return shortCircuitErr 1558 } 1559 1560 // Iterate through each constituent label. 1561 for constituent := range constituents { 1562 begTKey := NewTKey(constituent, minZYX.ToIZYXString()) 1563 endTKey := NewTKey(constituent, maxZYX.ToIZYXString()) 1564 err := store.ProcessRange(ctx, begTKey, endTKey, &storage.ChunkOp{}, f) 1565 if err == shortCircuitErr { 1566 return true, nil 1567 } 1568 if err != nil { 1569 return false, err 1570 } 1571 } 1572 dvid.Debugf("Found no sparse volume with label %d, composed of labels %s\n", label, constituents) 1573 return false, nil 1574 } 1575 1576 // GetSparseVol returns an encoded sparse volume given a label. The encoding has the 1577 // following format where integers are little endian: 1578 // byte Payload descriptor: 1579 // Bit 0 (LSB) - 8-bit grayscale 1580 // Bit 1 - 16-bit grayscale 1581 // Bit 2 - 16-bit normal 1582 // ... 1583 // uint8 Number of dimensions 1584 // uint8 Dimension of run (typically 0 = X) 1585 // byte Reserved (to be used later) 1586 // uint32 # Voxels 1587 // uint32 # Spans 1588 // Repeating unit of: 1589 // int32 Coordinate of run start (dimension 0) 1590 // int32 Coordinate of run start (dimension 1) 1591 // int32 Coordinate of run start (dimension 2) 1592 // int32 Length of run 1593 // bytes Optional payload dependent on first byte descriptor 1594 // 1595 func (d *Data) GetSparseVol(ctx *datastore.VersionedCtx, label uint64, bounds dvid.Bounds) ([]byte, error) { 1596 iv := d.getMergeIV(ctx.VersionID()) 1597 mapping := labels.LabelMap(iv) 1598 if mapping != nil { 1599 // Check if this label has been merged. 1600 if mapped, found := mapping.FinalLabel(label); found { 1601 dvid.Debugf("Label %d has already been merged into label %d. Skipping sparse vol retrieval.\n", label, mapped) 1602 return nil, nil 1603 } 1604 } 1605 1606 store, err := datastore.GetOrderedKeyValueDB(d) 1607 if err != nil { 1608 return nil, fmt.Errorf("Data type labelvol had error initializing store: %v\n", err) 1609 } 1610 1611 // Create the sparse volume header 1612 buf := new(bytes.Buffer) 1613 buf.WriteByte(dvid.EncodingBinary) 1614 binary.Write(buf, binary.LittleEndian, uint8(3)) // # of dimensions 1615 binary.Write(buf, binary.LittleEndian, byte(0)) // dimension of run (X = 0) 1616 buf.WriteByte(byte(0)) // reserved for later 1617 binary.Write(buf, binary.LittleEndian, uint32(0)) // Placeholder for # voxels 1618 binary.Write(buf, binary.LittleEndian, uint32(0)) // Placeholder for # spans 1619 1620 // Get the start/end indices for this body's KeyLabelSpatialMap (b + s) keys. 1621 minZYX := dvid.MinIndexZYX 1622 maxZYX := dvid.MaxIndexZYX 1623 if minZ, ok := bounds.Block.MinZ(); ok { 1624 minZYX[2] = minZ 1625 } 1626 if maxZ, ok := bounds.Block.MaxZ(); ok { 1627 maxZYX[2] = maxZ 1628 } 1629 // Process all the b+s keys and their values, which contain RLE runs for that label. 1630 // TODO -- Make processing asynchronous so can overlap with range disk read now that 1631 // there could be more processing due to bounding calcs. 1632 var numRuns, numBlocks uint32 1633 encoding := buf.Bytes() 1634 1635 var f storage.ChunkFunc = func(chunk *storage.Chunk) error { 1636 numBlocks++ 1637 1638 // Make sure this block is within the optinonal bounding. 1639 if bounds.Block.BoundedX() || bounds.Block.BoundedY() { 1640 _, blockStr, err := DecodeTKey(chunk.K) 1641 if err != nil { 1642 return fmt.Errorf("Error decoding sparse volume key (%v): %v\n", chunk.K, err) 1643 } 1644 blockX, blockY, _, err := blockStr.Unpack() 1645 if err != nil { 1646 return fmt.Errorf("Error decoding block %v: %v\n", blockStr, err) 1647 } 1648 if bounds.Block.OutsideX(blockX) || bounds.Block.OutsideY(blockY) { 1649 return nil 1650 } 1651 } 1652 1653 // Adjust RLEs within block if we are bounded. 1654 var rles []byte 1655 var err error 1656 if bounds.Exact && bounds.Voxel.IsSet() { 1657 rles, err = boundRLEs(chunk.V, bounds.Voxel) 1658 if err != nil { 1659 return fmt.Errorf("Error in adjusting RLEs to bounds: %v\n", err) 1660 } 1661 } else { 1662 rles = chunk.V 1663 } 1664 1665 numRuns += uint32(len(rles) / 16) 1666 if int64(len(encoding))+int64(len(rles)) > server.MaxDataRequest { 1667 return fmt.Errorf("Sparse volume read aborted because length exceeds %d bytes", server.MaxDataRequest) 1668 } 1669 encoding = append(encoding, rles...) 1670 return nil 1671 } 1672 1673 // Iterate through each constituent label. 1674 if mapping == nil { 1675 begTKey := NewTKey(label, minZYX.ToIZYXString()) 1676 endTKey := NewTKey(label, maxZYX.ToIZYXString()) 1677 if err := store.ProcessRange(ctx, begTKey, endTKey, &storage.ChunkOp{}, f); err != nil { 1678 return nil, err 1679 } 1680 } else { 1681 // Find all labels merged into this label. 1682 constituents := mapping.ConstituentLabels(label) 1683 for constituent := range constituents { 1684 begTKey := NewTKey(constituent, minZYX.ToIZYXString()) 1685 endTKey := NewTKey(constituent, maxZYX.ToIZYXString()) 1686 if err := store.ProcessRange(ctx, begTKey, endTKey, &storage.ChunkOp{}, f); err != nil { 1687 return nil, err 1688 } 1689 } 1690 if len(constituents) > 1 { 1691 dvid.Debugf("Returning sparse volume for label %d, composed of labels %s\n", label, constituents) 1692 } 1693 } 1694 1695 binary.LittleEndian.PutUint32(encoding[8:12], numRuns) 1696 1697 dvid.Debugf("[%s] label %d: found %d blocks, %d runs\n", ctx, label, numBlocks, numRuns) 1698 if numBlocks == 0 { 1699 return nil, nil 1700 } 1701 return encoding, nil 1702 } 1703 1704 // GetSparseCoarseVol returns an encoded sparse volume given a label. The encoding has the 1705 // following format where integers are little endian: 1706 // byte Set to 0 1707 // uint8 Number of dimensions 1708 // uint8 Dimension of run (typically 0 = X) 1709 // byte Reserved (to be used later) 1710 // uint32 # Blocks [TODO. 0 for now] 1711 // uint32 # Spans 1712 // Repeating unit of: 1713 // int32 Block coordinate of run start (dimension 0) 1714 // int32 Block coordinate of run start (dimension 1) 1715 // int32 Block coordinate of run start (dimension 2) 1716 // int32 Length of run 1717 // 1718 func (d *Data) GetSparseCoarseVol(ctx *datastore.VersionedCtx, label uint64) ([]byte, error) { 1719 iv := d.getMergeIV(ctx.VersionID()) 1720 mapping := labels.LabelMap(iv) 1721 if mapping != nil { 1722 // Check if this label has been merged. 1723 if _, found := mapping.FinalLabel(label); found { 1724 return nil, nil 1725 } 1726 } 1727 1728 // Create the sparse volume header 1729 buf := new(bytes.Buffer) 1730 buf.WriteByte(dvid.EncodingBinary) 1731 binary.Write(buf, binary.LittleEndian, uint8(3)) // # of dimensions 1732 binary.Write(buf, binary.LittleEndian, byte(0)) // dimension of run (X = 0) 1733 buf.WriteByte(byte(0)) // reserved for later 1734 binary.Write(buf, binary.LittleEndian, uint32(0)) // Placeholder for # blocks 1735 encoding := buf.Bytes() 1736 1737 // Iterate through each constituent label, appending to the spans. 1738 var numBlocks uint32 1739 var spans dvid.Spans 1740 1741 if mapping == nil { 1742 var err error 1743 numBlocks, spans, err = getSparseVolBlocks(ctx, label) 1744 if err != nil { 1745 return nil, err 1746 } 1747 } else { 1748 // Find all labels merged into this label. 1749 constituents := mapping.ConstituentLabels(label) 1750 for constituent := range constituents { 1751 curBlocks, curSpans, err := getSparseVolBlocks(ctx, constituent) 1752 if err != nil { 1753 return nil, err 1754 } 1755 numBlocks += curBlocks 1756 spans = append(spans, curSpans...) 1757 } 1758 dvid.Debugf("Returning coarse sparse volume for label %d, composed of labels %s\n", label, constituents) 1759 } 1760 1761 if numBlocks == 0 { 1762 return nil, nil 1763 } 1764 1765 spansBytes, err := spans.MarshalBinary() 1766 if err != nil { 1767 return nil, err 1768 } 1769 encoding = append(encoding, spansBytes...) 1770 dvid.Debugf("[%s] coarse subvol for label %d: found %d blocks\n", ctx, label, numBlocks) 1771 return encoding, nil 1772 } 1773 1774 func getSparseVolBlocks(ctx *datastore.VersionedCtx, label uint64) (numBlocks uint32, spans dvid.Spans, err error) { 1775 store, err := ctx.GetOrderedKeyValueDB() 1776 if err != nil { 1777 return 0, nil, fmt.Errorf("Data type labelvol had error initializing store: %v\n", err) 1778 } 1779 1780 var span *dvid.Span 1781 1782 // Get the start/end indices for this body's KeyLabelSpatialMap (b + s) keys. 1783 begTKey := NewTKey(label, dvid.MinIndexZYX.ToIZYXString()) 1784 endTKey := NewTKey(label, dvid.MaxIndexZYX.ToIZYXString()) 1785 1786 // Process all the b+s keys and their values, which contain RLE runs for that label. 1787 keys, err := store.KeysInRange(ctx, begTKey, endTKey) 1788 if err != nil { 1789 return 0, nil, fmt.Errorf("Cannot get keys for coarse sparse volume: %v", err) 1790 } 1791 1792 for _, tk := range keys { 1793 numBlocks++ 1794 _, blockStr, err := DecodeTKey(tk) 1795 if err != nil { 1796 return 0, nil, fmt.Errorf("Error retrieving RLE runs for label %d: %v", label, err) 1797 } 1798 indexZYX, err := blockStr.IndexZYX() 1799 if err != nil { 1800 return 0, nil, fmt.Errorf("Error decoding block coordinate (%v) for sparse volume: %v\n", blockStr, err) 1801 } 1802 x, y, z := indexZYX.Unpack() 1803 if span == nil { 1804 span = &dvid.Span{z, y, x, x} 1805 } else if !span.Extends(x, y, z) { 1806 spans = append(spans, *span) 1807 span = &dvid.Span{z, y, x, x} 1808 } 1809 } 1810 if err != nil { 1811 return 0, nil, err 1812 } 1813 if span != nil { 1814 spans = append(spans, *span) 1815 } 1816 return 1817 } 1818 1819 func (d *Data) DeleteArea(ctx *datastore.VersionedCtx, label uint64, subvol *dvid.Subvolume) error { 1820 store, err := datastore.GetOrderedKeyValueDB(d) 1821 if err != nil { 1822 return fmt.Errorf("unable to get backing store for data %q: %v\n", d.DataName(), err) 1823 } 1824 batcher, ok := store.(storage.KeyValueBatcher) 1825 if !ok { 1826 return fmt.Errorf("Data type labelvol requires batch-enabled store, which %q is not\n", store) 1827 } 1828 1829 begVoxel, ok := subvol.StartPoint().(dvid.Chunkable) 1830 if !ok { 1831 return fmt.Errorf("delete area StartPoint() cannot handle Chunkable points.") 1832 } 1833 endVoxel, ok := subvol.EndPoint().(dvid.Chunkable) 1834 if !ok { 1835 return fmt.Errorf("delete area EndPoint() cannot handle Chunkable points.") 1836 } 1837 begChunk := begVoxel.Chunk(d.BlockSize).(dvid.ChunkPoint3d) 1838 endChunk := endVoxel.Chunk(d.BlockSize).(dvid.ChunkPoint3d) 1839 it := dvid.NewIndexZYXIterator(begChunk, endChunk) 1840 1841 batch := batcher.NewBatch(ctx) 1842 for { 1843 if !it.Valid() { 1844 dvid.Infof("Breaking\n") 1845 break 1846 } 1847 1848 indexBeg, indexEnd, err := it.IndexSpan() 1849 if err != nil { 1850 return err 1851 } 1852 begBlock := indexBeg.(*dvid.IndexZYX).ToIZYXString() 1853 endBlock := indexEnd.(*dvid.IndexZYX).ToIZYXString() 1854 dvid.Infof("Deleting from %s -> %s\n", begBlock, endBlock) 1855 1856 begTKey := NewTKey(label, begBlock) 1857 endTKey := NewTKey(label, endBlock) 1858 1859 err = store.ProcessRange(ctx, begTKey, endTKey, &storage.ChunkOp{}, func(c *storage.Chunk) error { 1860 if c == nil { 1861 return fmt.Errorf("received nil chunk in delete area for data %s", d.DataName()) 1862 } 1863 label, block, err := DecodeTKey(c.K) 1864 if err != nil { 1865 return err 1866 } 1867 dvid.Infof("Deleting label %d from %s\n", label, block) 1868 batch.Delete(c.K) 1869 return nil 1870 }) 1871 if err != nil { 1872 return err 1873 } 1874 it.NextSpan() 1875 } 1876 1877 if err := batch.Commit(); err != nil { 1878 return fmt.Errorf("Batch commit during delete area of %s label %d: %v\n", d.DataName(), label, err) 1879 } 1880 return nil 1881 } 1882 1883 // deletes prior sparsevol for label and recreates based on associated labelblk with the POSTed 1884 // blocks. 1885 func (d *Data) resyncLabel(ctx *datastore.VersionedCtx, relabel uint64, r io.ReadCloser) error { 1886 timedLog := dvid.NewTimeLog() 1887 store, err := datastore.GetOrderedKeyValueDB(d) 1888 if err != nil { 1889 return fmt.Errorf("data %q had error initializing store: %v\n", d.DataName(), err) 1890 } 1891 1892 blkdata, err := d.GetSyncedLabelblk() 1893 if err != nil { 1894 return err 1895 } 1896 1897 // Read the sparse volume from reader. 1898 var resyncROI dvid.RLEs 1899 resyncROI, err = dvid.ReadRLEs(r) 1900 if err != nil { 1901 return err 1902 } 1903 resyncNumBlocks, _ := resyncROI.Stats() 1904 1905 var numBlocks int 1906 for _, rle := range resyncROI { 1907 startPt := dvid.ChunkPoint3d(rle.StartPt()) 1908 var endPt dvid.ChunkPoint3d 1909 endPt[0] = startPt[0] + rle.Length() - 1 1910 endPt[1] = startPt[1] 1911 endPt[2] = startPt[2] 1912 1913 begTKey := NewTKey(relabel, startPt.ToIZYXString()) 1914 endTKey := NewTKey(relabel, endPt.ToIZYXString()) 1915 if err := store.DeleteRange(ctx, begTKey, endTKey); err != nil { 1916 return err 1917 } 1918 1919 blocks, err := blkdata.GetBlocks(ctx.VersionID(), startPt, int(rle.Length())) 1920 if err != nil { 1921 return err 1922 } 1923 numBlocks += len(blocks) 1924 1925 for _, labelBlock := range blocks { 1926 tk := NewTKey(relabel, labelBlock.Pos.ToIZYXString()) 1927 rleBytes, err := d.calcLabelRLEs(relabel, labelBlock) 1928 if err != nil { 1929 return err 1930 } 1931 if rleBytes == nil { 1932 continue 1933 } 1934 if err := store.Put(ctx, tk, rleBytes); err != nil { 1935 return err 1936 } 1937 } 1938 } 1939 timedLog.Infof("Finished resync label %d, %d of %d blocks within data %q\n", relabel, numBlocks, resyncNumBlocks, blkdata.DataName()) 1940 return nil 1941 } 1942 1943 func (d *Data) calcLabelRLEs(relabel uint64, block labelblk.Block) ([]byte, error) { 1944 blockBytes := len(block.Data) 1945 if blockBytes != int(d.BlockSize.Prod())*8 { 1946 return nil, fmt.Errorf("Deserialized label block %d bytes, not uint64 size times %d block elements\n", 1947 blockBytes, d.BlockSize.Prod()) 1948 } 1949 var labelRLEs dvid.RLEs 1950 firstPt := block.Pos.MinPoint(d.BlockSize) 1951 lastPt := block.Pos.MaxPoint(d.BlockSize) 1952 1953 var curStart dvid.Point3d 1954 var voxelLabel, curLabel uint64 1955 var z, y, x, curRun int32 1956 start := 0 1957 for z = firstPt.Value(2); z <= lastPt.Value(2); z++ { 1958 for y = firstPt.Value(1); y <= lastPt.Value(1); y++ { 1959 for x = firstPt.Value(0); x <= lastPt.Value(0); x++ { 1960 voxelLabel = binary.LittleEndian.Uint64(block.Data[start : start+8]) 1961 start += 8 1962 1963 // If we hit background or have switched label, save old run and start new one. 1964 if voxelLabel == 0 || voxelLabel != curLabel { 1965 // Save old run 1966 if curLabel == relabel && curRun > 0 { 1967 labelRLEs = append(labelRLEs, dvid.NewRLE(curStart, curRun)) 1968 } 1969 // Start new one if not zero label. 1970 if voxelLabel == relabel { 1971 curStart = dvid.Point3d{x, y, z} 1972 curRun = 1 1973 } else { 1974 curRun = 0 1975 } 1976 curLabel = voxelLabel 1977 } else if voxelLabel == relabel { 1978 curRun++ 1979 } 1980 } 1981 // Force break of any runs when we finish x scan. 1982 if curRun > 0 { 1983 labelRLEs = append(labelRLEs, dvid.NewRLE(curStart, curRun)) 1984 curLabel = 0 1985 curRun = 0 1986 } 1987 } 1988 } 1989 1990 return labelRLEs.MarshalBinary() 1991 } 1992 1993 // PutSparseVol stores an encoded sparse volume that stays within a given forward label. 1994 // Note that this encoded sparse volume is added to any existing sparse volume for 1995 // a body rather than replace it. To carve out a part of an existing sparse volume, 1996 // use the SplitLabels(). 1997 // 1998 // This function handles modification/deletion of all denormalized data touched by this 1999 // sparse label volume. 2000 // 2001 // EVENTS 2002 // 2003 // 2004 /* 2005 func (d *Data) PutSparseVol(v dvid.VersionID, label uint64, r io.Reader) error { 2006 store, err := storage.MutableStore() 2007 if err != nil { 2008 return fmt.Errorf("Data type labelvol had error initializing store: %v\n", err) 2009 } 2010 batcher, ok := store.(storage.KeyValueBatcher) 2011 if !ok { 2012 return fmt.Errorf("Data type labelvol requires batch-enabled store, which %q is not\n", store) 2013 } 2014 2015 // Mark the label as dirty until done. 2016 iv := dvid.InstanceVersion{d.DataName(), v} 2017 labels.DirtyCache.Incr(iv, label) 2018 defer labels.DirtyCache.Decr(iv, label) 2019 2020 // TODO -- Signal that we are starting a label modification. 2021 2022 // Read the sparse volume from reader. 2023 header := make([]byte, 8) 2024 if _, err = io.ReadFull(r, header); err != nil { 2025 return err 2026 } 2027 if header[0] != dvid.EncodingBinary { 2028 return fmt.Errorf("sparse vol for split has unknown encoding format: %v", header[0]) 2029 } 2030 var numSpans uint32 2031 if err = binary.Read(r, binary.LittleEndian, &numSpans); err != nil { 2032 return err 2033 } 2034 var mods dvid.RLEs 2035 if err = mods.UnmarshalBinaryReader(r, numSpans); err != nil { 2036 return err 2037 } 2038 2039 // Partition the mods spans into blocks. 2040 var modmap dvid.BlockRLEs 2041 modmap, err = mods.Partition(d.BlockSize) 2042 if err != nil { 2043 return err 2044 } 2045 2046 // Publish sparsevol mod event 2047 evt := datastore.SyncEvent{d.DataUUID(), labels.SparsevolModEvent} 2048 msg := datastore.SyncMessage{v, labels.DeltaSparsevol{label, modmap}} 2049 if err := datastore.NotifySubscribers(evt, msg); err != nil { 2050 return err 2051 } 2052 2053 // Get a sorted list of blocks that cover mods. 2054 modblks := modmap.SortedKeys() 2055 2056 ctx := datastore.NewVersionedCtx(d, v) 2057 batch := batcher.NewBatch(ctx) 2058 var voxelsAdded int64 2059 2060 for _, modblk := range modblks { 2061 2062 // Get original block 2063 tk := NewTKey(label, modblk) 2064 val, err := store.Get(ctx, tk) 2065 if err != nil { 2066 return err 2067 } 2068 2069 // If there's no original block, just write the mod block. 2070 if val == nil || len(val) == 0 { 2071 numVoxels, _ := modmap[modblk].Stats() 2072 voxelsAdded += int64(numVoxels) 2073 rleBytes, err := modmap[modblk].MarshalBinary() 2074 if err != nil { 2075 return fmt.Errorf("can't serialize modified RLEs for %d: %v\n", label, err) 2076 } 2077 batch.Put(tk, rleBytes) 2078 continue 2079 } 2080 2081 // if there's an original, integrate and write merged RLE. 2082 var rles dvid.RLEs 2083 if err := rles.UnmarshalBinary(val); err != nil { 2084 return fmt.Errorf("Unable to unmarshal RLE for original labels in block %s", modblk.Print()) 2085 } 2086 voxelsAdded += rles.Add(modmap[modblk]) 2087 rleBytes, err := rles.MarshalBinary() 2088 if err != nil { 2089 return fmt.Errorf("can't serialize modified RLEs of %d: %v\n", label, err) 2090 } 2091 batch.Put(tk, rleBytes) 2092 } 2093 2094 if err := batch.Commit(); err != nil { 2095 return fmt.Errorf("Batch commit during mod of %s label %d: %v\n", d.DataName(), label, err) 2096 } 2097 2098 // Publish change in label sizes. 2099 delta := labels.DeltaModSize{ 2100 Label: label, 2101 SizeChange: voxelsAdded, 2102 } 2103 evt = datastore.SyncEvent{d.DataUUID(), labels.ChangeSizeEvent} 2104 msg = datastore.SyncMessage{v, delta} 2105 if err := datastore.NotifySubscribers(evt, msg); err != nil { 2106 return err 2107 } 2108 2109 // TODO -- Publish label mod end 2110 2111 return nil 2112 } 2113 */ 2114 2115 type DumpFiles struct { 2116 files map[dvid.IZYXString]*os.File 2117 dir string 2118 2119 // debug vars 2120 tlog dvid.TimeLog 2121 numKV uint64 2122 lastTime time.Time 2123 lastBytes uint64 2124 totalBytes uint64 2125 } 2126 2127 func (df DumpFiles) close() { 2128 for _, f := range df.files { 2129 f.Close() 2130 } 2131 } 2132 2133 func (df *DumpFiles) process(kv *storage.TKeyValue, blockSize dvid.Point3d) error { 2134 if kv.K == nil { 2135 return fmt.Errorf("Had nil TKey in process() for labelvol dump!") 2136 } 2137 label, block, err := DecodeTKey(kv.K) 2138 if err != nil { 2139 return fmt.Errorf("Couldn't decode tkey from key %x: %v\n", kv.K, err) 2140 } 2141 2142 chunkPt, err := block.ToChunkPoint3d() 2143 if err != nil { 2144 return err 2145 } 2146 subvolPt := chunkPt 2147 subvolPt[0] /= (512 / blockSize[0]) 2148 subvolPt[1] /= (512 / blockSize[1]) 2149 subvolPt[2] /= (512 / blockSize[2]) 2150 subvolZYX := subvolPt.ToIZYXString() 2151 2152 f, found := df.files[subvolZYX] 2153 if !found { 2154 fname := filepath.Join(df.dir, fmt.Sprintf("subvols-%03d_%03d_%03d.dat", subvolPt[0], subvolPt[1], subvolPt[2])) 2155 f, err = os.OpenFile(fname, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0664) 2156 if err != nil { 2157 return fmt.Errorf("problem opening file for subvol %s: %v\n", subvolPt, err) 2158 } 2159 df.files[subvolZYX] = f 2160 } 2161 2162 var blockRLEs dvid.RLEs 2163 if err := blockRLEs.UnmarshalBinary(kv.V); err != nil { 2164 return fmt.Errorf("Unable to unmarshal RLEs for label %d, block %s: %v\n", label, subvolPt, err) 2165 } 2166 numSpans := int32(len(blockRLEs)) 2167 spanDataBytes := int32(numSpans * 16) 2168 2169 binary.Write(f, binary.LittleEndian, label) 2170 binary.Write(f, binary.LittleEndian, numSpans) 2171 n, err := f.Write(kv.V) 2172 if err != nil { 2173 return fmt.Errorf("Only wrote %d bytes out of %d for RLEs in block %s: %v\n", n, len(kv.V), subvolPt, err) 2174 } 2175 2176 curBytes := uint64(spanDataBytes + 12) 2177 df.lastBytes += curBytes 2178 df.totalBytes += curBytes 2179 df.numKV++ 2180 if elapsed := time.Since(df.lastTime); elapsed > time.Minute { 2181 mb := float64(df.lastBytes) / 1000000 2182 sec := elapsed.Seconds() 2183 throughput := mb / sec 2184 dvid.Debugf("Dump throughput: %5.2f MB/s (%s in %4.1f seconds). Total %s\n", throughput, humanize.Bytes(df.lastBytes), sec, humanize.Bytes(df.totalBytes)) 2185 2186 df.lastTime = time.Now() 2187 df.lastBytes = 0 2188 } 2189 return nil 2190 } 2191 2192 func (d *Data) DumpSubvols(uuid dvid.UUID, v dvid.VersionID, dirStr string) { 2193 df := DumpFiles{ 2194 files: make(map[dvid.IZYXString]*os.File), 2195 dir: dirStr, 2196 2197 tlog: dvid.NewTimeLog(), 2198 lastTime: time.Now(), 2199 } 2200 2201 store, err := datastore.GetOrderedKeyValueDB(d) 2202 if err != nil { 2203 dvid.Criticalf("unable to get backing store for data %q: %v\n", d.DataName(), err) 2204 return 2205 } 2206 ctx := datastore.NewVersionedCtx(d, v) 2207 2208 dataCh := make(chan *storage.TKeyValue, 1000) 2209 go func() { 2210 for { 2211 kv := <-dataCh 2212 if kv == nil { 2213 dvid.Infof("Finished subvol dump on instance %q.\n", d.DataName()) 2214 df.close() 2215 break 2216 } 2217 if err := df.process(kv, d.BlockSize); err != nil { 2218 dvid.Errorf("Subvol dump error: %v\n", err) 2219 df.close() 2220 break 2221 } 2222 } 2223 df.tlog.Infof("Finished %q subvol dump of %d kv pairs, %s", d.DataName(), df.numKV, humanize.Bytes(df.totalBytes)) 2224 }() 2225 2226 minTKey := storage.MinTKey(keyLabelBlockRLE) 2227 maxTKey := storage.MaxTKey(keyLabelBlockRLE) 2228 err = store.ProcessRange(ctx, minTKey, maxTKey, &storage.ChunkOp{}, func(c *storage.Chunk) error { 2229 if c == nil { 2230 return fmt.Errorf("received nil chunk in dump subvol for data %s", d.DataName()) 2231 } 2232 dataCh <- c.TKeyValue 2233 return nil 2234 }) 2235 dataCh <- nil 2236 if err != nil { 2237 dvid.Criticalf("error in dump subvol for data %q: %v", d.DataName(), err) 2238 } 2239 }