github.com/janelia-flyem/dvid@v1.0.0/datatype/labelmap/labelmap.go (about) 1 /* 2 Package labelmap handles both volumes of label data as well as indexing to 3 quickly find and generate sparse volumes of any particular label. It differs 4 from labelmap in using in-memory label maps to greatly decrease changes 5 to underlying label data key-value pairs, instead handling all merges through 6 changes in label maps. 7 */ 8 package labelmap 9 10 import ( 11 "bytes" 12 "crypto/md5" 13 "encoding/binary" 14 "encoding/gob" 15 "encoding/json" 16 "fmt" 17 "image" 18 "io/ioutil" 19 "net/http" 20 "net/url" 21 "os" 22 "strconv" 23 "strings" 24 "sync" 25 26 "github.com/janelia-flyem/dvid/datastore" 27 "github.com/janelia-flyem/dvid/datatype/common/labels" 28 "github.com/janelia-flyem/dvid/datatype/imageblk" 29 "github.com/janelia-flyem/dvid/dvid" 30 "github.com/janelia-flyem/dvid/server" 31 "github.com/janelia-flyem/dvid/storage" 32 ) 33 34 const ( 35 Version = "0.1" 36 RepoURL = "github.com/janelia-flyem/dvid/datatype/labelmap" 37 TypeName = "labelmap" 38 ) 39 40 const helpMessage = ` 41 API for label map data type (github.com/janelia-flyem/dvid/datatype/labelmap) 42 ============================================================================= 43 44 Note: UUIDs referenced below are strings that may either be a unique prefix of a 45 hexadecimal UUID string (e.g., 3FA22) or a branch leaf specification that adds 46 a colon (":") followed by the case-dependent branch name. In the case of a 47 branch leaf specification, the unique UUID prefix just identifies the repo of 48 the branch, and the UUID referenced is really the leaf of the branch name. 49 For example, if we have a DAG with root A -> B -> C where C is the current 50 HEAD or leaf of the "master" (default) branch, then asking for "B:master" is 51 the same as asking for "C". If we add another version so A -> B -> C -> D, then 52 references to "B:master" now return the data from "D". 53 54 ---- 55 56 Denormalizations like sparse volumes are *not* performed for the "0" label, which is 57 considered a special label useful for designating background. This allows users to define 58 sparse labeled structures in a large volume without requiring processing of entire volume. 59 60 61 Command-line: 62 63 $ dvid repo <UUID> new labelmap <data name> <settings...> 64 65 Adds newly named data of the 'type name' to repo with specified UUID. 66 67 Example (note anisotropic resolution specified instead of default 8 nm isotropic): 68 69 $ dvid repo 3f8c new labelmap superpixels VoxelSize=3.2,3.2,40.0 70 71 Arguments: 72 73 UUID Hexadecimal string with enough characters to uniquely identify a version node. 74 data name Name of data to create, e.g., "superpixels" 75 settings Configuration settings in "key=value" format separated by spaces. 76 77 Configuration Settings (case-insensitive keys) 78 79 BlockSize Size in voxels (default: %s) Should be multiples of 16. 80 VoxelSize Resolution of voxels (default: 8.0, 8.0, 8.0) 81 VoxelUnits Resolution units (default: "nanometers") 82 IndexedLabels "false" if no sparse volume support is required (default "true") 83 MaxDownresLevel The maximum down-res level supported. Each down-res is factor of 2. 84 85 $ dvid node <UUID> <data name> load <offset> <image glob> <settings...> 86 87 Initializes version node to a set of XY label images described by glob of filenames. 88 The DVID server must have access to the named files. Currently, XY images are required. 89 90 Example: 91 92 $ dvid node 3f8c superpixels load 0,0,100 "data/*.png" proc=noindex 93 94 Arguments: 95 96 UUID Hexadecimal string with enough characters to uniquely identify a version node. 97 data name Name of data to add. 98 offset 3d coordinate in the format "x,y,z". Gives coordinate of top upper left voxel. 99 image glob Filenames of label images, preferably in quotes, e.g., "foo-xy-*.png" 100 101 $ dvid node <UUID> <data name> composite <uint8 data name> <new rgba8 data name> 102 103 Creates a RGBA8 image where the RGB is a hash of the labels and the A is the 104 grayscale intensity. 105 106 Example: 107 108 $ dvid node 3f8c bodies composite grayscale bodyview 109 110 Arguments: 111 112 UUID Hexadecimal string with enough characters to uniquely identify a version node. 113 data name Name of data to add. 114 115 $ dvid node <UUID> <data name> dump <dump type> <file path> 116 117 Dumps the internal state of the specified version of labelmap data into a space-delimted file. 118 The format of the space-delimted files are as follows depending on the <dump type>: 119 120 "svcount": Supervoxel counts in a space-delimted file where each block in a supervoxel has a row: 121 <supervoxel id> <block z> <block y> <block x> <# voxels> 122 123 "mappings": Supervoxel to agglomerated label mappings in a space-delimted file where each supervoxel has a row: 124 <supervoxel id> <label> 125 Unlike the GET /mappings endpoint, the command-line version is consistent by default and will hold lock at 126 possible detriment to other users. 127 128 "indices": Label indices in a space-delimted file where each supervoxel has a row with its agglomerated label: 129 <label> <supervoxel id> <block z> <block y> <block x> <# voxels> 130 131 Note that the last two dumps can be used by a program to ingest all but the actual voxel labels, 132 using the POST /mappings and POST /indices endpoints. All three dumps can be used for quality 133 control. Sorting can be done after the dump by the linux "sort" command: 134 % sort -g -k1,1 -k2,2 -k3,3 -k4,4 svcount.csv > sorted-svcount.csv 135 136 Example: 137 138 $ dvid node 3f8c segmentation dump svcount /path/to/counts.csv 139 140 Arguments: 141 142 UUID Hexadecimal string with enough characters to uniquely identify a version node. 143 data name Name of data to add. 144 dump type One of "svcount", "mappings", or "indices". 145 file path Absolute path to a writable file that the dvid server has write privileges to. 146 147 $ dvid node <UUID> <data name> set-nextlabel <label> 148 149 Sets the counter for new labels repo-wide for the given labelmap instance. 150 Note that the next label will be one more than the given label, and the given 151 label must be 1 or more. If label is 0, then this next label setting will 152 be ignored and future labels will be determined by the repo-wide max label 153 as is the default. 154 155 This is a dangerous command if you set the next label to a low value because 156 it will not check if it starts to encroach higher label values, so use with 157 caution. 158 159 Example: 160 161 $ dvid node 3f8c segmentation set-nextlabel 999 162 163 The next new label, for example in a cleave, will be 1000. 164 165 Arguments: 166 167 UUID Hexadecimal string with enough characters to uniquely identify a version node. 168 data name Name of data to add. 169 label A uint64 label ID 170 171 ------------------ 172 173 HTTP API (Level 2 REST): 174 175 POST /api/repo/{uuid}/instance 176 177 Creates a new instance of the labelmap data type. Expects configuration data in JSON 178 as the body of the POST. Configuration data is a JSON object with each property 179 corresponding to a configuration keyword for the particular data type. 180 181 JSON name/value pairs: 182 183 REQUIRED "typename" Must be "labelmap" 184 REQUIRED "dataname" Name of the new instance 185 OPTIONAL "versioned" If "false" or "0", the data is unversioned and acts as if 186 all UUIDs within a repo become the root repo UUID. (True by default.) 187 OPTIONAL "BlockSize" Size in pixels (default: 64,64,64 and should be multiples of 16) 188 OPTIONAL "VoxelSize" Resolution of voxels (default: 8.0,8.0,8.0) 189 OPTIONAL "VoxelUnits" Resolution units (default: "nanometers") 190 OPTIONAL "IndexedLabels" "false" if no sparse volume support is required (default "true") 191 OPTIONAL "MaxDownresLevel" The maximum down-res level supported. Each down-res is factor of 2. 192 193 194 GET <api URL>/node/<UUID>/<data name>/help 195 196 Returns data-specific help message. 197 198 199 GET <api URL>/node/<UUID>/<data name>/info 200 POST <api URL>/node/<UUID>/<data name>/info 201 202 Retrieves or puts DVID-specific data properties for these voxels. 203 204 Example: 205 206 GET <api URL>/node/3f8c/segmentation/info 207 208 Returns or posts JSON of configuration settings with the following optional fields: 209 210 "BlockSize" Size in pixels (default: 64,64,64 and should be multiples of 16) 211 "VoxelSize" Resolution of voxels (default: 8.0,8.0,8.0) 212 "VoxelUnits" Resolution units (default: "nanometers") 213 "MinPoint" Minimum voxel coordinate as 3d point 214 "MaxPoint" Maximum voxel coordinate as 3d point 215 "GridStore" Store identifier in TOML config file that specifies precomputed store. 216 "MaxDownresLevel" The maximum down-res level supported. Each down-res is factor of 2. 217 218 Arguments: 219 220 UUID Hexadecimal string with enough characters to uniquely identify a version node. 221 data name Name of labelmap instance. 222 223 POST <api URL>/node/<UUID>/<data name>/extents 224 225 Sets the extents for the image volume. This is primarily used when POSTing from multiple 226 DVID servers not sharing common metadata to a shared backend. 227 228 Extents should be in JSON in the following format: 229 { 230 "MinPoint": [0,0,0], 231 "MaxPoint": [300,400,500] 232 } 233 234 POST <api URL>/node/<UUID>/<data name>/resolution 235 236 Sets the resolution for the image volume. 237 238 Extents should be in JSON in the following format: 239 [8,8,8] 240 241 POST <api URL>/node/<UUID>/<data name>/sync?<options> 242 243 Establishes labelvol data instances with which the annotations are synced. Expects JSON to be POSTed 244 with the following format: 245 246 { "sync": "bodies" } 247 248 To delete syncs, pass an empty string of names with query string "replace=true": 249 250 { "sync": "" } 251 252 The "sync" property should be followed by a comma-delimited list of data instances that MUST 253 already exist. Currently, syncs should be created before any annotations are pushed to 254 the server. If annotations already exist, these are currently not synced. 255 256 The labelmap data type accepts syncs to labelvol data instances. It also accepts syncs to 257 labelmap instances for multiscale. 258 259 Query-string Options: 260 261 replace Set to "true" if you want passed syncs to replace and not be appended to current syncs. 262 Default operation is false. 263 264 GET <api URL>/node/<UUID>/<data name>/tags 265 POST <api URL>/node/<UUID>/<data name>/tags?<options> 266 267 GET retrieves JSON of tags for this instance. 268 POST appends or replaces tags provided in POST body. Expects JSON to be POSTed 269 with the following format: 270 271 { "tag1": "anything you want", "tag2": "something else" } 272 273 To delete tags, pass an empty object with query string "replace=true". 274 275 POST Query-string Options: 276 277 replace Set to "true" if you want passed tags to replace and not be appended to current tags. 278 Default operation is false (append). 279 280 GET <api URL>/node/<UUID>/<data name>/metadata 281 282 Retrieves a JSON schema (application/vnd.dvid-nd-data+json) that describes the layout 283 of bytes returned for n-d images. 284 285 286 GET <api URL>/node/<UUID>/<data name>/specificblocks[?queryopts] 287 288 Retrieves blocks corresponding to those specified in the query string. This interface 289 is useful if the blocks retrieved are not consecutive or if the backend in non ordered. 290 291 Example: 292 293 GET <api URL>/node/3f8c/grayscale/specificblocks?blocks=x1,y1,z2,x2,y2,z2,x3,y3,z3 294 295 This will fetch blocks at position (x1,y1,z1), (x2,y2,z2), and (x3,y3,z3). 296 The returned byte stream has a list of blocks with a leading block 297 coordinate (3 x int32) plus int32 giving the # of bytes in this block, and then the 298 bytes for the value. If blocks are unset within the span, they will not appear in the stream, 299 so the returned data will be equal to or less than spanX blocks worth of data. 300 301 The returned data format has the following format where int32 is in little endian and the bytes 302 of block data have been compressed in the desired output format, according to the specification 303 in "compression" query string. 304 305 int32 Block 1 coordinate X (Note that this may not be starting block coordinate if it is unset.) 306 int32 Block 1 coordinate Y 307 int32 Block 1 coordinate Z 308 int32 # bytes for first block (N1) 309 byte0 Bytes of block data in compressed format. 310 byte1 311 ... 312 byteN1 313 314 int32 Block 2 coordinate X 315 int32 Block 2 coordinate Y 316 int32 Block 2 coordinate Z 317 int32 # bytes for second block (N2) 318 byte0 Bytes of block data in compressed format. 319 byte1 320 ... 321 byteN2 322 323 ... 324 325 If a block is not available, no data will be returned for it. 326 327 Arguments: 328 329 supervoxels If "true", returns unmapped supervoxels, disregarding any kind of merges. 330 UUID Hexadecimal string with enough characters to uniquely identify a version node. 331 data name Name of labelmap instance. 332 333 Query-string Options: 334 335 blocks x,y,z... block string 336 scale A number from 0 up to MaxDownresLevel where each level has 1/2 resolution of 337 previous level. Level 0 (default) is the highest resolution. 338 compression Allows retrieval of block data in "lz4", "gzip", "blocks" (native DVID 339 label blocks), or "uncompressed" (uint64 labels). Default is "blocks". 340 341 342 GET <api URL>/node/<UUID>/<data name>/isotropic/<dims>/<size>/<offset>[/<format>][?queryopts] 343 344 Retrieves either 2d images (PNG by default) or 3d binary data, depending on the dims parameter. 345 The 3d binary data response has "Content-type" set to "application/octet-stream" and is an array of 346 voxel values in ZYX order (X iterates most rapidly). 347 348 Example: 349 350 GET <api URL>/node/3f8c/segmentation/isotropic/0_1/512_256/0_0_100/jpg:80 351 352 Returns an isotropic XY slice (0th and 1st dimensions) with width (x) of 512 voxels and 353 height (y) of 256 voxels with offset (0,0,100) in JPG format with quality 80. 354 Additional processing is applied based on voxel resolutions to make sure the retrieved image 355 has isotropic pixels. For example, if an XZ image is requested and the image volume has 356 X resolution 3 nm and Z resolution 40 nm, the returned image's height will be magnified 40/3 357 relative to the raw data. 358 The example offset assumes the "grayscale" data in version node "3f8c" is 3d. 359 The "Content-type" of the HTTP response should agree with the requested format. 360 For example, returned PNGs will have "Content-type" of "image/png", and returned 361 nD data will be "application/octet-stream". 362 363 Arguments: 364 365 UUID Hexadecimal string with enough characters to uniquely identify a version node. 366 data name Name of labelmap instance. 367 dims The axes of data extraction in form "i_j_k,..." Example: "0_2" can be XZ. 368 Slice strings ("xy", "xz", or "yz") are also accepted. 369 size Size in voxels along each dimension specified in <dims>. 370 offset Gives coordinate of first voxel using dimensionality of data. 371 format Valid formats depend on the dimensionality of the request and formats 372 available in server implementation. 373 2D: "png", "jpg" (default: "png") 374 jpg allows lossy quality setting, e.g., "jpg:80" 375 nD: uses default "octet-stream". 376 377 Query-string Options: 378 379 roi Name of roi data instance used to mask the requested data. 380 scale A number from 0 up to MaxDownresLevel where each level beyond 0 has 1/2 resolution 381 of previous level. Level 0 is the highest resolution. 382 compression Allows retrieval or submission of 3d data in "lz4" and "gzip" 383 compressed format. The 2d data will ignore this and use 384 the image-based codec. 385 throttle Only works for 3d data requests. If "true", makes sure only N compute-intense operation 386 (all API calls that can be throttled) are handled. If the server can't initiate the API 387 call right away, a 503 (Service Unavailable) status code is returned. 388 389 390 GET <api URL>/node/<UUID>/<data name>/raw/<dims>/<size>/<offset>[/<format>][?queryopts] 391 392 Retrieves either 2d images (PNG by default) or 3d binary data, depending on the dims parameter. 393 The 3d binary data response has "Content-type" set to "application/octet-stream" and is a packed 394 array of voxel values (little-endian uint64 per voxel) in ZYX order (X iterates most rapidly). 395 396 Example: 397 398 GET <api URL>/node/3f8c/segmentation/raw/0_1/512_256/0_0_100/jpg:80 399 400 Returns a raw XY slice (0th and 1st dimensions) with width (x) of 512 voxels and 401 height (y) of 256 voxels with offset (0,0,100) in JPG format with quality 80. 402 By "raw", we mean that no additional processing is applied based on voxel 403 resolutions to make sure the retrieved image has isotropic pixels. 404 The example offset assumes the "grayscale" data in version node "3f8c" is 3d. 405 The "Content-type" of the HTTP response should agree with the requested format. 406 For example, returned PNGs will have "Content-type" of "image/png", and returned 407 nD data will be "application/octet-stream". 408 409 Arguments: 410 411 UUID Hexadecimal string with enough characters to uniquely identify a version node. 412 data name Name of labelmap instance. 413 dims The axes of data extraction in form "i_j_k,..." 414 Slice strings ("xy", "xz", or "yz") are also accepted. 415 Example: "0_2" is XZ, and "0_1_2" is a 3d subvolume. 416 size Size in voxels along each dimension specified in <dims>. 417 offset Gives coordinate of first voxel using dimensionality of data. 418 format Valid formats depend on the dimensionality of the request and formats 419 available in server implementation. 420 2D: "png", "jpg" (default: "png") 421 jpg allows lossy quality setting, e.g., "jpg:80" 422 nD: uses default "octet-stream". 423 424 Query-string Options: 425 426 supervoxels If "true", returns unmapped supervoxels, disregarding any kind of merges. 427 roi Name of roi data instance used to mask the requested data. 428 scale A number from 0 up to MaxDownresLevel where each level beyond 0 has 1/2 resolution 429 of previous level. Level 0 is the highest resolution. 430 compression Allows retrieval or submission of 3d data in "lz4","gzip", "google" 431 (neuroglancer compression format), "googlegzip" (google + gzip) 432 compressed format. The 2d data will ignore this and use 433 the image-based codec. 434 throttle Only works for 3d data requests. If "true", makes sure only N compute-intense operation 435 (all API calls that can be throttled) are handled. If the server can't initiate the API 436 call right away, a 503 (Service Unavailable) status code is returned. 437 438 439 POST <api URL>/node/<UUID>/<data name>/raw/0_1_2/<size>/<offset>[?queryopts] 440 441 Ingests block-aligned supervoxel data using the block sizes defined for this data instance. 442 For example, if the BlockSize = 32, offset and size must be multiples of 32. 443 The POST body should be a packed array of voxel values (little-endian uint64 per voxel) in ZYX order 444 (X iterates most rapidly). 445 446 Example: 447 448 POST <api URL>/node/3f8c/segmentation/raw/0_1_2/512_256_128/0_0_32 449 450 Arguments: 451 452 UUID Hexadecimal string with enough characters to uniquely identify a version node. 453 data name Name of labelmap instance. 454 size Size in voxels along each dimension specified in <dims>. 455 offset Gives coordinate of first voxel using dimensionality of data. 456 457 Query-string Options: 458 459 roi Name of roi data instance used to mask the requested data. 460 compression Allows retrieval or submission of 3d data in "lz4" and "gzip" 461 compressed format. 462 throttle If "true", makes sure only N compute-intense operation (all API calls that can 463 be throttled) are handled. If the server can't initiate the API call right away, 464 a 503 (Service Unavailable) status code is returned. 465 466 GET <api URL>/node/<UUID>/<data name>/pseudocolor/<dims>/<size>/<offset>[?queryopts] 467 468 Retrieves label data as pseudocolored 2D PNG color images where each label hashed to a different RGB. 469 470 Example: 471 472 GET <api URL>/node/3f8c/segmentation/pseudocolor/0_1/512_256/0_0_100 473 474 Returns an XY slice (0th and 1st dimensions) with width (x) of 512 voxels and 475 height (y) of 256 voxels with offset (0,0,100) in PNG format. 476 477 Arguments: 478 479 UUID Hexadecimal string with enough characters to uniquely identify a version node. 480 data name Name of labelmap instance. 481 dims The axes of data extraction. Example: "0_2" can be XZ. 482 Slice strings ("xy", "xz", or "yz") are also accepted. 483 size Size in voxels along each dimension specified in <dims>. 484 offset Gives coordinate of first voxel using dimensionality of data. 485 486 Query-string Options: 487 488 roi Name of roi data instance used to mask the requested data. 489 compression Allows retrieval or submission of 3d data in "lz4" and "gzip" 490 compressed format. 491 throttle If "true", makes sure only N compute-intense operation (all API calls that can be throttled) 492 are handled. If the server can't initiate the API call right away, a 503 (Service Unavailable) 493 status code is returned. 494 495 GET <api URL>/node/<UUID>/<data name>/label/<coord>[?queryopts] 496 497 Returns JSON for the label at the given coordinate: 498 { "Label": 23 } 499 500 Arguments: 501 UUID Hexadecimal string with enough characters to uniquely identify a version node. 502 data name Name of labelmap instance. 503 coord Coordinate of voxel with underscore as separator, e.g., 10_20_30 504 505 Query-string Options: 506 507 supervoxels If "true", returns unmapped supervoxel label, disregarding any kind of merges. 508 scale A number from 0 up to MaxDownresLevel where each level beyond 0 has 1/2 resolution 509 of previous level. Level 0 is the highest resolution. 510 511 GET <api URL>/node/<UUID>/<data name>/labels[?queryopts] 512 513 Returns JSON for the labels at a list of coordinates. Expects JSON in GET body: 514 515 [ [x0, y0, z0], [x1, y1, z1], ...] 516 517 Returns for each POSTed coordinate the corresponding label: 518 519 [ 23, 911, ...] 520 521 Arguments: 522 UUID Hexadecimal string with enough characters to uniquely identify a version node. 523 data name Name of label data. 524 525 Query-string Options: 526 527 supervoxels If "true", returns unmapped supervoxel label, disregarding any kind of merges. 528 scale A number from 0 up to MaxDownresLevel where each level beyond 0 has 1/2 resolution 529 of previous level. Level 0 is the highest resolution. 530 hash MD5 hash of request body content in hexidecimal string format. 531 532 GET <api URL>/node/<UUID>/<data name>/history/<label>/<from UUID>/<to UUID> 533 534 Returns JSON for mutations involving labels in "from UUID" version that correspond to the 535 supervoxels in the "to UUID" target label. 536 537 Arguments: 538 UUID Hexadecimal string with enough characters to uniquely identify a version node. 539 data name Name of labelmap instance. 540 label The label ID as exists in the later version specified by <to UUID>. 541 from UUID The UUID of the earlier version in time range. 542 to UUID The UUID of the later version in time range. 543 544 545 GET <api URL>/node/<UUID>/<data name>/mapping[?queryopts] 546 547 Returns JSON for mapped uint64 identifiers (labels). The mapping holds not only the 548 unique IDs of supervoxels but also newly created IDs for renumbered & cleaved bodies 549 that will never overlap with supervoxel IDs. 550 551 Expects JSON in GET body: 552 553 [ label1, label2, label3, ...] 554 555 Returns for each POSTed label the corresponding mapped label: 556 557 [ 23, 0, 911, ...] 558 559 The mapped label can be 0 in the following circumstances: 560 * The label was a supervoxel ID that was split into two different unique IDs. 561 * The label is used for a newly generated ID that will be a new renumbered label. 562 * The label is used for a newly generated ID that will represent a cleaved body ID. 563 564 Arguments: 565 UUID Hexadecimal string with enough characters to uniquely identify a version node. 566 data name Name of label data. 567 568 Query-string Options: 569 570 nolookup if "true", dvid won't verify that a supervoxel actually exists by looking up 571 the label indices. Only use this if supervoxels were known to exist at some time. 572 hash MD5 hash of request body content in hexidecimal string format. 573 574 GET <api URL>/node/<UUID>/<data name>/supervoxel-splits 575 576 Returns JSON for all supervoxel splits that have occured up to this version of the 577 labelmap instance. The returned JSON is of format: 578 579 [ 580 "abc123", 581 [[<mutid>, <old>, <remain>, <split>], 582 [<mutid>, <old>, <remain>, <split>], 583 [<mutid>, <old>, <remain>, <split>]], 584 "bcd234", 585 [[<mutid>, <old>, <remain>, <split>], 586 [<mutid>, <old>, <remain>, <split>], 587 [<mutid>, <old>, <remain>, <split>]] 588 ] 589 590 The UUID examples above, "abc123" and "bcd234", would be full UUID strings and are in order 591 of proximity to the given UUID. So the first UUID would be the version of interest, then 592 its parent, and so on. 593 594 Arguments: 595 UUID Hexadecimal string with enough characters to uniquely identify a version node. 596 data name Name of label data. 597 598 599 GET <api URL>/node/<UUID>/<data name>/blocks/<size>/<offset>[?queryopts] 600 601 Gets blocks corresponding to the extents specified by the size and offset. The 602 subvolume request must be block aligned. This is the most server-efficient way of 603 retrieving the labelmap data, where data read from the underlying storage engine is 604 written directly to the HTTP connection possibly after recompression to match the given 605 query-string compression option. The default labelmap compression 606 is gzip on compressed DVID label Block serialization ("blocks" option). 607 608 Example: 609 610 GET <api URL>/node/3f8c/segmentation/blocks/64_64_64/0_0_0 611 612 If block size is 32x32x32, this call retrieves up to 8 blocks where the first potential 613 block is at 0, 0, 0. The returned byte stream has a list of blocks with a leading block 614 coordinate (3 x int32) plus int32 giving the # of bytes in this block, and then the 615 bytes for the value. If blocks are unset within the span, they will not appear in the stream, 616 so the returned data will be equal to or less than spanX blocks worth of data. 617 618 The returned data format has the following format where int32 is in little endian and the 619 bytes of block data have been compressed in the desired output format. 620 621 int32 Block 1 coordinate X (Note that this may not be starting block coordinate if it is unset.) 622 int32 Block 1 coordinate Y 623 int32 Block 1 coordinate Z 624 int32 # bytes for first block (N1) 625 byte0 Block N1 serialization using chosen compression format (see "compression" option below) 626 byte1 627 ... 628 byteN1 629 630 int32 Block 2 coordinate X 631 int32 Block 2 coordinate Y 632 int32 Block 2 coordinate Z 633 int32 # bytes for second block (N2) 634 byte0 Block N2 serialization using chosen compression format (see "compression" option below) 635 byte1 636 ... 637 byteN2 638 639 ... 640 641 If a block is not available, no data will be returned for it. 642 643 Arguments: 644 645 UUID Hexadecimal string with enough characters to uniquely identify a version node. 646 data name Name of data to add. 647 size Size in voxels along each dimension specified in <dims>. 648 offset Gives coordinate of first voxel using dimensionality of data. 649 650 Query-string Options: 651 652 supervoxels If "true", returns unmapped supervoxels, disregarding any kind of merges. 653 scale A number from 0 up to MaxDownresLevel where each level beyond 0 has 1/2 resolution 654 of previous level. Level 0 is the highest resolution. 655 compression Allows retrieval of block data in "lz4" (default), "gzip", blocks" (native DVID 656 label blocks) or "uncompressed" (uint64 labels). 657 throttle If "true", makes sure only N compute-intense operation (all API calls that can be 658 throttled) are handled. If the server can't initiate the API call right away, a 503 659 (Service Unavailable) status code is returned. 660 661 662 POST <api URL>/node/<UUID>/<data name>/blocks[?queryopts] 663 664 Puts properly-sized supervoxel block data. This is the most server-efficient way of 665 storing labelmap data if you want DVID to also handle indexing and downres computation. 666 If you are calculating indices and downres supervoxel blocks offline for ingesting into 667 DVID, use the "POST /ingest-supervoxels" endpoint, since it is even faster. 668 669 It's suggested that downres supervoxel blocks should be calculated outside DVID and then 670 ingested for anything larger than small (Gigavoxel) volumes. Currently, the downres computation 671 is not robust for non-cubic chunk sizes and because this endpoint must consider parallel 672 requests using overlapping blocks, a mutex is employed that limits the overall throughput. 673 Still data read from the HTTP stream is written directly to the underlying storage. The 674 default (and currently only supported) compression is gzip on compressed DVID label Block 675 serialization. 676 677 Note that maximum label and extents are automatically handled during these calls. 678 If the optional "scale" query is greater than 0, these ingestions will not trigger 679 syncing with associated annotations, etc. 680 681 Example: 682 683 POST <api URL>/node/3f8c/segmentation/blocks 684 685 The posted data format should be in the following format where int32 is in little endian and 686 the bytes of block data have been compressed in the desired output format. 687 688 int32 Block 1 coordinate X (Note that this may not be starting block coordinate if it is unset.) 689 int32 Block 1 coordinate Y 690 int32 Block 1 coordinate Z 691 int32 # bytes for first block (N1) 692 byte0 Block N1 serialization using chosen compression format (see "compression" option below) 693 byte1 694 ... 695 byteN1 696 697 int32 Block 2 coordinate X 698 int32 Block 2 coordinate Y 699 int32 Block 2 coordinate Z 700 int32 # bytes for second block (N2) 701 byte0 Block N2 serialization using chosen compression format (see "compression" option below) 702 byte1 703 ... 704 byteN2 705 706 ... 707 708 The Block serialization format is as follows: 709 710 3 * uint32 values of gx, gy, and gz 711 uint32 # of labels (N), cannot exceed uint32. 712 N * uint64 packed labels in little-endian format. Label 0 can be used to represent 713 deleted labels, e.g., after a merge operation to avoid changing all 714 sub-block indices. 715 716 ----- Data below is only included if N > 1, otherwise it is a solid block. 717 Nsb = # sub-blocks = gx * gy * gz 718 719 Nsb * uint16 # of labels for sub-blocks. Each uint16 Ns[i] = # labels for sub-block i. 720 If Ns[i] == 0, the sub-block has no data (uninitialized), which 721 is useful for constructing Blocks with sparse data. 722 723 Nsb * Ns * uint32 label indices for sub-blocks where Ns = sum of Ns[i] over all sub-blocks. 724 For each sub-block i, we have Ns[i] label indices of lBits. 725 726 Nsb * values sub-block indices for each voxel. 727 Data encompasses 512 * ceil(log2(Ns[i])) bits, padded so no two 728 sub-blocks have indices in the same byte. 729 At most we use 9 bits per voxel for up to the 512 labels in sub-block. 730 A value gives the sub-block index which points to the index into 731 the N labels. If Ns[i] <= 1, there are no values. If Ns[i] = 0, 732 the 8x8x8 voxels are set to label 0. If Ns[i] = 1, all voxels 733 are the given label index. 734 735 Arguments: 736 737 UUID Hexadecimal string with enough characters to uniquely identify a version node. 738 data name Name of data to add. 739 740 Query-string Options: 741 742 scale A number from 0 up to MaxDownresLevel where each level beyond 0 has 1/2 resolution 743 of previous level. Level 0 is the highest resolution. 744 downres "false" (default) or "true", specifies whether the given blocks should be 745 down-sampled to lower resolution. If "true", scale must be "0" or absent. 746 compression Specifies compression format of block data: default and only option currently is 747 "blocks" (native DVID label blocks). 748 noindexing If "true" (default "false"), will not compute label indices from the received voxel data. 749 Use this in conjunction with POST /index and /affinities endpoint for faster ingestion. 750 throttle If "true", makes sure only N compute-intense operation (all API calls that can be 751 throttled) are handled. If the server can't initiate the API call right away, a 503 752 (Service Unavailable) status code is returned. 753 754 755 POST <api URL>/node/<UUID>/<data name>/ingest-supervoxels[?queryopts] 756 757 Ingests properly-sized supervoxel block data under the assumption that parallel calls to this 758 endpoint do not use overlapping blocks. This is the most server-efficient way of storing labelmap data, 759 where data read from the HTTP stream is written directly to the underlying storage. The default 760 (and currently only supported) compression is gzip on compressed DVID label Block serialization. 761 Unlike the "POST /blocks" endpoint, this endpoint assumes that the following operations will be done separately: 762 * label indexing 763 * syncing to other instances (e.g., annotations) 764 * calculation and POST /maxlabel 765 * calculation and POST /extents 766 * downres calculations of different scales and the POST of the downres supervoxel block data. 767 768 This endpoint maximizes write throughput and assumes parallel POST requests will not use overlapping blocks. 769 No goroutines are spawned so the number of write goroutines is directly related to the number of parallel 770 calls to this endpoint. 771 772 Example: 773 774 POST <api URL>/node/3f8c/segmentation/ingest-supervoxels?scale=1 775 776 The posted data format should be in the following format where int32 is in little endian and 777 the bytes of block data have been compressed in the desired output format. 778 779 int32 Block 1 coordinate X (Note that this may not be starting block coordinate if it is unset.) 780 int32 Block 1 coordinate Y 781 int32 Block 1 coordinate Z 782 int32 # bytes for first block (N1) 783 byte0 Block N1 serialization using chosen compression format (see "compression" option below) 784 byte1 785 ... 786 byteN1 787 788 int32 Block 2 coordinate X 789 int32 Block 2 coordinate Y 790 int32 Block 2 coordinate Z 791 int32 # bytes for second block (N2) 792 byte0 Block N2 serialization using chosen compression format (see "compression" option below) 793 byte1 794 ... 795 byteN2 796 797 ... 798 799 The Block serialization format is as follows: 800 801 3 * uint32 values of gx, gy, and gz 802 uint32 # of labels (N), cannot exceed uint32. 803 N * uint64 packed labels in little-endian format. Label 0 can be used to represent 804 deleted labels, e.g., after a merge operation to avoid changing all 805 sub-block indices. 806 807 ----- Data below is only included if N > 1, otherwise it is a solid block. 808 Nsb = # sub-blocks = gx * gy * gz 809 810 Nsb * uint16 # of labels for sub-blocks. Each uint16 Ns[i] = # labels for sub-block i. 811 If Ns[i] == 0, the sub-block has no data (uninitialized), which 812 is useful for constructing Blocks with sparse data. 813 814 Nsb * Ns * uint32 label indices for sub-blocks where Ns = sum of Ns[i] over all sub-blocks. 815 For each sub-block i, we have Ns[i] label indices of lBits. 816 817 Nsb * values sub-block indices for each voxel. 818 Data encompasses 512 * ceil(log2(Ns[i])) bits, padded so no two 819 sub-blocks have indices in the same byte. 820 At most we use 9 bits per voxel for up to the 512 labels in sub-block. 821 A value gives the sub-block index which points to the index into 822 the N labels. If Ns[i] <= 1, there are no values. If Ns[i] = 0, 823 the 8x8x8 voxels are set to label 0. If Ns[i] = 1, all voxels 824 are the given label index. 825 826 Arguments: 827 828 UUID Hexadecimal string with enough characters to uniquely identify a version node. 829 data name Name of data to add. 830 831 Query-string Options: 832 833 scale A number from 0 up to MaxDownresLevel where each level beyond 0 has 1/2 resolution 834 of previous level. Level 0 (default) is the highest resolution. 835 836 837 GET <api URL>/node/<UUID>/<data name>/maxlabel 838 839 GET returns the maximum label for the version of data in JSON form: 840 841 { "maxlabel": <label #> } 842 843 POST <api URL>/node/<UUID>/<data name>/maxlabel/<max label> 844 845 Sets the maximum label for the version of data specified by the UUID. This maximum label will be 846 ignored if it is not greater than the current maximum label. This value is purely informative 847 (i.e., not used for establishing new labels on split) and can be used to distinguish new labels 848 in remote stores that may collide with local ones. 849 850 If Kafka is enabled, a log message will be posted: 851 { 852 "Action": "post-maxlabel", 853 "Max Label": label, 854 "UUID": uuid, 855 "Timestamp": time.Now().String(), 856 } 857 858 GET <api URL>/node/<UUID>/<data name>/nextlabel 859 860 GET returns what would be a new label for the version of data in JSON form assuming the version 861 has not been committed: 862 863 { "nextlabel": <label #> } 864 865 POST <api URL>/node/<UUID>/<data name>/nextlabel/<desired # of labels> 866 867 POST allows the client to request some # of labels that will be reserved. 868 This is used if the client wants to introduce new labels. 869 870 Response: 871 872 { "start": <starting label #>, "end": <ending label #> } 873 874 Unlike POST /maxlabel, which can set the maximum label arbitrarily high, this 875 endpoint gives incremental new label ids. 876 877 If Kafka is enabled, a log message will be posted: 878 { 879 "Action": "post-nextlabel", 880 "Start Label": start, 881 "End Label": end, 882 "UUID": uuid, 883 "Timestamp": time.Now().String(), 884 } 885 886 887 ------------------------------------------------------------------------------------------------------- 888 --- The following endpoints require the labelmap data instance to have IndexedLabels set to true. --- 889 ------------------------------------------------------------------------------------------------------- 890 891 GET <api URL>/node/<UUID>/<data name>/lastmod/<label> 892 893 Returns last modification metadata for a label in JSON: 894 895 { "mutation id": 2314, "last mod user": "johndoe", "last mod time": "2000-02-01 12:13:14 +0000 UTC", "last mod app": "Neu3" } 896 897 Time is returned in RFC3339 string format. Returns a status code 404 (Not Found) 898 if label does not exist. 899 900 Arguments: 901 UUID Hexadecimal string with enough characters to uniquely identify a version node. 902 data name Name of labelmap instance. 903 label A 64-bit integer label id 904 905 GET <api URL>/node/<UUID>/<data name>/supervoxels/<label> 906 907 Returns JSON for the supervoxels that have been agglomerated into the given label: 908 909 [ 23, 911, ...] 910 911 Returns a status code 404 (Not Found) if label does not exist. 912 913 Arguments: 914 UUID Hexadecimal string with enough characters to uniquely identify a version node. 915 data name Name of labelmap instance. 916 label A 64-bit integer label id 917 918 GET <api URL>/node/<UUID>/<data name>/size/<label>[?supervoxels=true] 919 920 Returns the size in voxels for the given label (or supervoxel) in JSON: 921 922 { "voxels": 2314 } 923 924 Returns a status code 404 (Not Found) if label does not exist. 925 926 Arguments: 927 UUID Hexadecimal string with enough characters to uniquely identify a version node. 928 data name Name of labelmap instance. 929 label A 64-bit integer label id 930 931 Query-string Options: 932 933 supervoxels If "true", interprets the given label as a supervoxel id, not a possibly merged label. 934 935 GET <api URL>/node/<UUID>/<data name>/sizes[?supervoxels=true] 936 937 Returns the sizes in voxels for a list of labels (or supervoxels) in JSON. Expects JSON 938 for the list of labels (or supervoxels) in the body of the request: 939 940 [ 1, 2, 3, ... ] 941 942 Returns JSON of the sizes for each of the above labels: 943 944 [ 19381, 308, 586, ... ] 945 946 Returns a size of 0 if label does not exist. 947 948 Arguments: 949 UUID Hexadecimal string with enough characters to uniquely identify a version node. 950 data name Name of labelmap instance. 951 952 Query-string Options: 953 954 supervoxels If "true", interprets the given labels as a supervoxel ids. 955 hash MD5 hash of request body content in hexidecimal string format. 956 957 GET <api URL>/node/<UUID>/<data name>/supervoxel-sizes/<label> 958 959 Returns the supervoxels and their sizes for the given label in JSON. 960 Although the position in the lists will match supervoxel label and size, 961 the supervoxels may not be ordered: 962 { 963 "supervoxels": [1,2,4,3,...], 964 "sizes": [100,200,400,300,...] 965 } 966 967 Returns a status code 404 (Not Found) if label does not exist. 968 969 Arguments: 970 UUID Hexadecimal string with enough characters to uniquely identify a version node. 971 data name Name of labelmap instance. 972 label A 64-bit integer label id 973 974 975 GET <api URL>/node/<UUID>/<data name>/sparsevol-size/<label>[?supervoxels=true] 976 977 Returns JSON giving the number of voxels, number of native blocks and the coarse bounding box in DVID 978 coordinates (voxel space): 979 980 { "voxels": 231387, numblocks": 1081, "minvoxel": [0, 11, 23], "maxvoxel": [1723, 1279, 4855]} 981 982 Returns a status code 404 (Not Found) if label does not exist. 983 984 Note that the minvoxel and maxvoxel coordinates are voxel coordinates that are 985 accurate to the block, not the voxel. 986 987 Query-string Options: 988 989 supervoxels If "true", interprets the given label as a supervoxel id, not a possibly merged label. 990 991 GET <api URL>/node/<UUID>/<data name>/sparsevol/<label>?<options> 992 993 Returns a sparse volume with voxels of the given label in encoded RLE format. The returned 994 data can be optionally compressed using the "compression" option below. 995 996 Returns a status code 404 (Not Found) if label does not exist. 997 998 The encoding has the following possible format where integers are little endian and the order 999 of data is exactly as specified below: 1000 1001 Legacy RLEs ("rles") : 1002 1003 byte Payload descriptor: 1004 Bit 0 (LSB) - 8-bit grayscale 1005 Bit 1 - 16-bit grayscale 1006 Bit 2 - 16-bit normal 1007 If set to all 0, there is no payload and it's a binary sparse volume. 1008 uint8 Number of dimensions 1009 uint8 Dimension of run (typically 0 = X) 1010 byte Reserved (to be used later) 1011 uint32 # Voxels [TODO. 0 for now] 1012 uint32 # Spans 1013 Repeating unit of: 1014 int32 Coordinate of run start (dimension 0) 1015 int32 Coordinate of run start (dimension 1) 1016 int32 Coordinate of run start (dimension 2) 1017 int32 Length of run 1018 bytes Optional payload dependent on first byte descriptor 1019 ... 1020 1021 Streaming RLEs ("srles"): 1022 1023 Repeating unit of: 1024 int32 Coordinate of run start (dimension 0) 1025 int32 Coordinate of run start (dimension 1) 1026 int32 Coordinate of run start (dimension 2) 1027 int32 Length of run 1028 1029 Streaming Binary Blocks ("blocks"): 1030 1031 3 * uint32 values of gx, gy, and gz -- the # of sub-blocks along each dimension in a Block. 1032 uint64 foreground label 1033 1034 Stream of blocks. Each block has the following data: 1035 1036 3 * int32 offset of first voxel of Block in DVID space (x, y, z) 1037 byte content flag: 1038 0 = background ONLY (no more data for this block) 1039 1 = foreground ONLY (no more data for this block) 1040 2 = both background and foreground so stream of sub-blocks required. 1041 1042 If content is both background and foreground, stream of gx * gy * gz sub-blocks with the following data: 1043 1044 byte content flag: 1045 0 = background ONLY (no more data for this sub-block) 1046 1 = foreground ONLY (no more data for this sub-block) 1047 2 = both background and foreground so mask data required. 1048 mask 64 byte bitmask where each voxel is 0 (background) or 1 (foreground) 1049 1050 GET Query-string Options: 1051 1052 format One of the following: 1053 "rles" (default) - legacy RLEs with header including # spans.Data 1054 "srles" - streaming RLEs with each RLE composed of 4 int32 (16 bytes) for x, y, z, run 1055 "blocks" - binary Block stream 1056 1057 minx Spans must be >= this minimum x voxel coordinate at given scale 1058 maxx Spans must be <= this maximum x voxel coordinate at given scale. 1059 miny Spans must be >= this minimum y voxel coordinate at given scale. 1060 maxy Spans must be <= this maximum y voxel coordinate at given scale. 1061 minz Spans must be >= this minimum z voxel coordinate at given scale. 1062 maxz Spans must be <= this maximum z voxel coordinate at given scale. 1063 exact "false" if RLEs can extend a bit outside voxel bounds within border blocks. 1064 This will give slightly faster responses. 1065 1066 compression "lz4" and "gzip" compressed format; only applies to "rles" format for now. 1067 scale A number from 0 (default highest res) to MaxDownresLevel where each level 1068 beyond 0 has 1/2 resolution of previous level. 1069 supervoxels If "true", interprets the given label as a supervoxel id. 1070 1071 1072 HEAD <api URL>/node/<UUID>/<data name>/sparsevol/<label>[?supervoxels=true] 1073 1074 Returns: 1075 200 (OK) if a sparse volume of the given label exists within any optional bounds. 1076 204 (No Content) if there is no sparse volume for the given label within any optional bounds. 1077 1078 Note that for speed, the optional bounds are always expanded to the block-aligned containing 1079 subvolume, i.e., it's as if exact=false for the corresponding GET. 1080 1081 GET Query-string Options: 1082 1083 minx Spans must be equal to or larger than this minimum x voxel coordinate. 1084 maxx Spans must be equal to or smaller than this maximum x voxel coordinate. 1085 miny Spans must be equal to or larger than this minimum y voxel coordinate. 1086 maxy Spans must be equal to or smaller than this maximum y voxel coordinate. 1087 minz Spans must be equal to or larger than this minimum z voxel coordinate. 1088 maxz Spans must be equal to or smaller than this maximum z voxel coordinate. 1089 1090 Query-string Options: 1091 1092 supervoxels If "true", interprets the given label as a supervoxel id, not a possibly merged label. 1093 1094 GET <api URL>/node/<UUID>/<data name>/sparsevol-by-point/<coord>[?supervoxels=true] 1095 1096 Returns a sparse volume with voxels that pass through a given voxel. 1097 The encoding is described in the "sparsevol" request above. 1098 1099 Arguments: 1100 1101 UUID Hexadecimal string with enough characters to uniquely identify a version node. 1102 data name Name of mapping data. 1103 coord Coordinate of voxel with underscore as separator, e.g., 10_20_30 1104 1105 Query-string Options: 1106 1107 supervoxels If "true", returns the sparsevol of the supervoxel designated by the point. 1108 1109 GET <api URL>/node/<UUID>/<data name>/sparsevol-coarse/<label>?<options> 1110 1111 Returns a sparse volume with blocks of the given label in encoded RLE format. 1112 The encoding has the following format where integers are little endian and the order 1113 of data is exactly as specified below: 1114 1115 byte Set to 0 1116 uint8 Number of dimensions 1117 uint8 Dimension of run (typically 0 = X) 1118 byte Reserved (to be used later) 1119 uint32 # Blocks [TODO. 0 for now] 1120 uint32 # Spans 1121 Repeating unit of: 1122 int32 Block coordinate of run start (dimension 0) 1123 int32 Block coordinate of run start (dimension 1) 1124 int32 Block coordinate of run start (dimension 2) 1125 ... 1126 int32 Length of run 1127 1128 Note that the above format is the RLE encoding of sparsevol, where voxel coordinates 1129 have been replaced by block coordinates. 1130 1131 Returns a status code 404 (Not Found) if label does not exist. 1132 1133 GET Query-string Options: 1134 1135 supervoxels If "true", interprets the given label as a supervoxel id. 1136 1137 minx Spans must be equal to or larger than this minimum x voxel coordinate. 1138 maxx Spans must be equal to or smaller than this maximum x voxel coordinate. 1139 miny Spans must be equal to or larger than this minimum y voxel coordinate. 1140 maxy Spans must be equal to or smaller than this maximum y voxel coordinate. 1141 minz Spans must be equal to or larger than this minimum z voxel coordinate. 1142 maxz Spans must be equal to or smaller than this maximum z voxel coordinate. 1143 1144 1145 GET <api URL>/node/<UUID>/<data name>/sparsevols-coarse/<start label>/<end label> 1146 1147 Note: this request does not reflect ongoing merges/splits but is meant to be used 1148 for various batch operations on a static node. 1149 1150 Returns a stream of sparse volumes with blocks of the given label in encoded RLE format: 1151 1152 uint64 label 1153 <coarse sparse vol as given below> 1154 1155 uint64 label 1156 <coarse sparse vol as given below> 1157 1158 ... 1159 1160 The coarse sparse vol has the following format where integers are little endian and the order 1161 of data is exactly as specified below: 1162 1163 int32 # Spans 1164 Repeating unit of: 1165 int32 Block coordinate of run start (dimension 0) 1166 int32 Block coordinate of run start (dimension 1) 1167 int32 Block coordinate of run start (dimension 2) 1168 int32 Length of run 1169 1170 1171 POST <api URL>/node/<UUID>/<data name>/renumber 1172 1173 Renumbers labels. Requires JSON in request body using the following format: 1174 1175 [newLabel1, oldLabel1, newLabel2, oldLabel2, ...] 1176 1177 For each pair of numbers passed in, the oldLabel is renumbered to the newLabel. 1178 So if we pass [21, 1, 22, 2] then label 1 is renumbered to 21, and label 2 is 1179 renumbered to 22. 1180 1181 Kafka JSON message generated by each renumbering pair in this request: 1182 { 1183 "Action": "renumber", 1184 "NewLabel": <new label>, 1185 "OrigLabel": <old label>, 1186 "UUID": <UUID on which renumber was done>, 1187 "MutationID": <unique id for mutation>, 1188 "Timestamp": <time at start of operation> 1189 } 1190 1191 After completion of each renumbering op, the following Kafka JSON message is published: 1192 { 1193 "Action": "renumber-complete", 1194 "NewLabel": <new label>, 1195 "OrigLabel": <old label>, 1196 "UUID": <UUID on which renumber was done> 1197 "MutationID": <unique id for mutation>, 1198 "Timestamp": <time at end of operation> 1199 } 1200 1201 POST <api URL>/node/<UUID>/<data name>/merge 1202 1203 Merges labels (not supervoxels). Requires JSON in request body using the 1204 following format: 1205 1206 [toLabel1, fromLabel1, fromLabel2, fromLabel3, ...] 1207 1208 Returns JSON: 1209 { 1210 "MutationID": <unique id for mutation> 1211 } 1212 1213 The first element of the JSON array specifies the label to be used as the merge result. 1214 Note that it's computationally more efficient to group a number of merges into the 1215 same toLabel as a single merge request instead of multiple merge requests. 1216 1217 Kafka JSON message generated by this request: 1218 { 1219 "Action": "merge", 1220 "Target": <to label>, 1221 "Labels": [<to merge label 1>, <to merge label2>, ...], 1222 "UUID": <UUID on which merge was done>, 1223 "MutationID": <unique id for mutation>, 1224 "Timestamp": <time at start of operation> 1225 } 1226 1227 After completion of the merge op, the following JSON message is published: 1228 { 1229 "Action": "merge-complete", 1230 "Target": <to label>, 1231 "Labels": [<to merge label 1>, <to merge label2>, ...], 1232 "UUID": <UUID on which merge was done> 1233 "MutationID": <unique id for mutation>, 1234 "Timestamp": <time at end of operation> 1235 } 1236 1237 POST <api URL>/node/<UUID>/<data name>/cleave/<label> 1238 1239 Cleaves a label given supervoxels to be cleaved. Requires JSON in request body 1240 using the following format: 1241 1242 [supervoxel1, supervoxel2, ...] 1243 1244 Each element of the JSON array is a supervoxel to be cleaved from the label and is given 1245 a new unique label that's provided in the returned JSON. Note that unlike the 1246 target label to be cleved, the POSTed data should be supervoxel IDs, not potentially 1247 merged labels. 1248 1249 A bad request error (status 400) will be returned if you attempt to cleve on a 1250 non-existent body or attempt to cleve all the supervoxels from a label, i.e., you 1251 are not allowed to create empty labels from the cleave operation. 1252 1253 Returns the following JSON: 1254 1255 { 1256 "CleavedLabel": <new or optionally assigned label of cleaved portion>, 1257 "MutationID": <unique id for mutation> 1258 } 1259 1260 1261 Kafka JSON message generated by this request at its initiation: 1262 { 1263 "Action": "cleave", 1264 "OrigLabel": <original label>, 1265 "CleavedLabel": <cleaved label>, 1266 "CleavedSupervoxels": [<supervoxel 1>, <supervoxel 2>, ...], 1267 "UUID": <UUID on which cleave was done>, 1268 "MutationID": <unique id for mutation>, 1269 "Timestamp": <time at start of operation> 1270 } 1271 1272 After the cleave is successfully completed, the following JSON message is logged with Kafka: 1273 { 1274 "Action": "cleave-complete", 1275 "OrigLabel": <original label>, 1276 "CleavedLabel": <cleaved label>, 1277 "CleavedSupervoxels": [<supervoxel 1>, <supervoxel 2>, ...], 1278 "CleavedSize": <number of voxels cleaved>, 1279 "RemainSize": <number of voxels remaining>, 1280 "UUID": <UUID on which cleave was done>, 1281 "MutationID": <unique id for mutation>, 1282 "Timestamp": <time at end of operation> 1283 } 1284 1285 1286 POST <api URL>/node/<UUID>/<data name>/split-supervoxel/<supervoxel>?<options> 1287 1288 Splits a portion of a supervoxel into two new supervoxels, both of which will still be mapped 1289 to the original label. Returns the following JSON: 1290 1291 { 1292 "SplitSupervoxel": <new label of split portion>, 1293 "RemainSupervoxel": <new label of remainder>, 1294 "MutationID": <unique id for mutation> 1295 } 1296 1297 This request requires a binary sparse volume in the POSTed body with the following 1298 encoded RLE format, which is compatible with the format returned by a GET on the 1299 "sparsevol" endpoint described above: 1300 1301 All integers are in little-endian format. 1302 1303 byte Payload descriptor: 1304 Set to 0 to indicate it's a binary sparse volume. 1305 uint8 Number of dimensions 1306 uint8 Dimension of run (typically 0 = X) 1307 byte Reserved (to be used later) 1308 uint32 # Voxels [TODO. 0 for now] 1309 uint32 # Spans 1310 Repeating unit of: 1311 int32 Coordinate of run start (dimension 0) 1312 int32 Coordinate of run start (dimension 1) 1313 int32 Coordinate of run start (dimension 2) 1314 ... 1315 int32 Length of run 1316 1317 NOTE: The POSTed split sparse volume should be a subset of the given supervoxel. Any 1318 region that falls out of the given supervoxel will be ignored. 1319 1320 Kafka JSON message generated by this request: 1321 { 1322 "Action": "split-supervoxel", 1323 "Supervoxel": <supervoxel label>, 1324 "Body": <label of body containing the split supervoxel>, 1325 "SplitSupervoxel": <new label of split portion>, 1326 "RemainSupervoxel": <new label of remainder> 1327 "Split": <string for reference to split data in serialized RLE format>, 1328 "UUID": <UUID on which split was done>, 1329 "MutationID": <unique mutation ID corresponding to this operation>, 1330 "Timestamp": <time at start of operation> 1331 } 1332 1333 The split reference above can be used to download the split binary data by calling 1334 this data instance's BlobStore API. See the node-level HTTP API documentation. 1335 1336 GET /api/node/{uuid}/{data name}/blobstore/{reference} 1337 1338 After completion of the split op, since it can take some time to process all blocks, 1339 the following JSON message is published: 1340 { 1341 "Action": "split-supervoxel-complete", 1342 "Supervoxel": <supervoxel label>, 1343 "Body": <label of body containing the split supervoxel>, 1344 "SplitSupervoxel": <new label of split portion>, 1345 "RemainSupervoxel": <new label of remainder> 1346 "Split": <string for reference to split data in serialized RLE format>, 1347 "SplitSize": <size in voxels of the split supervoxel>, 1348 "RemainSize": <size in voxels of the remaining supervoxel>, 1349 "MutationID": <unique id for mutation>, 1350 "UUID": <UUID on which split was done>, 1351 "Timestamp": <time at end of operation> 1352 } 1353 1354 1355 POST Query-string Options: 1356 1357 split Label id that should be used for split voxels. 1358 remain Label id that should be used for remaining (unsplit) voxels. 1359 downres Defaults to "true" where all lower-res scales will be computed. 1360 Use "false" if you plan on supplying lower-res scales via POST /blocks. 1361 1362 POST <api URL>/node/<UUID>/<data name>/split/<label> 1363 1364 Splits a portion of a label's voxels into a new supervoxel with a new label. 1365 Returns the following JSON: 1366 1367 { 1368 "label": <new label>, 1369 "MutationID": <unique id for mutation> 1370 } 1371 1372 This request requires a binary sparse volume in the POSTed body with the following 1373 encoded RLE format, which is compatible with the format returned by a GET on the 1374 "sparsevol" endpoint described above: 1375 1376 All integers are in little-endian format. 1377 1378 byte Payload descriptor: 1379 Set to 0 to indicate it's a binary sparse volume. 1380 uint8 Number of dimensions 1381 uint8 Dimension of run (typically 0 = X) 1382 byte Reserved (to be used later) 1383 uint32 # Voxels [TODO. 0 for now] 1384 uint32 # Spans 1385 Repeating unit of: 1386 int32 Coordinate of run start (dimension 0) 1387 int32 Coordinate of run start (dimension 1) 1388 int32 Coordinate of run start (dimension 2) 1389 ... 1390 int32 Length of run 1391 1392 NOTE: The POSTed split sparse volume must be a subset of the given label's voxels. You cannot 1393 give an arbitrary sparse volume that may span multiple labels. An error will be returned if 1394 the POSTed split sparsevol isn't contained within the split label. 1395 1396 Kafka JSON message generated by this request: 1397 { 1398 "Action": "split", 1399 "Target": <from label>, 1400 "NewLabel": <to label>, 1401 "Split": <string for reference to split data in serialized RLE format>, 1402 "MutationID": <unique id for mutation>, 1403 "UUID": <UUID on which split was done>, 1404 "SVSplit": {23918: {"Remain": 481273, "Split": 481274}, 1839109: {"Remain":...}}, 1405 "Timestamp": <time at start of operation> 1406 } 1407 1408 The SVSplit field above gives the new remain and split labels for any given split supervoxel. 1409 The split reference above can be used to download the split binary data by calling 1410 this data instance's BlobStore API. See the node-level HTTP API documentation. 1411 1412 GET /api/node/{uuid}/{data name}/blobstore/{reference} 1413 1414 After completion of the split op, the following JSON message is published: 1415 { 1416 "Action": "split-complete", 1417 "Target": <from label>, 1418 "NewLabel": <to label>, 1419 "Split": <string for reference to split data in serialized RLE format>, 1420 "MutationID": <unique id for mutation>, 1421 "UUID": <UUID on which split was done>, 1422 "SVSplit": {23918: {"Remain": 481273, "Split": 481274}, 1839109: {"Remain":...}}, 1423 "Timestamp": <time at end of operation> 1424 } 1425 1426 GET <api URL>/node/<UUID>/<data name>/proximity/<label 1 (target)>,<label 2a>,<label 2b>,... 1427 1428 Determines proximity of a number of labels with a target label returning the 1429 following JSON giving blocks in which they intersect and the min euclidean 1430 voxel distance between the labels within that block: 1431 [ 1432 { 1433 "Block": [<x1>, <y1>, <z1>], 1434 "Label": <label 2a>, 1435 "Distance": <min euclidean voxel distance in block 1 to label 2a> 1436 }, 1437 { 1438 "Block": [<x1>, <y1>, <z1>], 1439 "Label": <label 2b> 1440 "Distance": <min euclidean voxel distance in block 1 to label 2b> 1441 }, 1442 ... 1443 ] 1444 If only two labels are specified, the "Label" property is omitted. 1445 1446 Note that this is an approximation in that we assume two labels aren't perfectly 1447 separated across block boundaries. This could be the case for supervoxels that 1448 are artificially split along block boundaries but won't be true for agglomerated 1449 labels. 1450 1451 GET <api URL>/node/<UUID>/<data name>/index/<label>?mutid=<uint64> 1452 POST <api URL>/node/<UUID>/<data name>/index/<label> 1453 1454 Allows direct retrieval or storing of an index (blocks per supervoxel and their voxel count) 1455 for a label. Typically, these indices are computed on-the-fly during ingestion of 1456 of blocks of label voxels. If there are cluster systems capable of computing label 1457 blocks, indices, and affinities directly, though, it's more efficient to simply POST 1458 them into dvid. 1459 1460 The returned (GET) or sent (POST) protobuf serialization of a LabelIndex message is defined by: 1461 1462 message SVCount { 1463 map<uint64, uint32> counts = 1; 1464 } 1465 1466 message LabelIndex { 1467 map<uint64, SVCount> blocks = 1; // key is encoded block coord ZYX (packed little-endian 21-bit numbers where MSB is sign flag) 1468 uint64 label = 2; 1469 uint64 last_mut_id = 3; 1470 string last_mod_time = 4; // string is time in RFC 3339 format 1471 string last_mod_user = 5; 1472 string last_mod_app = 6; 1473 } 1474 1475 If the blocks map is empty on a POST, the label index is deleted. 1476 1477 Query-string options: 1478 1479 metadata-only: if "true" (default "false"), returns JSON object of num_voxels, last_mutid, last_mod_time, last_mod_user. 1480 mutid: (Read only) Returns the label index prior to completing the given mutation id in the current branch. If 1481 no cached label index is found prior to the given mutation id in the branch of the specified UUID, 1482 a 404 not found is returned. 1483 1484 1485 GET <api URL>/node/<UUID>/<data name>/listlabels[?queryopts] 1486 1487 Streams labels and optionally their voxel counts in numerical order up to a maximum 1488 of 10 million labels. The GET returns a stream of little-endian uint64. If label sizes are 1489 requested, the stream is <label id i> <voxels in label i> <label id i+1> <voxels in label i+1> ... 1490 If label sizes aren't requested, the stream is simply <label id i> <label id i+1> ... 1491 1492 Note that because the data is streamed over HTTP, an error code cannot be sent once data is 1493 in flight, so any error is marked by four consecutive uint64 with value 0. 1494 1495 The query strings allow you to page through vast amounts of labels by changing the start. 1496 For example: 1497 GET /api/node/37af8/segmentation/listlabels 1498 which is equivalent to 1499 GET /api/node/37af8/segmentation/listlabels?start=0 --> say it returns up to label 10,281,384 due to some gaps in labeling. 1500 and then you can call 1501 GET /api/node/37af8/segmentation/listlabels?start=10281385 --> to return next batch, start with last received label + 1 1502 1503 Note that the start is a label identifier and not a position or index. Since labels are 1504 stored consecutively, you can omit the start query string (default is start=0) to guarantee 1505 you will get the smallest label identifier, which should be non-zero since 0 is used only for background. 1506 1507 Query-string options: 1508 1509 start: starting label id (use 0 to start at very beginning since labels are stored consecutively) 1510 number: number of labels to return (if not specified, returns maximum of 10,000,000 labels) 1511 sizes: if "true", returns the number of voxels for each label. 1512 1513 GET <api URL>/node/<UUID>/<data name>/indices 1514 1515 Note: For more optimized bulk index retrieval, see /indices-compressed. 1516 Allows bulk GET of indices (blocks per supervoxel and their voxel count) by 1517 including in the GET body a JSON array of requested labels (max 50,000 labels): 1518 1519 [ 1028193, 883177046, ... ] 1520 1521 The GET returns a protobuf serialization of a LabelIndices message defined by: 1522 1523 message LabelIndices { 1524 repeated LabelIndex indices = 1; 1525 } 1526 1527 where LabelIndex is defined by the protobuf above in the /index documentation. 1528 1529 GET <api URL>/node/<UUID>/<data name>/indices-compressed 1530 1531 The fastest and most efficient way for bulk index retrieval. Streams lz4 1532 compressed LabelIndex protobuf messages with minimal processing server-side. 1533 This allows lz4 uncompression to be done in parallel on clients as well as 1534 immediate processing of each label index as it is received. 1535 Up to 50,000 labels may be requested at once by sending a JSON array of 1536 requested labels in GET body: 1537 1538 [ 1028193, 883177046, ... ] 1539 1540 The GET returns a stream of data with the following format: 1541 1542 first label index compressed size in bytes (uint64 little endian) 1543 first label id (uint64 little endian) 1544 first label index protobuf (see definition in /index doc), lz4 compressed 1545 second label index compressed size in bytes (uint64 little endian) 1546 second label id (uint64 little endian) 1547 second label index protobuf, lz4 compressed 1548 ... 1549 1550 Although the label id is included in the protobuf, the stream includes the label 1551 of the record in case there is corruption of the record or other issues. So this 1552 call allows verifying the records are properly stored. 1553 1554 Missing labels will have 0 size and no bytes for the protobuf data. 1555 1556 If an error occurs, zeros will be transmitted for a label index size and a label id. 1557 1558 POST <api URL>/node/<UUID>/<data name>/indices 1559 1560 Allows bulk storage of indices (blocks per supervoxel and their voxel count) for any 1561 particular label. Typically, these indices are computed on-the-fly during ingestion of 1562 of blocks of label voxels. If there are cluster systems capable of computing label 1563 blocks, indices, and affinities directly, though, it's more efficient to simply load 1564 them into dvid. 1565 1566 The POST expects a protobuf serialization of a LabelIndices message defined by: 1567 1568 message LabelIndices { 1569 repeated LabelIndex indices = 1; 1570 } 1571 1572 where LabelIndex is defined by the protobuf in the /index endpoint documentation. 1573 A label index can be deleted as per the POST /index documentation by having an empty 1574 blocks map. 1575 1576 GET <api URL>/node/<UUID>/<data name>/mutations[?queryopts] 1577 1578 Returns JSON of the successfully completed mutations for the given version 1579 for this data instance. The format is equivalent to the JSON provided to 1580 the Kafka mutation log. For example, a merge mutation would be: 1581 1582 { 1583 "Action": "merge", 1584 "Target": <label ID>, 1585 "Labels": [<merged label 1>, <merged label 2>, ...], 1586 "UUID": "28841c8277e044a7b187dda03e18da13", 1587 "MutationID": <uint64>, 1588 "Timestamp": <string>, 1589 "User": <user id string if supplied during mutation>, 1590 "App": <app id string if supplied during mutation> 1591 } 1592 1593 Query-string options: 1594 1595 userid: Limit returned mutations to the given User ID. Note that 1596 this should be distinguished from the "u" query string which 1597 is the requester's User ID (not necessarily the same as the 1598 User ID whose mutations are being requested). 1599 1600 GET <api URL>/node/<UUID>/<data name>/mappings[?queryopts] 1601 1602 Streams space-delimited mappings for the given UUID, one mapping per line: 1603 1604 supervoxel0 mappedTo0 1605 supervoxel1 mappedTo1 1606 supervoxel2 mappedTo2 1607 ... 1608 supervoxelN mappedToN 1609 1610 Note that only non-identity mappings are presented unless the mapping is 1611 associated with some kind of mutation. The mapping contains not only the 1612 unique IDs of supervoxels but also newly created IDs for renumbered & 1613 cleaved bodies that will never overlap with supervoxel IDs. 1614 1615 The mapped label can be 0 in the following circumstances: 1616 * The label was a supervoxel ID that was split into two different unique IDs. 1617 * The label is used for a newly generated ID that will be a new renumbered label. 1618 * The label is used for a newly generated ID that will represent a cleaved body ID. 1619 1620 Query-string Options: 1621 1622 format: If format=binary, the data is returned as little-endian binary uint64 pairs, 1623 in the same order as shown in the CSV format above. 1624 1625 POST <api URL>/node/<UUID>/<data name>/mappings 1626 1627 Allows direct storing of merge maps for a particular UUID. Typically, merge 1628 maps are stored as part of POST /merge calls, which actually modify label 1629 index data. If there are cluster systems capable of computing label 1630 blocks, indices, mappings (to represent agglomerations) and affinities directly, 1631 though, it's more efficient to simply load them into dvid. 1632 1633 See the description of mappings in GET /mapping and /mappings. 1634 1635 The POST expects a protobuf serialization of a MergeOps message defined by: 1636 1637 message MappingOp { 1638 uint64 mutid = 1; 1639 uint64 mapped = 2; 1640 repeated uint64 original = 3; 1641 } 1642 1643 message MappingOps { 1644 repeated MappingOp mappings = 1; 1645 } 1646 1647 GET <api URL>/node/<UUID>/<data name>/map-stats 1648 1649 Returns JSON describing in-memory mapping stats. 1650 ` 1651 1652 var ( 1653 dtype Type 1654 encodeFormat dvid.DataValues 1655 1656 zeroLabelBytes = make([]byte, 8) 1657 1658 DefaultBlockSize int32 = 64 1659 DefaultRes float32 = imageblk.DefaultRes 1660 DefaultUnits = imageblk.DefaultUnits 1661 ) 1662 1663 // SparseVolFormat indicates the type of encoding used for sparse volume representation. 1664 type SparseVolFormat uint8 1665 1666 const ( 1667 // FormatLegacyRLE is Legacy RLE encoding with header that gives # spans. 1668 FormatLegacyRLE SparseVolFormat = iota 1669 1670 // FormatStreamingRLE specifies Streaming RLE encoding 1671 FormatStreamingRLE 1672 1673 // FormatBinaryBlocks specifies a streaming set of binary Blocks 1674 FormatBinaryBlocks 1675 ) 1676 1677 // returns default legacy RLE if no options set. 1678 func svformatFromQueryString(r *http.Request) SparseVolFormat { 1679 switch r.URL.Query().Get("format") { 1680 case "srles": 1681 return FormatStreamingRLE 1682 case "blocks": 1683 return FormatBinaryBlocks 1684 default: 1685 return FormatLegacyRLE 1686 } 1687 } 1688 1689 func init() { 1690 encodeFormat = dvid.DataValues{ 1691 { 1692 T: dvid.T_uint64, 1693 Label: TypeName, 1694 }, 1695 } 1696 interpolable := false 1697 dtype = Type{imageblk.NewType(encodeFormat, interpolable)} 1698 dtype.Type.Name = TypeName 1699 dtype.Type.URL = RepoURL 1700 dtype.Type.Version = Version 1701 1702 // See doc for package on why channels are segregated instead of interleaved. 1703 // Data types must be registered with the datastore to be used. 1704 datastore.Register(&dtype) 1705 1706 // Need to register types that will be used to fulfill interfaces. 1707 gob.Register(&Type{}) 1708 gob.Register(&Data{}) 1709 } 1710 1711 type bulkLoadInfo struct { 1712 filenames []string 1713 versionID dvid.VersionID 1714 offset dvid.Point 1715 extentChanged dvid.Bool 1716 } 1717 1718 // ZeroBytes returns a slice of bytes that represents the zero label. 1719 func ZeroBytes() []byte { 1720 return zeroLabelBytes 1721 } 1722 1723 func EncodeFormat() dvid.DataValues { 1724 return encodeFormat 1725 } 1726 1727 // --- Labels64 Datatype ----- 1728 1729 // Type uses imageblk data type by composition. 1730 type Type struct { 1731 imageblk.Type 1732 } 1733 1734 // --- TypeService interface --- 1735 1736 func (dtype *Type) NewDataService(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (datastore.DataService, error) { 1737 return NewData(uuid, id, name, c) 1738 } 1739 1740 func (dtype *Type) Help() string { 1741 return helpMessage 1742 } 1743 1744 // ------- 1745 1746 // GetByDataUUID returns a pointer to labelmap data given a data UUID. Returns error if not found. 1747 func GetByDataUUID(dataUUID dvid.UUID) (*Data, error) { 1748 source, err := datastore.GetDataByDataUUID(dataUUID) 1749 if err != nil { 1750 return nil, err 1751 } 1752 data, ok := source.(*Data) 1753 if !ok { 1754 return nil, fmt.Errorf("instance '%s' is not a labelmap datatype", source.DataName()) 1755 } 1756 return data, nil 1757 } 1758 1759 // GetByUUIDName returns a pointer to labelmap data given a UUID and data name. 1760 func GetByUUIDName(uuid dvid.UUID, name dvid.InstanceName) (*Data, error) { 1761 source, err := datastore.GetDataByUUIDName(uuid, name) 1762 if err != nil { 1763 return nil, err 1764 } 1765 data, ok := source.(*Data) 1766 if !ok { 1767 return nil, fmt.Errorf("instance '%s' is not a labelmap datatype", name) 1768 } 1769 return data, nil 1770 } 1771 1772 // GetByVersionName returns a pointer to labelmap data given a version and data name. 1773 func GetByVersionName(v dvid.VersionID, name dvid.InstanceName) (*Data, error) { 1774 source, err := datastore.GetDataByVersionName(v, name) 1775 if err != nil { 1776 return nil, err 1777 } 1778 data, ok := source.(*Data) 1779 if !ok { 1780 return nil, fmt.Errorf("instance '%s' is not a labelmap datatype", name) 1781 } 1782 return data, nil 1783 } 1784 1785 // ------- ExtData interface implementation ------------- 1786 1787 // Labels are voxels that have uint64 labels. 1788 type Labels struct { 1789 *imageblk.Voxels 1790 } 1791 1792 func (l *Labels) String() string { 1793 return fmt.Sprintf("Labels of size %s @ offset %s", l.Size(), l.StartPoint()) 1794 } 1795 1796 func (l *Labels) Interpolable() bool { 1797 return false 1798 } 1799 1800 // Data of labelmap type is an extended form of imageblk Data 1801 type Data struct { 1802 *imageblk.Data 1803 datastore.Updater 1804 1805 // The maximum label id found in each version of this instance. 1806 // Can be unset if no new label was added at that version, in which case 1807 // you must traverse DAG to find max label of parent. 1808 MaxLabel map[dvid.VersionID]uint64 1809 1810 // The maximum label for this instance in the entire repo. This allows us to do 1811 // conflict-free merges without any relabeling. Note that relabeling (rebasing) 1812 // is required if we move data between repos, e.g., when pushing remote nodes, 1813 // since we have no control over which labels were created remotely and there 1814 // could be conflicts between the local and remote repos. When mutations only 1815 // occur within a single repo, however, this atomic label allows us to prevent 1816 // conflict across all versions within this repo. 1817 MaxRepoLabel uint64 1818 1819 // The next label ID to be returned for new labels. This can override MaxRepoLabel 1820 // if it is non-zero. If zero, it is not used. This was added to allow new 1821 // labels to be assigned that have lower IDs than existing labels. 1822 NextLabel uint64 1823 1824 // True if sparse volumes (split, merge, sparse volume optimized GET) are supported 1825 // for this data instance. (Default true) 1826 IndexedLabels bool 1827 1828 // Maximum down-resolution level supported. Each down-res level is 2x scope of 1829 // the higher level. 1830 MaxDownresLevel uint8 1831 1832 updates []uint32 // tracks updating to each scale of labelmap [0:MaxDownresLevel+1] 1833 updateMu sync.RWMutex 1834 1835 mlMu sync.RWMutex // For atomic access of MaxLabel and MaxRepoLabel 1836 1837 voxelMu sync.Mutex // Only allow voxel-level label mutation ops sequentially. 1838 } 1839 1840 // --- Override of DataService interface --- 1841 1842 func (d *Data) modifyConfig(config dvid.Config) error { 1843 if err := d.Data.ModifyConfig(config); err != nil { 1844 return err 1845 } 1846 s, found, err := config.GetString("MaxDownresLevel") 1847 if err != nil { 1848 return err 1849 } 1850 if found { 1851 maxDownresLevel, err := strconv.ParseUint(s, 10, 8) 1852 if err != nil { 1853 return err 1854 } 1855 d.MaxDownresLevel = uint8(maxDownresLevel) 1856 } 1857 return nil 1858 } 1859 1860 // --- LogReadable interface --- 1861 1862 func (d *Data) ReadLogRequired() bool { 1863 return true 1864 } 1865 1866 // --- LogWritable interface --- 1867 1868 func (d *Data) WriteLogRequired() bool { 1869 return true 1870 } 1871 1872 // GetMaxDownresLevel returns the number of down-res levels, where level 0 = high-resolution 1873 // and each subsequent level has one-half the resolution. 1874 func (d *Data) GetMaxDownresLevel() uint8 { 1875 return d.MaxDownresLevel 1876 } 1877 1878 func (d *Data) StartScaleUpdate(scale uint8) { 1879 d.updateMu.Lock() 1880 d.updates[scale]++ 1881 d.updateMu.Unlock() 1882 } 1883 1884 func (d *Data) StopScaleUpdate(scale uint8) { 1885 d.updateMu.Lock() 1886 if d.updates[scale] == 0 { 1887 dvid.Criticalf("StopScaleUpdate(%d) called more than StartScaleUpdate.\n", scale) 1888 } 1889 d.updates[scale]-- 1890 d.updateMu.Unlock() 1891 } 1892 1893 func (d *Data) ScaleUpdating(scale uint8) bool { 1894 d.updateMu.RLock() 1895 updating := d.updates[scale] > 0 1896 d.updateMu.RUnlock() 1897 return updating 1898 } 1899 1900 func (d *Data) AnyScaleUpdating() bool { 1901 d.updateMu.RLock() 1902 for scale := uint8(0); scale < d.MaxDownresLevel; scale++ { 1903 if d.updates[scale] > 0 { 1904 d.updateMu.RUnlock() 1905 return true 1906 } 1907 } 1908 d.updateMu.RUnlock() 1909 return false 1910 } 1911 1912 // CopyPropertiesFrom copies the data instance-specific properties from a given 1913 // data instance into the receiver's properties. Fulfills the datastore.PropertyCopier interface. 1914 func (d *Data) CopyPropertiesFrom(src datastore.DataService, fs storage.FilterSpec) error { 1915 d2, ok := src.(*Data) 1916 if !ok { 1917 return fmt.Errorf("unable to copy properties from non-labelmap data %q", src.DataName()) 1918 } 1919 1920 // TODO -- Handle mutable data that could be potentially altered by filter. 1921 d.MaxLabel = make(map[dvid.VersionID]uint64, len(d2.MaxLabel)) 1922 for k, v := range d2.MaxLabel { 1923 d.MaxLabel[k] = v 1924 } 1925 d.MaxRepoLabel = d2.MaxRepoLabel 1926 d.NextLabel = d2.NextLabel 1927 1928 d.IndexedLabels = d2.IndexedLabels 1929 d.MaxDownresLevel = d2.MaxDownresLevel 1930 1931 return d.Data.CopyPropertiesFrom(d2.Data, fs) 1932 } 1933 1934 // NewData returns a pointer to labelmap data. 1935 func NewData(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (*Data, error) { 1936 blockSizeStr, found, err := c.GetString("BlockSize") 1937 if err != nil { 1938 return nil, err 1939 } 1940 if found { 1941 var nx, ny, nz int 1942 fmt.Sscanf(blockSizeStr, "%d,%d,%d", &nx, &ny, &nz) 1943 if nx%16 != 0 || ny%16 != 0 || nz%16 != 0 { 1944 return nil, fmt.Errorf("BlockSize must be multiples of 16") 1945 } 1946 } else { 1947 c.Set("BlockSize", fmt.Sprintf("%d,%d,%d", DefaultBlockSize, DefaultBlockSize, DefaultBlockSize)) 1948 } 1949 if _, found := c.Get("Compression"); !found { 1950 c.Set("Compression", "gzip") 1951 } 1952 // Note that labelmap essentially piggybacks off the associated imageblk Data. 1953 imgblkData, err := dtype.Type.NewData(uuid, id, name, c) 1954 if err != nil { 1955 return nil, err 1956 } 1957 if imgblkData.GetWriteLog() == nil { 1958 return nil, fmt.Errorf("labelmap %q instance needs a writable log yet has none assigned", name) 1959 } 1960 if imgblkData.GetReadLog() == nil { 1961 return nil, fmt.Errorf("labelmap %q instance needs a readable log yet has none assigned", name) 1962 } 1963 1964 data := &Data{ 1965 Data: imgblkData, 1966 } 1967 indexedLabels := true 1968 b, found, err := c.GetBool("IndexedLabels") 1969 if err != nil { 1970 return nil, err 1971 } 1972 if found { 1973 indexedLabels = b 1974 } 1975 1976 var downresLevels uint8 1977 levels, found, err := c.GetInt("MaxDownresLevel") 1978 if err != nil { 1979 return nil, err 1980 } 1981 if found { 1982 if levels < 0 || levels > 255 { 1983 return nil, fmt.Errorf("illegal number of down-res levels specified (%d): must be 0 <= n <= 255", levels) 1984 } 1985 downresLevels = uint8(levels) 1986 } 1987 data.updates = make([]uint32, downresLevels+1) 1988 1989 data.MaxLabel = make(map[dvid.VersionID]uint64) 1990 data.IndexedLabels = indexedLabels 1991 data.MaxDownresLevel = downresLevels 1992 1993 data.Initialize() 1994 return data, nil 1995 } 1996 1997 type propsJSON struct { 1998 imageblk.Properties 1999 MaxLabel map[dvid.VersionID]uint64 2000 MaxRepoLabel uint64 2001 NextLabel uint64 2002 IndexedLabels bool 2003 MaxDownresLevel uint8 2004 } 2005 2006 func (d *Data) MarshalJSON() ([]byte, error) { 2007 vctx, err := datastore.NewVersionedCtxMasterLeaf(d) 2008 if err != nil { 2009 return json.Marshal(struct { 2010 Base *datastore.Data 2011 Extended propsJSON 2012 }{ 2013 d.Data.Data, 2014 propsJSON{ 2015 Properties: d.Data.Properties, 2016 MaxLabel: d.MaxLabel, 2017 MaxRepoLabel: d.MaxRepoLabel, 2018 NextLabel: d.NextLabel, 2019 IndexedLabels: d.IndexedLabels, 2020 MaxDownresLevel: d.MaxDownresLevel, 2021 }, 2022 }) 2023 } 2024 return d.MarshalJSONExtents(vctx) 2025 } 2026 2027 func (d *Data) MarshalJSONExtents(ctx *datastore.VersionedCtx) ([]byte, error) { 2028 // grab extent property and load 2029 extents, err := d.GetExtents(ctx) 2030 if err != nil { 2031 return nil, err 2032 } 2033 2034 var extentsJSON imageblk.ExtentsJSON 2035 extentsJSON.MinPoint = extents.MinPoint 2036 extentsJSON.MaxPoint = extents.MaxPoint 2037 2038 props, err := d.PropertiesWithExtents(ctx) 2039 if err != nil { 2040 return nil, err 2041 } 2042 return json.Marshal(struct { 2043 Base *datastore.Data 2044 Extended propsJSON 2045 Extents imageblk.ExtentsJSON 2046 }{ 2047 d.Data.Data, 2048 propsJSON{ 2049 Properties: props, 2050 MaxLabel: d.MaxLabel, 2051 MaxRepoLabel: d.MaxRepoLabel, 2052 NextLabel: d.NextLabel, 2053 IndexedLabels: d.IndexedLabels, 2054 MaxDownresLevel: d.MaxDownresLevel, 2055 }, 2056 extentsJSON, 2057 }) 2058 } 2059 2060 func (d *Data) GobDecode(b []byte) error { 2061 buf := bytes.NewBuffer(b) 2062 dec := gob.NewDecoder(buf) 2063 if err := dec.Decode(&(d.Data)); err != nil { 2064 return err 2065 } 2066 if err := dec.Decode(&(d.IndexedLabels)); err != nil { 2067 dvid.Errorf("Decoding labelmap %q: no IndexedLabels, setting to true", d.DataName()) 2068 d.IndexedLabels = true 2069 } 2070 if err := dec.Decode(&(d.MaxDownresLevel)); err != nil { 2071 dvid.Errorf("Decoding labelmap %q: no MaxDownresLevel, setting to 7", d.DataName()) 2072 d.MaxDownresLevel = 7 2073 } 2074 d.updates = make([]uint32, d.MaxDownresLevel+1) 2075 return nil 2076 } 2077 2078 func (d *Data) GobEncode() ([]byte, error) { 2079 var buf bytes.Buffer 2080 enc := gob.NewEncoder(&buf) 2081 if err := enc.Encode(d.Data); err != nil { 2082 return nil, err 2083 } 2084 if err := enc.Encode(d.IndexedLabels); err != nil { 2085 return nil, err 2086 } 2087 if err := enc.Encode(d.MaxDownresLevel); err != nil { 2088 return nil, err 2089 } 2090 return buf.Bytes(), nil 2091 } 2092 2093 // setResolution loads JSON data to set VoxelSize property. 2094 func (d *Data) setResolution(uuid dvid.UUID, jsonBytes []byte) error { 2095 config := make(dvid.NdFloat32, 3) 2096 if err := json.Unmarshal(jsonBytes, &config); err != nil { 2097 return err 2098 } 2099 d.Properties.VoxelSize = config 2100 return datastore.SaveDataByUUID(uuid, d) 2101 } 2102 2103 // makes database call for any update 2104 func (d *Data) updateMaxLabel(v dvid.VersionID, label uint64) (changed bool, err error) { 2105 d.mlMu.RLock() 2106 curMax, found := d.MaxLabel[v] 2107 if !found || curMax < label { 2108 changed = true 2109 } 2110 d.mlMu.RUnlock() 2111 if !changed { 2112 return 2113 } 2114 2115 d.mlMu.Lock() 2116 defer d.mlMu.Unlock() 2117 2118 d.MaxLabel[v] = label 2119 if err = d.persistMaxLabel(v); err != nil { 2120 err = fmt.Errorf("updateMaxLabel of data %q: %v", d.DataName(), err) 2121 return 2122 } 2123 if label > d.MaxRepoLabel { 2124 d.MaxRepoLabel = label 2125 if err = d.persistMaxRepoLabel(); err != nil { 2126 err = fmt.Errorf("updateMaxLabel of data %q: %v", d.DataName(), err) 2127 return 2128 } 2129 } 2130 return 2131 } 2132 2133 // makes database call for any update 2134 func (d *Data) updateBlockMaxLabel(v dvid.VersionID, block *labels.Block) { 2135 var changed bool 2136 d.mlMu.RLock() 2137 curMax, found := d.MaxLabel[v] 2138 d.mlMu.RUnlock() 2139 if !found { 2140 curMax = 0 2141 } 2142 for _, label := range block.Labels { 2143 if label > curMax { 2144 curMax = label 2145 changed = true 2146 } 2147 } 2148 if changed { 2149 d.mlMu.Lock() 2150 d.MaxLabel[v] = curMax 2151 if err := d.persistMaxLabel(v); err != nil { 2152 dvid.Errorf("updateBlockMaxLabel of data %q: %v\n", d.DataName(), err) 2153 } 2154 if curMax > d.MaxRepoLabel { 2155 d.MaxRepoLabel = curMax 2156 if err := d.persistMaxRepoLabel(); err != nil { 2157 dvid.Errorf("updateBlockMaxLabel of data %q: %v\n", d.DataName(), err) 2158 } 2159 } 2160 d.mlMu.Unlock() 2161 } 2162 } 2163 2164 func (d *Data) Equals(d2 *Data) bool { 2165 return d.Data.Equals(d2.Data) 2166 } 2167 2168 func (d *Data) persistMaxLabel(v dvid.VersionID) error { 2169 store, err := datastore.GetOrderedKeyValueDB(d) 2170 if err != nil { 2171 return err 2172 } 2173 if len(d.MaxLabel) == 0 { 2174 return fmt.Errorf("bad attempt to save non-existent max label for version %d", v) 2175 } 2176 buf := make([]byte, 8) 2177 binary.LittleEndian.PutUint64(buf, d.MaxLabel[v]) 2178 ctx := datastore.NewVersionedCtx(d, v) 2179 return store.Put(ctx, maxLabelTKey, buf) 2180 } 2181 2182 func (d *Data) persistMaxRepoLabel() error { 2183 store, err := datastore.GetOrderedKeyValueDB(d) 2184 if err != nil { 2185 return err 2186 } 2187 buf := make([]byte, 8) 2188 binary.LittleEndian.PutUint64(buf, d.MaxRepoLabel) 2189 ctx := storage.NewDataContext(d, 0) 2190 return store.Put(ctx, maxRepoLabelTKey, buf) 2191 } 2192 2193 func (d *Data) persistNextLabel() error { 2194 store, err := datastore.GetOrderedKeyValueDB(d) 2195 if err != nil { 2196 return err 2197 } 2198 buf := make([]byte, 8) 2199 binary.LittleEndian.PutUint64(buf, d.NextLabel) 2200 ctx := storage.NewDataContext(d, 0) 2201 return store.Put(ctx, nextLabelTKey, buf) 2202 } 2203 2204 // newLabel returns a new label for the given version. 2205 func (d *Data) newLabel(v dvid.VersionID) (uint64, error) { 2206 d.mlMu.Lock() 2207 defer d.mlMu.Unlock() 2208 2209 // Increment and store if we don't have an ephemeral new label start ID. 2210 if d.NextLabel != 0 { 2211 d.NextLabel++ 2212 if err := d.persistNextLabel(); err != nil { 2213 return d.NextLabel, err 2214 } 2215 return d.NextLabel, nil 2216 } 2217 d.MaxRepoLabel++ 2218 d.MaxLabel[v] = d.MaxRepoLabel 2219 if err := d.persistMaxLabel(v); err != nil { 2220 return d.MaxRepoLabel, err 2221 } 2222 if err := d.persistMaxRepoLabel(); err != nil { 2223 return d.MaxRepoLabel, err 2224 } 2225 return d.MaxRepoLabel, nil 2226 } 2227 2228 // newLabels returns a span of new labels for the given version 2229 func (d *Data) newLabels(v dvid.VersionID, numLabels uint64) (begin, end uint64, err error) { 2230 if numLabels <= 0 { 2231 err = fmt.Errorf("cannot request %d new labels, must be 1 or more", numLabels) 2232 } 2233 d.mlMu.Lock() 2234 defer d.mlMu.Unlock() 2235 2236 // Increment and store. 2237 if d.NextLabel != 0 { 2238 begin = d.NextLabel + 1 2239 end = d.NextLabel + numLabels 2240 d.NextLabel = end 2241 if err = d.persistNextLabel(); err != nil { 2242 return 2243 } 2244 return 2245 } 2246 begin = d.MaxRepoLabel + 1 2247 end = d.MaxRepoLabel + numLabels 2248 d.MaxRepoLabel = end 2249 d.MaxLabel[v] = d.MaxRepoLabel 2250 if err = d.persistMaxLabel(v); err != nil { 2251 return 2252 } 2253 if err = d.persistMaxRepoLabel(); err != nil { 2254 return 2255 } 2256 return 2257 } 2258 2259 // SetNextLabelStart sets the next label ID for this labelmap instance across 2260 // the entire repo. 2261 func (d *Data) SetNextLabelStart(nextLabelID uint64) error { 2262 d.NextLabel = nextLabelID 2263 if err := d.persistNextLabel(); err != nil { 2264 return err 2265 } 2266 return nil 2267 } 2268 2269 // --- datastore.InstanceMutator interface ----- 2270 2271 // LoadMutable loads mutable properties of label volumes like the maximum labels 2272 // for each version. Note that we load these max and next labels from key-value pairs 2273 // rather than data instance properties persistence, because in the case of a crash, 2274 // the actually stored repo data structure may be out-of-date compared to the guaranteed 2275 // up-to-date key-value pairs for max labels. 2276 func (d *Data) LoadMutable(root dvid.VersionID, storedVersion, expectedVersion uint64) (bool, error) { 2277 ctx := storage.NewDataContext(d, 0) 2278 store, err := datastore.GetOrderedKeyValueDB(d) 2279 if err != nil { 2280 return false, fmt.Errorf("Data type labelmap had error initializing store: %v", err) 2281 } 2282 2283 wg := new(sync.WaitGroup) 2284 wg.Add(1) 2285 ch := make(chan *storage.KeyValue) 2286 2287 // Start appropriate migration function if any. 2288 var saveRequired bool 2289 go d.loadLabelIDs(wg, ch) 2290 2291 // Send the max label data per version 2292 minKey, err := ctx.MinVersionKey(maxLabelTKey) 2293 if err != nil { 2294 return false, err 2295 } 2296 maxKey, err := ctx.MaxVersionKey(maxLabelTKey) 2297 if err != nil { 2298 return false, err 2299 } 2300 keysOnly := false 2301 if err = store.RawRangeQuery(minKey, maxKey, keysOnly, ch, nil); err != nil { 2302 return false, err 2303 } 2304 wg.Wait() 2305 2306 dvid.Infof("Loaded max label values for labelmap %q with repo-wide max %d\n", d.DataName(), d.MaxRepoLabel) 2307 if d.NextLabel != 0 { 2308 dvid.Infof("Loaded next label id for labelmap %q: %d\n", d.DataName(), d.NextLabel) 2309 } 2310 return saveRequired, nil 2311 } 2312 2313 const veryLargeLabel = 10000000000 // 10 billion 2314 2315 func (d *Data) loadLabelIDs(wg *sync.WaitGroup, ch chan *storage.KeyValue) { 2316 ctx := storage.NewDataContext(d, 0) 2317 var repoMax uint64 2318 d.MaxLabel = make(map[dvid.VersionID]uint64) 2319 for { 2320 kv := <-ch 2321 if kv == nil { 2322 break 2323 } 2324 v, err := ctx.VersionFromKey(kv.K) 2325 if err != nil { 2326 dvid.Errorf("Can't decode key when loading mutable data for %s", d.DataName()) 2327 continue 2328 } 2329 var label uint64 = veryLargeLabel 2330 if len(kv.V) != 8 { 2331 dvid.Errorf("Got bad value. Expected 64-bit label, got %v", kv.V) 2332 } else { 2333 label = binary.LittleEndian.Uint64(kv.V) 2334 } 2335 d.MaxLabel[v] = label 2336 if label > repoMax { 2337 repoMax = label 2338 } 2339 } 2340 2341 // Load in the repo-wide max label. 2342 store, err := datastore.GetOrderedKeyValueDB(d) 2343 if err != nil { 2344 dvid.Errorf("Data type labelmap had error initializing store: %v\n", err) 2345 return 2346 } 2347 data, err := store.Get(ctx, maxRepoLabelTKey) 2348 if err != nil { 2349 dvid.Errorf("Error getting repo-wide max label: %v\n", err) 2350 return 2351 } 2352 if data == nil || len(data) != 8 { 2353 dvid.Errorf("Could not load repo-wide max label for instance %q. Only got %d bytes, not 64-bit label.\n", d.DataName(), len(data)) 2354 if repoMax == 0 { 2355 repoMax = veryLargeLabel 2356 } 2357 dvid.Errorf("Using max label across versions: %d\n", repoMax) 2358 d.MaxRepoLabel = repoMax 2359 } else { 2360 d.MaxRepoLabel = binary.LittleEndian.Uint64(data) 2361 if d.MaxRepoLabel < repoMax { 2362 dvid.Errorf("Saved repo-wide max for instance %q was %d, changed to largest version max %d\n", d.DataName(), d.MaxRepoLabel, repoMax) 2363 d.MaxRepoLabel = repoMax 2364 } 2365 } 2366 2367 // Load in next label if set. 2368 data, err = store.Get(ctx, nextLabelTKey) 2369 if err != nil { 2370 dvid.Errorf("Error getting repo-wide next label: %v\n", err) 2371 return 2372 } 2373 if data == nil || len(data) != 8 { 2374 dvid.Errorf("Could not load repo-wide next label for instance %q. No next label override of max label.\n", d.DataName()) 2375 } else { 2376 d.NextLabel = binary.LittleEndian.Uint64(data) 2377 } 2378 2379 wg.Done() 2380 } 2381 2382 // --- imageblk.IntData interface ------------- 2383 2384 func (d *Data) BlockSize() dvid.Point { 2385 return d.Properties.BlockSize 2386 } 2387 2388 func (d *Data) Extents() *dvid.Extents { 2389 return &(d.Properties.Extents) 2390 } 2391 2392 // NewLabels returns labelmap Labels, a representation of externally usable subvolume 2393 // or slice data, given some geometry and optional image data. 2394 // If img is passed in, the function will initialize Voxels with data from the image. 2395 // Otherwise, it will allocate a zero buffer of appropriate size. 2396 func (d *Data) NewLabels(geom dvid.Geometry, img interface{}) (*Labels, error) { 2397 bytesPerVoxel := d.Properties.Values.BytesPerElement() 2398 stride := geom.Size().Value(0) * bytesPerVoxel 2399 var data []byte 2400 2401 if img == nil { 2402 numVoxels := geom.NumVoxels() 2403 if numVoxels <= 0 { 2404 return nil, fmt.Errorf("illegal geometry requested: %s", geom) 2405 } 2406 requestSize := int64(bytesPerVoxel) * numVoxels 2407 if requestSize > server.MaxDataRequest { 2408 return nil, fmt.Errorf("requested payload (%d bytes) exceeds this DVID server's set limit (%d)", 2409 requestSize, server.MaxDataRequest) 2410 } 2411 data = make([]byte, requestSize) 2412 } else { 2413 switch t := img.(type) { 2414 case image.Image: 2415 var inputBytesPerVoxel, actualStride int32 2416 var err error 2417 data, inputBytesPerVoxel, actualStride, err = dvid.ImageData(t) 2418 if err != nil { 2419 return nil, err 2420 } 2421 if actualStride != stride { 2422 data, err = d.convertTo64bit(geom, data, int(inputBytesPerVoxel), int(actualStride)) 2423 if err != nil { 2424 return nil, err 2425 } 2426 } 2427 case []byte: 2428 data = t 2429 actualLen := int64(len(data)) 2430 expectedLen := int64(bytesPerVoxel) * geom.NumVoxels() 2431 if actualLen != expectedLen { 2432 return nil, fmt.Errorf("labels data was %d bytes, expected %d bytes for %s", 2433 actualLen, expectedLen, geom) 2434 } 2435 default: 2436 return nil, fmt.Errorf("unexpected image type given to NewVoxels(): %T", t) 2437 } 2438 } 2439 2440 labels := &Labels{ 2441 imageblk.NewVoxels(geom, d.Properties.Values, data, stride), 2442 } 2443 return labels, nil 2444 } 2445 2446 // Convert raw image data into a 2d array of 64-bit labels 2447 func (d *Data) convertTo64bit(geom dvid.Geometry, data []uint8, bytesPerVoxel, stride int) ([]byte, error) { 2448 nx := int(geom.Size().Value(0)) 2449 ny := int(geom.Size().Value(1)) 2450 numBytes := nx * ny * 8 2451 data64 := make([]byte, numBytes) 2452 2453 var byteOrder binary.ByteOrder 2454 if geom.DataShape().ShapeDimensions() == 2 { 2455 byteOrder = binary.BigEndian // This is the default for PNG 2456 } else { 2457 byteOrder = binary.LittleEndian 2458 } 2459 2460 switch bytesPerVoxel { 2461 case 1: 2462 dstI := 0 2463 for y := 0; y < ny; y++ { 2464 srcI := y * stride 2465 for x := 0; x < nx; x++ { 2466 binary.LittleEndian.PutUint64(data64[dstI:dstI+8], uint64(data[srcI])) 2467 srcI++ 2468 dstI += 8 2469 } 2470 } 2471 case 2: 2472 dstI := 0 2473 for y := 0; y < ny; y++ { 2474 srcI := y * stride 2475 for x := 0; x < nx; x++ { 2476 value := byteOrder.Uint16(data[srcI : srcI+2]) 2477 binary.LittleEndian.PutUint64(data64[dstI:dstI+8], uint64(value)) 2478 srcI += 2 2479 dstI += 8 2480 } 2481 } 2482 case 4: 2483 dstI := 0 2484 for y := 0; y < ny; y++ { 2485 srcI := y * stride 2486 for x := 0; x < nx; x++ { 2487 value := byteOrder.Uint32(data[srcI : srcI+4]) 2488 binary.LittleEndian.PutUint64(data64[dstI:dstI+8], uint64(value)) 2489 srcI += 4 2490 dstI += 8 2491 } 2492 } 2493 case 8: 2494 dstI := 0 2495 for y := 0; y < ny; y++ { 2496 srcI := y * stride 2497 for x := 0; x < nx; x++ { 2498 value := byteOrder.Uint64(data[srcI : srcI+8]) 2499 binary.LittleEndian.PutUint64(data64[dstI:dstI+8], uint64(value)) 2500 srcI += 8 2501 dstI += 8 2502 } 2503 } 2504 default: 2505 return nil, fmt.Errorf("could not convert to 64-bit label given %d bytes/voxel", bytesPerVoxel) 2506 } 2507 return data64, nil 2508 } 2509 2510 // --- datastore.DataService interface --------- 2511 2512 // PushData pushes labelmap data to a remote DVID. 2513 func (d *Data) PushData(p *datastore.PushSession) error { 2514 // Delegate to imageblk's implementation. 2515 return d.Data.PushData(p) 2516 } 2517 2518 // DoRPC acts as a switchboard for RPC commands. 2519 func (d *Data) DoRPC(req datastore.Request, reply *datastore.Response) error { 2520 switch req.TypeCommand() { 2521 case "set-nextlabel": 2522 if len(req.Command) < 5 { 2523 return fmt.Errorf("poorly formatted set-nextlabel command, see command-line help") 2524 } 2525 2526 // Parse the request 2527 var uuidStr, dataName, cmdStr, labelStr string 2528 req.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &labelStr) 2529 2530 uuid, _, err := datastore.MatchingUUID(uuidStr) 2531 if err != nil { 2532 return err 2533 } 2534 2535 dataservice, err := datastore.GetDataByUUIDName(uuid, dvid.InstanceName(dataName)) 2536 if err != nil { 2537 return err 2538 } 2539 lmData, ok := dataservice.(*Data) 2540 if !ok { 2541 return fmt.Errorf("instance %q of uuid %s was not a labelmap instance", dataName, uuid) 2542 } 2543 2544 nextLabelID, err := strconv.ParseUint(labelStr, 10, 64) 2545 if err != nil { 2546 return err 2547 } 2548 2549 if err := lmData.SetNextLabelStart(nextLabelID); err != nil { 2550 return err 2551 } 2552 2553 reply.Text = fmt.Sprintf("Set next label ID to %d.\n", nextLabelID) 2554 return nil 2555 2556 case "load": 2557 if len(req.Command) < 5 { 2558 return fmt.Errorf("poorly formatted load command, see command-line help") 2559 } 2560 // Parse the request 2561 var uuidStr, dataName, cmdStr, offsetStr string 2562 filenames, err := req.FilenameArgs(1, &uuidStr, &dataName, &cmdStr, &offsetStr) 2563 if err != nil { 2564 return err 2565 } 2566 if len(filenames) == 0 { 2567 return fmt.Errorf("need to include at least one file to add: %s", req) 2568 } 2569 2570 offset, err := dvid.StringToPoint(offsetStr, ",") 2571 if err != nil { 2572 return fmt.Errorf("illegal offset specification: %s: %v", offsetStr, err) 2573 } 2574 2575 var addedFiles string 2576 if len(filenames) == 1 { 2577 addedFiles = filenames[0] 2578 } else { 2579 addedFiles = fmt.Sprintf("filenames: %s [%d more]", filenames[0], len(filenames)-1) 2580 } 2581 dvid.Debugf(addedFiles + "\n") 2582 2583 uuid, versionID, err := datastore.MatchingUUID(uuidStr) 2584 if err != nil { 2585 return err 2586 } 2587 if err = datastore.AddToNodeLog(uuid, []string{req.Command.String()}); err != nil { 2588 return err 2589 } 2590 go func() { 2591 if err = d.LoadImages(versionID, offset, filenames); err != nil { 2592 dvid.Errorf("Cannot load images into data instance %q @ node %s: %v\n", dataName, uuidStr, err) 2593 } 2594 if err := datastore.SaveDataByUUID(uuid, d); err != nil { 2595 dvid.Errorf("Could not store metadata changes into data instance %q @ node %s: %v\n", dataName, uuidStr, err) 2596 } 2597 }() 2598 reply.Text = fmt.Sprintf("Asynchronously loading %d files into data instance %q @ node %s (errors will be printed in server log) ...\n", len(filenames), dataName, uuidStr) 2599 return nil 2600 2601 case "composite": 2602 if len(req.Command) < 6 { 2603 return fmt.Errorf("poorly formatted composite command. See command-line help") 2604 } 2605 return d.createComposite(req, reply) 2606 2607 case "dump": 2608 if len(req.Command) < 6 { 2609 return fmt.Errorf("poorly formatted dump command. See command-line help") 2610 } 2611 // Parse the request 2612 var uuidStr, dataName, cmdStr, dumpType, outPath string 2613 req.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &dumpType, &outPath) 2614 2615 uuid, v, err := datastore.MatchingUUID(uuidStr) 2616 if err != nil { 2617 return err 2618 } 2619 2620 // Setup output file 2621 f, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) 2622 if err != nil { 2623 return err 2624 } 2625 switch dumpType { 2626 case "svcount": 2627 go d.writeSVCounts(f, outPath, v) 2628 reply.Text = fmt.Sprintf("Asynchronously writing supervoxel counts for data %q, uuid %s to file: %s\n", d.DataName(), uuid, outPath) 2629 case "mappings": 2630 go d.writeFileMappings(f, outPath, v) 2631 reply.Text = fmt.Sprintf("Asynchronously writing mappings for data %q, uuid %s to file: %s\n", d.DataName(), uuid, outPath) 2632 case "indices": 2633 go d.writeIndices(f, outPath, v) 2634 reply.Text = fmt.Sprintf("Asynchronously writing label indices for data %q, uuid %s to file: %s\n", d.DataName(), uuid, outPath) 2635 default: 2636 } 2637 return nil 2638 2639 default: 2640 return fmt.Errorf("unknown command. Data type '%s' [%s] does not support '%s' command", 2641 d.DataName(), d.TypeName(), req.TypeCommand()) 2642 } 2643 } 2644 2645 func colorImage(labels *dvid.Image) (image.Image, error) { 2646 if labels == nil || labels.Which != 3 || labels.NRGBA64 == nil { 2647 return nil, fmt.Errorf("writePseudoColor can't use labels image with wrong format: %v", labels) 2648 } 2649 src := labels.NRGBA64 2650 srcRect := src.Bounds() 2651 srcW := srcRect.Dx() 2652 srcH := srcRect.Dy() 2653 2654 dst := image.NewNRGBA(image.Rect(0, 0, srcW, srcH)) 2655 2656 for y := 0; y < srcH; y++ { 2657 srcI := src.PixOffset(0, y) 2658 dstI := dst.PixOffset(0, y) 2659 for x := 0; x < srcW; x++ { 2660 murmurhash3(src.Pix[srcI:srcI+8], dst.Pix[dstI:dstI+4]) 2661 dst.Pix[dstI+3] = 255 2662 2663 srcI += 8 2664 dstI += 4 2665 } 2666 } 2667 return dst, nil 2668 } 2669 2670 // if hash is not empty, make sure it is hash of data. 2671 func checkContentHash(hash string, data []byte) error { 2672 if hash == "" { 2673 return nil 2674 } 2675 hexHash := fmt.Sprintf("%x", md5.Sum(data)) 2676 if hexHash != hash { 2677 return fmt.Errorf("content hash incorrect. expected %s, got %s", hash, hexHash) 2678 } 2679 return nil 2680 } 2681 2682 func getScale(queryStrings url.Values) (scale uint8, err error) { 2683 scaleStr := queryStrings.Get("scale") 2684 if scaleStr != "" { 2685 var scaleInt int 2686 scaleInt, err = strconv.Atoi(scaleStr) 2687 if err != nil { 2688 return 2689 } 2690 scale = uint8(scaleInt) 2691 } 2692 return 2693 } 2694 2695 // ServeHTTP handles all incoming HTTP requests for this data. 2696 func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) (activity map[string]interface{}) { 2697 // Get the action (GET, POST) 2698 action := strings.ToLower(r.Method) 2699 2700 // Break URL request into arguments 2701 url := r.URL.Path[len(server.WebAPIPath):] 2702 parts := strings.Split(url, "/") 2703 if len(parts[len(parts)-1]) == 0 { 2704 parts = parts[:len(parts)-1] 2705 } 2706 2707 if len(parts) < 4 { 2708 server.BadRequest(w, r, "Incomplete API request") 2709 return 2710 } 2711 2712 // Prevent use of APIs that require IndexedLabels when it is not set. 2713 if !d.IndexedLabels { 2714 switch parts[3] { 2715 case "sparsevol", "sparsevol-by-point", "sparsevol-coarse", "maxlabel", "nextlabel", "split-supervoxel", "cleave", "merge": 2716 server.BadRequest(w, r, "data %q is not label indexed (IndexedLabels=false): %q endpoint is not supported", d.DataName(), parts[3]) 2717 return 2718 } 2719 } 2720 2721 // Handle all requests 2722 switch parts[3] { 2723 case "help": 2724 w.Header().Set("Content-Type", "text/plain") 2725 fmt.Fprintln(w, dtype.Help()) 2726 2727 case "metadata": 2728 jsonStr, err := d.NdDataMetadata(ctx) 2729 if err != nil { 2730 server.BadRequest(w, r, err) 2731 return 2732 } 2733 w.Header().Set("Content-Type", "application/vnd.dvid-nd-data+json") 2734 fmt.Fprintln(w, jsonStr) 2735 2736 case "extents": 2737 jsonBytes, err := ioutil.ReadAll(r.Body) 2738 if err != nil { 2739 server.BadRequest(w, r, err) 2740 return 2741 } 2742 if err := d.SetExtents(ctx, uuid, jsonBytes); err != nil { 2743 server.BadRequest(w, r, err) 2744 return 2745 } 2746 2747 case "resolution": 2748 jsonBytes, err := ioutil.ReadAll(r.Body) 2749 if err != nil { 2750 server.BadRequest(w, r, err) 2751 return 2752 } 2753 if err := d.setResolution(uuid, jsonBytes); err != nil { 2754 server.BadRequest(w, r, err) 2755 return 2756 } 2757 2758 case "map-stats": 2759 if action != "get" { 2760 server.BadRequest(w, r, "only GET available for endpoint /map-stats") 2761 return 2762 } 2763 jsonBytes, err := d.GetMapStats(ctx) 2764 if err != nil { 2765 server.BadRequest(w, r, err) 2766 return 2767 } 2768 w.Header().Set("Content-Type", "application/json") 2769 fmt.Fprint(w, string(jsonBytes)) 2770 2771 case "info": 2772 if action == "post" { 2773 config, err := server.DecodeJSON(r) 2774 if err != nil { 2775 server.BadRequest(w, r, err) 2776 return 2777 } 2778 if err := d.modifyConfig(config); err != nil { 2779 server.BadRequest(w, r, err) 2780 return 2781 } 2782 if err := datastore.SaveDataByUUID(uuid, d); err != nil { 2783 server.BadRequest(w, r, err) 2784 return 2785 } 2786 fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config) 2787 return 2788 } else { 2789 jsonBytes, err := d.MarshalJSONExtents(ctx) 2790 if err != nil { 2791 server.BadRequest(w, r, err) 2792 return 2793 } 2794 w.Header().Set("Content-Type", "application/json") 2795 fmt.Fprint(w, string(jsonBytes)) 2796 } 2797 2798 case "tags": 2799 if action == "post" { 2800 replace := r.URL.Query().Get("replace") == "true" 2801 if err := datastore.SetTagsByJSON(d, uuid, replace, r.Body); err != nil { 2802 server.BadRequest(w, r, err) 2803 return 2804 } 2805 } else { 2806 jsonBytes, err := d.MarshalJSONTags() 2807 if err != nil { 2808 server.BadRequest(w, r, err) 2809 return 2810 } 2811 w.Header().Set("Content-Type", "application/json") 2812 fmt.Fprint(w, string(jsonBytes)) 2813 } 2814 2815 case "specificblocks": 2816 // GET <api URL>/node/<UUID>/<data name>/specificblocks?blocks=x,y,z,x,y,z... 2817 queryStrings := r.URL.Query() 2818 blocklist := queryStrings.Get("blocks") 2819 supervoxels := queryStrings.Get("supervoxels") == "true" 2820 compression := queryStrings.Get("compression") 2821 scale, err := getScale(queryStrings) 2822 if err != nil { 2823 server.BadRequest(w, r, err) 2824 return 2825 } 2826 if action == "get" { 2827 timedLog := dvid.NewTimeLog() 2828 numBlocks, err := d.sendBlocksSpecific(ctx, w, supervoxels, compression, blocklist, scale) 2829 if err != nil { 2830 server.BadRequest(w, r, err) 2831 return 2832 } 2833 timedLog.Infof("HTTP %s: %s", r.Method, r.URL) 2834 activity = map[string]interface{}{ 2835 "num_blocks": numBlocks, 2836 } 2837 } else { 2838 server.BadRequest(w, r, "DVID does not accept the %s action on the 'specificblocks' endpoint", action) 2839 return 2840 } 2841 2842 case "sync": 2843 if action != "post" { 2844 server.BadRequest(w, r, "Only POST allowed to sync endpoint") 2845 return 2846 } 2847 replace := r.URL.Query().Get("replace") == "true" 2848 if err := datastore.SetSyncByJSON(d, uuid, replace, r.Body); err != nil { 2849 server.BadRequest(w, r, err) 2850 return 2851 } 2852 2853 case "label": 2854 d.handleLabel(ctx, w, r, parts) 2855 2856 case "labels": 2857 d.handleLabels(ctx, w, r) 2858 2859 case "listlabels": 2860 d.handleListLabels(ctx, w, r) 2861 2862 case "mapping": 2863 d.handleMapping(ctx, w, r) 2864 2865 case "supervoxel-splits": 2866 d.handleSupervoxelSplits(ctx, w, r) 2867 2868 case "blocks": 2869 d.handleBlocks(ctx, w, r, parts) 2870 2871 case "ingest-supervoxels": 2872 d.handleIngest(ctx, w, r) 2873 2874 case "pseudocolor": 2875 d.handlePseudocolor(ctx, w, r, parts) 2876 2877 case "raw", "isotropic": 2878 d.handleDataRequest(ctx, w, r, parts) 2879 2880 // endpoints after this must have data instance IndexedLabels = true 2881 2882 case "lastmod": 2883 d.handleLabelmod(ctx, w, r, parts) 2884 2885 case "supervoxels": 2886 d.handleSupervoxels(ctx, w, r, parts) 2887 2888 case "supervoxel-sizes": 2889 d.handleSupervoxelSizes(ctx, w, r, parts) 2890 2891 case "size": 2892 d.handleSize(ctx, w, r, parts) 2893 2894 case "sizes": 2895 d.handleSizes(ctx, w, r) 2896 2897 case "sparsevol-size": 2898 d.handleSparsevolSize(ctx, w, r, parts) 2899 2900 case "sparsevol": 2901 d.handleSparsevol(ctx, w, r, parts) 2902 2903 case "sparsevol-by-point": 2904 d.handleSparsevolByPoint(ctx, w, r, parts) 2905 2906 case "sparsevol-coarse": 2907 d.handleSparsevolCoarse(ctx, w, r, parts) 2908 2909 case "sparsevols-coarse": 2910 d.handleSparsevolsCoarse(ctx, w, r, parts) 2911 2912 case "maxlabel": 2913 d.handleMaxlabel(ctx, w, r, parts) 2914 2915 case "nextlabel": 2916 d.handleNextlabel(ctx, w, r, parts) 2917 2918 case "split-supervoxel": 2919 d.handleSplitSupervoxel(ctx, w, r, parts) 2920 2921 case "cleave": 2922 d.handleCleave(ctx, w, r, parts) 2923 2924 case "split": 2925 d.handleSplit(ctx, w, r, parts) 2926 2927 case "merge": 2928 d.handleMerge(ctx, w, r, parts) 2929 2930 case "renumber": 2931 d.handleRenumber(ctx, w, r) 2932 2933 case "proximity": 2934 d.handleProximity(ctx, w, r, parts) 2935 2936 case "index": 2937 d.handleIndex(ctx, w, r, parts) 2938 2939 case "indices": 2940 d.handleIndices(ctx, w, r) 2941 2942 case "indices-compressed": 2943 d.handleIndicesCompressed(ctx, w, r) 2944 2945 case "mappings": 2946 d.handleMappings(ctx, w, r) 2947 2948 case "history": 2949 d.handleHistory(ctx, w, r, parts) 2950 2951 case "mutations": 2952 d.handleMutations(ctx, w, r) 2953 2954 default: 2955 server.BadAPIRequest(w, r, d) 2956 } 2957 return 2958 }