github.com/janelia-flyem/dvid@v1.0.0/datatype/labelvol/labelvol.go (about)

     1  /*
     2  	Package labelvol supports label-specific sparse volumes.  It can be synced
     3  	with labelblk and is a different view of 64-bit label data.
     4  */
     5  package labelvol
     6  
     7  import (
     8  	"bytes"
     9  	"compress/gzip"
    10  	"encoding/binary"
    11  	"encoding/gob"
    12  	"encoding/json"
    13  	"fmt"
    14  	"io"
    15  	"io/ioutil"
    16  	"net/http"
    17  	"os"
    18  	"path/filepath"
    19  	"strconv"
    20  	"strings"
    21  	"sync"
    22  	"time"
    23  
    24  	humanize "github.com/dustin/go-humanize"
    25  
    26  	"github.com/janelia-flyem/dvid/datastore"
    27  	"github.com/janelia-flyem/dvid/datatype/common/labels"
    28  	"github.com/janelia-flyem/dvid/datatype/labelblk"
    29  	"github.com/janelia-flyem/dvid/dvid"
    30  	"github.com/janelia-flyem/dvid/server"
    31  	"github.com/janelia-flyem/dvid/storage"
    32  
    33  	lz4 "github.com/janelia-flyem/go/golz4-updated"
    34  )
    35  
    36  const (
    37  	Version  = "0.1"
    38  	RepoURL  = "github.com/janelia-flyem/dvid/datatype/labelvol"
    39  	TypeName = "labelvol"
    40  )
    41  
    42  const helpMessage = `
    43  API for label sparse volume data type (github.com/janelia-flyem/dvid/datatype/labelvol)
    44  =======================================================================================
    45  
    46  Note: UUIDs referenced below are strings that may either be a unique prefix of a
    47  hexadecimal UUID string (e.g., 3FA22) or a branch leaf specification that adds
    48  a colon (":") followed by the case-dependent branch name.  In the case of a
    49  branch leaf specification, the unique UUID prefix just identifies the repo of
    50  the branch, and the UUID referenced is really the leaf of the branch name.
    51  For example, if we have a DAG with root A -> B -> C where C is the current
    52  HEAD or leaf of the "master" (default) branch, then asking for "B:master" is
    53  the same as asking for "C".  If we add another version so A -> B -> C -> D, then
    54  references to "B:master" now return the data from "D".
    55  
    56  -----
    57  
    58  Denormalizations like sparse volumes are *not* performed for the "0" label, which is
    59  considered a special label useful for designating background.  This allows users to define
    60  sparse labeled structures in a large volume without requiring processing of entire volume.
    61  
    62  
    63  Command-line:
    64  
    65  $ dvid repo <UUID> new labelvol <data name> <settings...>
    66  
    67  	Adds newly named data of the 'type name' to repo with specified UUID.
    68  
    69  	Example:
    70  
    71  	$ dvid repo 3f8c new labelvol sparsevols
    72  
    73      Arguments:
    74  
    75      UUID           Hexadecimal string with enough characters to uniquely identify a version node.
    76      data name      Name of data to create, e.g., "sparsevols"
    77      settings       Configuration settings in "key=value" format separated by spaces.
    78  
    79      Configuration Settings (case-insensitive keys)
    80  
    81      BlockSize      Size in pixels  (default: %s)
    82      VoxelSize      Resolution of voxels (default: 8.0, 8.0, 8.0)
    83      VoxelUnits     Resolution units (default: "nanometers")
    84  
    85  $ dvid node <UUID> <data name> dump <server-accessible directory>
    86  
    87  	Dumps files, one per 512^3 voxel subvolume, for all block RLEs for each label within the subvolume.
    88  	File names will be formatted as subvols-X_Y_Z.dat, where (X,Y,Z) is the subvolume index; for example,
    89  	the file subvols-0_1_2.dat has all label block RLEs for the subvolume with smallest voxel coordinate
    90  	at (0, 512, 1024).	The encoding has the following format where integers are little endian and the 
    91  	order of data is exactly as specified below:
    92  
    93  	    uint64    Label ID
    94  	    uint32    # Spans
    95  	    Repeating unit of:
    96  	        int32   Coordinate of run start (dimension 0)
    97  	        int32   Coordinate of run start (dimension 1)
    98  	        int32   Coordinate of run start (dimension 2)
    99  	        int32   Length of run
   100  			  ...
   101  
   102  	This is similar to the voxel RLEs returned by the sparsevol endpoint, except the initial 8 byte header
   103  	is replaced with a label identifier.  Spans are always in X direction.
   104  	
   105  
   106  ------------------
   107  
   108  HTTP API (Level 2 REST):
   109  
   110  GET  <api URL>/node/<UUID>/<data name>/help
   111  
   112  	Returns data-specific help message.
   113  
   114  
   115  GET  <api URL>/node/<UUID>/<data name>/info
   116  POST <api URL>/node/<UUID>/<data name>/info
   117  
   118      Retrieves or puts DVID-specific data properties for these voxels.
   119  
   120      Example: 
   121  
   122      GET <api URL>/node/3f8c/bodies/info
   123  
   124      Returns JSON with configuration settings that include location in DVID space and
   125      min/max block indices.
   126  
   127      Arguments:
   128  
   129      UUID          Hexadecimal string with enough characters to uniquely identify a version node.
   130      data name     Name of labelvol data.
   131  
   132  
   133  POST <api URL>/node/<UUID>/<data name>/sync?<options>
   134  
   135      Establishes labelblk data instances with which the annotations are synced.  Expects JSON to be POSTed
   136      with the following format:
   137  
   138      { "sync": "labels" }
   139  
   140  	To delete syncs, pass an empty string of names with query string "replace=true":
   141  
   142  	{ "sync": "" }
   143  
   144      The "sync" property should be followed by a comma-delimited list of data instances that MUST
   145      already exist.  Currently, syncs should be created before any annotations are pushed to
   146      the server.  If annotations already exist, these are currently not synced.
   147  
   148      The labelvol data type only accepts syncs to labelblk data instances.
   149  
   150      GET Query-string Options:
   151  
   152      replace    Set to "true" if you want passed syncs to replace and not be appended to current syncs.
   153  			   Default operation is false.
   154  
   155  
   156  GET  <api URL>/node/<UUID>/<data name>/sparsevol/<label>?<options>
   157  
   158  	Returns a sparse volume with voxels of the given label in encoded RLE format.  The returned
   159  	data can be optionally compressed using the "compression" option below.
   160  
   161  	The encoding has the following format where integers are little endian and the order
   162  	of data is exactly as specified below:
   163  
   164  	    byte     Payload descriptor:
   165  	               Bit 0 (LSB) - 8-bit grayscale
   166  	               Bit 1 - 16-bit grayscale
   167  	               Bit 2 - 16-bit normal
   168  	               If set to all 0, there is no payload and it's a binary sparse volume.
   169  	    uint8    Number of dimensions
   170  	    uint8    Dimension of run (typically 0 = X)
   171  	    byte     Reserved (to be used later)
   172  	    uint32    # Voxels [TODO.  0 for now]
   173  	    uint32    # Spans
   174  	    Repeating unit of:
   175  	        int32   Coordinate of run start (dimension 0)
   176  	        int32   Coordinate of run start (dimension 1)
   177  	        int32   Coordinate of run start (dimension 2)
   178  	        int32   Length of run
   179  	        bytes   Optional payload dependent on first byte descriptor
   180  			  ...
   181  
   182      GET Query-string Options:
   183  
   184      minx    Spans must be equal to or larger than this minimum x voxel coordinate.
   185      maxx    Spans must be equal to or smaller than this maximum x voxel coordinate.
   186      miny    Spans must be equal to or larger than this minimum y voxel coordinate.
   187      maxy    Spans must be equal to or smaller than this maximum y voxel coordinate.
   188      minz    Spans must be equal to or larger than this minimum z voxel coordinate.
   189      maxz    Spans must be equal to or smaller than this maximum z voxel coordinate.
   190      exact   "false" if RLEs can extend a bit outside voxel bounds within border blocks.
   191              This will give slightly faster responses. 
   192  
   193      compression   Allows retrieval of data in "lz4" and "gzip"
   194                    compressed format.
   195  
   196  
   197  HEAD <api URL>/node/<UUID>/<data name>/sparsevol/<label>?<options>
   198  
   199  	Returns:
   200  		200 (OK) if a sparse volume of the given label exists within any optional bounds.
   201  		204 (No Content) if there is no sparse volume for the given label within any optional bounds.
   202  		404 (Not found) if the label has no sparse volume.
   203  
   204  	Note that for speed, the optional bounds are always expanded to the block-aligned containing
   205  	subvolume, i.e., it's as if exact=false for the corresponding GET.
   206  
   207      GET Query-string Options:
   208  
   209      minx    Spans must be equal to or larger than this minimum x voxel coordinate.
   210      maxx    Spans must be equal to or smaller than this maximum x voxel coordinate.
   211      miny    Spans must be equal to or larger than this minimum y voxel coordinate.
   212      maxy    Spans must be equal to or smaller than this maximum y voxel coordinate.
   213      minz    Spans must be equal to or larger than this minimum z voxel coordinate.
   214      maxz    Spans must be equal to or smaller than this maximum z voxel coordinate.
   215  
   216  
   217  GET <api URL>/node/<UUID>/<data name>/sparsevol-by-point/<coord>
   218  
   219  	Returns a sparse volume with voxels that pass through a given voxel.
   220  	The encoding is described in the "sparsevol" request above.
   221  	
   222      Arguments:
   223  
   224      UUID          Hexadecimal string with enough characters to uniquely identify a version node.
   225      data name     Name of mapping data.
   226      coord     	  Coordinate of voxel with underscore as separator, e.g., 10_20_30
   227  
   228  
   229  GET <api URL>/node/<UUID>/<data name>/sparsevol-coarse/<label>
   230  
   231  	Returns a sparse volume with blocks of the given label in encoded RLE format.
   232  	The encoding has the following format where integers are little endian and the order
   233  	of data is exactly as specified below:
   234  
   235  	    byte     Set to 0
   236  	    uint8    Number of dimensions
   237  	    uint8    Dimension of run (typically 0 = X)
   238  	    byte     Reserved (to be used later)
   239  	    uint32    # Blocks [TODO.  0 for now]
   240  	    uint32    # Spans
   241  	    Repeating unit of:
   242  	        int32   Block coordinate of run start (dimension 0)
   243  	        int32   Block coordinate of run start (dimension 1)
   244  	        int32   Block coordinate of run start (dimension 2)
   245  			  ...
   246  	        int32   Length of run
   247  
   248  	Note that the above format is the RLE encoding of sparsevol, where voxel coordinates
   249  	have been replaced by block coordinates.
   250  
   251  
   252  GET <api URL>/node/<UUID>/<data name>/maxlabel
   253  
   254  	GET returns the maximum label for the version of data in JSON form:
   255  
   256  		{ "maxlabel": <label #> }
   257  
   258  
   259  GET <api URL>/node/<UUID>/<data name>/nextlabel
   260  POST <api URL>/node/<UUID>/<data name>/nextlabel
   261  
   262  	GET returns the next label for the version of data in JSON form:
   263  
   264  		{ "nextlabel": <label #> }
   265  
   266  	POST allows the client to request some # of labels that will be reserved.
   267  	This is used if the client wants to introduce new labels.
   268  
   269  	The request:
   270  
   271  		{ "needed": <# of labels> }
   272  
   273  	Response:
   274  
   275  		{ "start": <starting label #>, "end": <ending label #> }
   276  
   277  
   278  POST <api URL>/node/<UUID>/<data name>/merge
   279  
   280  	Merges labels.  Requires JSON in request body using the following format:
   281  
   282  	[toLabel1, fromLabel1, fromLabel2, fromLabel3, ...]
   283  
   284  	The first element of the JSON array specifies the label to be used as the merge result.
   285  	Note that it's computationally more efficient to group a number of merges into the
   286  	same toLabel as a single merge request instead of multiple merge requests.
   287  
   288  
   289  POST <api URL>/node/<UUID>/<data name>/split/<label>[?splitlabel=X]
   290  
   291  	Splits a portion of a label's voxels into a new label or, if "splitlabel" is specified
   292  	as an optional query string, the given split label.  Returns the following JSON:
   293  
   294  		{ "label": <new label> }
   295  
   296  	This request requires a binary sparse volume in the POSTed body with the following 
   297  	encoded RLE format, which is compatible with the format returned by a GET on the 
   298  	"sparsevol" endpoint described above:
   299  
   300  		All integers are in little-endian format.
   301  
   302  	    byte     Payload descriptor:
   303  	               Set to 0 to indicate it's a binary sparse volume.
   304  	    uint8    Number of dimensions
   305  	    uint8    Dimension of run (typically 0 = X)
   306  	    byte     Reserved (to be used later)
   307  	    uint32    # Voxels [TODO.  0 for now]
   308  	    uint32    # Spans
   309  	    Repeating unit of:
   310  	        int32   Coordinate of run start (dimension 0)
   311  	        int32   Coordinate of run start (dimension 1)
   312  	        int32   Coordinate of run start (dimension 2)
   313  			  ...
   314  	        int32   Length of run
   315  
   316  	NOTE 1: The POSTed split sparse volume must be a subset of the given label's voxels.  You cannot
   317  	give an arbitrary sparse volume that may span multiple labels.
   318  
   319  	NOTE 2: If a split label is specified, it is the client's responsibility to make sure the given
   320  	label will not create conflict with labels in other versions.  It should primarily be used in
   321  	chain operations like "split-coarse" followed by "split" using voxels, where the new label
   322  	created by the split coarse is used as the split label for the smaller, higher-res "split".
   323  
   324  POST <api URL>/node/<UUID>/<data name>/split-coarse/<label>[?splitlabel=X]
   325  
   326  	Splits a portion of a label's blocks into a new label or, if "splitlabel" is specified
   327  	as an optional query string, the given split label.  Returns the following JSON:
   328  
   329  		{ "label": <new label> }
   330  
   331  	This request requires a binary sparse volume in the POSTed body with the following 
   332  	encoded RLE format, which is similar to the "split" request format but uses block
   333  	instead of voxel coordinates:
   334  
   335  		All integers are in little-endian format.
   336  
   337  	    byte     Payload descriptor:
   338  	               Set to 0 to indicate it's a binary sparse volume.
   339  	    uint8    Number of dimensions
   340  	    uint8    Dimension of run (typically 0 = X)
   341  	    byte     Reserved (to be used later)
   342  	    uint32    # Blocks [TODO.  0 for now]
   343  	    uint32    # Spans
   344  	    Repeating unit of:
   345  	        int32   Coordinate of run start (dimension 0)
   346  	        int32   Coordinate of run start (dimension 1)
   347  	        int32   Coordinate of run start (dimension 2)
   348  			  ...
   349  	        int32   Length of run
   350  
   351  	The Notes for "split" endpoint above are applicable to this "split-coarse" endpoint.
   352  
   353  POST <api URL>/node/<UUID>/<data name>/resync/<label>
   354  
   355  	Regenerates the sparse volume for the given label from its synced labelblk.
   356  	This is used to repair databases that have been inadvertantly shutdown or
   357  	crashed while undergoing merges/splits.
   358  
   359  	This request requires a binary sparse volume in the POSTed body with the following 
   360  	encoded RLE format, which is similar to the "split" request format but uses block
   361  	instead of voxel coordinates:
   362  
   363  		All integers are in little-endian format.
   364  
   365  	    byte     Payload descriptor:
   366  	               Set to 0 to indicate it's a binary sparse volume.
   367  	    uint8    Number of dimensions
   368  	    uint8    Dimension of run (typically 0 = X)
   369  	    byte     Reserved (to be used later)
   370  	    uint32    # Blocks [TODO.  0 for now]
   371  	    uint32    # Spans
   372  	    Repeating unit of:
   373  	        int32   Coordinate of run start (dimension 0)
   374  	        int32   Coordinate of run start (dimension 1)
   375  	        int32   Coordinate of run start (dimension 2)
   376  			  ...
   377  	        int32   Length of run
   378  
   379  DELETE <api URL>/node/<UUID>/<data name>/area/<label>/<size>/<offset>
   380  
   381  	NOTE: Does not honor syncs and is intended purely for low-level mods.
   382  	Deletes all of a label's sparse volume within a given 3d volume from a 3d offset.
   383  
   384  	Example: 
   385  
   386  	DELETE <api URL>/node/3f8c/bodies/area/23/512_512_512/0_0_128
   387  
   388  	The above example deletes all sparsevol associated with label 23 in subvolume
   389  	that is 512 x 512 x 512 starting at offset (0,0,128).
   390  `
   391  
   392  var (
   393  	dtype *Type
   394  
   395  	DefaultBlockSize int32   = labelblk.DefaultBlockSize
   396  	DefaultRes       float32 = labelblk.DefaultRes
   397  	DefaultUnits             = labelblk.DefaultUnits
   398  )
   399  
   400  func init() {
   401  	dtype = new(Type)
   402  	dtype.Type = datastore.Type{
   403  		Name:    TypeName,
   404  		URL:     RepoURL,
   405  		Version: Version,
   406  		Requirements: &storage.Requirements{
   407  			Batcher: true,
   408  		},
   409  	}
   410  
   411  	// See doc for package on why channels are segregated instead of interleaved.
   412  	// Data types must be registered with the datastore to be used.
   413  	datastore.Register(dtype)
   414  
   415  	// Need to register types that will be used to fulfill interfaces.
   416  	gob.Register(&Type{})
   417  	gob.Register(&Data{})
   418  }
   419  
   420  // NewData returns a pointer to labelvol data.
   421  func NewData(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (*Data, error) {
   422  	// Initialize the Data for this data type
   423  	basedata, err := datastore.NewDataService(dtype, uuid, id, name, c)
   424  	if err != nil {
   425  		return nil, err
   426  	}
   427  	props := new(Properties)
   428  	props.setDefault()
   429  	if err := props.setByConfig(c); err != nil {
   430  		return nil, err
   431  	}
   432  	data := &Data{
   433  		Data:       basedata,
   434  		Properties: *props,
   435  	}
   436  	data.Properties.MaxLabel = make(map[dvid.VersionID]uint64)
   437  	return data, nil
   438  }
   439  
   440  // --- Labelvol Datatype -----
   441  
   442  type Type struct {
   443  	datastore.Type
   444  }
   445  
   446  // --- TypeService interface ---
   447  
   448  func (dtype *Type) NewDataService(uuid dvid.UUID, id dvid.InstanceID, name dvid.InstanceName, c dvid.Config) (datastore.DataService, error) {
   449  	return NewData(uuid, id, name, c)
   450  }
   451  
   452  func (dtype *Type) Help() string {
   453  	return helpMessage
   454  }
   455  
   456  // Properties are additional properties for data beyond those in standard datastore.Data.
   457  type Properties struct {
   458  	Resolution dvid.Resolution
   459  
   460  	// Block size for this repo
   461  	BlockSize dvid.Point3d
   462  
   463  	mlMu sync.RWMutex // For atomic access of MaxLabel and MaxRepoLabel
   464  
   465  	// The maximum label id found in each version of this instance.
   466  	// Can be unset if no new label was added at that version, in which case
   467  	// you must traverse DAG to find max label of parent.
   468  	MaxLabel map[dvid.VersionID]uint64
   469  
   470  	// The maximum label for this instance in the entire repo.  This allows us to do
   471  	// conflict-free merges without any relabeling.  Note that relabeling (rebasing)
   472  	// is required if we move data between repos, e.g., when pushing remote nodes,
   473  	// since we have no control over which labels were created remotely and there
   474  	// could be conflicts between the local and remote repos.  When mutations only
   475  	// occur within a single repo, however, this atomic label allows us to prevent
   476  	// conflict across all versions within this repo.
   477  	MaxRepoLabel uint64
   478  }
   479  
   480  // CopyPropertiesFrom copies the data instance-specific properties from a given
   481  // data instance into the receiver's properties.  Fulfills the datastore.PropertyCopier interface.
   482  func (d *Data) CopyPropertiesFrom(src datastore.DataService, fs storage.FilterSpec) error {
   483  	d2, ok := src.(*Data)
   484  	if !ok {
   485  		return fmt.Errorf("unable to copy properties from non-labelvol data %q", src.DataName())
   486  	}
   487  	d.Properties.copyImmutable(&(d2.Properties))
   488  
   489  	// TODO -- Handle mutable data that could be potentially altered by filter.
   490  	d.MaxLabel = make(map[dvid.VersionID]uint64, len(d2.MaxLabel))
   491  	for k, v := range d2.MaxLabel {
   492  		d.MaxLabel[k] = v
   493  	}
   494  	d.MaxRepoLabel = d2.MaxRepoLabel
   495  
   496  	return nil
   497  }
   498  
   499  func (p *Properties) copyImmutable(p2 *Properties) {
   500  	p.BlockSize = p2.BlockSize
   501  
   502  	p.Resolution.VoxelSize = make(dvid.NdFloat32, 3)
   503  	copy(p.Resolution.VoxelSize, p2.Resolution.VoxelSize)
   504  	p.Resolution.VoxelUnits = make(dvid.NdString, 3)
   505  	copy(p.Resolution.VoxelUnits, p2.Resolution.VoxelUnits)
   506  }
   507  
   508  func (p Properties) MarshalJSON() ([]byte, error) {
   509  	maxLabels := make(map[string]uint64)
   510  	for v, max := range p.MaxLabel {
   511  		uuid, err := datastore.UUIDFromVersion(v)
   512  		if err != nil {
   513  			return nil, err
   514  		}
   515  		maxLabels[string(uuid)] = max
   516  	}
   517  	return json.Marshal(struct {
   518  		dvid.Resolution
   519  		BlockSize dvid.Point3d
   520  		MaxLabel  map[string]uint64
   521  	}{
   522  		p.Resolution,
   523  		p.BlockSize,
   524  		maxLabels,
   525  	})
   526  }
   527  
   528  func (p *Properties) setDefault() {
   529  	for d := 0; d < 3; d++ {
   530  		p.BlockSize[d] = DefaultBlockSize
   531  	}
   532  	p.Resolution.VoxelSize = make(dvid.NdFloat32, 3)
   533  	for d := 0; d < 3; d++ {
   534  		p.Resolution.VoxelSize[d] = DefaultRes
   535  	}
   536  	p.Resolution.VoxelUnits = make(dvid.NdString, 3)
   537  	for d := 0; d < 3; d++ {
   538  		p.Resolution.VoxelUnits[d] = DefaultUnits
   539  	}
   540  }
   541  
   542  func (p *Properties) setByConfig(config dvid.Config) error {
   543  	s, found, err := config.GetString("BlockSize")
   544  	if err != nil {
   545  		return err
   546  	}
   547  	if found {
   548  		p.BlockSize, err = dvid.StringToPoint3d(s, ",")
   549  		if err != nil {
   550  			return err
   551  		}
   552  	}
   553  	s, found, err = config.GetString("VoxelSize")
   554  	if err != nil {
   555  		return err
   556  	}
   557  	if found {
   558  		dvid.Infof("Changing resolution of voxels to %s\n", s)
   559  		p.Resolution.VoxelSize, err = dvid.StringToNdFloat32(s, ",")
   560  		if err != nil {
   561  			return err
   562  		}
   563  	}
   564  	s, found, err = config.GetString("VoxelUnits")
   565  	if err != nil {
   566  		return err
   567  	}
   568  	if found {
   569  		p.Resolution.VoxelUnits, err = dvid.StringToNdString(s, ",")
   570  		if err != nil {
   571  			return err
   572  		}
   573  	}
   574  	return nil
   575  }
   576  
   577  // Data instance of labelvol, label sparse volumes.
   578  type Data struct {
   579  	*datastore.Data
   580  	Properties
   581  
   582  	// Keep track of sync operations that could be updating the data.
   583  	// TODO: Think about making this per label since sync status is pessimistic, assuming
   584  	// all labels are being updated.
   585  	datastore.Updater
   586  
   587  	// channels for mutations and downres caching.
   588  	syncCh   chan datastore.SyncMessage
   589  	syncDone chan *sync.WaitGroup
   590  }
   591  
   592  // RemapVersions modifies internal data instance properties that rely
   593  // on server-specific version ids.  This is necessary after DVID-to-DVID
   594  // transmission.
   595  func (d *Data) RemapVersions(vmap dvid.VersionMap) error {
   596  	maxLabels := make(map[dvid.VersionID]uint64, len(d.Properties.MaxLabel))
   597  	for oldv, label := range d.Properties.MaxLabel {
   598  		newv, found := vmap[oldv]
   599  		if !found {
   600  			dvid.Infof("No version %d in labelvol %q... discarding max label", oldv, d.DataName())
   601  			continue
   602  		}
   603  		maxLabels[newv] = label
   604  	}
   605  	d.Properties.MaxLabel = maxLabels
   606  	return nil
   607  }
   608  
   609  // GetSyncedLabelblk returns the synced labelblk data instance or returns
   610  // an error if there is no synced labelblk.
   611  func (d *Data) GetSyncedLabelblk() (*labelblk.Data, error) {
   612  	// Go through all synced names, and checking if there's a valid source.
   613  	for dataUUID := range d.SyncedData() {
   614  		source, err := labelblk.GetByDataUUID(dataUUID)
   615  		if err == nil {
   616  			return source, nil
   617  		}
   618  	}
   619  	return nil, fmt.Errorf("no labelblk data is syncing with %s", d.DataName())
   620  }
   621  
   622  // GetByDataUUID returns a pointer to labelvol data given a data UUID.
   623  func GetByDataUUID(dataUUID dvid.UUID) (*Data, error) {
   624  	source, err := datastore.GetDataByDataUUID(dataUUID)
   625  	if err != nil {
   626  		return nil, err
   627  	}
   628  	data, ok := source.(*Data)
   629  	if !ok {
   630  		return nil, fmt.Errorf("Instance '%s' is not a labelvol datatype!", source.DataName())
   631  	}
   632  	return data, nil
   633  }
   634  
   635  // GetByUUIDName returns a pointer to labelvol data given a version (UUID) and data name.
   636  func GetByUUIDName(uuid dvid.UUID, name dvid.InstanceName) (*Data, error) {
   637  	source, err := datastore.GetDataByUUIDName(uuid, name)
   638  	if err != nil {
   639  		return nil, err
   640  	}
   641  	data, ok := source.(*Data)
   642  	if !ok {
   643  		return nil, fmt.Errorf("Instance '%s' is not a labelvol datatype!", name)
   644  	}
   645  	return data, nil
   646  }
   647  
   648  // GetByVersionName returns a pointer to labelblk data given a UUID and data name.
   649  func GetByVersionName(v dvid.VersionID, name dvid.InstanceName) (*Data, error) {
   650  	source, err := datastore.GetDataByVersionName(v, name)
   651  	if err != nil {
   652  		return nil, err
   653  	}
   654  	data, ok := source.(*Data)
   655  	if !ok {
   656  		return nil, fmt.Errorf("Instance '%s' is not a labelvol datatype!", name)
   657  	}
   658  	return data, nil
   659  }
   660  
   661  // --- datastore.InstanceMutator interface -----
   662  
   663  // LoadMutable loads mutable properties of label volumes like the maximum labels
   664  // for each version.  Note that we load these max labels from key-value pairs
   665  // rather than data instance properties persistence, because in the case of a crash,
   666  // the actually stored repo data structure may be out-of-date compared to the guaranteed
   667  // up-to-date key-value pairs for max labels.
   668  func (d *Data) LoadMutable(root dvid.VersionID, storedVersion, expectedVersion uint64) (bool, error) {
   669  	ctx := storage.NewDataContext(d, 0)
   670  	store, err := datastore.GetOrderedKeyValueDB(d)
   671  	if err != nil {
   672  		return false, fmt.Errorf("Data type labelvol had error initializing store: %v\n", err)
   673  	}
   674  
   675  	wg := new(sync.WaitGroup)
   676  	wg.Add(1)
   677  	ch := make(chan *storage.KeyValue)
   678  
   679  	// Start appropriate migration function if any.
   680  	var saveRequired bool
   681  
   682  	switch storedVersion {
   683  	case 0:
   684  		// Need to update all max labels and set repo-level max label.
   685  		saveRequired = true
   686  		dvid.Infof("Migrating old version of labelvol %q to new version\n", d.DataName())
   687  		go d.migrateMaxLabels(root, wg, ch)
   688  	default:
   689  		// Load in each version max label without migration.
   690  		go d.loadMaxLabels(wg, ch)
   691  	}
   692  
   693  	// Send the max label data per version
   694  	minKey, err := ctx.MinVersionKey(maxLabelTKey)
   695  	if err != nil {
   696  		return false, err
   697  	}
   698  	maxKey, err := ctx.MaxVersionKey(maxLabelTKey)
   699  	if err != nil {
   700  		return false, err
   701  	}
   702  	keysOnly := false
   703  	if err = store.RawRangeQuery(minKey, maxKey, keysOnly, ch, nil); err != nil {
   704  		return false, err
   705  	}
   706  	wg.Wait()
   707  
   708  	dvid.Infof("Loaded max label values for labelvol %q with repo-wide max %d\n", d.DataName(), d.MaxRepoLabel)
   709  	return saveRequired, nil
   710  }
   711  
   712  func (d *Data) migrateMaxLabels(root dvid.VersionID, wg *sync.WaitGroup, ch chan *storage.KeyValue) {
   713  	ctx := storage.NewDataContext(d, 0)
   714  	store, err := datastore.GetOrderedKeyValueDB(d)
   715  	if err != nil {
   716  		dvid.Errorf("Can't initialize store for labelvol %q: %v\n", d.DataName(), err)
   717  	}
   718  
   719  	var maxRepoLabel uint64
   720  	d.MaxLabel = make(map[dvid.VersionID]uint64)
   721  	for {
   722  		kv := <-ch
   723  		if kv == nil {
   724  			break
   725  		}
   726  		v, err := ctx.VersionFromKey(kv.K)
   727  		if err != nil {
   728  			dvid.Errorf("Can't decode key when loading mutable data for %s", d.DataName())
   729  			continue
   730  		}
   731  		if len(kv.V) != 8 {
   732  			dvid.Errorf("Got bad value.  Expected 64-bit label, got %v", kv.V)
   733  			continue
   734  		}
   735  		label := binary.LittleEndian.Uint64(kv.V)
   736  		d.MaxLabel[v] = label
   737  		if label > maxRepoLabel {
   738  			maxRepoLabel = label
   739  		}
   740  	}
   741  
   742  	// Adjust the MaxLabel data to make sure we correct for any case of child max < parent max.
   743  	d.adjustMaxLabels(store, root)
   744  
   745  	// Set the repo-wide max label.
   746  	d.MaxRepoLabel = maxRepoLabel
   747  
   748  	buf := make([]byte, 8)
   749  	binary.LittleEndian.PutUint64(buf, maxRepoLabel)
   750  	store.Put(ctx, maxRepoLabelTKey, buf)
   751  
   752  	wg.Done()
   753  	return
   754  }
   755  
   756  func (d *Data) adjustMaxLabels(store storage.KeyValueSetter, root dvid.VersionID) error {
   757  	buf := make([]byte, 8)
   758  
   759  	parentMax, ok := d.MaxLabel[root]
   760  	if !ok {
   761  		return fmt.Errorf("can't adjust version id %d since none exists in metadata", root)
   762  	}
   763  	childIDs, err := datastore.GetChildrenByVersion(root)
   764  	if err != nil {
   765  		return err
   766  	}
   767  	for _, childID := range childIDs {
   768  		var save bool
   769  		childMax, ok := d.MaxLabel[childID]
   770  		if !ok {
   771  			// set to parent max
   772  			d.MaxLabel[childID] = parentMax
   773  			save = true
   774  		} else if childMax < parentMax {
   775  			d.MaxLabel[childID] = parentMax + childMax + 1
   776  			save = true
   777  		}
   778  
   779  		// save the key-value
   780  		if save {
   781  			binary.LittleEndian.PutUint64(buf, d.MaxLabel[childID])
   782  			ctx := datastore.NewVersionedCtx(d, childID)
   783  			store.Put(ctx, maxLabelTKey, buf)
   784  		}
   785  
   786  		// recurse for depth-first
   787  		if err := d.adjustMaxLabels(store, childID); err != nil {
   788  			return err
   789  		}
   790  	}
   791  	return nil
   792  }
   793  
   794  func (d *Data) loadMaxLabels(wg *sync.WaitGroup, ch chan *storage.KeyValue) {
   795  	ctx := storage.NewDataContext(d, 0)
   796  	var repoMax uint64
   797  	d.MaxLabel = make(map[dvid.VersionID]uint64)
   798  	for {
   799  		kv := <-ch
   800  		if kv == nil {
   801  			break
   802  		}
   803  		v, err := ctx.VersionFromKey(kv.K)
   804  		if err != nil {
   805  			dvid.Errorf("Can't decode key when loading mutable data for %s", d.DataName())
   806  			continue
   807  		}
   808  		if len(kv.V) != 8 {
   809  			dvid.Errorf("Got bad value.  Expected 64-bit label, got %v", kv.V)
   810  			continue
   811  		}
   812  		label := binary.LittleEndian.Uint64(kv.V)
   813  		d.MaxLabel[v] = label
   814  		if label > repoMax {
   815  			repoMax = label
   816  		}
   817  	}
   818  
   819  	// Load in the repo-wide max label.
   820  	store, err := datastore.GetOrderedKeyValueDB(d)
   821  	if err != nil {
   822  		dvid.Errorf("Data type labelvol had error initializing store: %v\n", err)
   823  		return
   824  	}
   825  	data, err := store.Get(ctx, maxRepoLabelTKey)
   826  	if err != nil {
   827  		dvid.Errorf("Error getting repo-wide max label: %v\n", err)
   828  		return
   829  	}
   830  	if data == nil || len(data) != 8 {
   831  		dvid.Errorf("Could not load repo-wide max label for instance %q.  Only got %d bytes, not 64-bit label.\n", d.DataName(), len(data))
   832  		dvid.Errorf("Using max label across versions: %d\n", repoMax)
   833  		d.MaxRepoLabel = repoMax
   834  	} else {
   835  		d.MaxRepoLabel = binary.LittleEndian.Uint64(data)
   836  		if d.MaxRepoLabel < repoMax {
   837  			dvid.Errorf("Saved repo-wide max for instance %q was %d, changed to largest version max %d\n", d.DataName(), d.MaxRepoLabel, repoMax)
   838  			d.MaxRepoLabel = repoMax
   839  		}
   840  	}
   841  	wg.Done()
   842  }
   843  
   844  // --- datastore.DataService interface ---------
   845  
   846  func (d *Data) Help() string {
   847  	return helpMessage
   848  }
   849  
   850  func (d *Data) MarshalJSON() ([]byte, error) {
   851  	return json.Marshal(struct {
   852  		Base     *datastore.Data
   853  		Extended Properties
   854  	}{
   855  		d.Data,
   856  		d.Properties,
   857  	})
   858  }
   859  
   860  func (d *Data) GobDecode(b []byte) error {
   861  	buf := bytes.NewBuffer(b)
   862  	dec := gob.NewDecoder(buf)
   863  	if err := dec.Decode(&(d.Data)); err != nil {
   864  		return err
   865  	}
   866  	if err := dec.Decode(&(d.Properties)); err != nil {
   867  		return err
   868  	}
   869  	return nil
   870  }
   871  
   872  func (d *Data) GobEncode() ([]byte, error) {
   873  	var buf bytes.Buffer
   874  	enc := gob.NewEncoder(&buf)
   875  	if err := enc.Encode(d.Data); err != nil {
   876  		return nil, err
   877  	}
   878  	if err := enc.Encode(d.Properties); err != nil {
   879  		return nil, err
   880  	}
   881  	return buf.Bytes(), nil
   882  }
   883  
   884  // DoRPC acts as a switchboard for RPC commands.
   885  func (d *Data) DoRPC(req datastore.Request, reply *datastore.Response) error {
   886  	switch req.TypeCommand() {
   887  	case "dump":
   888  		if len(req.Command) < 5 {
   889  			return fmt.Errorf("Poorly formatted dump command.  See command-line help.")
   890  		}
   891  		// Parse the request
   892  		var uuidStr, dataName, cmdStr, dirStr string
   893  		req.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &dirStr)
   894  
   895  		uuid, versionID, err := datastore.MatchingUUID(uuidStr)
   896  		if err != nil {
   897  			return err
   898  		}
   899  		go func() {
   900  			d.DumpSubvols(uuid, versionID, dirStr)
   901  		}()
   902  		reply.Text = fmt.Sprintf("Asynchronously dumping sparse volumes in directory: %s\n", dirStr)
   903  
   904  	default:
   905  		return fmt.Errorf("Unknown command.  Data type '%s' [%s] does not support '%s' command.",
   906  			d.DataName(), d.TypeName(), req.TypeCommand())
   907  	}
   908  	return nil
   909  }
   910  
   911  // ServeHTTP handles all incoming HTTP requests for this data.
   912  func (d *Data) ServeHTTP(uuid dvid.UUID, ctx *datastore.VersionedCtx, w http.ResponseWriter, r *http.Request) (activity map[string]interface{}) {
   913  	timedLog := dvid.NewTimeLog()
   914  	versionID := ctx.VersionID()
   915  
   916  	// Get the action (GET, POST)
   917  	action := strings.ToLower(r.Method)
   918  
   919  	// Break URL request into arguments
   920  	url := r.URL.Path[len(server.WebAPIPath):]
   921  	parts := strings.Split(url, "/")
   922  	if len(parts[len(parts)-1]) == 0 {
   923  		parts = parts[:len(parts)-1]
   924  	}
   925  
   926  	// Handle POST on data -> setting of configuration
   927  	if len(parts) == 3 && action == "put" {
   928  		config, err := server.DecodeJSON(r)
   929  		if err != nil {
   930  			server.BadRequest(w, r, err)
   931  			return
   932  		}
   933  		if err := d.ModifyConfig(config); err != nil {
   934  			server.BadRequest(w, r, err)
   935  			return
   936  		}
   937  		if err := datastore.SaveDataByUUID(uuid, d); err != nil {
   938  			server.BadRequest(w, r, err)
   939  			return
   940  		}
   941  		fmt.Fprintf(w, "Changed '%s' based on received configuration:\n%s\n", d.DataName(), config)
   942  		return
   943  	}
   944  
   945  	if len(parts) < 4 {
   946  		server.BadRequest(w, r, "Incomplete API request")
   947  		return
   948  	}
   949  
   950  	// Process help and info.
   951  	switch parts[3] {
   952  	case "help":
   953  		w.Header().Set("Content-Type", "text/plain")
   954  		fmt.Fprintln(w, dtype.Help())
   955  
   956  	case "info":
   957  		jsonBytes, err := d.MarshalJSON()
   958  		if err != nil {
   959  			server.BadRequest(w, r, err)
   960  			return
   961  		}
   962  		w.Header().Set("Content-Type", "application/json")
   963  		fmt.Fprintf(w, string(jsonBytes))
   964  
   965  	case "sync":
   966  		if action != "post" {
   967  			server.BadRequest(w, r, "Only POST allowed to sync endpoint")
   968  			return
   969  		}
   970  		replace := r.URL.Query().Get("replace") == "true"
   971  		if err := datastore.SetSyncByJSON(d, uuid, replace, r.Body); err != nil {
   972  			server.BadRequest(w, r, err)
   973  			return
   974  		}
   975  
   976  	case "sparsevol":
   977  		// GET <api URL>/node/<UUID>/<data name>/sparsevol/<label>
   978  		// POST <api URL>/node/<UUID>/<data name>/sparsevol/<label>
   979  		// HEAD <api URL>/node/<UUID>/<data name>/sparsevol/<label>
   980  		if len(parts) < 5 {
   981  			server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'sparsevol' command")
   982  			return
   983  		}
   984  		label, err := strconv.ParseUint(parts[4], 10, 64)
   985  		if err != nil {
   986  			server.BadRequest(w, r, err)
   987  			return
   988  		}
   989  		if label == 0 {
   990  			server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n")
   991  			return
   992  		}
   993  		queryStrings := r.URL.Query()
   994  		var b dvid.Bounds
   995  		b.Voxel, err = dvid.OptionalBoundsFromQueryString(r)
   996  		if err != nil {
   997  			server.BadRequest(w, r, "Error parsing bounds from query string: %v\n", err)
   998  			return
   999  		}
  1000  		b.Block = b.Voxel.Divide(d.BlockSize)
  1001  		b.Exact = true
  1002  		if queryStrings.Get("exact") == "false" {
  1003  			b.Exact = false
  1004  		}
  1005  
  1006  		compression := queryStrings.Get("compression")
  1007  
  1008  		switch action {
  1009  		case "get":
  1010  			data, err := d.GetSparseVol(ctx, label, b)
  1011  			if err != nil {
  1012  				server.BadRequest(w, r, err)
  1013  				return
  1014  			}
  1015  			fmt.Printf("Sparsevol on %d returned %d bytes\n", label, len(data))
  1016  			if data == nil {
  1017  				w.WriteHeader(http.StatusNotFound)
  1018  				return
  1019  			}
  1020  			w.Header().Set("Content-type", "application/octet-stream")
  1021  			switch compression {
  1022  			case "":
  1023  				_, err = w.Write(data)
  1024  				if err != nil {
  1025  					server.BadRequest(w, r, err)
  1026  					return
  1027  				}
  1028  			case "lz4":
  1029  				compressed := make([]byte, lz4.CompressBound(data))
  1030  				var n, outSize int
  1031  				if outSize, err = lz4.Compress(data, compressed); err != nil {
  1032  					server.BadRequest(w, r, err)
  1033  					return
  1034  				}
  1035  				compressed = compressed[:outSize]
  1036  				if n, err = w.Write(compressed); err != nil {
  1037  					server.BadRequest(w, r, err)
  1038  					return
  1039  				}
  1040  				if n != outSize {
  1041  					errmsg := fmt.Sprintf("Only able to write %d of %d lz4 compressed bytes\n", n, outSize)
  1042  					dvid.Errorf(errmsg)
  1043  					server.BadRequest(w, r, errmsg)
  1044  					return
  1045  				}
  1046  			case "gzip":
  1047  				gw := gzip.NewWriter(w)
  1048  				if _, err = gw.Write(data); err != nil {
  1049  					server.BadRequest(w, r, err)
  1050  					return
  1051  				}
  1052  				if err = gw.Close(); err != nil {
  1053  					server.BadRequest(w, r, err)
  1054  					return
  1055  				}
  1056  			default:
  1057  				server.BadRequest(w, r, "unknown compression type %q", compression)
  1058  				return
  1059  			}
  1060  
  1061  		case "head":
  1062  			w.Header().Set("Content-type", "text/html")
  1063  			found, err := d.FoundSparseVol(ctx, label, b)
  1064  			if err != nil {
  1065  				server.BadRequest(w, r, err)
  1066  				return
  1067  			}
  1068  			if found {
  1069  				w.WriteHeader(http.StatusOK)
  1070  			} else {
  1071  				w.WriteHeader(http.StatusNoContent)
  1072  			}
  1073  			return
  1074  
  1075  		case "post":
  1076  			server.BadRequest(w, r, "POST of sparsevol not currently implemented\n")
  1077  			return
  1078  			// if err := d.PutSparseVol(versionID, label, r.Body); err != nil {
  1079  			// 	server.BadRequest(w, r, err)
  1080  			// 	return
  1081  			// }
  1082  		default:
  1083  			server.BadRequest(w, r, "Unable to handle HTTP action %s on sparsevol endpoint", action)
  1084  			return
  1085  		}
  1086  		timedLog.Infof("HTTP %s: sparsevol on label %d (%s)", r.Method, label, r.URL)
  1087  
  1088  	case "sparsevol-by-point":
  1089  		// GET <api URL>/node/<UUID>/<data name>/sparsevol-by-point/<coord>
  1090  		if len(parts) < 5 {
  1091  			server.BadRequest(w, r, "ERROR: DVID requires coord to follow 'sparsevol-by-point' command")
  1092  			return
  1093  		}
  1094  		coord, err := dvid.StringToPoint(parts[4], "_")
  1095  		if err != nil {
  1096  			server.BadRequest(w, r, err)
  1097  			return
  1098  		}
  1099  		source, err := d.GetSyncedLabelblk()
  1100  		if err != nil {
  1101  			server.BadRequest(w, r, err)
  1102  			return
  1103  		}
  1104  		label, err := source.GetLabelAtPoint(ctx.VersionID(), coord)
  1105  		if err != nil {
  1106  			server.BadRequest(w, r, err)
  1107  			return
  1108  		}
  1109  		if label == 0 {
  1110  			server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n")
  1111  			return
  1112  		}
  1113  		data, err := d.GetSparseVol(ctx, label, dvid.Bounds{})
  1114  		if err != nil {
  1115  			server.BadRequest(w, r, err)
  1116  			return
  1117  		}
  1118  		if data == nil {
  1119  			w.WriteHeader(http.StatusNotFound)
  1120  			return
  1121  		}
  1122  		w.Header().Set("Content-type", "application/octet-stream")
  1123  		_, err = w.Write(data)
  1124  		if err != nil {
  1125  			server.BadRequest(w, r, err)
  1126  			return
  1127  		}
  1128  		timedLog.Infof("HTTP %s: sparsevol-by-point at %s (%s)", r.Method, coord, r.URL)
  1129  
  1130  	case "sparsevol-coarse":
  1131  		// GET <api URL>/node/<UUID>/<data name>/sparsevol-coarse/<label>
  1132  		if len(parts) < 5 {
  1133  			server.BadRequest(w, r, "DVID requires label ID to follow 'sparsevol-coarse' command")
  1134  			return
  1135  		}
  1136  		label, err := strconv.ParseUint(parts[4], 10, 64)
  1137  		if err != nil {
  1138  			server.BadRequest(w, r, err)
  1139  			return
  1140  		}
  1141  		if label == 0 {
  1142  			server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n")
  1143  			return
  1144  		}
  1145  		data, err := d.GetSparseCoarseVol(ctx, label)
  1146  		if err != nil {
  1147  			server.BadRequest(w, r, err)
  1148  			return
  1149  		}
  1150  		if data == nil {
  1151  			w.WriteHeader(http.StatusNotFound)
  1152  			return
  1153  		}
  1154  		w.Header().Set("Content-type", "application/octet-stream")
  1155  		_, err = w.Write(data)
  1156  		if err != nil {
  1157  			server.BadRequest(w, r, err)
  1158  			return
  1159  		}
  1160  		timedLog.Infof("HTTP %s: sparsevol-coarse on label %d (%s)", r.Method, label, r.URL)
  1161  
  1162  	case "area":
  1163  		// DELETE <api URL>/node/<UUID>/<data name>/area/<label>/<size>/<offset>
  1164  		if len(parts) < 6 {
  1165  			server.BadRequest(w, r, "DVID requires label ID, size, and offset to follow 'area' endpoint")
  1166  			return
  1167  		}
  1168  		label, err := strconv.ParseUint(parts[4], 10, 64)
  1169  		if err != nil {
  1170  			server.BadRequest(w, r, err)
  1171  			return
  1172  		}
  1173  		if label == 0 {
  1174  			server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n")
  1175  			return
  1176  		}
  1177  		sizeStr, offsetStr := parts[5], parts[6]
  1178  
  1179  		subvol, err := dvid.NewSubvolumeFromStrings(offsetStr, sizeStr, "_")
  1180  		if err != nil {
  1181  			server.BadRequest(w, r, err)
  1182  			return
  1183  		}
  1184  		if subvol.StartPoint().NumDims() != 3 || subvol.Size().NumDims() != 3 {
  1185  			server.BadRequest(w, r, "must specify 3D subvolumes", subvol.StartPoint(), subvol.EndPoint())
  1186  			return
  1187  		}
  1188  
  1189  		// Make sure subvolume gets align with blocks
  1190  		if !dvid.BlockAligned(subvol, d.BlockSize) {
  1191  			server.BadRequest(w, r, "cannot delete sparsevol in non-block aligned geometry %s -> %s", subvol.StartPoint(), subvol.EndPoint())
  1192  			return
  1193  		}
  1194  
  1195  		if action != "delete" {
  1196  			server.BadRequest(w, r, "DVID does not accept the %s action on the 'blocks' endpoint", action)
  1197  			return
  1198  		}
  1199  
  1200  		if err := d.DeleteArea(ctx, label, subvol); err != nil {
  1201  			server.BadRequest(w, r, err)
  1202  			return
  1203  		}
  1204  		timedLog.Infof("HTTP %s: area %s (%s)", r.Method, subvol, r.URL)
  1205  
  1206  	case "maxlabel":
  1207  		// GET <api URL>/node/<UUID>/<data name>/maxlabel
  1208  		w.Header().Set("Content-Type", "application/json")
  1209  		switch action {
  1210  		case "get":
  1211  			maxlabel, ok := d.MaxLabel[versionID]
  1212  			if !ok {
  1213  				server.BadRequest(w, r, "No maximum label found for %s version %d\n", d.DataName(), versionID)
  1214  				return
  1215  			}
  1216  			fmt.Fprintf(w, "{%q: %d}", "maxlabel", maxlabel)
  1217  		default:
  1218  			server.BadRequest(w, r, "Unknown action %q requested: %s\n", action, r.URL)
  1219  			return
  1220  		}
  1221  		timedLog.Infof("HTTP maxlabel request (%s)", r.URL)
  1222  
  1223  	case "nextlabel":
  1224  		// GET <api URL>/node/<UUID>/<data name>/nextlabel
  1225  		// POST <api URL>/node/<UUID>/<data name>/nextlabel
  1226  		w.Header().Set("Content-Type", "application/json")
  1227  		switch action {
  1228  		case "get":
  1229  			fmt.Fprintf(w, "{%q: %d}", "nextlabel", d.MaxRepoLabel+1)
  1230  		case "post":
  1231  			server.BadRequest(w, r, "POST on maxlabel is not supported yet.\n")
  1232  			return
  1233  		default:
  1234  			server.BadRequest(w, r, "Unknown action %q requested: %s\n", action, r.URL)
  1235  			return
  1236  		}
  1237  		timedLog.Infof("HTTP maxlabel request (%s)", r.URL)
  1238  
  1239  	case "split":
  1240  		// POST <api URL>/node/<UUID>/<data name>/split/<label>[?splitlabel=X]
  1241  		if action != "post" {
  1242  			server.BadRequest(w, r, "Split requests must be POST actions.")
  1243  			return
  1244  		}
  1245  		if len(parts) < 5 {
  1246  			server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'split' command")
  1247  			return
  1248  		}
  1249  		fromLabel, err := strconv.ParseUint(parts[4], 10, 64)
  1250  		if err != nil {
  1251  			server.BadRequest(w, r, err)
  1252  			return
  1253  		}
  1254  		if fromLabel == 0 {
  1255  			server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n")
  1256  			return
  1257  		}
  1258  		var splitLabel uint64
  1259  		queryStrings := r.URL.Query()
  1260  		splitStr := queryStrings.Get("splitlabel")
  1261  		if splitStr != "" {
  1262  			splitLabel, err = strconv.ParseUint(splitStr, 10, 64)
  1263  			if err != nil {
  1264  				server.BadRequest(w, r, "Bad parameter for 'splitlabel' query string (%q).  Must be uint64.\n", splitStr)
  1265  			}
  1266  		}
  1267  		toLabel, err := d.SplitLabels(ctx.VersionID(), fromLabel, splitLabel, r.Body)
  1268  		if err != nil {
  1269  			server.BadRequest(w, r, fmt.Sprintf("split: %v", err))
  1270  			return
  1271  		}
  1272  		w.Header().Set("Content-Type", "application/json")
  1273  		fmt.Fprintf(w, "{%q: %d}", "label", toLabel)
  1274  		timedLog.Infof("HTTP split request (%s)", r.URL)
  1275  
  1276  	case "split-coarse":
  1277  		// POST <api URL>/node/<UUID>/<data name>/split-coarse/<label>[?splitlabel=X]
  1278  		if action != "post" {
  1279  			server.BadRequest(w, r, "Split-coarse requests must be POST actions.")
  1280  			return
  1281  		}
  1282  		if len(parts) < 5 {
  1283  			server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'split' command")
  1284  			return
  1285  		}
  1286  		fromLabel, err := strconv.ParseUint(parts[4], 10, 64)
  1287  		if err != nil {
  1288  			server.BadRequest(w, r, err)
  1289  			return
  1290  		}
  1291  		if fromLabel == 0 {
  1292  			server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n")
  1293  			return
  1294  		}
  1295  		var splitLabel uint64
  1296  		queryStrings := r.URL.Query()
  1297  		splitStr := queryStrings.Get("splitlabel")
  1298  		if splitStr != "" {
  1299  			splitLabel, err = strconv.ParseUint(splitStr, 10, 64)
  1300  			if err != nil {
  1301  				server.BadRequest(w, r, "Bad parameter for 'splitlabel' query string (%q).  Must be uint64.\n", splitStr)
  1302  			}
  1303  		}
  1304  		toLabel, err := d.SplitCoarseLabels(ctx.VersionID(), fromLabel, splitLabel, r.Body)
  1305  		if err != nil {
  1306  			server.BadRequest(w, r, fmt.Sprintf("split-coarse: %v", err))
  1307  			return
  1308  		}
  1309  		w.Header().Set("Content-Type", "application/json")
  1310  		fmt.Fprintf(w, "{%q: %d}", "label", toLabel)
  1311  		timedLog.Infof("HTTP split-coarse request (%s)", r.URL)
  1312  
  1313  	case "merge":
  1314  		// POST <api URL>/node/<UUID>/<data name>/merge
  1315  		if action != "post" {
  1316  			server.BadRequest(w, r, "Merge requests must be POST actions.")
  1317  			return
  1318  		}
  1319  		data, err := ioutil.ReadAll(r.Body)
  1320  		if err != nil {
  1321  			server.BadRequest(w, r, "Bad POSTed data for merge.  Should be JSON.")
  1322  			return
  1323  		}
  1324  		var tuple labels.MergeTuple
  1325  		if err := json.Unmarshal(data, &tuple); err != nil {
  1326  			server.BadRequest(w, r, fmt.Sprintf("Bad merge op JSON: %v", err))
  1327  			return
  1328  		}
  1329  		mergeOp, err := tuple.Op()
  1330  		if err != nil {
  1331  			server.BadRequest(w, r, err)
  1332  			return
  1333  		}
  1334  		mergeOp.MutID = d.NewMutationID()
  1335  		if err := d.MergeLabels(ctx.VersionID(), mergeOp); err != nil {
  1336  			server.BadRequest(w, r, fmt.Sprintf("Error on merge: %v", err))
  1337  			return
  1338  		}
  1339  		timedLog.Infof("HTTP merge request (%s)", r.URL)
  1340  
  1341  	case "resync":
  1342  		// POST <api URL>/node/<UUID>/<data name>/resync/<label>
  1343  		if action != "post" {
  1344  			server.BadRequest(w, r, "resync requests must be POST actions.")
  1345  			return
  1346  		}
  1347  		if len(parts) < 5 {
  1348  			server.BadRequest(w, r, "ERROR: DVID requires label ID to follow 'resync' command")
  1349  			return
  1350  		}
  1351  		relabel, err := strconv.ParseUint(parts[4], 10, 64)
  1352  		if err != nil {
  1353  			server.BadRequest(w, r, err)
  1354  			return
  1355  		}
  1356  		if relabel == 0 {
  1357  			server.BadRequest(w, r, "Label 0 is protected background value and cannot be used as sparse volume.\n")
  1358  			return
  1359  		}
  1360  		d.StartUpdate()
  1361  		go func() {
  1362  			if err := d.resyncLabel(ctx, relabel, r.Body); err != nil {
  1363  				dvid.Errorf("Error on resync of data %q, label %d: %v\n", d.DataName(), relabel, err)
  1364  			}
  1365  			d.StopUpdate()
  1366  		}()
  1367  		timedLog.Infof("HTTP resync %d started (%s)", relabel, r.URL)
  1368  
  1369  	default:
  1370  		server.BadAPIRequest(w, r, d)
  1371  	}
  1372  	return
  1373  }
  1374  
  1375  // Given a stored label, make sure our max label tracking is updated.
  1376  func (d *Data) casMaxLabel(batch storage.Batch, v dvid.VersionID, label uint64) {
  1377  	d.mlMu.Lock()
  1378  	defer d.mlMu.Unlock()
  1379  
  1380  	save := false
  1381  	maxLabel, found := d.MaxLabel[v]
  1382  	if !found {
  1383  		dvid.Infof("Bad max label of version %d -- none found!\n", v)
  1384  		maxLabel = 1
  1385  	}
  1386  	if maxLabel < label {
  1387  		maxLabel = label
  1388  		save = true
  1389  	}
  1390  	if save {
  1391  		buf := make([]byte, 8)
  1392  		binary.LittleEndian.PutUint64(buf, maxLabel)
  1393  		batch.Put(maxLabelTKey, buf)
  1394  		d.MaxLabel[v] = maxLabel
  1395  
  1396  		if d.MaxRepoLabel < maxLabel {
  1397  			d.MaxRepoLabel = maxLabel
  1398  			ctx := storage.NewDataContext(d, 0)
  1399  			store, err := datastore.GetOrderedKeyValueDB(d)
  1400  			if err != nil {
  1401  				dvid.Errorf("Data type labelvol had error initializing store: %v\n", err)
  1402  			} else {
  1403  				store.Put(ctx, maxRepoLabelTKey, buf)
  1404  			}
  1405  		}
  1406  	}
  1407  	if err := batch.Commit(); err != nil {
  1408  		dvid.Errorf("batch put: %v\n", err)
  1409  		return
  1410  	}
  1411  }
  1412  
  1413  // NewLabel returns a new label for the given version.
  1414  func (d *Data) NewLabel(v dvid.VersionID) (uint64, error) {
  1415  	d.mlMu.Lock()
  1416  	defer d.mlMu.Unlock()
  1417  
  1418  	// Make sure we aren't trying to increment a label on a locked node.
  1419  	locked, err := datastore.LockedVersion(v)
  1420  	if err != nil {
  1421  		return 0, err
  1422  	}
  1423  	if locked {
  1424  		return 0, fmt.Errorf("can't ask for new label in a locked version id %d", v)
  1425  	}
  1426  
  1427  	// Increment and store.
  1428  	d.MaxRepoLabel++
  1429  	d.MaxLabel[v] = d.MaxRepoLabel
  1430  
  1431  	store, err := datastore.GetOrderedKeyValueDB(d)
  1432  	if err != nil {
  1433  		return 0, err
  1434  	}
  1435  	buf := make([]byte, 8)
  1436  	binary.LittleEndian.PutUint64(buf, d.MaxRepoLabel)
  1437  	ctx := datastore.NewVersionedCtx(d, v)
  1438  	if err := store.Put(ctx, maxLabelTKey, buf); err != nil {
  1439  		return 0, err
  1440  	}
  1441  
  1442  	ctx2 := storage.NewDataContext(d, 0)
  1443  	if err := store.Put(ctx2, maxRepoLabelTKey, buf); err != nil {
  1444  		return 0, err
  1445  	}
  1446  
  1447  	return d.MaxRepoLabel, nil
  1448  }
  1449  
  1450  // Returns RLEs for a given label where the key of the returned map is the block index
  1451  // in string format.
  1452  func (d *Data) GetLabelRLEs(v dvid.VersionID, label uint64) (dvid.BlockRLEs, error) {
  1453  	store, err := datastore.GetOrderedKeyValueDB(d)
  1454  	if err != nil {
  1455  		return nil, fmt.Errorf("Data type labelvol had error initializing store: %v\n", err)
  1456  	}
  1457  
  1458  	// Get the start/end indices for this body's KeyLabelSpatialMap (b + s) keys.
  1459  	begIndex := NewTKey(label, dvid.MinIndexZYX.ToIZYXString())
  1460  	endIndex := NewTKey(label, dvid.MaxIndexZYX.ToIZYXString())
  1461  
  1462  	// Process all the b+s keys and their values, which contain RLE runs for that label.
  1463  	labelRLEs := dvid.BlockRLEs{}
  1464  	var f storage.ChunkFunc = func(chunk *storage.Chunk) error {
  1465  		// Get the block index where the fromLabel is present
  1466  		_, blockStr, err := DecodeTKey(chunk.K)
  1467  		if err != nil {
  1468  			return fmt.Errorf("Can't recover block index with chunk key %v: %v\n", chunk.K, err)
  1469  		}
  1470  
  1471  		var blockRLEs dvid.RLEs
  1472  		if err := blockRLEs.UnmarshalBinary(chunk.V); err != nil {
  1473  			return fmt.Errorf("Unable to unmarshal RLE for label in block %v", chunk.K)
  1474  		}
  1475  		labelRLEs[blockStr] = blockRLEs
  1476  		return nil
  1477  	}
  1478  	ctx := datastore.NewVersionedCtx(d, v)
  1479  	err = store.ProcessRange(ctx, begIndex, endIndex, &storage.ChunkOp{}, f)
  1480  	if err != nil {
  1481  		return nil, err
  1482  	}
  1483  	return labelRLEs, nil
  1484  }
  1485  
  1486  type sparseOp struct {
  1487  	versionID dvid.VersionID
  1488  	encoding  []byte
  1489  	numBlocks uint32
  1490  	numRuns   uint32
  1491  	//numVoxels int32
  1492  }
  1493  
  1494  // Alter serialized RLEs by the bounds.
  1495  func boundRLEs(b []byte, bounds *dvid.OptionalBounds) ([]byte, error) {
  1496  	var oldRLEs dvid.RLEs
  1497  	err := oldRLEs.UnmarshalBinary(b)
  1498  	if err != nil {
  1499  		return nil, err
  1500  	}
  1501  	newRLEs := oldRLEs.FitToBounds(bounds)
  1502  	return newRLEs.MarshalBinary()
  1503  }
  1504  
  1505  // FoundSparseVol returns true if a sparse volume is found for the given label
  1506  // within the given bounds.
  1507  func (d *Data) FoundSparseVol(ctx *datastore.VersionedCtx, label uint64, bounds dvid.Bounds) (bool, error) {
  1508  	store, err := datastore.GetOrderedKeyValueDB(d)
  1509  	if err != nil {
  1510  		return false, fmt.Errorf("Data type labelvol had error initializing store: %v\n", err)
  1511  	}
  1512  
  1513  	// Get the start/end indices for this body's KeyLabelSpatialMap (b + s) keys.
  1514  	minZYX := dvid.MinIndexZYX
  1515  	maxZYX := dvid.MaxIndexZYX
  1516  	if minZ, ok := bounds.Block.MinZ(); ok {
  1517  		minZYX[2] = minZ
  1518  	}
  1519  	if maxZ, ok := bounds.Block.MaxZ(); ok {
  1520  		maxZYX[2] = maxZ
  1521  	}
  1522  
  1523  	// Scan through all keys coming from label blocks to see if we have any hits.
  1524  	var constituents labels.Set
  1525  	iv := d.getMergeIV(ctx.VersionID())
  1526  	mapping := labels.LabelMap(iv)
  1527  
  1528  	if mapping == nil {
  1529  		constituents = labels.Set{label: struct{}{}}
  1530  	} else {
  1531  		// Check if this label has been merged.
  1532  		if _, found := mapping.Get(label); found {
  1533  			return false, nil
  1534  		}
  1535  
  1536  		// If not, see if labels have been merged into it.
  1537  		constituents = mapping.ConstituentLabels(label)
  1538  	}
  1539  
  1540  	shortCircuitErr := fmt.Errorf("Found data, aborting.")
  1541  	var f storage.ChunkFunc = func(chunk *storage.Chunk) error {
  1542  		// Make sure this block is within the optinonal bounding.
  1543  		if bounds.Block.BoundedX() || bounds.Block.BoundedY() {
  1544  			_, blockStr, err := DecodeTKey(chunk.K)
  1545  			if err != nil {
  1546  				return fmt.Errorf("Error decoding sparse volume key (%v): %v\n", chunk.K, err)
  1547  			}
  1548  			indexZYX, err := blockStr.IndexZYX()
  1549  			if err != nil {
  1550  				return fmt.Errorf("Error decoding block coordinate (%v) for sparse volume: %v\n", blockStr, err)
  1551  			}
  1552  			blockX, blockY, _ := indexZYX.Unpack()
  1553  			if bounds.Block.OutsideX(blockX) || bounds.Block.OutsideY(blockY) {
  1554  				return nil
  1555  			}
  1556  		}
  1557  		return shortCircuitErr
  1558  	}
  1559  
  1560  	// Iterate through each constituent label.
  1561  	for constituent := range constituents {
  1562  		begTKey := NewTKey(constituent, minZYX.ToIZYXString())
  1563  		endTKey := NewTKey(constituent, maxZYX.ToIZYXString())
  1564  		err := store.ProcessRange(ctx, begTKey, endTKey, &storage.ChunkOp{}, f)
  1565  		if err == shortCircuitErr {
  1566  			return true, nil
  1567  		}
  1568  		if err != nil {
  1569  			return false, err
  1570  		}
  1571  	}
  1572  	dvid.Debugf("Found no sparse volume with label %d, composed of labels %s\n", label, constituents)
  1573  	return false, nil
  1574  }
  1575  
  1576  // GetSparseVol returns an encoded sparse volume given a label.  The encoding has the
  1577  // following format where integers are little endian:
  1578  //    byte     Payload descriptor:
  1579  //               Bit 0 (LSB) - 8-bit grayscale
  1580  //               Bit 1 - 16-bit grayscale
  1581  //               Bit 2 - 16-bit normal
  1582  //               ...
  1583  //    uint8    Number of dimensions
  1584  //    uint8    Dimension of run (typically 0 = X)
  1585  //    byte     Reserved (to be used later)
  1586  //    uint32    # Voxels
  1587  //    uint32    # Spans
  1588  //    Repeating unit of:
  1589  //        int32   Coordinate of run start (dimension 0)
  1590  //        int32   Coordinate of run start (dimension 1)
  1591  //        int32   Coordinate of run start (dimension 2)
  1592  //        int32   Length of run
  1593  //        bytes   Optional payload dependent on first byte descriptor
  1594  //
  1595  func (d *Data) GetSparseVol(ctx *datastore.VersionedCtx, label uint64, bounds dvid.Bounds) ([]byte, error) {
  1596  	iv := d.getMergeIV(ctx.VersionID())
  1597  	mapping := labels.LabelMap(iv)
  1598  	if mapping != nil {
  1599  		// Check if this label has been merged.
  1600  		if mapped, found := mapping.FinalLabel(label); found {
  1601  			dvid.Debugf("Label %d has already been merged into label %d.  Skipping sparse vol retrieval.\n", label, mapped)
  1602  			return nil, nil
  1603  		}
  1604  	}
  1605  
  1606  	store, err := datastore.GetOrderedKeyValueDB(d)
  1607  	if err != nil {
  1608  		return nil, fmt.Errorf("Data type labelvol had error initializing store: %v\n", err)
  1609  	}
  1610  
  1611  	// Create the sparse volume header
  1612  	buf := new(bytes.Buffer)
  1613  	buf.WriteByte(dvid.EncodingBinary)
  1614  	binary.Write(buf, binary.LittleEndian, uint8(3))  // # of dimensions
  1615  	binary.Write(buf, binary.LittleEndian, byte(0))   // dimension of run (X = 0)
  1616  	buf.WriteByte(byte(0))                            // reserved for later
  1617  	binary.Write(buf, binary.LittleEndian, uint32(0)) // Placeholder for # voxels
  1618  	binary.Write(buf, binary.LittleEndian, uint32(0)) // Placeholder for # spans
  1619  
  1620  	// Get the start/end indices for this body's KeyLabelSpatialMap (b + s) keys.
  1621  	minZYX := dvid.MinIndexZYX
  1622  	maxZYX := dvid.MaxIndexZYX
  1623  	if minZ, ok := bounds.Block.MinZ(); ok {
  1624  		minZYX[2] = minZ
  1625  	}
  1626  	if maxZ, ok := bounds.Block.MaxZ(); ok {
  1627  		maxZYX[2] = maxZ
  1628  	}
  1629  	// Process all the b+s keys and their values, which contain RLE runs for that label.
  1630  	// TODO -- Make processing asynchronous so can overlap with range disk read now that
  1631  	//   there could be more processing due to bounding calcs.
  1632  	var numRuns, numBlocks uint32
  1633  	encoding := buf.Bytes()
  1634  
  1635  	var f storage.ChunkFunc = func(chunk *storage.Chunk) error {
  1636  		numBlocks++
  1637  
  1638  		// Make sure this block is within the optinonal bounding.
  1639  		if bounds.Block.BoundedX() || bounds.Block.BoundedY() {
  1640  			_, blockStr, err := DecodeTKey(chunk.K)
  1641  			if err != nil {
  1642  				return fmt.Errorf("Error decoding sparse volume key (%v): %v\n", chunk.K, err)
  1643  			}
  1644  			blockX, blockY, _, err := blockStr.Unpack()
  1645  			if err != nil {
  1646  				return fmt.Errorf("Error decoding block %v: %v\n", blockStr, err)
  1647  			}
  1648  			if bounds.Block.OutsideX(blockX) || bounds.Block.OutsideY(blockY) {
  1649  				return nil
  1650  			}
  1651  		}
  1652  
  1653  		// Adjust RLEs within block if we are bounded.
  1654  		var rles []byte
  1655  		var err error
  1656  		if bounds.Exact && bounds.Voxel.IsSet() {
  1657  			rles, err = boundRLEs(chunk.V, bounds.Voxel)
  1658  			if err != nil {
  1659  				return fmt.Errorf("Error in adjusting RLEs to bounds: %v\n", err)
  1660  			}
  1661  		} else {
  1662  			rles = chunk.V
  1663  		}
  1664  
  1665  		numRuns += uint32(len(rles) / 16)
  1666  		if int64(len(encoding))+int64(len(rles)) > server.MaxDataRequest {
  1667  			return fmt.Errorf("Sparse volume read aborted because length exceeds %d bytes", server.MaxDataRequest)
  1668  		}
  1669  		encoding = append(encoding, rles...)
  1670  		return nil
  1671  	}
  1672  
  1673  	// Iterate through each constituent label.
  1674  	if mapping == nil {
  1675  		begTKey := NewTKey(label, minZYX.ToIZYXString())
  1676  		endTKey := NewTKey(label, maxZYX.ToIZYXString())
  1677  		if err := store.ProcessRange(ctx, begTKey, endTKey, &storage.ChunkOp{}, f); err != nil {
  1678  			return nil, err
  1679  		}
  1680  	} else {
  1681  		// Find all labels merged into this label.
  1682  		constituents := mapping.ConstituentLabels(label)
  1683  		for constituent := range constituents {
  1684  			begTKey := NewTKey(constituent, minZYX.ToIZYXString())
  1685  			endTKey := NewTKey(constituent, maxZYX.ToIZYXString())
  1686  			if err := store.ProcessRange(ctx, begTKey, endTKey, &storage.ChunkOp{}, f); err != nil {
  1687  				return nil, err
  1688  			}
  1689  		}
  1690  		if len(constituents) > 1 {
  1691  			dvid.Debugf("Returning sparse volume for label %d, composed of labels %s\n", label, constituents)
  1692  		}
  1693  	}
  1694  
  1695  	binary.LittleEndian.PutUint32(encoding[8:12], numRuns)
  1696  
  1697  	dvid.Debugf("[%s] label %d: found %d blocks, %d runs\n", ctx, label, numBlocks, numRuns)
  1698  	if numBlocks == 0 {
  1699  		return nil, nil
  1700  	}
  1701  	return encoding, nil
  1702  }
  1703  
  1704  // GetSparseCoarseVol returns an encoded sparse volume given a label.  The encoding has the
  1705  // following format where integers are little endian:
  1706  // 		byte     Set to 0
  1707  // 		uint8    Number of dimensions
  1708  // 		uint8    Dimension of run (typically 0 = X)
  1709  // 		byte     Reserved (to be used later)
  1710  // 		uint32    # Blocks [TODO.  0 for now]
  1711  // 		uint32    # Spans
  1712  // 		Repeating unit of:
  1713  //     		int32   Block coordinate of run start (dimension 0)
  1714  //     		int32   Block coordinate of run start (dimension 1)
  1715  //     		int32   Block coordinate of run start (dimension 2)
  1716  //     		int32   Length of run
  1717  //
  1718  func (d *Data) GetSparseCoarseVol(ctx *datastore.VersionedCtx, label uint64) ([]byte, error) {
  1719  	iv := d.getMergeIV(ctx.VersionID())
  1720  	mapping := labels.LabelMap(iv)
  1721  	if mapping != nil {
  1722  		// Check if this label has been merged.
  1723  		if _, found := mapping.FinalLabel(label); found {
  1724  			return nil, nil
  1725  		}
  1726  	}
  1727  
  1728  	// Create the sparse volume header
  1729  	buf := new(bytes.Buffer)
  1730  	buf.WriteByte(dvid.EncodingBinary)
  1731  	binary.Write(buf, binary.LittleEndian, uint8(3))  // # of dimensions
  1732  	binary.Write(buf, binary.LittleEndian, byte(0))   // dimension of run (X = 0)
  1733  	buf.WriteByte(byte(0))                            // reserved for later
  1734  	binary.Write(buf, binary.LittleEndian, uint32(0)) // Placeholder for # blocks
  1735  	encoding := buf.Bytes()
  1736  
  1737  	// Iterate through each constituent label, appending to the spans.
  1738  	var numBlocks uint32
  1739  	var spans dvid.Spans
  1740  
  1741  	if mapping == nil {
  1742  		var err error
  1743  		numBlocks, spans, err = getSparseVolBlocks(ctx, label)
  1744  		if err != nil {
  1745  			return nil, err
  1746  		}
  1747  	} else {
  1748  		// Find all labels merged into this label.
  1749  		constituents := mapping.ConstituentLabels(label)
  1750  		for constituent := range constituents {
  1751  			curBlocks, curSpans, err := getSparseVolBlocks(ctx, constituent)
  1752  			if err != nil {
  1753  				return nil, err
  1754  			}
  1755  			numBlocks += curBlocks
  1756  			spans = append(spans, curSpans...)
  1757  		}
  1758  		dvid.Debugf("Returning coarse sparse volume for label %d, composed of labels %s\n", label, constituents)
  1759  	}
  1760  
  1761  	if numBlocks == 0 {
  1762  		return nil, nil
  1763  	}
  1764  
  1765  	spansBytes, err := spans.MarshalBinary()
  1766  	if err != nil {
  1767  		return nil, err
  1768  	}
  1769  	encoding = append(encoding, spansBytes...)
  1770  	dvid.Debugf("[%s] coarse subvol for label %d: found %d blocks\n", ctx, label, numBlocks)
  1771  	return encoding, nil
  1772  }
  1773  
  1774  func getSparseVolBlocks(ctx *datastore.VersionedCtx, label uint64) (numBlocks uint32, spans dvid.Spans, err error) {
  1775  	store, err := ctx.GetOrderedKeyValueDB()
  1776  	if err != nil {
  1777  		return 0, nil, fmt.Errorf("Data type labelvol had error initializing store: %v\n", err)
  1778  	}
  1779  
  1780  	var span *dvid.Span
  1781  
  1782  	// Get the start/end indices for this body's KeyLabelSpatialMap (b + s) keys.
  1783  	begTKey := NewTKey(label, dvid.MinIndexZYX.ToIZYXString())
  1784  	endTKey := NewTKey(label, dvid.MaxIndexZYX.ToIZYXString())
  1785  
  1786  	// Process all the b+s keys and their values, which contain RLE runs for that label.
  1787  	keys, err := store.KeysInRange(ctx, begTKey, endTKey)
  1788  	if err != nil {
  1789  		return 0, nil, fmt.Errorf("Cannot get keys for coarse sparse volume: %v", err)
  1790  	}
  1791  
  1792  	for _, tk := range keys {
  1793  		numBlocks++
  1794  		_, blockStr, err := DecodeTKey(tk)
  1795  		if err != nil {
  1796  			return 0, nil, fmt.Errorf("Error retrieving RLE runs for label %d: %v", label, err)
  1797  		}
  1798  		indexZYX, err := blockStr.IndexZYX()
  1799  		if err != nil {
  1800  			return 0, nil, fmt.Errorf("Error decoding block coordinate (%v) for sparse volume: %v\n", blockStr, err)
  1801  		}
  1802  		x, y, z := indexZYX.Unpack()
  1803  		if span == nil {
  1804  			span = &dvid.Span{z, y, x, x}
  1805  		} else if !span.Extends(x, y, z) {
  1806  			spans = append(spans, *span)
  1807  			span = &dvid.Span{z, y, x, x}
  1808  		}
  1809  	}
  1810  	if err != nil {
  1811  		return 0, nil, err
  1812  	}
  1813  	if span != nil {
  1814  		spans = append(spans, *span)
  1815  	}
  1816  	return
  1817  }
  1818  
  1819  func (d *Data) DeleteArea(ctx *datastore.VersionedCtx, label uint64, subvol *dvid.Subvolume) error {
  1820  	store, err := datastore.GetOrderedKeyValueDB(d)
  1821  	if err != nil {
  1822  		return fmt.Errorf("unable to get backing store for data %q: %v\n", d.DataName(), err)
  1823  	}
  1824  	batcher, ok := store.(storage.KeyValueBatcher)
  1825  	if !ok {
  1826  		return fmt.Errorf("Data type labelvol requires batch-enabled store, which %q is not\n", store)
  1827  	}
  1828  
  1829  	begVoxel, ok := subvol.StartPoint().(dvid.Chunkable)
  1830  	if !ok {
  1831  		return fmt.Errorf("delete area StartPoint() cannot handle Chunkable points.")
  1832  	}
  1833  	endVoxel, ok := subvol.EndPoint().(dvid.Chunkable)
  1834  	if !ok {
  1835  		return fmt.Errorf("delete area EndPoint() cannot handle Chunkable points.")
  1836  	}
  1837  	begChunk := begVoxel.Chunk(d.BlockSize).(dvid.ChunkPoint3d)
  1838  	endChunk := endVoxel.Chunk(d.BlockSize).(dvid.ChunkPoint3d)
  1839  	it := dvid.NewIndexZYXIterator(begChunk, endChunk)
  1840  
  1841  	batch := batcher.NewBatch(ctx)
  1842  	for {
  1843  		if !it.Valid() {
  1844  			dvid.Infof("Breaking\n")
  1845  			break
  1846  		}
  1847  
  1848  		indexBeg, indexEnd, err := it.IndexSpan()
  1849  		if err != nil {
  1850  			return err
  1851  		}
  1852  		begBlock := indexBeg.(*dvid.IndexZYX).ToIZYXString()
  1853  		endBlock := indexEnd.(*dvid.IndexZYX).ToIZYXString()
  1854  		dvid.Infof("Deleting from %s -> %s\n", begBlock, endBlock)
  1855  
  1856  		begTKey := NewTKey(label, begBlock)
  1857  		endTKey := NewTKey(label, endBlock)
  1858  
  1859  		err = store.ProcessRange(ctx, begTKey, endTKey, &storage.ChunkOp{}, func(c *storage.Chunk) error {
  1860  			if c == nil {
  1861  				return fmt.Errorf("received nil chunk in delete area for data %s", d.DataName())
  1862  			}
  1863  			label, block, err := DecodeTKey(c.K)
  1864  			if err != nil {
  1865  				return err
  1866  			}
  1867  			dvid.Infof("Deleting label %d from %s\n", label, block)
  1868  			batch.Delete(c.K)
  1869  			return nil
  1870  		})
  1871  		if err != nil {
  1872  			return err
  1873  		}
  1874  		it.NextSpan()
  1875  	}
  1876  
  1877  	if err := batch.Commit(); err != nil {
  1878  		return fmt.Errorf("Batch commit during delete area of %s label %d: %v\n", d.DataName(), label, err)
  1879  	}
  1880  	return nil
  1881  }
  1882  
  1883  // deletes prior sparsevol for label and recreates based on associated labelblk with the POSTed
  1884  // blocks.
  1885  func (d *Data) resyncLabel(ctx *datastore.VersionedCtx, relabel uint64, r io.ReadCloser) error {
  1886  	timedLog := dvid.NewTimeLog()
  1887  	store, err := datastore.GetOrderedKeyValueDB(d)
  1888  	if err != nil {
  1889  		return fmt.Errorf("data %q had error initializing store: %v\n", d.DataName(), err)
  1890  	}
  1891  
  1892  	blkdata, err := d.GetSyncedLabelblk()
  1893  	if err != nil {
  1894  		return err
  1895  	}
  1896  
  1897  	// Read the sparse volume from reader.
  1898  	var resyncROI dvid.RLEs
  1899  	resyncROI, err = dvid.ReadRLEs(r)
  1900  	if err != nil {
  1901  		return err
  1902  	}
  1903  	resyncNumBlocks, _ := resyncROI.Stats()
  1904  
  1905  	var numBlocks int
  1906  	for _, rle := range resyncROI {
  1907  		startPt := dvid.ChunkPoint3d(rle.StartPt())
  1908  		var endPt dvid.ChunkPoint3d
  1909  		endPt[0] = startPt[0] + rle.Length() - 1
  1910  		endPt[1] = startPt[1]
  1911  		endPt[2] = startPt[2]
  1912  
  1913  		begTKey := NewTKey(relabel, startPt.ToIZYXString())
  1914  		endTKey := NewTKey(relabel, endPt.ToIZYXString())
  1915  		if err := store.DeleteRange(ctx, begTKey, endTKey); err != nil {
  1916  			return err
  1917  		}
  1918  
  1919  		blocks, err := blkdata.GetBlocks(ctx.VersionID(), startPt, int(rle.Length()))
  1920  		if err != nil {
  1921  			return err
  1922  		}
  1923  		numBlocks += len(blocks)
  1924  
  1925  		for _, labelBlock := range blocks {
  1926  			tk := NewTKey(relabel, labelBlock.Pos.ToIZYXString())
  1927  			rleBytes, err := d.calcLabelRLEs(relabel, labelBlock)
  1928  			if err != nil {
  1929  				return err
  1930  			}
  1931  			if rleBytes == nil {
  1932  				continue
  1933  			}
  1934  			if err := store.Put(ctx, tk, rleBytes); err != nil {
  1935  				return err
  1936  			}
  1937  		}
  1938  	}
  1939  	timedLog.Infof("Finished resync label %d, %d of %d blocks within data %q\n", relabel, numBlocks, resyncNumBlocks, blkdata.DataName())
  1940  	return nil
  1941  }
  1942  
  1943  func (d *Data) calcLabelRLEs(relabel uint64, block labelblk.Block) ([]byte, error) {
  1944  	blockBytes := len(block.Data)
  1945  	if blockBytes != int(d.BlockSize.Prod())*8 {
  1946  		return nil, fmt.Errorf("Deserialized label block %d bytes, not uint64 size times %d block elements\n",
  1947  			blockBytes, d.BlockSize.Prod())
  1948  	}
  1949  	var labelRLEs dvid.RLEs
  1950  	firstPt := block.Pos.MinPoint(d.BlockSize)
  1951  	lastPt := block.Pos.MaxPoint(d.BlockSize)
  1952  
  1953  	var curStart dvid.Point3d
  1954  	var voxelLabel, curLabel uint64
  1955  	var z, y, x, curRun int32
  1956  	start := 0
  1957  	for z = firstPt.Value(2); z <= lastPt.Value(2); z++ {
  1958  		for y = firstPt.Value(1); y <= lastPt.Value(1); y++ {
  1959  			for x = firstPt.Value(0); x <= lastPt.Value(0); x++ {
  1960  				voxelLabel = binary.LittleEndian.Uint64(block.Data[start : start+8])
  1961  				start += 8
  1962  
  1963  				// If we hit background or have switched label, save old run and start new one.
  1964  				if voxelLabel == 0 || voxelLabel != curLabel {
  1965  					// Save old run
  1966  					if curLabel == relabel && curRun > 0 {
  1967  						labelRLEs = append(labelRLEs, dvid.NewRLE(curStart, curRun))
  1968  					}
  1969  					// Start new one if not zero label.
  1970  					if voxelLabel == relabel {
  1971  						curStart = dvid.Point3d{x, y, z}
  1972  						curRun = 1
  1973  					} else {
  1974  						curRun = 0
  1975  					}
  1976  					curLabel = voxelLabel
  1977  				} else if voxelLabel == relabel {
  1978  					curRun++
  1979  				}
  1980  			}
  1981  			// Force break of any runs when we finish x scan.
  1982  			if curRun > 0 {
  1983  				labelRLEs = append(labelRLEs, dvid.NewRLE(curStart, curRun))
  1984  				curLabel = 0
  1985  				curRun = 0
  1986  			}
  1987  		}
  1988  	}
  1989  
  1990  	return labelRLEs.MarshalBinary()
  1991  }
  1992  
  1993  // PutSparseVol stores an encoded sparse volume that stays within a given forward label.
  1994  // Note that this encoded sparse volume is added to any existing sparse volume for
  1995  // a body rather than replace it.  To carve out a part of an existing sparse volume,
  1996  // use the SplitLabels().
  1997  //
  1998  // This function handles modification/deletion of all denormalized data touched by this
  1999  // sparse label volume.
  2000  //
  2001  // EVENTS
  2002  //
  2003  //
  2004  /*
  2005  func (d *Data) PutSparseVol(v dvid.VersionID, label uint64, r io.Reader) error {
  2006  	store, err := storage.MutableStore()
  2007  	if err != nil {
  2008  		return fmt.Errorf("Data type labelvol had error initializing store: %v\n", err)
  2009  	}
  2010  	batcher, ok := store.(storage.KeyValueBatcher)
  2011  	if !ok {
  2012  		return fmt.Errorf("Data type labelvol requires batch-enabled store, which %q is not\n", store)
  2013  	}
  2014  
  2015  	// Mark the label as dirty until done.
  2016  	iv := dvid.InstanceVersion{d.DataName(), v}
  2017  	labels.DirtyCache.Incr(iv, label)
  2018  	defer labels.DirtyCache.Decr(iv, label)
  2019  
  2020  	// TODO -- Signal that we are starting a label modification.
  2021  
  2022  	// Read the sparse volume from reader.
  2023  	header := make([]byte, 8)
  2024  	if _, err = io.ReadFull(r, header); err != nil {
  2025  		return err
  2026  	}
  2027  	if header[0] != dvid.EncodingBinary {
  2028  		return fmt.Errorf("sparse vol for split has unknown encoding format: %v", header[0])
  2029  	}
  2030  	var numSpans uint32
  2031  	if err = binary.Read(r, binary.LittleEndian, &numSpans); err != nil {
  2032  		return err
  2033  	}
  2034  	var mods dvid.RLEs
  2035  	if err = mods.UnmarshalBinaryReader(r, numSpans); err != nil {
  2036  		return err
  2037  	}
  2038  
  2039  	// Partition the mods spans into blocks.
  2040  	var modmap dvid.BlockRLEs
  2041  	modmap, err = mods.Partition(d.BlockSize)
  2042  	if err != nil {
  2043  		return err
  2044  	}
  2045  
  2046  	// Publish sparsevol mod event
  2047  	evt := datastore.SyncEvent{d.DataUUID(), labels.SparsevolModEvent}
  2048  	msg := datastore.SyncMessage{v, labels.DeltaSparsevol{label, modmap}}
  2049  	if err := datastore.NotifySubscribers(evt, msg); err != nil {
  2050  		return err
  2051  	}
  2052  
  2053  	// Get a sorted list of blocks that cover mods.
  2054  	modblks := modmap.SortedKeys()
  2055  
  2056  	ctx := datastore.NewVersionedCtx(d, v)
  2057  	batch := batcher.NewBatch(ctx)
  2058  	var voxelsAdded int64
  2059  
  2060  	for _, modblk := range modblks {
  2061  
  2062  		// Get original block
  2063  		tk := NewTKey(label, modblk)
  2064  		val, err := store.Get(ctx, tk)
  2065  		if err != nil {
  2066  			return err
  2067  		}
  2068  
  2069  		// If there's no original block, just write the mod block.
  2070  		if val == nil || len(val) == 0 {
  2071  			numVoxels, _ := modmap[modblk].Stats()
  2072  			voxelsAdded += int64(numVoxels)
  2073  			rleBytes, err := modmap[modblk].MarshalBinary()
  2074  			if err != nil {
  2075  				return fmt.Errorf("can't serialize modified RLEs for %d: %v\n", label, err)
  2076  			}
  2077  			batch.Put(tk, rleBytes)
  2078  			continue
  2079  		}
  2080  
  2081  		// if there's an original, integrate and write merged RLE.
  2082  		var rles dvid.RLEs
  2083  		if err := rles.UnmarshalBinary(val); err != nil {
  2084  			return fmt.Errorf("Unable to unmarshal RLE for original labels in block %s", modblk.Print())
  2085  		}
  2086  		voxelsAdded += rles.Add(modmap[modblk])
  2087  		rleBytes, err := rles.MarshalBinary()
  2088  		if err != nil {
  2089  			return fmt.Errorf("can't serialize modified RLEs of %d: %v\n", label, err)
  2090  		}
  2091  		batch.Put(tk, rleBytes)
  2092  	}
  2093  
  2094  	if err := batch.Commit(); err != nil {
  2095  		return fmt.Errorf("Batch commit during mod of %s label %d: %v\n", d.DataName(), label, err)
  2096  	}
  2097  
  2098  	// Publish change in label sizes.
  2099  	delta := labels.DeltaModSize{
  2100  		Label:      label,
  2101  		SizeChange: voxelsAdded,
  2102  	}
  2103  	evt = datastore.SyncEvent{d.DataUUID(), labels.ChangeSizeEvent}
  2104  	msg = datastore.SyncMessage{v, delta}
  2105  	if err := datastore.NotifySubscribers(evt, msg); err != nil {
  2106  		return err
  2107  	}
  2108  
  2109  	// TODO -- Publish label mod end
  2110  
  2111  	return nil
  2112  }
  2113  */
  2114  
  2115  type DumpFiles struct {
  2116  	files map[dvid.IZYXString]*os.File
  2117  	dir   string
  2118  
  2119  	// debug vars
  2120  	tlog       dvid.TimeLog
  2121  	numKV      uint64
  2122  	lastTime   time.Time
  2123  	lastBytes  uint64
  2124  	totalBytes uint64
  2125  }
  2126  
  2127  func (df DumpFiles) close() {
  2128  	for _, f := range df.files {
  2129  		f.Close()
  2130  	}
  2131  }
  2132  
  2133  func (df *DumpFiles) process(kv *storage.TKeyValue, blockSize dvid.Point3d) error {
  2134  	if kv.K == nil {
  2135  		return fmt.Errorf("Had nil TKey in process() for labelvol dump!")
  2136  	}
  2137  	label, block, err := DecodeTKey(kv.K)
  2138  	if err != nil {
  2139  		return fmt.Errorf("Couldn't decode tkey from key %x: %v\n", kv.K, err)
  2140  	}
  2141  
  2142  	chunkPt, err := block.ToChunkPoint3d()
  2143  	if err != nil {
  2144  		return err
  2145  	}
  2146  	subvolPt := chunkPt
  2147  	subvolPt[0] /= (512 / blockSize[0])
  2148  	subvolPt[1] /= (512 / blockSize[1])
  2149  	subvolPt[2] /= (512 / blockSize[2])
  2150  	subvolZYX := subvolPt.ToIZYXString()
  2151  
  2152  	f, found := df.files[subvolZYX]
  2153  	if !found {
  2154  		fname := filepath.Join(df.dir, fmt.Sprintf("subvols-%03d_%03d_%03d.dat", subvolPt[0], subvolPt[1], subvolPt[2]))
  2155  		f, err = os.OpenFile(fname, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0664)
  2156  		if err != nil {
  2157  			return fmt.Errorf("problem opening file for subvol %s: %v\n", subvolPt, err)
  2158  		}
  2159  		df.files[subvolZYX] = f
  2160  	}
  2161  
  2162  	var blockRLEs dvid.RLEs
  2163  	if err := blockRLEs.UnmarshalBinary(kv.V); err != nil {
  2164  		return fmt.Errorf("Unable to unmarshal RLEs for label %d, block %s: %v\n", label, subvolPt, err)
  2165  	}
  2166  	numSpans := int32(len(blockRLEs))
  2167  	spanDataBytes := int32(numSpans * 16)
  2168  
  2169  	binary.Write(f, binary.LittleEndian, label)
  2170  	binary.Write(f, binary.LittleEndian, numSpans)
  2171  	n, err := f.Write(kv.V)
  2172  	if err != nil {
  2173  		return fmt.Errorf("Only wrote %d bytes out of %d for RLEs in block %s: %v\n", n, len(kv.V), subvolPt, err)
  2174  	}
  2175  
  2176  	curBytes := uint64(spanDataBytes + 12)
  2177  	df.lastBytes += curBytes
  2178  	df.totalBytes += curBytes
  2179  	df.numKV++
  2180  	if elapsed := time.Since(df.lastTime); elapsed > time.Minute {
  2181  		mb := float64(df.lastBytes) / 1000000
  2182  		sec := elapsed.Seconds()
  2183  		throughput := mb / sec
  2184  		dvid.Debugf("Dump throughput: %5.2f MB/s (%s in %4.1f seconds).  Total %s\n", throughput, humanize.Bytes(df.lastBytes), sec, humanize.Bytes(df.totalBytes))
  2185  
  2186  		df.lastTime = time.Now()
  2187  		df.lastBytes = 0
  2188  	}
  2189  	return nil
  2190  }
  2191  
  2192  func (d *Data) DumpSubvols(uuid dvid.UUID, v dvid.VersionID, dirStr string) {
  2193  	df := DumpFiles{
  2194  		files: make(map[dvid.IZYXString]*os.File),
  2195  		dir:   dirStr,
  2196  
  2197  		tlog:     dvid.NewTimeLog(),
  2198  		lastTime: time.Now(),
  2199  	}
  2200  
  2201  	store, err := datastore.GetOrderedKeyValueDB(d)
  2202  	if err != nil {
  2203  		dvid.Criticalf("unable to get backing store for data %q: %v\n", d.DataName(), err)
  2204  		return
  2205  	}
  2206  	ctx := datastore.NewVersionedCtx(d, v)
  2207  
  2208  	dataCh := make(chan *storage.TKeyValue, 1000)
  2209  	go func() {
  2210  		for {
  2211  			kv := <-dataCh
  2212  			if kv == nil {
  2213  				dvid.Infof("Finished subvol dump on instance %q.\n", d.DataName())
  2214  				df.close()
  2215  				break
  2216  			}
  2217  			if err := df.process(kv, d.BlockSize); err != nil {
  2218  				dvid.Errorf("Subvol dump error: %v\n", err)
  2219  				df.close()
  2220  				break
  2221  			}
  2222  		}
  2223  		df.tlog.Infof("Finished %q subvol dump of %d kv pairs, %s", d.DataName(), df.numKV, humanize.Bytes(df.totalBytes))
  2224  	}()
  2225  
  2226  	minTKey := storage.MinTKey(keyLabelBlockRLE)
  2227  	maxTKey := storage.MaxTKey(keyLabelBlockRLE)
  2228  	err = store.ProcessRange(ctx, minTKey, maxTKey, &storage.ChunkOp{}, func(c *storage.Chunk) error {
  2229  		if c == nil {
  2230  			return fmt.Errorf("received nil chunk in dump subvol for data %s", d.DataName())
  2231  		}
  2232  		dataCh <- c.TKeyValue
  2233  		return nil
  2234  	})
  2235  	dataCh <- nil
  2236  	if err != nil {
  2237  		dvid.Criticalf("error in dump subvol for data %q: %v", d.DataName(), err)
  2238  	}
  2239  }