github.com/janelia-flyem/dvid@v1.0.0/datatype/common/labels/labels.go (about)

     1  /*
     2  	Package labels supports label-based data types like labelblk, labelvol, labelsurf, labelsz, etc.
     3  	Basic 64-bit label data and deltas are kept here so all label-based data types can use them without
     4  	cyclic package dependencies, especially when writing code to synchronize across data instances.
     5  */
     6  package labels
     7  
     8  import (
     9  	"fmt"
    10  	"sync"
    11  
    12  	"github.com/janelia-flyem/dvid/dvid"
    13  )
    14  
    15  var (
    16  	mc              mergeCache
    17  	labelsMerging   dirtyCache
    18  	labelsSplitting dirtyCache
    19  )
    20  
    21  const (
    22  	// MaxAllowedLabel is the largest label that should be allowed by DVID if we want
    23  	// to take into account the maximum integer size within Javascript (due to its
    24  	// underlying use of a double float for numbers, leading to max int = 2^53 - 1).
    25  	// This would circumvent the need to use strings within JSON (e.g., the Google
    26  	// solution) to represent integer labels that could exceed the max javascript
    27  	// number.  It would require adding a value check on each label voxel of a
    28  	// mutation request, which might be too much of a hit to handle an edge case.
    29  	MaxAllowedLabel = 9007199254740991
    30  )
    31  
    32  // LabelMap returns a label mapping for a version of a data instance.
    33  // If no label mapping is available, a nil is returned.
    34  func LabelMap(iv dvid.InstanceVersion) *Mapping {
    35  	return mc.LabelMap(iv)
    36  }
    37  
    38  // MergeStart handles label map caches during an active merge operation.  Note that if there are
    39  // multiple synced label instances, the InstanceVersion will always be the labelblk instance.
    40  // Multiple merges into a single label are allowed, but chained merges are not.  For example,
    41  // you can merge label 1, 2, and 3 into 4, then later merge 6 into 4.  However, you cannot
    42  // concurrently merge label 4 into some other label because there can be a race condition between
    43  // 3 -> 4 and 4 -> X.
    44  func MergeStart(iv dvid.InstanceVersion, op MergeOp) error {
    45  	// Don't allow a merge to start in the middle of a concurrent merge/split.
    46  	if labelsSplitting.IsDirty(iv, op.Target) { // we might be able to relax this one.
    47  		return fmt.Errorf("can't merge into label %d while it has an ongoing split", op.Target)
    48  	}
    49  	if mc.MergingToOther(iv, op.Target) {
    50  		dvid.Errorf("can't merge label %d while it is currently merging into another label", op.Target)
    51  		return fmt.Errorf("can't merge label %d while it is currently merging into another label", op.Target)
    52  	}
    53  	for merged := range op.Merged {
    54  		if labelsSplitting.IsDirty(iv, merged) {
    55  			return fmt.Errorf("can't merge label %d while it has an ongoing split", merged)
    56  		}
    57  		if labelsMerging.IsDirty(iv, merged) {
    58  			dvid.Errorf("can't merge label %d while it is currently involved in a merge", merged)
    59  			return fmt.Errorf("can't merge label %d while it is currently involved in a merge", merged)
    60  		}
    61  	}
    62  
    63  	// Add the merge to the mapping.
    64  	if err := mc.Add(iv, op); err != nil {
    65  		return err
    66  	}
    67  
    68  	// Adjust the dirty counts on the involved labels.
    69  	labelsMerging.AddMerge(iv, op)
    70  
    71  	return nil
    72  }
    73  
    74  // MergeStop marks the end of a merge operation.
    75  func MergeStop(iv dvid.InstanceVersion, op MergeOp) {
    76  	// Adjust the dirty counts on the involved labels.
    77  	labelsMerging.RemoveMerge(iv, op)
    78  
    79  	// Remove the merge from the mapping.
    80  	mc.Remove(iv, op)
    81  
    82  	// If the instance version's dirty cache is empty, we can delete the merge cache.
    83  	if labelsMerging.Empty(iv) {
    84  		dvid.Debugf("Merge cache now empty for %s\n", iv)
    85  		mc.DeleteMap(iv)
    86  	}
    87  }
    88  
    89  // SplitStart checks current label map to see if the split conflicts.
    90  func SplitStart(iv dvid.InstanceVersion, op DeltaSplitStart) error {
    91  	if labelsMerging.IsDirty(iv, op.NewLabel) {
    92  		return fmt.Errorf("can't split into label %d while it is undergoing a merge", op.NewLabel)
    93  	}
    94  	if labelsMerging.IsDirty(iv, op.OldLabel) {
    95  		return fmt.Errorf("can't split label %d while it is undergoing a merge", op.OldLabel)
    96  	}
    97  	labelsSplitting.Incr(iv, op.NewLabel)
    98  	labelsSplitting.Incr(iv, op.OldLabel)
    99  	return nil
   100  }
   101  
   102  // SplitStop marks the end of a split operation.
   103  func SplitStop(iv dvid.InstanceVersion, op DeltaSplitEnd) {
   104  	labelsSplitting.Decr(iv, op.NewLabel)
   105  	labelsSplitting.Decr(iv, op.OldLabel)
   106  }
   107  
   108  type mergeCache struct {
   109  	sync.RWMutex
   110  	m map[dvid.InstanceVersion]*Mapping
   111  }
   112  
   113  // Add adds a merge operation to the given InstanceVersion's cache.
   114  func (mc *mergeCache) Add(iv dvid.InstanceVersion, op MergeOp) error {
   115  	mc.Lock()
   116  	defer mc.Unlock()
   117  
   118  	if mc.m == nil {
   119  		mc.m = make(map[dvid.InstanceVersion]*Mapping)
   120  	}
   121  	mapping, found := mc.m[iv]
   122  	if !found {
   123  		mapping = &Mapping{
   124  			f: make(map[uint64]uint64, len(op.Merged)),
   125  			r: make(map[uint64]Set),
   126  		}
   127  		mc.m[iv] = mapping
   128  	}
   129  	for merged := range op.Merged {
   130  		if err := mapping.set(merged, op.Target); err != nil {
   131  			return err
   132  		}
   133  	}
   134  	return nil
   135  }
   136  
   137  // Remove removes a merge operation from the given InstanceVersion's cache,
   138  // allowing us to return no mapping if it's already been processed.
   139  func (mc *mergeCache) Remove(iv dvid.InstanceVersion, op MergeOp) {
   140  	mc.Lock()
   141  	defer mc.Unlock()
   142  
   143  	if mc.m == nil {
   144  		dvid.Errorf("mergeCache.Remove() called with iv %s, op %v there are no mappings for any iv.\n", iv, op)
   145  		return
   146  	}
   147  	mapping, found := mc.m[iv]
   148  	if !found {
   149  		dvid.Errorf("mergeCache.Remove() called with iv %s, op %v and there is no mapping for that iv.\n", iv, op)
   150  		return
   151  	}
   152  	for merged := range op.Merged {
   153  		mapping.delete(merged)
   154  	}
   155  }
   156  
   157  // LabelMap returns a label mapping for a version of a data instance.
   158  // If no label mapping is available, a nil is returned.
   159  func (mc *mergeCache) LabelMap(iv dvid.InstanceVersion) *Mapping {
   160  	mc.RLock()
   161  	defer mc.RUnlock()
   162  
   163  	if mc.m == nil {
   164  		return nil
   165  	}
   166  	mapping, found := mc.m[iv]
   167  	if found {
   168  		if len(mapping.f) == 0 {
   169  			return nil
   170  		}
   171  		return mapping
   172  	}
   173  	return nil
   174  }
   175  
   176  // MergingToOther returns true if the label is currently being merged into another label.
   177  func (mc *mergeCache) MergingToOther(iv dvid.InstanceVersion, label uint64) bool {
   178  	mc.RLock()
   179  	defer mc.RUnlock()
   180  
   181  	if mc.m == nil {
   182  		return false
   183  	}
   184  	mapping, found := mc.m[iv]
   185  	if found {
   186  		_, merging := mapping.f[label]
   187  		return merging
   188  	}
   189  	return false
   190  }
   191  
   192  // DeleteMap removes a mapping of the given InstanceVersion.
   193  func (mc *mergeCache) DeleteMap(iv dvid.InstanceVersion) {
   194  	mc.Lock()
   195  	defer mc.Unlock()
   196  
   197  	if mc.m != nil {
   198  		delete(mc.m, iv)
   199  	}
   200  }
   201  
   202  // SVSplit provides labels after a supervoxel split
   203  type SVSplit struct {
   204  	Split  uint64 // label corresponding to split sparse volume
   205  	Remain uint64 // relabeling of supervoxel that remains after split
   206  }
   207  
   208  // SVSplitCount provides both labels and the # voxels after a supervoxel split.
   209  type SVSplitCount struct {
   210  	SVSplit
   211  	Voxels uint32 // number of voxels split
   212  }
   213  
   214  // SVSplitMap is a thread-safe mapping of supervoxels labels to their new split labels.
   215  type SVSplitMap struct {
   216  	sync.RWMutex
   217  	Splits map[uint64]SVSplit
   218  }
   219  
   220  // returns a new mapped label or the previously mapped one.
   221  func (m *SVSplitMap) getMapping(label uint64, newLabelFunc func() (uint64, error)) (relabel SVSplit, found bool, err error) {
   222  	m.Lock()
   223  	defer m.Unlock()
   224  	if m.Splits == nil {
   225  		m.Splits = make(map[uint64]SVSplit)
   226  	} else {
   227  		relabel, found = m.Splits[label]
   228  	}
   229  	if !found {
   230  		if relabel.Split, err = newLabelFunc(); err != nil {
   231  			return
   232  		}
   233  		if relabel.Remain, err = newLabelFunc(); err != nil {
   234  			return
   235  		}
   236  		m.Splits[label] = relabel
   237  	}
   238  	return
   239  }
   240  
   241  // Mapping is a thread-safe, mapping of labels to labels in both forward and backward direction.
   242  // Mutation of a Mapping instance can only be done through labels.MergeCache.
   243  type Mapping struct {
   244  	sync.RWMutex
   245  	f map[uint64]uint64
   246  	r map[uint64]Set
   247  }
   248  
   249  // ConstituentLabels returns a set of labels that will be mapped to the given label.
   250  // The set will always include the given label.
   251  func (m *Mapping) ConstituentLabels(final uint64) Set {
   252  	m.RLock()
   253  	defer m.RUnlock()
   254  
   255  	if m.r == nil {
   256  		return Set{final: struct{}{}}
   257  	}
   258  
   259  	// We need to return all labels that will eventually have the given final label
   260  	// including any intermediate ones that were subsequently merged.
   261  	constituents := Set{}
   262  	toCheck := []uint64{final}
   263  	for {
   264  		endI := len(toCheck) - 1
   265  		label := toCheck[endI]
   266  		toCheck = toCheck[:endI]
   267  
   268  		constituents[label] = struct{}{}
   269  
   270  		s, found := m.r[label]
   271  		if found {
   272  			// push these labels onto stack
   273  			for c := range s {
   274  				toCheck = append(toCheck, c)
   275  			}
   276  		}
   277  		if len(toCheck) == 0 {
   278  			break
   279  		}
   280  	}
   281  	return constituents
   282  }
   283  
   284  // FinalLabel follows mappings from a start label until
   285  // a final mapped label is reached.
   286  func (m *Mapping) FinalLabel(start uint64) (uint64, bool) {
   287  	m.RLock()
   288  	defer m.RUnlock()
   289  
   290  	if m.f == nil {
   291  		return start, false
   292  	}
   293  	cur := start
   294  	found := false
   295  	for {
   296  		v, ok := m.f[cur]
   297  		if !ok {
   298  			break
   299  		}
   300  		cur = v
   301  		found = true
   302  	}
   303  	return cur, found
   304  }
   305  
   306  // Get returns the mapping or false if no mapping exists.
   307  func (m *Mapping) Get(label uint64) (uint64, bool) {
   308  	m.RLock()
   309  	defer m.RUnlock()
   310  
   311  	if m.f == nil {
   312  		return 0, false
   313  	}
   314  	mapped, found := m.f[label]
   315  	if found {
   316  		return mapped, true
   317  	}
   318  	return 0, false
   319  }
   320  
   321  // set returns error if b is currently being mapped to another label.
   322  func (m *Mapping) set(a, b uint64) error {
   323  	m.Lock()
   324  	defer m.Unlock()
   325  
   326  	if m.f == nil {
   327  		m.f = make(map[uint64]uint64)
   328  		m.r = make(map[uint64]Set)
   329  	} else {
   330  		if c, found := m.f[b]; found {
   331  			return fmt.Errorf("label %d is currently getting merged into label %d", b, c)
   332  		}
   333  	}
   334  	m.f[a] = b
   335  	s, found := m.r[b]
   336  	if found {
   337  		s[a] = struct{}{}
   338  		m.r[b] = s
   339  	} else {
   340  		m.r[b] = Set{a: struct{}{}}
   341  	}
   342  	return nil
   343  }
   344  
   345  func (m *Mapping) delete(label uint64) {
   346  	m.Lock()
   347  	defer m.Unlock()
   348  
   349  	if m.f != nil {
   350  		mapped, found := m.f[label]
   351  		if !found {
   352  			return
   353  		}
   354  		delete(m.f, label)
   355  		s, found := m.r[mapped]
   356  		if found {
   357  			delete(s, label)
   358  			m.r[mapped] = s
   359  		}
   360  	}
   361  }
   362  
   363  // Set is a set of labels.
   364  type Set map[uint64]struct{}
   365  
   366  // Merge returns a set made of the given labels.
   367  func NewSet(lbls ...uint64) Set {
   368  	s := make(Set, len(lbls))
   369  	for _, label := range lbls {
   370  		s[label] = struct{}{}
   371  	}
   372  	return s
   373  }
   374  
   375  // Merge adds the elements in the given set to the receiver.
   376  func (s Set) Merge(s2 Set) {
   377  	for label := range s2 {
   378  		s[label] = struct{}{}
   379  	}
   380  }
   381  
   382  // Exists returns true if the given uint64 is present in the Set.
   383  func (s Set) Exists(i uint64) bool {
   384  	if s == nil {
   385  		return false
   386  	}
   387  	_, found := s[i]
   388  	return found
   389  }
   390  
   391  // Copy returns a duplicate of the Set.
   392  func (s Set) Copy() Set {
   393  	dup := make(Set, len(s))
   394  	for k := range s {
   395  		dup[k] = struct{}{}
   396  	}
   397  	return dup
   398  }
   399  
   400  func (s Set) String() string {
   401  	var str string
   402  	i := 1
   403  	for k := range s {
   404  		str += fmt.Sprintf("%d", k)
   405  		if i < len(s) {
   406  			str += ", "
   407  		}
   408  		i++
   409  	}
   410  	return str
   411  }
   412  
   413  // Counts is a thread-safe type for counting label references.
   414  type Counts struct {
   415  	sync.RWMutex
   416  	m map[uint64]int
   417  }
   418  
   419  // Incr increments the count for a label.
   420  func (c *Counts) Incr(label uint64) {
   421  	if c.m == nil {
   422  		c.m = make(map[uint64]int)
   423  	}
   424  	c.Lock()
   425  	defer c.Unlock()
   426  	c.m[label] = c.m[label] + 1
   427  }
   428  
   429  // Decr decrements the count for a label.
   430  func (c *Counts) Decr(label uint64) {
   431  	if c.m == nil {
   432  		c.m = make(map[uint64]int)
   433  	}
   434  	c.Lock()
   435  	defer c.Unlock()
   436  	c.m[label] = c.m[label] - 1
   437  	if c.m[label] == 0 {
   438  		delete(c.m, label)
   439  	}
   440  }
   441  
   442  // Value returns the count for a label.
   443  func (c *Counts) Value(label uint64) int {
   444  	if c.m == nil {
   445  		return 0
   446  	}
   447  	c.RLock()
   448  	defer c.RUnlock()
   449  	return c.m[label]
   450  }
   451  
   452  // Empty returns true if there are no counts.
   453  func (c *Counts) Empty() bool {
   454  	if len(c.m) == 0 {
   455  		return true
   456  	}
   457  	return false
   458  }
   459  
   460  // dirtyCache is a thread-safe cache for tracking dirty labels across versions, which is necessary when we
   461  // don't know exactly how a label is being transformed.  For example, when merging
   462  // we can easily track what a label will be, however during a split, we don't know whether
   463  // a particular voxel with label X will become label Y unless we also store the split
   464  // voxels.  So DirtyCache is good for tracking "changing" status in splits while MergeCache
   465  // can give us complete label transformation of non-dirty labels.
   466  type dirtyCache struct {
   467  	sync.RWMutex
   468  	dirty map[dvid.InstanceVersion]*Counts
   469  }
   470  
   471  func (d *dirtyCache) Incr(iv dvid.InstanceVersion, label uint64) {
   472  	d.Lock()
   473  	defer d.Unlock()
   474  
   475  	if d.dirty == nil {
   476  		d.dirty = make(map[dvid.InstanceVersion]*Counts)
   477  	}
   478  	d.incr(iv, label)
   479  }
   480  
   481  func (d *dirtyCache) Decr(iv dvid.InstanceVersion, label uint64) {
   482  	d.Lock()
   483  	defer d.Unlock()
   484  
   485  	if d.dirty == nil {
   486  		d.dirty = make(map[dvid.InstanceVersion]*Counts)
   487  	}
   488  	d.decr(iv, label)
   489  }
   490  
   491  func (d *dirtyCache) IsDirty(iv dvid.InstanceVersion, label uint64) bool {
   492  	d.RLock()
   493  	defer d.RUnlock()
   494  
   495  	if d.dirty == nil {
   496  		return false
   497  	}
   498  
   499  	cnts, found := d.dirty[iv]
   500  	if !found || cnts == nil {
   501  		return false
   502  	}
   503  	if cnts.Value(label) == 0 {
   504  		return false
   505  	}
   506  	return true
   507  }
   508  
   509  func (d *dirtyCache) Empty(iv dvid.InstanceVersion) bool {
   510  	d.RLock()
   511  	defer d.RUnlock()
   512  
   513  	if len(d.dirty) == 0 {
   514  		return true
   515  	}
   516  	cnts, found := d.dirty[iv]
   517  	if !found || cnts == nil {
   518  		return true
   519  	}
   520  	return cnts.Empty()
   521  }
   522  
   523  func (d *dirtyCache) AddMerge(iv dvid.InstanceVersion, op MergeOp) {
   524  	d.Lock()
   525  	defer d.Unlock()
   526  
   527  	if d.dirty == nil {
   528  		d.dirty = make(map[dvid.InstanceVersion]*Counts)
   529  	}
   530  
   531  	d.incr(iv, op.Target)
   532  	for label := range op.Merged {
   533  		d.incr(iv, label)
   534  	}
   535  }
   536  
   537  func (d *dirtyCache) RemoveMerge(iv dvid.InstanceVersion, op MergeOp) {
   538  	d.Lock()
   539  	defer d.Unlock()
   540  
   541  	if d.dirty == nil {
   542  		d.dirty = make(map[dvid.InstanceVersion]*Counts)
   543  	}
   544  
   545  	d.decr(iv, op.Target)
   546  	for label := range op.Merged {
   547  		d.decr(iv, label)
   548  	}
   549  }
   550  
   551  func (d *dirtyCache) incr(iv dvid.InstanceVersion, label uint64) {
   552  	cnts, found := d.dirty[iv]
   553  	if !found || cnts == nil {
   554  		cnts = new(Counts)
   555  		d.dirty[iv] = cnts
   556  	}
   557  	cnts.Incr(label)
   558  }
   559  
   560  func (d *dirtyCache) decr(iv dvid.InstanceVersion, label uint64) {
   561  	cnts, found := d.dirty[iv]
   562  	if !found || cnts == nil {
   563  		dvid.Errorf("decremented non-existent count for label %d, version %v\n", label, iv)
   564  		return
   565  	}
   566  	cnts.Decr(label)
   567  }