github.com/janelia-flyem/dvid@v1.0.0/datatype/common/labels/labels.go (about) 1 /* 2 Package labels supports label-based data types like labelblk, labelvol, labelsurf, labelsz, etc. 3 Basic 64-bit label data and deltas are kept here so all label-based data types can use them without 4 cyclic package dependencies, especially when writing code to synchronize across data instances. 5 */ 6 package labels 7 8 import ( 9 "fmt" 10 "sync" 11 12 "github.com/janelia-flyem/dvid/dvid" 13 ) 14 15 var ( 16 mc mergeCache 17 labelsMerging dirtyCache 18 labelsSplitting dirtyCache 19 ) 20 21 const ( 22 // MaxAllowedLabel is the largest label that should be allowed by DVID if we want 23 // to take into account the maximum integer size within Javascript (due to its 24 // underlying use of a double float for numbers, leading to max int = 2^53 - 1). 25 // This would circumvent the need to use strings within JSON (e.g., the Google 26 // solution) to represent integer labels that could exceed the max javascript 27 // number. It would require adding a value check on each label voxel of a 28 // mutation request, which might be too much of a hit to handle an edge case. 29 MaxAllowedLabel = 9007199254740991 30 ) 31 32 // LabelMap returns a label mapping for a version of a data instance. 33 // If no label mapping is available, a nil is returned. 34 func LabelMap(iv dvid.InstanceVersion) *Mapping { 35 return mc.LabelMap(iv) 36 } 37 38 // MergeStart handles label map caches during an active merge operation. Note that if there are 39 // multiple synced label instances, the InstanceVersion will always be the labelblk instance. 40 // Multiple merges into a single label are allowed, but chained merges are not. For example, 41 // you can merge label 1, 2, and 3 into 4, then later merge 6 into 4. However, you cannot 42 // concurrently merge label 4 into some other label because there can be a race condition between 43 // 3 -> 4 and 4 -> X. 44 func MergeStart(iv dvid.InstanceVersion, op MergeOp) error { 45 // Don't allow a merge to start in the middle of a concurrent merge/split. 46 if labelsSplitting.IsDirty(iv, op.Target) { // we might be able to relax this one. 47 return fmt.Errorf("can't merge into label %d while it has an ongoing split", op.Target) 48 } 49 if mc.MergingToOther(iv, op.Target) { 50 dvid.Errorf("can't merge label %d while it is currently merging into another label", op.Target) 51 return fmt.Errorf("can't merge label %d while it is currently merging into another label", op.Target) 52 } 53 for merged := range op.Merged { 54 if labelsSplitting.IsDirty(iv, merged) { 55 return fmt.Errorf("can't merge label %d while it has an ongoing split", merged) 56 } 57 if labelsMerging.IsDirty(iv, merged) { 58 dvid.Errorf("can't merge label %d while it is currently involved in a merge", merged) 59 return fmt.Errorf("can't merge label %d while it is currently involved in a merge", merged) 60 } 61 } 62 63 // Add the merge to the mapping. 64 if err := mc.Add(iv, op); err != nil { 65 return err 66 } 67 68 // Adjust the dirty counts on the involved labels. 69 labelsMerging.AddMerge(iv, op) 70 71 return nil 72 } 73 74 // MergeStop marks the end of a merge operation. 75 func MergeStop(iv dvid.InstanceVersion, op MergeOp) { 76 // Adjust the dirty counts on the involved labels. 77 labelsMerging.RemoveMerge(iv, op) 78 79 // Remove the merge from the mapping. 80 mc.Remove(iv, op) 81 82 // If the instance version's dirty cache is empty, we can delete the merge cache. 83 if labelsMerging.Empty(iv) { 84 dvid.Debugf("Merge cache now empty for %s\n", iv) 85 mc.DeleteMap(iv) 86 } 87 } 88 89 // SplitStart checks current label map to see if the split conflicts. 90 func SplitStart(iv dvid.InstanceVersion, op DeltaSplitStart) error { 91 if labelsMerging.IsDirty(iv, op.NewLabel) { 92 return fmt.Errorf("can't split into label %d while it is undergoing a merge", op.NewLabel) 93 } 94 if labelsMerging.IsDirty(iv, op.OldLabel) { 95 return fmt.Errorf("can't split label %d while it is undergoing a merge", op.OldLabel) 96 } 97 labelsSplitting.Incr(iv, op.NewLabel) 98 labelsSplitting.Incr(iv, op.OldLabel) 99 return nil 100 } 101 102 // SplitStop marks the end of a split operation. 103 func SplitStop(iv dvid.InstanceVersion, op DeltaSplitEnd) { 104 labelsSplitting.Decr(iv, op.NewLabel) 105 labelsSplitting.Decr(iv, op.OldLabel) 106 } 107 108 type mergeCache struct { 109 sync.RWMutex 110 m map[dvid.InstanceVersion]*Mapping 111 } 112 113 // Add adds a merge operation to the given InstanceVersion's cache. 114 func (mc *mergeCache) Add(iv dvid.InstanceVersion, op MergeOp) error { 115 mc.Lock() 116 defer mc.Unlock() 117 118 if mc.m == nil { 119 mc.m = make(map[dvid.InstanceVersion]*Mapping) 120 } 121 mapping, found := mc.m[iv] 122 if !found { 123 mapping = &Mapping{ 124 f: make(map[uint64]uint64, len(op.Merged)), 125 r: make(map[uint64]Set), 126 } 127 mc.m[iv] = mapping 128 } 129 for merged := range op.Merged { 130 if err := mapping.set(merged, op.Target); err != nil { 131 return err 132 } 133 } 134 return nil 135 } 136 137 // Remove removes a merge operation from the given InstanceVersion's cache, 138 // allowing us to return no mapping if it's already been processed. 139 func (mc *mergeCache) Remove(iv dvid.InstanceVersion, op MergeOp) { 140 mc.Lock() 141 defer mc.Unlock() 142 143 if mc.m == nil { 144 dvid.Errorf("mergeCache.Remove() called with iv %s, op %v there are no mappings for any iv.\n", iv, op) 145 return 146 } 147 mapping, found := mc.m[iv] 148 if !found { 149 dvid.Errorf("mergeCache.Remove() called with iv %s, op %v and there is no mapping for that iv.\n", iv, op) 150 return 151 } 152 for merged := range op.Merged { 153 mapping.delete(merged) 154 } 155 } 156 157 // LabelMap returns a label mapping for a version of a data instance. 158 // If no label mapping is available, a nil is returned. 159 func (mc *mergeCache) LabelMap(iv dvid.InstanceVersion) *Mapping { 160 mc.RLock() 161 defer mc.RUnlock() 162 163 if mc.m == nil { 164 return nil 165 } 166 mapping, found := mc.m[iv] 167 if found { 168 if len(mapping.f) == 0 { 169 return nil 170 } 171 return mapping 172 } 173 return nil 174 } 175 176 // MergingToOther returns true if the label is currently being merged into another label. 177 func (mc *mergeCache) MergingToOther(iv dvid.InstanceVersion, label uint64) bool { 178 mc.RLock() 179 defer mc.RUnlock() 180 181 if mc.m == nil { 182 return false 183 } 184 mapping, found := mc.m[iv] 185 if found { 186 _, merging := mapping.f[label] 187 return merging 188 } 189 return false 190 } 191 192 // DeleteMap removes a mapping of the given InstanceVersion. 193 func (mc *mergeCache) DeleteMap(iv dvid.InstanceVersion) { 194 mc.Lock() 195 defer mc.Unlock() 196 197 if mc.m != nil { 198 delete(mc.m, iv) 199 } 200 } 201 202 // SVSplit provides labels after a supervoxel split 203 type SVSplit struct { 204 Split uint64 // label corresponding to split sparse volume 205 Remain uint64 // relabeling of supervoxel that remains after split 206 } 207 208 // SVSplitCount provides both labels and the # voxels after a supervoxel split. 209 type SVSplitCount struct { 210 SVSplit 211 Voxels uint32 // number of voxels split 212 } 213 214 // SVSplitMap is a thread-safe mapping of supervoxels labels to their new split labels. 215 type SVSplitMap struct { 216 sync.RWMutex 217 Splits map[uint64]SVSplit 218 } 219 220 // returns a new mapped label or the previously mapped one. 221 func (m *SVSplitMap) getMapping(label uint64, newLabelFunc func() (uint64, error)) (relabel SVSplit, found bool, err error) { 222 m.Lock() 223 defer m.Unlock() 224 if m.Splits == nil { 225 m.Splits = make(map[uint64]SVSplit) 226 } else { 227 relabel, found = m.Splits[label] 228 } 229 if !found { 230 if relabel.Split, err = newLabelFunc(); err != nil { 231 return 232 } 233 if relabel.Remain, err = newLabelFunc(); err != nil { 234 return 235 } 236 m.Splits[label] = relabel 237 } 238 return 239 } 240 241 // Mapping is a thread-safe, mapping of labels to labels in both forward and backward direction. 242 // Mutation of a Mapping instance can only be done through labels.MergeCache. 243 type Mapping struct { 244 sync.RWMutex 245 f map[uint64]uint64 246 r map[uint64]Set 247 } 248 249 // ConstituentLabels returns a set of labels that will be mapped to the given label. 250 // The set will always include the given label. 251 func (m *Mapping) ConstituentLabels(final uint64) Set { 252 m.RLock() 253 defer m.RUnlock() 254 255 if m.r == nil { 256 return Set{final: struct{}{}} 257 } 258 259 // We need to return all labels that will eventually have the given final label 260 // including any intermediate ones that were subsequently merged. 261 constituents := Set{} 262 toCheck := []uint64{final} 263 for { 264 endI := len(toCheck) - 1 265 label := toCheck[endI] 266 toCheck = toCheck[:endI] 267 268 constituents[label] = struct{}{} 269 270 s, found := m.r[label] 271 if found { 272 // push these labels onto stack 273 for c := range s { 274 toCheck = append(toCheck, c) 275 } 276 } 277 if len(toCheck) == 0 { 278 break 279 } 280 } 281 return constituents 282 } 283 284 // FinalLabel follows mappings from a start label until 285 // a final mapped label is reached. 286 func (m *Mapping) FinalLabel(start uint64) (uint64, bool) { 287 m.RLock() 288 defer m.RUnlock() 289 290 if m.f == nil { 291 return start, false 292 } 293 cur := start 294 found := false 295 for { 296 v, ok := m.f[cur] 297 if !ok { 298 break 299 } 300 cur = v 301 found = true 302 } 303 return cur, found 304 } 305 306 // Get returns the mapping or false if no mapping exists. 307 func (m *Mapping) Get(label uint64) (uint64, bool) { 308 m.RLock() 309 defer m.RUnlock() 310 311 if m.f == nil { 312 return 0, false 313 } 314 mapped, found := m.f[label] 315 if found { 316 return mapped, true 317 } 318 return 0, false 319 } 320 321 // set returns error if b is currently being mapped to another label. 322 func (m *Mapping) set(a, b uint64) error { 323 m.Lock() 324 defer m.Unlock() 325 326 if m.f == nil { 327 m.f = make(map[uint64]uint64) 328 m.r = make(map[uint64]Set) 329 } else { 330 if c, found := m.f[b]; found { 331 return fmt.Errorf("label %d is currently getting merged into label %d", b, c) 332 } 333 } 334 m.f[a] = b 335 s, found := m.r[b] 336 if found { 337 s[a] = struct{}{} 338 m.r[b] = s 339 } else { 340 m.r[b] = Set{a: struct{}{}} 341 } 342 return nil 343 } 344 345 func (m *Mapping) delete(label uint64) { 346 m.Lock() 347 defer m.Unlock() 348 349 if m.f != nil { 350 mapped, found := m.f[label] 351 if !found { 352 return 353 } 354 delete(m.f, label) 355 s, found := m.r[mapped] 356 if found { 357 delete(s, label) 358 m.r[mapped] = s 359 } 360 } 361 } 362 363 // Set is a set of labels. 364 type Set map[uint64]struct{} 365 366 // Merge returns a set made of the given labels. 367 func NewSet(lbls ...uint64) Set { 368 s := make(Set, len(lbls)) 369 for _, label := range lbls { 370 s[label] = struct{}{} 371 } 372 return s 373 } 374 375 // Merge adds the elements in the given set to the receiver. 376 func (s Set) Merge(s2 Set) { 377 for label := range s2 { 378 s[label] = struct{}{} 379 } 380 } 381 382 // Exists returns true if the given uint64 is present in the Set. 383 func (s Set) Exists(i uint64) bool { 384 if s == nil { 385 return false 386 } 387 _, found := s[i] 388 return found 389 } 390 391 // Copy returns a duplicate of the Set. 392 func (s Set) Copy() Set { 393 dup := make(Set, len(s)) 394 for k := range s { 395 dup[k] = struct{}{} 396 } 397 return dup 398 } 399 400 func (s Set) String() string { 401 var str string 402 i := 1 403 for k := range s { 404 str += fmt.Sprintf("%d", k) 405 if i < len(s) { 406 str += ", " 407 } 408 i++ 409 } 410 return str 411 } 412 413 // Counts is a thread-safe type for counting label references. 414 type Counts struct { 415 sync.RWMutex 416 m map[uint64]int 417 } 418 419 // Incr increments the count for a label. 420 func (c *Counts) Incr(label uint64) { 421 if c.m == nil { 422 c.m = make(map[uint64]int) 423 } 424 c.Lock() 425 defer c.Unlock() 426 c.m[label] = c.m[label] + 1 427 } 428 429 // Decr decrements the count for a label. 430 func (c *Counts) Decr(label uint64) { 431 if c.m == nil { 432 c.m = make(map[uint64]int) 433 } 434 c.Lock() 435 defer c.Unlock() 436 c.m[label] = c.m[label] - 1 437 if c.m[label] == 0 { 438 delete(c.m, label) 439 } 440 } 441 442 // Value returns the count for a label. 443 func (c *Counts) Value(label uint64) int { 444 if c.m == nil { 445 return 0 446 } 447 c.RLock() 448 defer c.RUnlock() 449 return c.m[label] 450 } 451 452 // Empty returns true if there are no counts. 453 func (c *Counts) Empty() bool { 454 if len(c.m) == 0 { 455 return true 456 } 457 return false 458 } 459 460 // dirtyCache is a thread-safe cache for tracking dirty labels across versions, which is necessary when we 461 // don't know exactly how a label is being transformed. For example, when merging 462 // we can easily track what a label will be, however during a split, we don't know whether 463 // a particular voxel with label X will become label Y unless we also store the split 464 // voxels. So DirtyCache is good for tracking "changing" status in splits while MergeCache 465 // can give us complete label transformation of non-dirty labels. 466 type dirtyCache struct { 467 sync.RWMutex 468 dirty map[dvid.InstanceVersion]*Counts 469 } 470 471 func (d *dirtyCache) Incr(iv dvid.InstanceVersion, label uint64) { 472 d.Lock() 473 defer d.Unlock() 474 475 if d.dirty == nil { 476 d.dirty = make(map[dvid.InstanceVersion]*Counts) 477 } 478 d.incr(iv, label) 479 } 480 481 func (d *dirtyCache) Decr(iv dvid.InstanceVersion, label uint64) { 482 d.Lock() 483 defer d.Unlock() 484 485 if d.dirty == nil { 486 d.dirty = make(map[dvid.InstanceVersion]*Counts) 487 } 488 d.decr(iv, label) 489 } 490 491 func (d *dirtyCache) IsDirty(iv dvid.InstanceVersion, label uint64) bool { 492 d.RLock() 493 defer d.RUnlock() 494 495 if d.dirty == nil { 496 return false 497 } 498 499 cnts, found := d.dirty[iv] 500 if !found || cnts == nil { 501 return false 502 } 503 if cnts.Value(label) == 0 { 504 return false 505 } 506 return true 507 } 508 509 func (d *dirtyCache) Empty(iv dvid.InstanceVersion) bool { 510 d.RLock() 511 defer d.RUnlock() 512 513 if len(d.dirty) == 0 { 514 return true 515 } 516 cnts, found := d.dirty[iv] 517 if !found || cnts == nil { 518 return true 519 } 520 return cnts.Empty() 521 } 522 523 func (d *dirtyCache) AddMerge(iv dvid.InstanceVersion, op MergeOp) { 524 d.Lock() 525 defer d.Unlock() 526 527 if d.dirty == nil { 528 d.dirty = make(map[dvid.InstanceVersion]*Counts) 529 } 530 531 d.incr(iv, op.Target) 532 for label := range op.Merged { 533 d.incr(iv, label) 534 } 535 } 536 537 func (d *dirtyCache) RemoveMerge(iv dvid.InstanceVersion, op MergeOp) { 538 d.Lock() 539 defer d.Unlock() 540 541 if d.dirty == nil { 542 d.dirty = make(map[dvid.InstanceVersion]*Counts) 543 } 544 545 d.decr(iv, op.Target) 546 for label := range op.Merged { 547 d.decr(iv, label) 548 } 549 } 550 551 func (d *dirtyCache) incr(iv dvid.InstanceVersion, label uint64) { 552 cnts, found := d.dirty[iv] 553 if !found || cnts == nil { 554 cnts = new(Counts) 555 d.dirty[iv] = cnts 556 } 557 cnts.Incr(label) 558 } 559 560 func (d *dirtyCache) decr(iv dvid.InstanceVersion, label uint64) { 561 cnts, found := d.dirty[iv] 562 if !found || cnts == nil { 563 dvid.Errorf("decremented non-existent count for label %d, version %v\n", label, iv) 564 return 565 } 566 cnts.Decr(label) 567 }