github.com/janelia-flyem/dvid@v1.0.0/datatype/common/labels/mutlog.go (about)

     1  /* Handles writing to mutation log for ops on labels. */
     2  
     3  package labels
     4  
     5  import (
     6  	pb "google.golang.org/protobuf/proto"
     7  
     8  	"github.com/janelia-flyem/dvid/datastore"
     9  	"github.com/janelia-flyem/dvid/datatype/common/proto"
    10  	"github.com/janelia-flyem/dvid/dvid"
    11  	"github.com/janelia-flyem/dvid/storage"
    12  )
    13  
    14  // LogSplit logs the split of a set of voxels from the underlying label.
    15  func LogSplit(d dvid.Data, v dvid.VersionID, op SplitOp) error {
    16  	uuid, err := datastore.UUIDFromVersion(v)
    17  	if err != nil {
    18  		return err
    19  	}
    20  	logable, ok := d.(storage.LogWritable)
    21  	if !ok {
    22  		return nil // skip logging
    23  	}
    24  	log := logable.GetWriteLog()
    25  	if log == nil {
    26  		return nil
    27  	}
    28  	data, err := serializeSplit(op)
    29  	if err != nil {
    30  		return err
    31  	}
    32  	msg := storage.LogMessage{EntryType: proto.SplitOpType, Data: data}
    33  	return log.Append(d.DataUUID(), uuid, msg)
    34  }
    35  
    36  // LogSupervoxelSplit logs the split of a supervoxel into two separate supervoxels.
    37  func LogSupervoxelSplit(d dvid.Data, v dvid.VersionID, op SplitSupervoxelOp) error {
    38  	uuid, err := datastore.UUIDFromVersion(v)
    39  	if err != nil {
    40  		return err
    41  	}
    42  	logable, ok := d.(storage.LogWritable)
    43  	if !ok {
    44  		return nil // skip logging
    45  	}
    46  	log := logable.GetWriteLog()
    47  	if log == nil {
    48  		return nil
    49  	}
    50  	pop := &proto.SupervoxelSplitOp{
    51  		Mutid:       op.MutID,
    52  		Supervoxel:  op.Supervoxel,
    53  		Splitlabel:  op.SplitSupervoxel,
    54  		Remainlabel: op.RemainSupervoxel,
    55  	}
    56  	serialization, err := pb.Marshal(pop)
    57  
    58  	if err != nil {
    59  		return err
    60  	}
    61  
    62  	msg := storage.LogMessage{EntryType: proto.SupervoxelSplitType, Data: serialization}
    63  	return log.Append(d.DataUUID(), uuid, msg)
    64  }
    65  
    66  // LogMerge logs the merge of supervoxels to a label.
    67  func LogMerge(d dvid.Data, v dvid.VersionID, op MergeOp) error {
    68  	uuid, err := datastore.UUIDFromVersion(v)
    69  	if err != nil {
    70  		return err
    71  	}
    72  	logable, ok := d.(storage.LogWritable)
    73  	if !ok {
    74  		return nil // skip logging
    75  	}
    76  	log := logable.GetWriteLog()
    77  	if log == nil {
    78  		return nil
    79  	}
    80  	data, err := serializeMerge(op)
    81  	if err != nil {
    82  		return err
    83  	}
    84  	msg := storage.LogMessage{EntryType: proto.MergeOpType, Data: data}
    85  	return log.Append(d.DataUUID(), uuid, msg)
    86  }
    87  
    88  // LogRenumber logs the merge of supervoxels to a label.
    89  func LogRenumber(d dvid.Data, v dvid.VersionID, mutID, origLabel, newLabel uint64) error {
    90  	uuid, err := datastore.UUIDFromVersion(v)
    91  	if err != nil {
    92  		return err
    93  	}
    94  	logable, ok := d.(storage.LogWritable)
    95  	if !ok {
    96  		return nil // skip logging
    97  	}
    98  	log := logable.GetWriteLog()
    99  	if log == nil {
   100  		return nil
   101  	}
   102  	pop := &proto.RenumberOp{
   103  		Mutid:    mutID,
   104  		Target:   origLabel,
   105  		Newlabel: newLabel,
   106  	}
   107  	data, err := pb.Marshal(pop)
   108  	if err != nil {
   109  		return err
   110  	}
   111  
   112  	msg := storage.LogMessage{EntryType: proto.RenumberOpType, Data: data}
   113  	return log.Append(d.DataUUID(), uuid, msg)
   114  }
   115  
   116  // LogCleave logs the cleave of supervoxels to a label.
   117  func LogCleave(d dvid.Data, v dvid.VersionID, op CleaveOp) error {
   118  	uuid, err := datastore.UUIDFromVersion(v)
   119  	if err != nil {
   120  		return err
   121  	}
   122  	logable, ok := d.(storage.LogWritable)
   123  	if !ok {
   124  		return nil // skip logging
   125  	}
   126  	log := logable.GetWriteLog()
   127  	if log == nil {
   128  		return nil
   129  	}
   130  	data, err := serializeCleave(op)
   131  	if err != nil {
   132  		return err
   133  	}
   134  	msg := storage.LogMessage{EntryType: proto.CleaveOpType, Data: data}
   135  	return log.Append(d.DataUUID(), uuid, msg)
   136  }
   137  
   138  // LogMapping logs the mapping of supervoxels to a label.
   139  func LogMapping(d dvid.Data, v dvid.VersionID, op MappingOp) error {
   140  	uuid, err := datastore.UUIDFromVersion(v)
   141  	if err != nil {
   142  		return err
   143  	}
   144  	logable, ok := d.(storage.LogWritable)
   145  	if !ok {
   146  		return nil // skip logging
   147  	}
   148  	log := logable.GetWriteLog()
   149  	if log == nil {
   150  		return nil
   151  	}
   152  	data, err := op.Marshal()
   153  	if err != nil {
   154  		return err
   155  	}
   156  	msg := storage.LogMessage{EntryType: proto.MappingOpType, Data: data}
   157  	return log.Append(d.DataUUID(), uuid, msg)
   158  }
   159  
   160  // LogMappings logs a collection of mapping operations to a UUID.
   161  func LogMappings(d dvid.Data, v dvid.VersionID, ops *proto.MappingOps) error {
   162  	uuid, err := datastore.UUIDFromVersion(v)
   163  	if err != nil {
   164  		return err
   165  	}
   166  	logable, ok := d.(storage.LogWritable)
   167  	if !ok {
   168  		return nil // skip logging
   169  	}
   170  	log := logable.GetWriteLog()
   171  	if log == nil {
   172  		return nil
   173  	}
   174  	for _, op := range ops.Mappings {
   175  		data, err := pb.Marshal(op)
   176  		if err != nil {
   177  			return err
   178  		}
   179  		msg := storage.LogMessage{EntryType: proto.MappingOpType, Data: data}
   180  		if err := log.Append(d.DataUUID(), uuid, msg); err != nil {
   181  			return err
   182  		}
   183  	}
   184  	return nil
   185  }
   186  
   187  func ReadMappingLog(d dvid.Data, v dvid.VersionID) ([]MappingOp, error) {
   188  	uuid, err := datastore.UUIDFromVersion(v)
   189  	if err != nil {
   190  		return nil, err
   191  	}
   192  	logreadable, ok := d.(storage.LogReadable)
   193  	if !ok {
   194  		return nil, nil
   195  	}
   196  	rl := logreadable.GetReadLog()
   197  	if rl == nil {
   198  		return nil, nil
   199  	}
   200  	msgs, err := rl.ReadAll(d.DataUUID(), uuid)
   201  	if err != nil {
   202  		return nil, err
   203  	}
   204  	if len(msgs) == 0 {
   205  		return nil, nil
   206  	}
   207  	mappingOps := make([]MappingOp, len(msgs))
   208  	var numMappings int
   209  	for i, msg := range msgs {
   210  		if msg.EntryType != proto.MappingOpType {
   211  			continue
   212  		}
   213  
   214  		var op proto.MappingOp
   215  		if err := pb.Unmarshal(msg.Data, &op); err != nil {
   216  			return nil, err
   217  		}
   218  		mappingOps[i].Mapped = op.GetMapped()
   219  		original := op.GetOriginal()
   220  		mappingOps[i].Original = make(Set, len(original))
   221  		for _, label := range original {
   222  			mappingOps[i].Original[label] = struct{}{}
   223  		}
   224  		numMappings++
   225  	}
   226  	mappingOps = mappingOps[:numMappings]
   227  	return mappingOps, nil
   228  }
   229  
   230  func StreamLog(d dvid.Data, v dvid.VersionID, ch chan storage.LogMessage) error {
   231  	uuid, err := datastore.UUIDFromVersion(v)
   232  	if err != nil {
   233  		return err
   234  	}
   235  	logreadable, ok := d.(storage.LogReadable)
   236  	if !ok {
   237  		close(ch)
   238  		return nil
   239  	}
   240  	rl := logreadable.GetReadLog()
   241  	if rl == nil {
   242  		close(ch)
   243  		return nil
   244  	}
   245  	return rl.StreamAll(d.DataUUID(), uuid, ch)
   246  }
   247  
   248  func LogAffinity(d dvid.Data, v dvid.VersionID, aff Affinity) error {
   249  	uuid, err := datastore.UUIDFromVersion(v)
   250  	if err != nil {
   251  		return err
   252  	}
   253  	logable, ok := d.(storage.LogWritable)
   254  	if !ok {
   255  		return nil // skip logging
   256  	}
   257  	log := logable.GetWriteLog()
   258  	if log == nil {
   259  		return nil
   260  	}
   261  	data, err := serializeAffinity(aff)
   262  	if err != nil {
   263  		return err
   264  	}
   265  	msg := storage.LogMessage{EntryType: proto.AffinityType, Data: data}
   266  	return log.Append(d.DataUUID(), uuid, msg)
   267  }
   268  
   269  func serializeSplit(op SplitOp) (serialization []byte, err error) {
   270  	rlesBytes, err := op.RLEs.MarshalBinary()
   271  	if err != nil {
   272  		return nil, err
   273  	}
   274  	svsplits := make(map[uint64]*proto.SVSplit, len(op.SplitMap))
   275  	for supervoxel, split := range op.SplitMap {
   276  		svsplit := new(proto.SVSplit)
   277  		svsplit.Splitlabel = split.Split
   278  		svsplit.Remainlabel = split.Remain
   279  		svsplits[supervoxel] = svsplit
   280  	}
   281  	pop := &proto.SplitOp{
   282  		Mutid:    op.MutID,
   283  		Target:   op.Target,
   284  		Newlabel: op.NewLabel,
   285  		Coarse:   op.Coarse,
   286  		Rles:     rlesBytes,
   287  		Svsplits: svsplits,
   288  	}
   289  	return pb.Marshal(pop)
   290  }
   291  
   292  func serializeMerge(op MergeOp) (serialization []byte, err error) {
   293  	merged := make([]uint64, len(op.Merged))
   294  	var i int
   295  	for label := range op.Merged {
   296  		merged[i] = label
   297  		i++
   298  	}
   299  	pop := &proto.MergeOp{
   300  		Mutid:  op.MutID,
   301  		Target: op.Target,
   302  		Merged: merged,
   303  	}
   304  	return pb.Marshal(pop)
   305  }
   306  
   307  func serializeCleave(op CleaveOp) (serialization []byte, err error) {
   308  	pop := &proto.CleaveOp{
   309  		Mutid:        op.MutID,
   310  		Target:       op.Target,
   311  		Cleavedlabel: op.CleavedLabel,
   312  		Cleaved:      op.CleavedSupervoxels,
   313  	}
   314  	return pb.Marshal(pop)
   315  }
   316  
   317  func serializeAffinity(aff Affinity) (serialization []byte, err error) {
   318  	pop := &proto.Affinity{
   319  		Label1: aff.Label1,
   320  		Label2: aff.Label2,
   321  		Value:  aff.Value,
   322  	}
   323  	return pb.Marshal(pop)
   324  }