github.com/janelia-flyem/dvid@v1.0.0/datatype/common/labels/mutlog.go (about) 1 /* Handles writing to mutation log for ops on labels. */ 2 3 package labels 4 5 import ( 6 pb "google.golang.org/protobuf/proto" 7 8 "github.com/janelia-flyem/dvid/datastore" 9 "github.com/janelia-flyem/dvid/datatype/common/proto" 10 "github.com/janelia-flyem/dvid/dvid" 11 "github.com/janelia-flyem/dvid/storage" 12 ) 13 14 // LogSplit logs the split of a set of voxels from the underlying label. 15 func LogSplit(d dvid.Data, v dvid.VersionID, op SplitOp) error { 16 uuid, err := datastore.UUIDFromVersion(v) 17 if err != nil { 18 return err 19 } 20 logable, ok := d.(storage.LogWritable) 21 if !ok { 22 return nil // skip logging 23 } 24 log := logable.GetWriteLog() 25 if log == nil { 26 return nil 27 } 28 data, err := serializeSplit(op) 29 if err != nil { 30 return err 31 } 32 msg := storage.LogMessage{EntryType: proto.SplitOpType, Data: data} 33 return log.Append(d.DataUUID(), uuid, msg) 34 } 35 36 // LogSupervoxelSplit logs the split of a supervoxel into two separate supervoxels. 37 func LogSupervoxelSplit(d dvid.Data, v dvid.VersionID, op SplitSupervoxelOp) error { 38 uuid, err := datastore.UUIDFromVersion(v) 39 if err != nil { 40 return err 41 } 42 logable, ok := d.(storage.LogWritable) 43 if !ok { 44 return nil // skip logging 45 } 46 log := logable.GetWriteLog() 47 if log == nil { 48 return nil 49 } 50 pop := &proto.SupervoxelSplitOp{ 51 Mutid: op.MutID, 52 Supervoxel: op.Supervoxel, 53 Splitlabel: op.SplitSupervoxel, 54 Remainlabel: op.RemainSupervoxel, 55 } 56 serialization, err := pb.Marshal(pop) 57 58 if err != nil { 59 return err 60 } 61 62 msg := storage.LogMessage{EntryType: proto.SupervoxelSplitType, Data: serialization} 63 return log.Append(d.DataUUID(), uuid, msg) 64 } 65 66 // LogMerge logs the merge of supervoxels to a label. 67 func LogMerge(d dvid.Data, v dvid.VersionID, op MergeOp) error { 68 uuid, err := datastore.UUIDFromVersion(v) 69 if err != nil { 70 return err 71 } 72 logable, ok := d.(storage.LogWritable) 73 if !ok { 74 return nil // skip logging 75 } 76 log := logable.GetWriteLog() 77 if log == nil { 78 return nil 79 } 80 data, err := serializeMerge(op) 81 if err != nil { 82 return err 83 } 84 msg := storage.LogMessage{EntryType: proto.MergeOpType, Data: data} 85 return log.Append(d.DataUUID(), uuid, msg) 86 } 87 88 // LogRenumber logs the merge of supervoxels to a label. 89 func LogRenumber(d dvid.Data, v dvid.VersionID, mutID, origLabel, newLabel uint64) error { 90 uuid, err := datastore.UUIDFromVersion(v) 91 if err != nil { 92 return err 93 } 94 logable, ok := d.(storage.LogWritable) 95 if !ok { 96 return nil // skip logging 97 } 98 log := logable.GetWriteLog() 99 if log == nil { 100 return nil 101 } 102 pop := &proto.RenumberOp{ 103 Mutid: mutID, 104 Target: origLabel, 105 Newlabel: newLabel, 106 } 107 data, err := pb.Marshal(pop) 108 if err != nil { 109 return err 110 } 111 112 msg := storage.LogMessage{EntryType: proto.RenumberOpType, Data: data} 113 return log.Append(d.DataUUID(), uuid, msg) 114 } 115 116 // LogCleave logs the cleave of supervoxels to a label. 117 func LogCleave(d dvid.Data, v dvid.VersionID, op CleaveOp) error { 118 uuid, err := datastore.UUIDFromVersion(v) 119 if err != nil { 120 return err 121 } 122 logable, ok := d.(storage.LogWritable) 123 if !ok { 124 return nil // skip logging 125 } 126 log := logable.GetWriteLog() 127 if log == nil { 128 return nil 129 } 130 data, err := serializeCleave(op) 131 if err != nil { 132 return err 133 } 134 msg := storage.LogMessage{EntryType: proto.CleaveOpType, Data: data} 135 return log.Append(d.DataUUID(), uuid, msg) 136 } 137 138 // LogMapping logs the mapping of supervoxels to a label. 139 func LogMapping(d dvid.Data, v dvid.VersionID, op MappingOp) error { 140 uuid, err := datastore.UUIDFromVersion(v) 141 if err != nil { 142 return err 143 } 144 logable, ok := d.(storage.LogWritable) 145 if !ok { 146 return nil // skip logging 147 } 148 log := logable.GetWriteLog() 149 if log == nil { 150 return nil 151 } 152 data, err := op.Marshal() 153 if err != nil { 154 return err 155 } 156 msg := storage.LogMessage{EntryType: proto.MappingOpType, Data: data} 157 return log.Append(d.DataUUID(), uuid, msg) 158 } 159 160 // LogMappings logs a collection of mapping operations to a UUID. 161 func LogMappings(d dvid.Data, v dvid.VersionID, ops *proto.MappingOps) error { 162 uuid, err := datastore.UUIDFromVersion(v) 163 if err != nil { 164 return err 165 } 166 logable, ok := d.(storage.LogWritable) 167 if !ok { 168 return nil // skip logging 169 } 170 log := logable.GetWriteLog() 171 if log == nil { 172 return nil 173 } 174 for _, op := range ops.Mappings { 175 data, err := pb.Marshal(op) 176 if err != nil { 177 return err 178 } 179 msg := storage.LogMessage{EntryType: proto.MappingOpType, Data: data} 180 if err := log.Append(d.DataUUID(), uuid, msg); err != nil { 181 return err 182 } 183 } 184 return nil 185 } 186 187 func ReadMappingLog(d dvid.Data, v dvid.VersionID) ([]MappingOp, error) { 188 uuid, err := datastore.UUIDFromVersion(v) 189 if err != nil { 190 return nil, err 191 } 192 logreadable, ok := d.(storage.LogReadable) 193 if !ok { 194 return nil, nil 195 } 196 rl := logreadable.GetReadLog() 197 if rl == nil { 198 return nil, nil 199 } 200 msgs, err := rl.ReadAll(d.DataUUID(), uuid) 201 if err != nil { 202 return nil, err 203 } 204 if len(msgs) == 0 { 205 return nil, nil 206 } 207 mappingOps := make([]MappingOp, len(msgs)) 208 var numMappings int 209 for i, msg := range msgs { 210 if msg.EntryType != proto.MappingOpType { 211 continue 212 } 213 214 var op proto.MappingOp 215 if err := pb.Unmarshal(msg.Data, &op); err != nil { 216 return nil, err 217 } 218 mappingOps[i].Mapped = op.GetMapped() 219 original := op.GetOriginal() 220 mappingOps[i].Original = make(Set, len(original)) 221 for _, label := range original { 222 mappingOps[i].Original[label] = struct{}{} 223 } 224 numMappings++ 225 } 226 mappingOps = mappingOps[:numMappings] 227 return mappingOps, nil 228 } 229 230 func StreamLog(d dvid.Data, v dvid.VersionID, ch chan storage.LogMessage) error { 231 uuid, err := datastore.UUIDFromVersion(v) 232 if err != nil { 233 return err 234 } 235 logreadable, ok := d.(storage.LogReadable) 236 if !ok { 237 close(ch) 238 return nil 239 } 240 rl := logreadable.GetReadLog() 241 if rl == nil { 242 close(ch) 243 return nil 244 } 245 return rl.StreamAll(d.DataUUID(), uuid, ch) 246 } 247 248 func LogAffinity(d dvid.Data, v dvid.VersionID, aff Affinity) error { 249 uuid, err := datastore.UUIDFromVersion(v) 250 if err != nil { 251 return err 252 } 253 logable, ok := d.(storage.LogWritable) 254 if !ok { 255 return nil // skip logging 256 } 257 log := logable.GetWriteLog() 258 if log == nil { 259 return nil 260 } 261 data, err := serializeAffinity(aff) 262 if err != nil { 263 return err 264 } 265 msg := storage.LogMessage{EntryType: proto.AffinityType, Data: data} 266 return log.Append(d.DataUUID(), uuid, msg) 267 } 268 269 func serializeSplit(op SplitOp) (serialization []byte, err error) { 270 rlesBytes, err := op.RLEs.MarshalBinary() 271 if err != nil { 272 return nil, err 273 } 274 svsplits := make(map[uint64]*proto.SVSplit, len(op.SplitMap)) 275 for supervoxel, split := range op.SplitMap { 276 svsplit := new(proto.SVSplit) 277 svsplit.Splitlabel = split.Split 278 svsplit.Remainlabel = split.Remain 279 svsplits[supervoxel] = svsplit 280 } 281 pop := &proto.SplitOp{ 282 Mutid: op.MutID, 283 Target: op.Target, 284 Newlabel: op.NewLabel, 285 Coarse: op.Coarse, 286 Rles: rlesBytes, 287 Svsplits: svsplits, 288 } 289 return pb.Marshal(pop) 290 } 291 292 func serializeMerge(op MergeOp) (serialization []byte, err error) { 293 merged := make([]uint64, len(op.Merged)) 294 var i int 295 for label := range op.Merged { 296 merged[i] = label 297 i++ 298 } 299 pop := &proto.MergeOp{ 300 Mutid: op.MutID, 301 Target: op.Target, 302 Merged: merged, 303 } 304 return pb.Marshal(pop) 305 } 306 307 func serializeCleave(op CleaveOp) (serialization []byte, err error) { 308 pop := &proto.CleaveOp{ 309 Mutid: op.MutID, 310 Target: op.Target, 311 Cleavedlabel: op.CleavedLabel, 312 Cleaved: op.CleavedSupervoxels, 313 } 314 return pb.Marshal(pop) 315 } 316 317 func serializeAffinity(aff Affinity) (serialization []byte, err error) { 318 pop := &proto.Affinity{ 319 Label1: aff.Label1, 320 Label2: aff.Label2, 321 Value: aff.Value, 322 } 323 return pb.Marshal(pop) 324 }