github.com/janelia-flyem/dvid@v1.0.0/datatype/labelmap/mutation_log.go (about) 1 package labelmap 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "net/http" 7 "os" 8 "sync" 9 10 pb "google.golang.org/protobuf/proto" 11 12 "github.com/janelia-flyem/dvid/datastore" 13 "github.com/janelia-flyem/dvid/datatype/common/labels" 14 "github.com/janelia-flyem/dvid/datatype/common/proto" 15 "github.com/janelia-flyem/dvid/dvid" 16 "github.com/janelia-flyem/dvid/storage" 17 ) 18 19 // DumpMutations makes a log of all mutations from the start UUID to the end UUID. 20 func (d *Data) DumpMutations(startUUID, endUUID dvid.UUID, filename string) (comment string, err error) { 21 rl := d.GetReadLog() 22 if rl == nil { 23 err = fmt.Errorf("no mutation log was available for data %q", d.DataName()) 24 return 25 } 26 var startV, endV dvid.VersionID 27 startV, err = datastore.VersionFromUUID(startUUID) 28 if err != nil { 29 return 30 } 31 endV, err = datastore.VersionFromUUID(endUUID) 32 if err != nil { 33 return 34 } 35 var rootToLeaf, leafToRoot []dvid.VersionID 36 leafToRoot, err = datastore.GetAncestry(endV) 37 if err != nil { 38 return 39 } 40 rootToLeaf = make([]dvid.VersionID, len(leafToRoot)) 41 42 // reverse it and screen on UUID start/stop 43 startPos := len(leafToRoot) - 1 44 for _, v := range leafToRoot { 45 rootToLeaf[startPos] = v 46 if v == startV { 47 break 48 } 49 startPos-- 50 } 51 52 // open up target log 53 var f *os.File 54 f, err = os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755) 55 if err != nil { 56 return 57 } 58 59 // go through the ancestors from root to leaf, appending data to target log 60 numLogs := 0 61 var uuid dvid.UUID 62 for i, ancestor := range rootToLeaf[startPos:] { 63 if uuid, err = datastore.UUIDFromVersion(ancestor); err != nil { 64 return 65 } 66 timedLog := dvid.NewTimeLog() 67 var data []byte 68 if data, err = rl.ReadBinary(d.DataUUID(), uuid); err != nil { 69 return 70 } 71 if len(data) == 0 { 72 timedLog.Infof("No mappings found for data %q, version %s", d.DataName(), uuid) 73 continue 74 } 75 if _, err = f.Write(data); err != nil { 76 return 77 } 78 timedLog.Infof("Loaded mappings #%d for data %q, version ID %s", i+1, d.DataName(), uuid) 79 numLogs++ 80 } 81 err = f.Close() 82 comment = fmt.Sprintf("Completed flattening of %d mutation logs to %s\n", numLogs, filename) 83 return 84 } 85 86 // GetMutationHistory writes JSON of the mutations that were done to the given label at toUUID version, 87 // where we delimit the time range of interest to [fromUUID, toUUID] versions. The mutations are written 88 // backwards in time toUUID -> fromUUID. 89 func (d *Data) GetMutationHistory(w http.ResponseWriter, fromUUID, toUUID dvid.UUID, target uint64) error { 90 fromV, err := datastore.VersionFromUUID(fromUUID) 91 if err != nil { 92 return err 93 } 94 toV, err := datastore.VersionFromUUID(toUUID) 95 if err != nil { 96 return err 97 } 98 99 // Get latest supervoxels assigned to target id. 100 supervoxelSet, err := d.GetSupervoxels(toV, target) 101 if err != nil { 102 return err 103 } 104 supervoxels := make([]uint64, len(supervoxelSet)) 105 i := 0 106 for sv := range supervoxelSet { 107 supervoxels[i] = sv 108 } 109 110 // Get the starting version labels mapped to the target's supervoxels -> origBodies set 111 mapped, _, err := d.GetMappedLabels(fromV, supervoxels) 112 if err != nil { 113 return err 114 } 115 origBodies := make(labels.Set, len(mapped)) 116 for _, label := range mapped { 117 origBodies[label] = struct{}{} 118 } 119 120 // Get all UUIDs in [fromUUID, toUUID] span -> versions 121 ancestors, err := datastore.GetAncestry(toV) 122 if err != nil { 123 return err 124 } 125 var record bool 126 var versions []dvid.VersionID 127 for i := len(ancestors) - 1; i >= 0; i-- { // work from root to toUUID 128 curV := ancestors[i] 129 if !record && curV == fromV { 130 record = true 131 } 132 if record { 133 versions = append(versions, curV) 134 } 135 if curV == toV { 136 break 137 } 138 } 139 140 // Write the mutations for any fromUUID label touched by supervoxels in the toUUID target label. 141 for _, v := range versions { 142 timedLog := dvid.NewTimeLog() 143 ch := make(chan storage.LogMessage, 100) 144 wg := new(sync.WaitGroup) 145 wg.Add(1) 146 go processMutationLogStream(w, v, ch, wg, origBodies, supervoxelSet) 147 if err = labels.StreamLog(d, v, ch); err != nil { 148 return fmt.Errorf("problem loading mutation log: %v", err) 149 } 150 wg.Wait() 151 timedLog.Infof("Wrote mutation history for data %q, from UUID %s to %s", d.DataName(), fromUUID, toUUID) 152 } 153 return nil 154 } 155 156 func processMutationLogStream(w http.ResponseWriter, v dvid.VersionID, ch chan storage.LogMessage, wg *sync.WaitGroup, origBodies, supervoxelSet labels.Set) { 157 numMsgs := 0 158 for msg := range ch { // expects channel to be closed on completion 159 numMsgs++ 160 var err error 161 var out []byte 162 switch msg.EntryType { 163 case proto.MergeOpType: 164 var op proto.MergeOp 165 if err := pb.Unmarshal(msg.Data, &op); err != nil { 166 dvid.Errorf("unable to unmarshal cleave message for version %d: %v\n", v, err) 167 wg.Done() 168 continue 169 } 170 if len(op.Merged) == 0 { 171 wg.Done() 172 continue 173 } 174 if _, found := origBodies[op.Target]; found { 175 out, err = json.Marshal(struct { 176 Action string 177 Target uint64 178 Labels []uint64 179 }{ 180 Action: "merge", 181 Target: op.Target, 182 Labels: op.Merged, 183 }) 184 if err != nil { 185 dvid.Errorf("unable to write merge message: %v\n", err) 186 } 187 } 188 189 case proto.CleaveOpType: 190 var op proto.CleaveOp 191 if err := pb.Unmarshal(msg.Data, &op); err != nil { 192 dvid.Errorf("unable to unmarshal cleave message for version %d: %v\n", v, err) 193 wg.Done() 194 continue 195 } 196 if len(op.Cleaved) == 0 { 197 wg.Done() 198 continue 199 } 200 if _, found := origBodies[op.Target]; found { 201 out, err = json.Marshal(struct { 202 Action string 203 OrigLabel uint64 204 CleavedLabel uint64 205 CleavedSupervoxels []uint64 206 }{ 207 Action: "cleave", 208 OrigLabel: op.Target, 209 CleavedLabel: op.Cleavedlabel, 210 CleavedSupervoxels: op.Cleaved, 211 }) 212 if err != nil { 213 dvid.Errorf("unable to write cleave message: %v\n", err) 214 } 215 } 216 217 case proto.SplitOpType: 218 var op proto.SplitOp 219 if err := pb.Unmarshal(msg.Data, &op); err != nil { 220 dvid.Errorf("unable to unmarshal split log message for version %d: %v\n", v, err) 221 wg.Done() 222 continue 223 } 224 if _, found := origBodies[op.Target]; found { 225 out, err = json.Marshal(struct { 226 Action string 227 Target uint64 228 NewLabel uint64 229 Coarse bool 230 }{ 231 Action: "split", 232 Target: op.Target, 233 NewLabel: op.Newlabel, 234 Coarse: op.Coarse, 235 }) 236 if err != nil { 237 dvid.Errorf("unable to write split message: %v\n", err) 238 } 239 } 240 241 case proto.SupervoxelSplitType: 242 var op proto.SupervoxelSplitOp 243 if err := pb.Unmarshal(msg.Data, &op); err != nil { 244 dvid.Errorf("unable to unmarshal split log message for version %d: %v\n", v, err) 245 wg.Done() 246 continue 247 } 248 if _, found := supervoxelSet[op.Supervoxel]; found { 249 out, err = json.Marshal(struct { 250 Action string 251 Supervoxel uint64 252 SplitSupervoxel uint64 253 RemainSupervoxel uint64 254 }{ 255 Action: "split-supervoxel", 256 Supervoxel: op.Supervoxel, 257 SplitSupervoxel: op.Splitlabel, 258 RemainSupervoxel: op.Remainlabel, 259 }) 260 if err != nil { 261 dvid.Errorf("unable to write split-supervoxel message: %v\n", err) 262 } 263 } 264 265 default: 266 } 267 if len(out) != 0 { 268 _, err := w.Write(out) 269 if err != nil { 270 dvid.Errorf("Error writing mutation history for %s: %v\n", origBodies, err) 271 } 272 } 273 } 274 wg.Done() 275 }