github.com/janelia-flyem/dvid@v1.0.0/datatype/labelmap/mutation_log.go (about)

     1  package labelmap
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"net/http"
     7  	"os"
     8  	"sync"
     9  
    10  	pb "google.golang.org/protobuf/proto"
    11  
    12  	"github.com/janelia-flyem/dvid/datastore"
    13  	"github.com/janelia-flyem/dvid/datatype/common/labels"
    14  	"github.com/janelia-flyem/dvid/datatype/common/proto"
    15  	"github.com/janelia-flyem/dvid/dvid"
    16  	"github.com/janelia-flyem/dvid/storage"
    17  )
    18  
    19  // DumpMutations makes a log of all mutations from the start UUID to the end UUID.
    20  func (d *Data) DumpMutations(startUUID, endUUID dvid.UUID, filename string) (comment string, err error) {
    21  	rl := d.GetReadLog()
    22  	if rl == nil {
    23  		err = fmt.Errorf("no mutation log was available for data %q", d.DataName())
    24  		return
    25  	}
    26  	var startV, endV dvid.VersionID
    27  	startV, err = datastore.VersionFromUUID(startUUID)
    28  	if err != nil {
    29  		return
    30  	}
    31  	endV, err = datastore.VersionFromUUID(endUUID)
    32  	if err != nil {
    33  		return
    34  	}
    35  	var rootToLeaf, leafToRoot []dvid.VersionID
    36  	leafToRoot, err = datastore.GetAncestry(endV)
    37  	if err != nil {
    38  		return
    39  	}
    40  	rootToLeaf = make([]dvid.VersionID, len(leafToRoot))
    41  
    42  	// reverse it and screen on UUID start/stop
    43  	startPos := len(leafToRoot) - 1
    44  	for _, v := range leafToRoot {
    45  		rootToLeaf[startPos] = v
    46  		if v == startV {
    47  			break
    48  		}
    49  		startPos--
    50  	}
    51  
    52  	// open up target log
    53  	var f *os.File
    54  	f, err = os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0755)
    55  	if err != nil {
    56  		return
    57  	}
    58  
    59  	// go through the ancestors from root to leaf, appending data to target log
    60  	numLogs := 0
    61  	var uuid dvid.UUID
    62  	for i, ancestor := range rootToLeaf[startPos:] {
    63  		if uuid, err = datastore.UUIDFromVersion(ancestor); err != nil {
    64  			return
    65  		}
    66  		timedLog := dvid.NewTimeLog()
    67  		var data []byte
    68  		if data, err = rl.ReadBinary(d.DataUUID(), uuid); err != nil {
    69  			return
    70  		}
    71  		if len(data) == 0 {
    72  			timedLog.Infof("No mappings found for data %q, version %s", d.DataName(), uuid)
    73  			continue
    74  		}
    75  		if _, err = f.Write(data); err != nil {
    76  			return
    77  		}
    78  		timedLog.Infof("Loaded mappings #%d for data %q, version ID %s", i+1, d.DataName(), uuid)
    79  		numLogs++
    80  	}
    81  	err = f.Close()
    82  	comment = fmt.Sprintf("Completed flattening of %d mutation logs to %s\n", numLogs, filename)
    83  	return
    84  }
    85  
    86  // GetMutationHistory writes JSON of the mutations that were done to the given label at toUUID version,
    87  // where we delimit the time range of interest to [fromUUID, toUUID] versions.  The mutations are written
    88  // backwards in time toUUID -> fromUUID.
    89  func (d *Data) GetMutationHistory(w http.ResponseWriter, fromUUID, toUUID dvid.UUID, target uint64) error {
    90  	fromV, err := datastore.VersionFromUUID(fromUUID)
    91  	if err != nil {
    92  		return err
    93  	}
    94  	toV, err := datastore.VersionFromUUID(toUUID)
    95  	if err != nil {
    96  		return err
    97  	}
    98  
    99  	// Get latest supervoxels assigned to target id.
   100  	supervoxelSet, err := d.GetSupervoxels(toV, target)
   101  	if err != nil {
   102  		return err
   103  	}
   104  	supervoxels := make([]uint64, len(supervoxelSet))
   105  	i := 0
   106  	for sv := range supervoxelSet {
   107  		supervoxels[i] = sv
   108  	}
   109  
   110  	// Get the starting version labels mapped to the target's supervoxels -> origBodies set
   111  	mapped, _, err := d.GetMappedLabels(fromV, supervoxels)
   112  	if err != nil {
   113  		return err
   114  	}
   115  	origBodies := make(labels.Set, len(mapped))
   116  	for _, label := range mapped {
   117  		origBodies[label] = struct{}{}
   118  	}
   119  
   120  	// Get all UUIDs in [fromUUID, toUUID] span -> versions
   121  	ancestors, err := datastore.GetAncestry(toV)
   122  	if err != nil {
   123  		return err
   124  	}
   125  	var record bool
   126  	var versions []dvid.VersionID
   127  	for i := len(ancestors) - 1; i >= 0; i-- { // work from root to toUUID
   128  		curV := ancestors[i]
   129  		if !record && curV == fromV {
   130  			record = true
   131  		}
   132  		if record {
   133  			versions = append(versions, curV)
   134  		}
   135  		if curV == toV {
   136  			break
   137  		}
   138  	}
   139  
   140  	// Write the mutations for any fromUUID label touched by supervoxels in the toUUID target label.
   141  	for _, v := range versions {
   142  		timedLog := dvid.NewTimeLog()
   143  		ch := make(chan storage.LogMessage, 100)
   144  		wg := new(sync.WaitGroup)
   145  		wg.Add(1)
   146  		go processMutationLogStream(w, v, ch, wg, origBodies, supervoxelSet)
   147  		if err = labels.StreamLog(d, v, ch); err != nil {
   148  			return fmt.Errorf("problem loading mutation log: %v", err)
   149  		}
   150  		wg.Wait()
   151  		timedLog.Infof("Wrote mutation history for data %q, from UUID %s to %s", d.DataName(), fromUUID, toUUID)
   152  	}
   153  	return nil
   154  }
   155  
   156  func processMutationLogStream(w http.ResponseWriter, v dvid.VersionID, ch chan storage.LogMessage, wg *sync.WaitGroup, origBodies, supervoxelSet labels.Set) {
   157  	numMsgs := 0
   158  	for msg := range ch { // expects channel to be closed on completion
   159  		numMsgs++
   160  		var err error
   161  		var out []byte
   162  		switch msg.EntryType {
   163  		case proto.MergeOpType:
   164  			var op proto.MergeOp
   165  			if err := pb.Unmarshal(msg.Data, &op); err != nil {
   166  				dvid.Errorf("unable to unmarshal cleave message for version %d: %v\n", v, err)
   167  				wg.Done()
   168  				continue
   169  			}
   170  			if len(op.Merged) == 0 {
   171  				wg.Done()
   172  				continue
   173  			}
   174  			if _, found := origBodies[op.Target]; found {
   175  				out, err = json.Marshal(struct {
   176  					Action string
   177  					Target uint64
   178  					Labels []uint64
   179  				}{
   180  					Action: "merge",
   181  					Target: op.Target,
   182  					Labels: op.Merged,
   183  				})
   184  				if err != nil {
   185  					dvid.Errorf("unable to write merge message: %v\n", err)
   186  				}
   187  			}
   188  
   189  		case proto.CleaveOpType:
   190  			var op proto.CleaveOp
   191  			if err := pb.Unmarshal(msg.Data, &op); err != nil {
   192  				dvid.Errorf("unable to unmarshal cleave message for version %d: %v\n", v, err)
   193  				wg.Done()
   194  				continue
   195  			}
   196  			if len(op.Cleaved) == 0 {
   197  				wg.Done()
   198  				continue
   199  			}
   200  			if _, found := origBodies[op.Target]; found {
   201  				out, err = json.Marshal(struct {
   202  					Action             string
   203  					OrigLabel          uint64
   204  					CleavedLabel       uint64
   205  					CleavedSupervoxels []uint64
   206  				}{
   207  					Action:             "cleave",
   208  					OrigLabel:          op.Target,
   209  					CleavedLabel:       op.Cleavedlabel,
   210  					CleavedSupervoxels: op.Cleaved,
   211  				})
   212  				if err != nil {
   213  					dvid.Errorf("unable to write cleave message: %v\n", err)
   214  				}
   215  			}
   216  
   217  		case proto.SplitOpType:
   218  			var op proto.SplitOp
   219  			if err := pb.Unmarshal(msg.Data, &op); err != nil {
   220  				dvid.Errorf("unable to unmarshal split log message for version %d: %v\n", v, err)
   221  				wg.Done()
   222  				continue
   223  			}
   224  			if _, found := origBodies[op.Target]; found {
   225  				out, err = json.Marshal(struct {
   226  					Action   string
   227  					Target   uint64
   228  					NewLabel uint64
   229  					Coarse   bool
   230  				}{
   231  					Action:   "split",
   232  					Target:   op.Target,
   233  					NewLabel: op.Newlabel,
   234  					Coarse:   op.Coarse,
   235  				})
   236  				if err != nil {
   237  					dvid.Errorf("unable to write split message: %v\n", err)
   238  				}
   239  			}
   240  
   241  		case proto.SupervoxelSplitType:
   242  			var op proto.SupervoxelSplitOp
   243  			if err := pb.Unmarshal(msg.Data, &op); err != nil {
   244  				dvid.Errorf("unable to unmarshal split log message for version %d: %v\n", v, err)
   245  				wg.Done()
   246  				continue
   247  			}
   248  			if _, found := supervoxelSet[op.Supervoxel]; found {
   249  				out, err = json.Marshal(struct {
   250  					Action           string
   251  					Supervoxel       uint64
   252  					SplitSupervoxel  uint64
   253  					RemainSupervoxel uint64
   254  				}{
   255  					Action:           "split-supervoxel",
   256  					Supervoxel:       op.Supervoxel,
   257  					SplitSupervoxel:  op.Splitlabel,
   258  					RemainSupervoxel: op.Remainlabel,
   259  				})
   260  				if err != nil {
   261  					dvid.Errorf("unable to write split-supervoxel message: %v\n", err)
   262  				}
   263  			}
   264  
   265  		default:
   266  		}
   267  		if len(out) != 0 {
   268  			_, err := w.Write(out)
   269  			if err != nil {
   270  				dvid.Errorf("Error writing mutation history for %s: %v\n", origBodies, err)
   271  			}
   272  		}
   273  	}
   274  	wg.Done()
   275  }