github.com/janelia-flyem/dvid@v1.0.0/datatype/common/downres/downres.go (about)

     1  /*
     2  	Package downres provides a system for computing multi-scale 3d arrays given mutations.
     3  	Two workflows are provided: (1) mutation-based with on-the-fly downres operations activated
     4  	by the end of a mutation, and (2) larger ingestions where it is not possible to
     5  	retain all changed data in memory.  #2 is TODO.
     6  */
     7  package downres
     8  
     9  import (
    10  	"fmt"
    11  	"sync"
    12  	"time"
    13  
    14  	"github.com/janelia-flyem/dvid/datastore"
    15  	"github.com/janelia-flyem/dvid/dvid"
    16  )
    17  
    18  type BlockMap map[dvid.IZYXString]interface{}
    19  
    20  type Updater interface {
    21  	StartScaleUpdate(scale uint8)
    22  	StopScaleUpdate(scale uint8)
    23  	ScaleUpdating(scale uint8) bool
    24  	AnyScaleUpdating() bool
    25  }
    26  
    27  type Downreser interface {
    28  	Updater
    29  
    30  	DataName() dvid.InstanceName
    31  	GetMaxDownresLevel() uint8
    32  
    33  	// StoreDownres computes and stores the down-res for the given blocks, returning
    34  	// the computed down-res blocks at 1/2 resolution.
    35  	StoreDownres(v dvid.VersionID, hiresScale uint8, hires BlockMap) (BlockMap, error)
    36  }
    37  
    38  // BlockOnUpdating blocks until the given data is not updating from any normal updates or
    39  // also downres operations.  Primarily used during testing.
    40  func BlockOnUpdating(uuid dvid.UUID, name dvid.InstanceName) error {
    41  	time.Sleep(100 * time.Millisecond)
    42  	if err := datastore.BlockOnUpdating(uuid, name); err != nil {
    43  		return err
    44  	}
    45  	d, err := datastore.GetDataByUUIDName(uuid, name)
    46  	if err != nil {
    47  		return err
    48  	}
    49  
    50  	updater, isUpdater := d.(Updater)
    51  	for isUpdater && updater.AnyScaleUpdating() {
    52  		time.Sleep(50 * time.Millisecond)
    53  	}
    54  	return nil
    55  }
    56  
    57  // Mutation is a stash of changes that will be handled in down-resolution scaling.
    58  type Mutation struct {
    59  	d          Downreser
    60  	v          dvid.VersionID
    61  	mutID      uint64
    62  	hiresCache BlockMap
    63  
    64  	sync.RWMutex
    65  }
    66  
    67  // NewMutation returns a new Mutation for stashing changes.
    68  func NewMutation(d Downreser, v dvid.VersionID, mutID uint64) *Mutation {
    69  	for scale := uint8(1); scale <= d.GetMaxDownresLevel(); scale++ {
    70  		d.StartScaleUpdate(scale)
    71  	}
    72  	m := Mutation{
    73  		d:          d,
    74  		v:          v,
    75  		mutID:      mutID,
    76  		hiresCache: make(BlockMap),
    77  	}
    78  	return &m
    79  }
    80  
    81  // MutationID returns the mutation ID associated with this mutation.
    82  func (m *Mutation) MutationID() uint64 {
    83  	if m == nil {
    84  		return 0
    85  	}
    86  	return m.mutID
    87  }
    88  
    89  // BlockMutated caches mutations at the highest resolution (scale 0)
    90  func (m *Mutation) BlockMutated(bcoord dvid.IZYXString, block interface{}) error {
    91  	if m.hiresCache == nil {
    92  		return fmt.Errorf("bad attempt to mutate block %s when mutation already closed", bcoord)
    93  	}
    94  	m.Lock()
    95  	m.hiresCache[bcoord] = block
    96  	m.Unlock()
    97  	return nil
    98  }
    99  
   100  // Execute computes lower-res scales for all altered blocks in a mutation.
   101  func (m *Mutation) Execute() error {
   102  	timedLog := dvid.NewTimeLog()
   103  	m.Lock()
   104  	bm := m.hiresCache
   105  	var err error
   106  	for scale := uint8(0); scale < m.d.GetMaxDownresLevel(); scale++ {
   107  		bm, err = m.d.StoreDownres(m.v, scale, bm)
   108  		if err != nil {
   109  			return fmt.Errorf("mutation %d for data %q: %v", m.mutID, m.d.DataName(), err)
   110  		}
   111  		m.d.StopScaleUpdate(scale + 1)
   112  	}
   113  	m.hiresCache = nil
   114  	m.Unlock()
   115  	timedLog.Debugf("Computed and stored downres for scale 1 to %d for data %q", m.d.GetMaxDownresLevel(), m.d.DataName())
   116  	return nil
   117  }