github.com/janelia-flyem/dvid@v1.0.0/datatype/labelmap/downres.go (about)

     1  package labelmap
     2  
     3  import (
     4  	"fmt"
     5  	"runtime"
     6  	"sync"
     7  
     8  	"github.com/janelia-flyem/dvid/datastore"
     9  	"github.com/janelia-flyem/dvid/datatype/common/downres"
    10  	"github.com/janelia-flyem/dvid/datatype/common/labels"
    11  	"github.com/janelia-flyem/dvid/dvid"
    12  	"github.com/janelia-flyem/dvid/storage"
    13  )
    14  
    15  // For any lores block, divide it into octants and see if we have mutated the corresponding higher-res blocks.
    16  type octantMap map[dvid.IZYXString][8]*labels.Block
    17  
    18  // Group hires blocks by octants so we see when we actually need to GET a lower-res block.
    19  func (d *Data) getHiresChanges(hires downres.BlockMap) (octantMap, error) {
    20  	octants := make(octantMap)
    21  
    22  	for hiresZYX, value := range hires {
    23  		block, ok := value.(*labels.Block)
    24  		if !ok {
    25  			return nil, fmt.Errorf("bad changing block %s: expected *labels.Block got %v", hiresZYX, value)
    26  		}
    27  		hresCoord, err := hiresZYX.ToChunkPoint3d()
    28  		if err != nil {
    29  			return nil, err
    30  		}
    31  		downresX := hresCoord[0] >> 1
    32  		downresY := hresCoord[1] >> 1
    33  		downresZ := hresCoord[2] >> 1
    34  		loresZYX := dvid.ChunkPoint3d{downresX, downresY, downresZ}.ToIZYXString()
    35  		octidx := ((hresCoord[2] % 2) << 2) + ((hresCoord[1] % 2) << 1) + (hresCoord[0] % 2)
    36  		oct, found := octants[loresZYX]
    37  		if !found {
    38  			oct = [8]*labels.Block{}
    39  		}
    40  		oct[octidx] = block
    41  		octants[loresZYX] = oct
    42  	}
    43  
    44  	return octants, nil
    45  }
    46  
    47  type octantMsg struct {
    48  	loresZYX dvid.IZYXString
    49  	octant   [8]*labels.Block
    50  }
    51  
    52  func (d *Data) downresOctant(v dvid.VersionID, hiresScale uint8, mu *sync.Mutex, downresBMap downres.BlockMap, batch storage.Batch, octantCh chan octantMsg, errCh chan error) {
    53  	blockSize, ok := d.BlockSize().(dvid.Point3d)
    54  	if !ok {
    55  		errCh <- fmt.Errorf("block size for data %q is not 3d: %v", d.DataName(), d.BlockSize())
    56  		return
    57  	}
    58  	for msg := range octantCh {
    59  		var numBlocks int
    60  		for _, block := range msg.octant {
    61  			if block != nil {
    62  				numBlocks++
    63  			}
    64  		}
    65  
    66  		var loresBlock *labels.Block
    67  		if numBlocks < 8 {
    68  			chunkPt, err := (msg.loresZYX).ToChunkPoint3d()
    69  			if err != nil {
    70  				errCh <- err
    71  				return
    72  			}
    73  			loresBlock, err = d.getSupervoxelBlock(v, chunkPt, hiresScale+1)
    74  			if err != nil {
    75  				errCh <- err
    76  				return
    77  			}
    78  		} else {
    79  			loresBlock = labels.MakeSolidBlock(0, blockSize)
    80  		}
    81  		if err := loresBlock.Downres(msg.octant); err != nil {
    82  			errCh <- err
    83  			return
    84  		}
    85  		mu.Lock()
    86  		downresBMap[msg.loresZYX] = loresBlock
    87  		mu.Unlock()
    88  
    89  		compressed, _ := loresBlock.MarshalBinary()
    90  		serialization, err := dvid.SerializeData(compressed, d.Compression(), d.Checksum())
    91  		if err != nil {
    92  			errCh <- fmt.Errorf("unable to serialize downres block in %q: %v", d.DataName(), err)
    93  			return
    94  		}
    95  		tk := NewBlockTKeyByCoord(hiresScale+1, msg.loresZYX)
    96  		dvid.Infof("Storing downres block %s (%d bytes) at scale %d\n", msg.loresZYX, len(compressed), hiresScale+1)
    97  		mu.Lock()
    98  		batch.Put(tk, serialization)
    99  		mu.Unlock()
   100  		errCh <- nil
   101  	}
   102  }
   103  
   104  // StoreDownres computes a downscale representation of a set of mutated blocks.
   105  func (d *Data) StoreDownres(v dvid.VersionID, hiresScale uint8, hires downres.BlockMap) (downres.BlockMap, error) {
   106  	timedLog := dvid.NewTimeLog()
   107  	if hiresScale >= d.MaxDownresLevel {
   108  		return nil, fmt.Errorf("can't downres %q scale %d since max downres scale is %d", d.DataName(), hiresScale, d.MaxDownresLevel)
   109  	}
   110  	octants, err := d.getHiresChanges(hires)
   111  	if err != nil {
   112  		return nil, err
   113  	}
   114  
   115  	batcher, err := datastore.GetKeyValueBatcher(d)
   116  	if err != nil {
   117  		return nil, err
   118  	}
   119  	ctx := datastore.NewVersionedCtx(d, v)
   120  	batch := batcher.NewBatch(ctx)
   121  
   122  	mu := new(sync.Mutex)
   123  	downresBMap := make(downres.BlockMap)
   124  	octantCh := make(chan octantMsg, len(octants))
   125  	errCh := make(chan error, len(octants))
   126  	defer func() {
   127  		close(octantCh)
   128  		close(errCh)
   129  	}()
   130  
   131  	numProcessors := runtime.NumCPU()
   132  	for i := 0; i < numProcessors; i++ {
   133  		go d.downresOctant(v, hiresScale, mu, downresBMap, batch, octantCh, errCh)
   134  	}
   135  	for loresZYX, octant := range octants {
   136  		octantCh <- octantMsg{loresZYX, octant}
   137  	}
   138  	for i := 0; i < len(octants); i++ {
   139  		err := <-errCh
   140  		if err != nil {
   141  			return nil, err
   142  		}
   143  	}
   144  	if err := batch.Commit(); err != nil {
   145  		return nil, fmt.Errorf("error on trying to write downres batch of scale %d->%d: %v", hiresScale, hiresScale+1, err)
   146  	}
   147  	timedLog.Infof("Computed down-resolution of %d octants at scale %d", len(octants), hiresScale)
   148  	return downresBMap, nil
   149  }