github.com/janelia-flyem/dvid@v1.0.0/datatype/labelmap/downres.go (about) 1 package labelmap 2 3 import ( 4 "fmt" 5 "runtime" 6 "sync" 7 8 "github.com/janelia-flyem/dvid/datastore" 9 "github.com/janelia-flyem/dvid/datatype/common/downres" 10 "github.com/janelia-flyem/dvid/datatype/common/labels" 11 "github.com/janelia-flyem/dvid/dvid" 12 "github.com/janelia-flyem/dvid/storage" 13 ) 14 15 // For any lores block, divide it into octants and see if we have mutated the corresponding higher-res blocks. 16 type octantMap map[dvid.IZYXString][8]*labels.Block 17 18 // Group hires blocks by octants so we see when we actually need to GET a lower-res block. 19 func (d *Data) getHiresChanges(hires downres.BlockMap) (octantMap, error) { 20 octants := make(octantMap) 21 22 for hiresZYX, value := range hires { 23 block, ok := value.(*labels.Block) 24 if !ok { 25 return nil, fmt.Errorf("bad changing block %s: expected *labels.Block got %v", hiresZYX, value) 26 } 27 hresCoord, err := hiresZYX.ToChunkPoint3d() 28 if err != nil { 29 return nil, err 30 } 31 downresX := hresCoord[0] >> 1 32 downresY := hresCoord[1] >> 1 33 downresZ := hresCoord[2] >> 1 34 loresZYX := dvid.ChunkPoint3d{downresX, downresY, downresZ}.ToIZYXString() 35 octidx := ((hresCoord[2] % 2) << 2) + ((hresCoord[1] % 2) << 1) + (hresCoord[0] % 2) 36 oct, found := octants[loresZYX] 37 if !found { 38 oct = [8]*labels.Block{} 39 } 40 oct[octidx] = block 41 octants[loresZYX] = oct 42 } 43 44 return octants, nil 45 } 46 47 type octantMsg struct { 48 loresZYX dvid.IZYXString 49 octant [8]*labels.Block 50 } 51 52 func (d *Data) downresOctant(v dvid.VersionID, hiresScale uint8, mu *sync.Mutex, downresBMap downres.BlockMap, batch storage.Batch, octantCh chan octantMsg, errCh chan error) { 53 blockSize, ok := d.BlockSize().(dvid.Point3d) 54 if !ok { 55 errCh <- fmt.Errorf("block size for data %q is not 3d: %v", d.DataName(), d.BlockSize()) 56 return 57 } 58 for msg := range octantCh { 59 var numBlocks int 60 for _, block := range msg.octant { 61 if block != nil { 62 numBlocks++ 63 } 64 } 65 66 var loresBlock *labels.Block 67 if numBlocks < 8 { 68 chunkPt, err := (msg.loresZYX).ToChunkPoint3d() 69 if err != nil { 70 errCh <- err 71 return 72 } 73 loresBlock, err = d.getSupervoxelBlock(v, chunkPt, hiresScale+1) 74 if err != nil { 75 errCh <- err 76 return 77 } 78 } else { 79 loresBlock = labels.MakeSolidBlock(0, blockSize) 80 } 81 if err := loresBlock.Downres(msg.octant); err != nil { 82 errCh <- err 83 return 84 } 85 mu.Lock() 86 downresBMap[msg.loresZYX] = loresBlock 87 mu.Unlock() 88 89 compressed, _ := loresBlock.MarshalBinary() 90 serialization, err := dvid.SerializeData(compressed, d.Compression(), d.Checksum()) 91 if err != nil { 92 errCh <- fmt.Errorf("unable to serialize downres block in %q: %v", d.DataName(), err) 93 return 94 } 95 tk := NewBlockTKeyByCoord(hiresScale+1, msg.loresZYX) 96 dvid.Infof("Storing downres block %s (%d bytes) at scale %d\n", msg.loresZYX, len(compressed), hiresScale+1) 97 mu.Lock() 98 batch.Put(tk, serialization) 99 mu.Unlock() 100 errCh <- nil 101 } 102 } 103 104 // StoreDownres computes a downscale representation of a set of mutated blocks. 105 func (d *Data) StoreDownres(v dvid.VersionID, hiresScale uint8, hires downres.BlockMap) (downres.BlockMap, error) { 106 timedLog := dvid.NewTimeLog() 107 if hiresScale >= d.MaxDownresLevel { 108 return nil, fmt.Errorf("can't downres %q scale %d since max downres scale is %d", d.DataName(), hiresScale, d.MaxDownresLevel) 109 } 110 octants, err := d.getHiresChanges(hires) 111 if err != nil { 112 return nil, err 113 } 114 115 batcher, err := datastore.GetKeyValueBatcher(d) 116 if err != nil { 117 return nil, err 118 } 119 ctx := datastore.NewVersionedCtx(d, v) 120 batch := batcher.NewBatch(ctx) 121 122 mu := new(sync.Mutex) 123 downresBMap := make(downres.BlockMap) 124 octantCh := make(chan octantMsg, len(octants)) 125 errCh := make(chan error, len(octants)) 126 defer func() { 127 close(octantCh) 128 close(errCh) 129 }() 130 131 numProcessors := runtime.NumCPU() 132 for i := 0; i < numProcessors; i++ { 133 go d.downresOctant(v, hiresScale, mu, downresBMap, batch, octantCh, errCh) 134 } 135 for loresZYX, octant := range octants { 136 octantCh <- octantMsg{loresZYX, octant} 137 } 138 for i := 0; i < len(octants); i++ { 139 err := <-errCh 140 if err != nil { 141 return nil, err 142 } 143 } 144 if err := batch.Commit(); err != nil { 145 return nil, fmt.Errorf("error on trying to write downres batch of scale %d->%d: %v", hiresScale, hiresScale+1, err) 146 } 147 timedLog.Infof("Computed down-resolution of %d octants at scale %d", len(octants), hiresScale) 148 return downresBMap, nil 149 }