github.com/janelia-flyem/dvid@v1.0.0/datatype/labelblk/composite.go (about) 1 /* 2 Functinos that support compositing label and image blocks into a pseudo-color representation. 3 */ 4 5 package labelblk 6 7 import ( 8 "bytes" 9 "encoding/binary" 10 "fmt" 11 "sync" 12 13 "github.com/janelia-flyem/dvid/datastore" 14 "github.com/janelia-flyem/dvid/datatype/imageblk" 15 "github.com/janelia-flyem/dvid/dvid" 16 "github.com/janelia-flyem/dvid/server" 17 "github.com/janelia-flyem/dvid/storage" 18 ) 19 20 type compositeOp struct { 21 grayscale *imageblk.Data 22 composite *imageblk.Data 23 versionID dvid.VersionID 24 } 25 26 // CreateComposite creates a new rgba8 image by combining hash of labels + the grayscale 27 func (d *Data) CreateComposite(request datastore.Request, reply *datastore.Response) error { 28 timedLog := dvid.NewTimeLog() 29 30 // Parse the request 31 var uuidStr, dataName, cmdStr, grayscaleName, destName string 32 request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &grayscaleName, &destName) 33 34 // Get the version 35 uuid, v, err := datastore.MatchingUUID(uuidStr) 36 if err != nil { 37 return err 38 } 39 40 // Log request 41 if err = datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil { 42 return err 43 } 44 45 // Get the grayscale data. 46 dataservice, err := datastore.GetDataByUUIDName(uuid, dvid.InstanceName(grayscaleName)) 47 if err != nil { 48 return err 49 } 50 grayscale, ok := dataservice.(*imageblk.Data) 51 if !ok { 52 return fmt.Errorf("%s is not the name of uint8 data", grayscaleName) 53 } 54 55 // Create a new rgba8blk data. 56 var compservice datastore.DataService 57 compservice, err = datastore.GetDataByUUIDName(uuid, dvid.InstanceName(destName)) 58 if err == nil { 59 return fmt.Errorf("Data instance with name %q already exists", destName) 60 } 61 typeService, err := datastore.TypeServiceByName("rgba8blk") 62 if err != nil { 63 return fmt.Errorf("Could not get rgba8 type service from DVID") 64 } 65 config := dvid.NewConfig() 66 compservice, err = datastore.NewData(uuid, typeService, dvid.InstanceName(destName), config) 67 if err != nil { 68 return err 69 } 70 composite, ok := compservice.(*imageblk.Data) 71 if !ok { 72 return fmt.Errorf("Error: %s was unable to be set to rgba8 data", destName) 73 } 74 75 // Iterate through all labels and grayscale chunks incrementally in Z, a layer at a time. 76 wg := new(sync.WaitGroup) 77 op := &compositeOp{grayscale, composite, v} 78 chunkOp := &storage.ChunkOp{op, wg} 79 80 store, err := datastore.GetOrderedKeyValueDB(d) 81 if err != nil { 82 return err 83 } 84 ctx := datastore.NewVersionedCtx(d, v) 85 extents := d.Extents() 86 blockBeg := imageblk.NewTKey(extents.MinIndex) 87 blockEnd := imageblk.NewTKey(extents.MaxIndex) 88 err = store.ProcessRange(ctx, blockBeg, blockEnd, chunkOp, storage.ChunkFunc(d.CreateCompositeChunk)) 89 wg.Wait() 90 91 // Set new mapped data to same extents. 92 composite.Properties.Extents = grayscale.Properties.Extents 93 if err := datastore.SaveDataByUUID(uuid, composite); err != nil { 94 dvid.Infof("Could not save new data '%s': %v\n", destName, err) 95 } 96 97 timedLog.Infof("Created composite of %s and %s", grayscaleName, destName) 98 return nil 99 } 100 101 // CreateCompositeChunk processes each chunk of labels and grayscale data, 102 // saving the composited result into an rgba8. 103 // Only some multiple of the # of CPU cores can be used for chunk handling before 104 // it waits for chunk processing to abate via the buffered server.HandlerToken channel. 105 func (d *Data) CreateCompositeChunk(chunk *storage.Chunk) error { 106 server.CheckChunkThrottling() 107 go d.createCompositeChunk(chunk) 108 return nil 109 } 110 111 var curZ int32 112 var curZMutex sync.Mutex 113 114 func (d *Data) createCompositeChunk(chunk *storage.Chunk) { 115 defer func() { 116 // After processing a chunk, return the token. 117 server.HandlerToken <- 1 118 119 // Notify the requestor that this chunk is done. 120 if chunk.Wg != nil { 121 chunk.Wg.Done() 122 } 123 }() 124 125 op := chunk.Op.(*compositeOp) 126 127 // Get the spatial index associated with this chunk. 128 zyx, err := imageblk.DecodeTKey(chunk.K) 129 if err != nil { 130 dvid.Errorf("Error in %s.ChunkApplyMap(): %v", d.Data.DataName(), err) 131 return 132 } 133 134 // Initialize the label buffers. For voxels, this data needs to be uncompressed and deserialized. 135 curZMutex.Lock() 136 if zyx[2] > curZ { 137 curZ = zyx[2] 138 minZ := zyx.MinPoint(d.BlockSize()).Value(2) 139 maxZ := zyx.MaxPoint(d.BlockSize()).Value(2) 140 dvid.Debugf("Now creating composite blocks for Z %d to %d\n", minZ, maxZ) 141 } 142 curZMutex.Unlock() 143 144 labelData, _, err := dvid.DeserializeData(chunk.V, true) 145 if err != nil { 146 dvid.Infof("Unable to deserialize block in '%s': %v\n", d.DataName(), err) 147 return 148 } 149 blockBytes := len(labelData) 150 if blockBytes%8 != 0 { 151 dvid.Infof("Retrieved, deserialized block is wrong size: %d bytes\n", blockBytes) 152 return 153 } 154 155 // Get the corresponding grayscale block. 156 store, err := datastore.GetOrderedKeyValueDB(op.grayscale) 157 if err != nil { 158 dvid.Errorf("Unable to retrieve store for %q: %v\n", op.grayscale.DataName(), err) 159 return 160 } 161 grayscaleCtx := datastore.NewVersionedCtx(op.grayscale, op.versionID) 162 blockData, err := store.Get(grayscaleCtx, chunk.K) 163 if err != nil { 164 dvid.Errorf("Error getting grayscale block for index %s\n", zyx) 165 return 166 } 167 grayscaleData, _, err := dvid.DeserializeData(blockData, true) 168 if err != nil { 169 dvid.Errorf("Unable to deserialize block in '%s': %v\n", op.grayscale.DataName(), err) 170 return 171 } 172 173 // Compute the composite block. 174 // TODO -- Exploit run lengths, use cache of hash? 175 compositeBytes := blockBytes / 2 176 compositeData := make([]byte, compositeBytes, compositeBytes) 177 compositeI := 0 178 labelI := 0 179 hashBuf := make([]byte, 4, 4) 180 for _, grayscale := range grayscaleData { 181 //murmurhash3(labelData[labelI:labelI+8], hashBuf) 182 //hashBuf[3] = grayscale 183 writePseudoColor(grayscale, labelData[labelI:labelI+8], hashBuf) 184 copy(compositeData[compositeI:compositeI+4], hashBuf) 185 compositeI += 4 186 labelI += 8 187 } 188 189 // Store the composite block into the rgba8 data. 190 serialization, err := dvid.SerializeData(compositeData, d.Compression(), d.Checksum()) 191 if err != nil { 192 dvid.Errorf("Unable to serialize composite block %s: %v\n", zyx, err) 193 return 194 } 195 compositeCtx := datastore.NewVersionedCtx(op.composite, op.versionID) 196 err = store.Put(compositeCtx, chunk.K, serialization) 197 if err != nil { 198 dvid.Errorf("Unable to PUT composite block %s: %v\n", zyx, err) 199 return 200 } 201 } 202 203 func writePseudoColor(grayscale uint8, in64bits, out32bits []byte) { 204 murmurhash3(in64bits, out32bits) 205 var t uint64 206 t = uint64(out32bits[0]) * uint64(grayscale) 207 t >>= 8 208 out32bits[0] = uint8(t) 209 t = uint64(out32bits[1]) * uint64(grayscale) 210 t >>= 8 211 out32bits[1] = uint8(t) 212 t = uint64(out32bits[2]) * uint64(grayscale) 213 t >>= 8 214 out32bits[2] = uint8(t) 215 out32bits[3] = 255 216 } 217 218 func murmurhash3(in64bits, out32bits []byte) { 219 length := len(in64bits) 220 var c1, c2 uint32 = 0xcc9e2d51, 0x1b873593 221 nblocks := length / 4 222 var h, k uint32 223 buf := bytes.NewBuffer(in64bits) 224 for i := 0; i < nblocks; i++ { 225 binary.Read(buf, binary.LittleEndian, &k) 226 k *= c1 227 k = (k << 15) | (k >> (32 - 15)) 228 k *= c2 229 h ^= k 230 h = (h << 13) | (h >> (32 - 13)) 231 h = (h * 5) + 0xe6546b64 232 } 233 k = 0 234 tailIndex := nblocks * 4 235 switch length & 3 { 236 case 3: 237 k ^= uint32(in64bits[tailIndex+2]) << 16 238 fallthrough 239 case 2: 240 k ^= uint32(in64bits[tailIndex+1]) << 8 241 fallthrough 242 case 1: 243 k ^= uint32(in64bits[tailIndex]) 244 k *= c1 245 k = (k << 15) | (k >> (32 - 15)) 246 k *= c2 247 h ^= k 248 } 249 h ^= uint32(length) 250 h ^= h >> 16 251 h *= 0x85ebca6b 252 h ^= h >> 13 253 h *= 0xc2b2ae35 254 h ^= h >> 16 255 binary.BigEndian.PutUint32(out32bits, h) 256 }