github.com/janelia-flyem/dvid@v1.0.0/datatype/labelblk/composite.go (about)

     1  /*
     2  	Functinos that support compositing label and image blocks into a pseudo-color representation.
     3  */
     4  
     5  package labelblk
     6  
     7  import (
     8  	"bytes"
     9  	"encoding/binary"
    10  	"fmt"
    11  	"sync"
    12  
    13  	"github.com/janelia-flyem/dvid/datastore"
    14  	"github.com/janelia-flyem/dvid/datatype/imageblk"
    15  	"github.com/janelia-flyem/dvid/dvid"
    16  	"github.com/janelia-flyem/dvid/server"
    17  	"github.com/janelia-flyem/dvid/storage"
    18  )
    19  
    20  type compositeOp struct {
    21  	grayscale *imageblk.Data
    22  	composite *imageblk.Data
    23  	versionID dvid.VersionID
    24  }
    25  
    26  // CreateComposite creates a new rgba8 image by combining hash of labels + the grayscale
    27  func (d *Data) CreateComposite(request datastore.Request, reply *datastore.Response) error {
    28  	timedLog := dvid.NewTimeLog()
    29  
    30  	// Parse the request
    31  	var uuidStr, dataName, cmdStr, grayscaleName, destName string
    32  	request.CommandArgs(1, &uuidStr, &dataName, &cmdStr, &grayscaleName, &destName)
    33  
    34  	// Get the version
    35  	uuid, v, err := datastore.MatchingUUID(uuidStr)
    36  	if err != nil {
    37  		return err
    38  	}
    39  
    40  	// Log request
    41  	if err = datastore.AddToNodeLog(uuid, []string{request.Command.String()}); err != nil {
    42  		return err
    43  	}
    44  
    45  	// Get the grayscale data.
    46  	dataservice, err := datastore.GetDataByUUIDName(uuid, dvid.InstanceName(grayscaleName))
    47  	if err != nil {
    48  		return err
    49  	}
    50  	grayscale, ok := dataservice.(*imageblk.Data)
    51  	if !ok {
    52  		return fmt.Errorf("%s is not the name of uint8 data", grayscaleName)
    53  	}
    54  
    55  	// Create a new rgba8blk data.
    56  	var compservice datastore.DataService
    57  	compservice, err = datastore.GetDataByUUIDName(uuid, dvid.InstanceName(destName))
    58  	if err == nil {
    59  		return fmt.Errorf("Data instance with name %q already exists", destName)
    60  	}
    61  	typeService, err := datastore.TypeServiceByName("rgba8blk")
    62  	if err != nil {
    63  		return fmt.Errorf("Could not get rgba8 type service from DVID")
    64  	}
    65  	config := dvid.NewConfig()
    66  	compservice, err = datastore.NewData(uuid, typeService, dvid.InstanceName(destName), config)
    67  	if err != nil {
    68  		return err
    69  	}
    70  	composite, ok := compservice.(*imageblk.Data)
    71  	if !ok {
    72  		return fmt.Errorf("Error: %s was unable to be set to rgba8 data", destName)
    73  	}
    74  
    75  	// Iterate through all labels and grayscale chunks incrementally in Z, a layer at a time.
    76  	wg := new(sync.WaitGroup)
    77  	op := &compositeOp{grayscale, composite, v}
    78  	chunkOp := &storage.ChunkOp{op, wg}
    79  
    80  	store, err := datastore.GetOrderedKeyValueDB(d)
    81  	if err != nil {
    82  		return err
    83  	}
    84  	ctx := datastore.NewVersionedCtx(d, v)
    85  	extents := d.Extents()
    86  	blockBeg := imageblk.NewTKey(extents.MinIndex)
    87  	blockEnd := imageblk.NewTKey(extents.MaxIndex)
    88  	err = store.ProcessRange(ctx, blockBeg, blockEnd, chunkOp, storage.ChunkFunc(d.CreateCompositeChunk))
    89  	wg.Wait()
    90  
    91  	// Set new mapped data to same extents.
    92  	composite.Properties.Extents = grayscale.Properties.Extents
    93  	if err := datastore.SaveDataByUUID(uuid, composite); err != nil {
    94  		dvid.Infof("Could not save new data '%s': %v\n", destName, err)
    95  	}
    96  
    97  	timedLog.Infof("Created composite of %s and %s", grayscaleName, destName)
    98  	return nil
    99  }
   100  
   101  // CreateCompositeChunk processes each chunk of labels and grayscale data,
   102  // saving the composited result into an rgba8.
   103  // Only some multiple of the # of CPU cores can be used for chunk handling before
   104  // it waits for chunk processing to abate via the buffered server.HandlerToken channel.
   105  func (d *Data) CreateCompositeChunk(chunk *storage.Chunk) error {
   106  	server.CheckChunkThrottling()
   107  	go d.createCompositeChunk(chunk)
   108  	return nil
   109  }
   110  
   111  var curZ int32
   112  var curZMutex sync.Mutex
   113  
   114  func (d *Data) createCompositeChunk(chunk *storage.Chunk) {
   115  	defer func() {
   116  		// After processing a chunk, return the token.
   117  		server.HandlerToken <- 1
   118  
   119  		// Notify the requestor that this chunk is done.
   120  		if chunk.Wg != nil {
   121  			chunk.Wg.Done()
   122  		}
   123  	}()
   124  
   125  	op := chunk.Op.(*compositeOp)
   126  
   127  	// Get the spatial index associated with this chunk.
   128  	zyx, err := imageblk.DecodeTKey(chunk.K)
   129  	if err != nil {
   130  		dvid.Errorf("Error in %s.ChunkApplyMap(): %v", d.Data.DataName(), err)
   131  		return
   132  	}
   133  
   134  	// Initialize the label buffers.  For voxels, this data needs to be uncompressed and deserialized.
   135  	curZMutex.Lock()
   136  	if zyx[2] > curZ {
   137  		curZ = zyx[2]
   138  		minZ := zyx.MinPoint(d.BlockSize()).Value(2)
   139  		maxZ := zyx.MaxPoint(d.BlockSize()).Value(2)
   140  		dvid.Debugf("Now creating composite blocks for Z %d to %d\n", minZ, maxZ)
   141  	}
   142  	curZMutex.Unlock()
   143  
   144  	labelData, _, err := dvid.DeserializeData(chunk.V, true)
   145  	if err != nil {
   146  		dvid.Infof("Unable to deserialize block in '%s': %v\n", d.DataName(), err)
   147  		return
   148  	}
   149  	blockBytes := len(labelData)
   150  	if blockBytes%8 != 0 {
   151  		dvid.Infof("Retrieved, deserialized block is wrong size: %d bytes\n", blockBytes)
   152  		return
   153  	}
   154  
   155  	// Get the corresponding grayscale block.
   156  	store, err := datastore.GetOrderedKeyValueDB(op.grayscale)
   157  	if err != nil {
   158  		dvid.Errorf("Unable to retrieve store for %q: %v\n", op.grayscale.DataName(), err)
   159  		return
   160  	}
   161  	grayscaleCtx := datastore.NewVersionedCtx(op.grayscale, op.versionID)
   162  	blockData, err := store.Get(grayscaleCtx, chunk.K)
   163  	if err != nil {
   164  		dvid.Errorf("Error getting grayscale block for index %s\n", zyx)
   165  		return
   166  	}
   167  	grayscaleData, _, err := dvid.DeserializeData(blockData, true)
   168  	if err != nil {
   169  		dvid.Errorf("Unable to deserialize block in '%s': %v\n", op.grayscale.DataName(), err)
   170  		return
   171  	}
   172  
   173  	// Compute the composite block.
   174  	// TODO -- Exploit run lengths, use cache of hash?
   175  	compositeBytes := blockBytes / 2
   176  	compositeData := make([]byte, compositeBytes, compositeBytes)
   177  	compositeI := 0
   178  	labelI := 0
   179  	hashBuf := make([]byte, 4, 4)
   180  	for _, grayscale := range grayscaleData {
   181  		//murmurhash3(labelData[labelI:labelI+8], hashBuf)
   182  		//hashBuf[3] = grayscale
   183  		writePseudoColor(grayscale, labelData[labelI:labelI+8], hashBuf)
   184  		copy(compositeData[compositeI:compositeI+4], hashBuf)
   185  		compositeI += 4
   186  		labelI += 8
   187  	}
   188  
   189  	// Store the composite block into the rgba8 data.
   190  	serialization, err := dvid.SerializeData(compositeData, d.Compression(), d.Checksum())
   191  	if err != nil {
   192  		dvid.Errorf("Unable to serialize composite block %s: %v\n", zyx, err)
   193  		return
   194  	}
   195  	compositeCtx := datastore.NewVersionedCtx(op.composite, op.versionID)
   196  	err = store.Put(compositeCtx, chunk.K, serialization)
   197  	if err != nil {
   198  		dvid.Errorf("Unable to PUT composite block %s: %v\n", zyx, err)
   199  		return
   200  	}
   201  }
   202  
   203  func writePseudoColor(grayscale uint8, in64bits, out32bits []byte) {
   204  	murmurhash3(in64bits, out32bits)
   205  	var t uint64
   206  	t = uint64(out32bits[0]) * uint64(grayscale)
   207  	t >>= 8
   208  	out32bits[0] = uint8(t)
   209  	t = uint64(out32bits[1]) * uint64(grayscale)
   210  	t >>= 8
   211  	out32bits[1] = uint8(t)
   212  	t = uint64(out32bits[2]) * uint64(grayscale)
   213  	t >>= 8
   214  	out32bits[2] = uint8(t)
   215  	out32bits[3] = 255
   216  }
   217  
   218  func murmurhash3(in64bits, out32bits []byte) {
   219  	length := len(in64bits)
   220  	var c1, c2 uint32 = 0xcc9e2d51, 0x1b873593
   221  	nblocks := length / 4
   222  	var h, k uint32
   223  	buf := bytes.NewBuffer(in64bits)
   224  	for i := 0; i < nblocks; i++ {
   225  		binary.Read(buf, binary.LittleEndian, &k)
   226  		k *= c1
   227  		k = (k << 15) | (k >> (32 - 15))
   228  		k *= c2
   229  		h ^= k
   230  		h = (h << 13) | (h >> (32 - 13))
   231  		h = (h * 5) + 0xe6546b64
   232  	}
   233  	k = 0
   234  	tailIndex := nblocks * 4
   235  	switch length & 3 {
   236  	case 3:
   237  		k ^= uint32(in64bits[tailIndex+2]) << 16
   238  		fallthrough
   239  	case 2:
   240  		k ^= uint32(in64bits[tailIndex+1]) << 8
   241  		fallthrough
   242  	case 1:
   243  		k ^= uint32(in64bits[tailIndex])
   244  		k *= c1
   245  		k = (k << 15) | (k >> (32 - 15))
   246  		k *= c2
   247  		h ^= k
   248  	}
   249  	h ^= uint32(length)
   250  	h ^= h >> 16
   251  	h *= 0x85ebca6b
   252  	h ^= h >> 13
   253  	h *= 0xc2b2ae35
   254  	h ^= h >> 16
   255  	binary.BigEndian.PutUint32(out32bits, h)
   256  }