github.com/janelia-flyem/dvid@v1.0.0/datatype/imageblk/uint8_test.go (about)

     1  package imageblk
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/json"
     6  	"fmt"
     7  	"log"
     8  	"math/rand"
     9  	"reflect"
    10  	"sync"
    11  	"testing"
    12  
    13  	"github.com/janelia-flyem/dvid/datastore"
    14  	"github.com/janelia-flyem/dvid/dvid"
    15  	"github.com/janelia-flyem/dvid/server"
    16  )
    17  
    18  var (
    19  	grayscaleT,
    20  	roiT datastore.TypeService
    21  
    22  	testMu sync.Mutex
    23  )
    24  
    25  // Sets package-level testRepo and TestVersionID
    26  func initTestRepo() (dvid.UUID, dvid.VersionID) {
    27  	testMu.Lock()
    28  	defer testMu.Unlock()
    29  	if grayscaleT == nil {
    30  		var err error
    31  		grayscaleT, err = datastore.TypeServiceByName("uint8blk")
    32  		if err != nil {
    33  			log.Fatalf("Can't get uint8blk type: %s\n", err)
    34  		}
    35  		floatimgT, err = datastore.TypeServiceByName("float32blk")
    36  		if err != nil {
    37  			log.Fatalf("Can't get float32blk type: %s\n", err)
    38  		}
    39  		uint16imgT, err = datastore.TypeServiceByName("uint16blk")
    40  		if err != nil {
    41  			log.Fatalf("Can't get uint16blk type: %s\n", err)
    42  		}
    43  		// uint32imgT, err = datastore.TypeServiceByName("uint32blk")
    44  		// if err != nil {
    45  		// 	log.Fatalf("Can't get uint32blk type: %s\n", err)
    46  		// }
    47  		// uint64imgT, err = datastore.TypeServiceByName("uint64blk")
    48  		// if err != nil {
    49  		// 	log.Fatalf("Can't get uint64blk type: %s\n", err)
    50  		// }
    51  		roiT, err = datastore.TypeServiceByName("roi")
    52  		if err != nil {
    53  			log.Fatalf("Can't get ROI type: %s\n", err)
    54  		}
    55  	}
    56  	return datastore.NewTestRepo()
    57  }
    58  
    59  // A slice of bytes representing a 3d float32 volume
    60  type testVolume struct {
    61  	data   []byte
    62  	offset dvid.Point3d
    63  	size   dvid.Point3d
    64  }
    65  
    66  // Put label data into given data instance.
    67  func (v *testVolume) put(t *testing.T, uuid dvid.UUID, name string) {
    68  	apiStr := fmt.Sprintf("%snode/%s/%s/raw/0_1_2/%d_%d_%d/%d_%d_%d?mutate=true", server.WebAPIPath,
    69  		uuid, name, v.size[0], v.size[1], v.size[2], v.offset[0], v.offset[1], v.offset[2])
    70  	server.TestHTTP(t, "POST", apiStr, bytes.NewBuffer(v.data))
    71  }
    72  
    73  // Data from which to construct repeatable 3d images where adjacent voxels have different values.
    74  var xdata = []byte{'\x01', '\x07', '\xAF', '\xFF', '\x70'}
    75  var ydata = []byte{'\x33', '\xB2', '\x77', '\xD0', '\x4F'}
    76  var zdata = []byte{'\x5C', '\x89', '\x40', '\x13', '\xCA'}
    77  
    78  // Make a 2d slice of bytes with top left corner at (ox,oy,oz) and size (nx,ny)
    79  func makeSlice(offset dvid.Point3d, size dvid.Point2d) []byte {
    80  	slice := make([]byte, size[0]*size[1], size[0]*size[1])
    81  	i := 0
    82  	modz := offset[2] % int32(len(zdata))
    83  	for y := int32(0); y < size[1]; y++ {
    84  		sy := y + offset[1]
    85  		mody := sy % int32(len(ydata))
    86  		sx := offset[0]
    87  		for x := int32(0); x < size[0]; x++ {
    88  			modx := sx % int32(len(xdata))
    89  			slice[i] = xdata[modx] ^ ydata[mody] ^ zdata[modz]
    90  			i++
    91  			sx++
    92  		}
    93  	}
    94  	return slice
    95  }
    96  
    97  // Make a 3d volume of bytes with top left corner at (ox,oy,oz) and size (nx, ny, nz)
    98  func makeVolume(offset, size dvid.Point3d) []byte {
    99  	sliceBytes := size[0] * size[1]
   100  	volumeBytes := sliceBytes * size[2]
   101  	volume := make([]byte, volumeBytes, volumeBytes)
   102  	var i int32
   103  	size2d := dvid.Point2d{size[0], size[1]}
   104  	startZ := offset[2]
   105  	endZ := startZ + size[2]
   106  	for z := startZ; z < endZ; z++ {
   107  		offset[2] = z
   108  		copy(volume[i:i+sliceBytes], makeSlice(offset, size2d))
   109  		i += sliceBytes
   110  	}
   111  	return volume
   112  }
   113  
   114  func makeGrayscale(uuid dvid.UUID, t *testing.T, name string) *Data {
   115  	config := dvid.NewConfig()
   116  	dataservice, err := datastore.NewData(uuid, grayscaleT, dvid.InstanceName(name), config)
   117  	if err != nil {
   118  		t.Errorf("Unable to create grayscale instance %q: %v\n", name, err)
   119  	}
   120  	grayscale, ok := dataservice.(*Data)
   121  	if !ok {
   122  		t.Errorf("Can't cast uint8blk data service into Data\n")
   123  	}
   124  	return grayscale
   125  }
   126  
   127  func TestVoxelsInstanceCreation(t *testing.T) {
   128  	if err := server.OpenTest(); err != nil {
   129  		t.Fatalf("can't open test server: %v\n", err)
   130  	}
   131  	defer server.CloseTest()
   132  
   133  	uuid, _ := datastore.NewTestRepo()
   134  
   135  	// Create new voxels instance with optional parameters
   136  	name := "grayscale"
   137  	metadata := fmt.Sprintf(`{
   138  		"typename": "uint8blk",
   139  		"dataname": %q,
   140  		"blocksize": "64,43,28",
   141  		"VoxelSize": "13.1, 14.2, 15.3",
   142  		"VoxelUnits": "picometers,nanometers,microns"
   143  	}`, name)
   144  	apiStr := fmt.Sprintf("%srepo/%s/instance", server.WebAPIPath, uuid)
   145  	server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString(metadata))
   146  
   147  	// Get metadata and make sure optional settings have been set.
   148  	apiStr = fmt.Sprintf("%snode/%s/%s/info", server.WebAPIPath, uuid, name)
   149  	result := server.TestHTTP(t, "GET", apiStr, nil)
   150  	var parsed = struct {
   151  		Base struct {
   152  			TypeName, Name string
   153  		}
   154  		Extended struct {
   155  			BlockSize  dvid.Point3d
   156  			VoxelSize  dvid.NdFloat32
   157  			VoxelUnits dvid.NdString
   158  		}
   159  	}{}
   160  	if err := json.Unmarshal(result, &parsed); err != nil {
   161  		t.Fatalf("Error parsing JSON response of new instance metadata: %v\n", err)
   162  	}
   163  	if parsed.Base.Name != name {
   164  		t.Errorf("Parsed new instance has unexpected name: %s != %s (expected)\n",
   165  			parsed.Base.Name, name)
   166  	}
   167  	if parsed.Base.TypeName != "uint8blk" {
   168  		t.Errorf("Parsed new instance has unexpected type name: %s != uint8blk (expected)\n",
   169  			parsed.Base.TypeName)
   170  	}
   171  	if !parsed.Extended.BlockSize.Equals(dvid.Point3d{64, 43, 28}) {
   172  		t.Errorf("Bad block size in new uint8blk instance: %s\n", parsed.Extended.BlockSize)
   173  	}
   174  	if !parsed.Extended.VoxelSize.Equals(dvid.NdFloat32{13.1, 14.2, 15.3}) {
   175  		t.Errorf("Bad voxel size in new uint8blk instance: %s\n", parsed.Extended.VoxelSize)
   176  	}
   177  	if parsed.Extended.VoxelUnits[0] != "picometers" {
   178  		t.Errorf("Got %q for X voxel units, not picometers\n", parsed.Extended.VoxelUnits[0])
   179  	}
   180  	if parsed.Extended.VoxelUnits[1] != "nanometers" {
   181  		t.Errorf("Got %q for X voxel units, not picometers\n", parsed.Extended.VoxelUnits[0])
   182  	}
   183  	if parsed.Extended.VoxelUnits[2] != "microns" {
   184  		t.Errorf("Got %q for X voxel units, not picometers\n", parsed.Extended.VoxelUnits[0])
   185  	}
   186  }
   187  
   188  func TestExtentsAndRes(t *testing.T) {
   189  	if err := server.OpenTest(); err != nil {
   190  		t.Fatalf("can't open test server: %v\n", err)
   191  	}
   192  	defer server.CloseTest()
   193  
   194  	uuid, _ := initTestRepo()
   195  	makeGrayscale(uuid, t, "grayscale")
   196  
   197  	extents := `{
   198  		"MinPoint": [68, 127, 210],
   199  		"MaxPoint": [1023, 4811, 12187]
   200  	}`
   201  	apiStr := fmt.Sprintf("%snode/%s/grayscale/extents", server.WebAPIPath, uuid)
   202  	server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString(extents))
   203  
   204  	resolution := `[4.0, 4.0, 4.0]`
   205  	apiStr = fmt.Sprintf("%snode/%s/grayscale/resolution", server.WebAPIPath, uuid)
   206  	server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString(resolution))
   207  
   208  	apiStr = fmt.Sprintf("%snode/%s/grayscale/info", server.WebAPIPath, uuid)
   209  	result := server.TestHTTP(t, "GET", apiStr, nil)
   210  	var parsed = struct {
   211  		Base struct {
   212  			TypeName, Name string
   213  		}
   214  		Extended struct {
   215  			BlockSize  dvid.Point3d
   216  			VoxelSize  dvid.NdFloat32
   217  			VoxelUnits dvid.NdString
   218  			MinPoint   dvid.Point3d
   219  			MaxPoint   dvid.Point3d
   220  			MinIndex   dvid.Point3d
   221  			MaxIndex   dvid.Point3d
   222  		}
   223  	}{}
   224  	if err := json.Unmarshal(result, &parsed); err != nil {
   225  		t.Fatalf("Error parsing JSON response of new instance metadata: %v\n", err)
   226  	}
   227  	if !parsed.Extended.MinPoint.Equals(dvid.Point3d{68, 127, 210}) {
   228  		t.Errorf("Bad MinPoint in new uint8blk instance: %s\n", parsed.Extended.MinPoint)
   229  	}
   230  	if !parsed.Extended.MaxPoint.Equals(dvid.Point3d{1023, 4811, 12187}) {
   231  		t.Errorf("Bad MaxPoint in new uint8blk instance: %s\n", parsed.Extended.MaxPoint)
   232  	}
   233  	if !parsed.Extended.MinIndex.Equals(dvid.Point3d{2, 3, 6}) {
   234  		t.Errorf("Bad MinIndex in new uint8blk instance: %s\n", parsed.Extended.MinIndex)
   235  	}
   236  	if !parsed.Extended.MaxIndex.Equals(dvid.Point3d{31, 150, 380}) {
   237  		t.Errorf("Bad MaxIndex in new uint8blk instance: %s\n", parsed.Extended.MaxIndex)
   238  	}
   239  	if !parsed.Extended.VoxelSize.Equals(dvid.NdFloat32{4.0, 4.0, 4.0}) {
   240  		t.Errorf("Bad voxel size in new uint8blk instance: %s\n", parsed.Extended.VoxelSize)
   241  	}
   242  }
   243  
   244  func TestForegroundROI(t *testing.T) {
   245  	if err := server.OpenTest(); err != nil {
   246  		t.Fatalf("can't open test server: %v\n", err)
   247  	}
   248  	defer server.CloseTest()
   249  
   250  	uuid, _ := initTestRepo()
   251  	grayscale := makeGrayscale(uuid, t, "grayscale")
   252  
   253  	// Create a fake 128^3 volume with inner 64^3 foreground and
   254  	// outer background split between 0 and 255 at z = 63,64
   255  	data := make([]uint8, 128*128*128, 128*128*128)
   256  	for z := 64; z < 128; z++ {
   257  		for y := 0; y < 128; y++ {
   258  			for x := 0; x < 128; x++ {
   259  				data[z*128*128+y*128+x] = 255
   260  			}
   261  		}
   262  	}
   263  	for z := 32; z < 96; z++ {
   264  		for y := 32; y < 96; y++ {
   265  			for x := 32; x < 96; x++ {
   266  				data[z*128*128+y*128+x] = 128
   267  			}
   268  		}
   269  	}
   270  
   271  	// Put the volume so it's block-aligned
   272  	buf := bytes.NewBuffer(data)
   273  	putRequest := fmt.Sprintf("%snode/%s/grayscale/raw/0_1_2/128_128_128/160_64_128", server.WebAPIPath, uuid)
   274  	server.TestHTTP(t, "POST", putRequest, buf)
   275  
   276  	// Request a foreground ROI
   277  	var reply datastore.Response
   278  	cmd := dvid.Command{"node", string(uuid), "grayscale", "roi", "foreground", "0,255"}
   279  	if err := grayscale.DoRPC(datastore.Request{Command: cmd}, &reply); err != nil {
   280  		t.Fatalf("Error running foreground ROI command: %v\n", err)
   281  	}
   282  
   283  	if err := datastore.BlockOnUpdating(uuid, "foreground"); err != nil {
   284  		t.Fatalf("Error blocking on foreground roi updating: %v\n", err)
   285  	}
   286  
   287  	// Check results, making sure it's valid (200).
   288  	roiRequest := fmt.Sprintf("%snode/%s/foreground/roi", server.WebAPIPath, uuid)
   289  	resp := server.TestHTTP(t, "GET", roiRequest, nil)
   290  
   291  	// Check results
   292  	// We have block-aligned 2^3 block foreground
   293  	//   from z = 32 -> 95, block coord 1 -> 2 + offset in z by 4 blocks = 5, 6
   294  	//   from y = 32 -> 95, offset in y by 2 blocks = 3, 4
   295  	//   from x = 32 -> 95, offset in x by 5 blocks = 6, 7
   296  	expected := "[[5,3,6,7],[5,4,6,7],[6,3,6,7],[6,4,6,7]]"
   297  	if string(resp) != expected {
   298  		t.Errorf("Expected the following foreground ROI:\n%s\nGot instead:\n%s\n", expected, string(resp))
   299  	}
   300  }
   301  
   302  func TestDirectCalls(t *testing.T) {
   303  	if err := server.OpenTest(); err != nil {
   304  		t.Fatalf("can't open test server: %v\n", err)
   305  	}
   306  	defer server.CloseTest()
   307  
   308  	uuid, versionID := initTestRepo()
   309  	grayscale := makeGrayscale(uuid, t, "grayscale")
   310  	grayscaleCtx := datastore.NewVersionedCtx(grayscale, versionID)
   311  
   312  	// Create a block-aligned 8-bit grayscale image
   313  	offset := dvid.Point3d{512, 32, 1024}
   314  	size := dvid.Point3d{128, 96, 64}
   315  	subvol := dvid.NewSubvolume(offset, size)
   316  	data := makeVolume(offset, size)
   317  	origData := make([]byte, len(data))
   318  	copy(origData, data)
   319  
   320  	// Store it into datastore at root
   321  	v, err := grayscale.NewVoxels(subvol, data)
   322  	if err != nil {
   323  		t.Fatalf("Unable to make new grayscale voxels: %v\n", err)
   324  	}
   325  	if err = grayscale.IngestVoxels(versionID, 1, v, ""); err != nil {
   326  		t.Errorf("Unable to put voxels for %s: %v\n", grayscaleCtx, err)
   327  	}
   328  	if v.NumVoxels() != int64(len(origData)) {
   329  		t.Errorf("# voxels (%d) after PutVoxels != # original voxels (%d)\n",
   330  			v.NumVoxels(), int64(len(origData)))
   331  	}
   332  
   333  	// Read the stored image
   334  	v2, err := grayscale.NewVoxels(subvol, nil)
   335  	if err != nil {
   336  		t.Errorf("Unable to make new grayscale ExtHandler: %v\n", err)
   337  	}
   338  	if err = grayscale.GetVoxels(versionID, v2, ""); err != nil {
   339  		t.Errorf("Unable to get voxels for %s: %v\n", grayscaleCtx, err)
   340  	}
   341  
   342  	// Make sure the retrieved image matches the original
   343  	if v.Stride() != v2.Stride() {
   344  		t.Errorf("Stride in retrieved subvol incorrect\n")
   345  	}
   346  	if v.Interpolable() != v2.Interpolable() {
   347  		t.Errorf("Interpolable bool in retrieved subvol incorrect\n")
   348  	}
   349  	if !reflect.DeepEqual(v.Size(), v2.Size()) {
   350  		t.Errorf("Size in retrieved subvol incorrect: %s vs expected %s\n",
   351  			v2.Size(), v.Size())
   352  	}
   353  	if v.NumVoxels() != v2.NumVoxels() {
   354  		t.Errorf("# voxels in retrieved is different: %d vs expected %d\n",
   355  			v2.NumVoxels(), v.NumVoxels())
   356  	}
   357  	data = v2.Data()
   358  	//dvid.PrintNonZero("original value", origData)
   359  	//dvid.PrintNonZero("returned value", data)
   360  	for i := int64(0); i < v2.NumVoxels(); i++ {
   361  		if data[i] != origData[i] {
   362  			t.Logf("Data returned != data stored for voxel %d\n", i)
   363  			t.Logf("Size of data: %d bytes from GET, %d bytes in PUT\n", len(data), len(origData))
   364  			t.Fatalf("GET subvol (%d) != PUT subvol (%d) @ index %d", data[i], origData[i], i)
   365  		}
   366  	}
   367  }
   368  
   369  func TestBlockAPI(t *testing.T) {
   370  	if err := server.OpenTest(); err != nil {
   371  		t.Fatalf("can't open test server: %v\n", err)
   372  	}
   373  	defer server.CloseTest()
   374  
   375  	uuid, _ := initTestRepo()
   376  	grayscale := makeGrayscale(uuid, t, "grayscale")
   377  
   378  	// construct random blocks of data.
   379  	numBlockBytes := int32(grayscale.BlockSize().Prod())
   380  	testBlocks := 1001
   381  	var blockData []byte
   382  	for i := 0; i < testBlocks; i++ {
   383  		blockData = append(blockData, dvid.RandomBytes(numBlockBytes)...)
   384  	}
   385  
   386  	// set start of span
   387  	x := rand.Int31()
   388  	y := rand.Int31()
   389  	z := rand.Int31()
   390  
   391  	// Test uncompressed roundtrip
   392  
   393  	// send the blocks
   394  	blockReq := fmt.Sprintf("%snode/%s/grayscale/blocks/%d_%d_%d/%d", server.WebAPIPath, uuid, x, y, z, testBlocks)
   395  	server.TestHTTP(t, "POST", blockReq, bytes.NewBuffer(blockData))
   396  
   397  	// read same span of blocks
   398  	if err := datastore.BlockOnUpdating(uuid, "grayscale"); err != nil {
   399  		t.Fatalf("Error blocking on POST of grayscale: %v\n", err)
   400  	}
   401  	returnedData := server.TestHTTP(t, "GET", blockReq, nil)
   402  
   403  	// make sure blocks are same
   404  	totBytes := testBlocks * int(numBlockBytes)
   405  	if len(returnedData) != totBytes {
   406  		t.Errorf("Returned %d bytes, expected %d bytes", len(returnedData), totBytes)
   407  	}
   408  	if !reflect.DeepEqual(returnedData, blockData) {
   409  		t.Errorf("Returned block data != original block data\n")
   410  	}
   411  
   412  	// We should get blank blocks at different z
   413  	z += 1
   414  	blockReq = fmt.Sprintf("%snode/%s/grayscale/blocks/%d_%d_%d/%d", server.WebAPIPath, uuid, x, y, z, testBlocks)
   415  	returnedData = server.TestHTTP(t, "GET", blockReq, nil)
   416  	if len(returnedData) != totBytes {
   417  		t.Errorf("Returned %d bytes, expected %d bytes", len(returnedData), totBytes)
   418  	}
   419  	for i, b := range returnedData {
   420  		if b != 0 {
   421  			t.Fatalf("Expected 0 at returned byte %d, got %d instead.\n", i, b)
   422  		}
   423  	}
   424  }
   425  
   426  func TestGrayscaleRepoPersistence(t *testing.T) {
   427  	if err := server.OpenTest(); err != nil {
   428  		t.Fatalf("can't open test server: %v\n", err)
   429  	}
   430  	defer server.CloseTest()
   431  
   432  	uuid, _ := initTestRepo()
   433  
   434  	// Make grayscale and set various properties
   435  	config := dvid.NewConfig()
   436  	config.Set("BlockSize", "12,13,14")
   437  	config.Set("VoxelSize", "1.1,2.8,11")
   438  	config.Set("VoxelUnits", "microns,millimeters,nanometers")
   439  	dataservice, err := datastore.NewData(uuid, grayscaleT, "mygrayscale", config)
   440  	if err != nil {
   441  		t.Errorf("Unable to create grayscale instance: %s\n", err)
   442  	}
   443  	grayscale, ok := dataservice.(*Data)
   444  	if !ok {
   445  		t.Errorf("Can't cast uint8 data service into Data\n")
   446  	}
   447  	oldData := *grayscale
   448  
   449  	// Restart test datastore and see if datasets are still there.
   450  	if err = datastore.SaveDataByUUID(uuid, grayscale); err != nil {
   451  		t.Fatalf("Unable to save repo during grayscale persistence test: %v\n", err)
   452  	}
   453  	datastore.CloseReopenTest()
   454  
   455  	dataservice2, err := datastore.GetDataByUUIDName(uuid, "mygrayscale")
   456  	if err != nil {
   457  		t.Fatalf("Can't get grayscale instance from reloaded test db: %v\n", err)
   458  	}
   459  	grayscale2, ok := dataservice2.(*Data)
   460  	if !ok {
   461  		t.Errorf("Returned new data instance 2 is not imageblk.Data\n")
   462  	}
   463  	if !oldData.Equals(grayscale2) {
   464  		t.Errorf("Expected %v, got %v\n", oldData, *grayscale2)
   465  	}
   466  }