github.com/janelia-flyem/dvid@v1.0.0/datatype/imageblk/uint8_test.go (about) 1 package imageblk 2 3 import ( 4 "bytes" 5 "encoding/json" 6 "fmt" 7 "log" 8 "math/rand" 9 "reflect" 10 "sync" 11 "testing" 12 13 "github.com/janelia-flyem/dvid/datastore" 14 "github.com/janelia-flyem/dvid/dvid" 15 "github.com/janelia-flyem/dvid/server" 16 ) 17 18 var ( 19 grayscaleT, 20 roiT datastore.TypeService 21 22 testMu sync.Mutex 23 ) 24 25 // Sets package-level testRepo and TestVersionID 26 func initTestRepo() (dvid.UUID, dvid.VersionID) { 27 testMu.Lock() 28 defer testMu.Unlock() 29 if grayscaleT == nil { 30 var err error 31 grayscaleT, err = datastore.TypeServiceByName("uint8blk") 32 if err != nil { 33 log.Fatalf("Can't get uint8blk type: %s\n", err) 34 } 35 floatimgT, err = datastore.TypeServiceByName("float32blk") 36 if err != nil { 37 log.Fatalf("Can't get float32blk type: %s\n", err) 38 } 39 uint16imgT, err = datastore.TypeServiceByName("uint16blk") 40 if err != nil { 41 log.Fatalf("Can't get uint16blk type: %s\n", err) 42 } 43 // uint32imgT, err = datastore.TypeServiceByName("uint32blk") 44 // if err != nil { 45 // log.Fatalf("Can't get uint32blk type: %s\n", err) 46 // } 47 // uint64imgT, err = datastore.TypeServiceByName("uint64blk") 48 // if err != nil { 49 // log.Fatalf("Can't get uint64blk type: %s\n", err) 50 // } 51 roiT, err = datastore.TypeServiceByName("roi") 52 if err != nil { 53 log.Fatalf("Can't get ROI type: %s\n", err) 54 } 55 } 56 return datastore.NewTestRepo() 57 } 58 59 // A slice of bytes representing a 3d float32 volume 60 type testVolume struct { 61 data []byte 62 offset dvid.Point3d 63 size dvid.Point3d 64 } 65 66 // Put label data into given data instance. 67 func (v *testVolume) put(t *testing.T, uuid dvid.UUID, name string) { 68 apiStr := fmt.Sprintf("%snode/%s/%s/raw/0_1_2/%d_%d_%d/%d_%d_%d?mutate=true", server.WebAPIPath, 69 uuid, name, v.size[0], v.size[1], v.size[2], v.offset[0], v.offset[1], v.offset[2]) 70 server.TestHTTP(t, "POST", apiStr, bytes.NewBuffer(v.data)) 71 } 72 73 // Data from which to construct repeatable 3d images where adjacent voxels have different values. 74 var xdata = []byte{'\x01', '\x07', '\xAF', '\xFF', '\x70'} 75 var ydata = []byte{'\x33', '\xB2', '\x77', '\xD0', '\x4F'} 76 var zdata = []byte{'\x5C', '\x89', '\x40', '\x13', '\xCA'} 77 78 // Make a 2d slice of bytes with top left corner at (ox,oy,oz) and size (nx,ny) 79 func makeSlice(offset dvid.Point3d, size dvid.Point2d) []byte { 80 slice := make([]byte, size[0]*size[1], size[0]*size[1]) 81 i := 0 82 modz := offset[2] % int32(len(zdata)) 83 for y := int32(0); y < size[1]; y++ { 84 sy := y + offset[1] 85 mody := sy % int32(len(ydata)) 86 sx := offset[0] 87 for x := int32(0); x < size[0]; x++ { 88 modx := sx % int32(len(xdata)) 89 slice[i] = xdata[modx] ^ ydata[mody] ^ zdata[modz] 90 i++ 91 sx++ 92 } 93 } 94 return slice 95 } 96 97 // Make a 3d volume of bytes with top left corner at (ox,oy,oz) and size (nx, ny, nz) 98 func makeVolume(offset, size dvid.Point3d) []byte { 99 sliceBytes := size[0] * size[1] 100 volumeBytes := sliceBytes * size[2] 101 volume := make([]byte, volumeBytes, volumeBytes) 102 var i int32 103 size2d := dvid.Point2d{size[0], size[1]} 104 startZ := offset[2] 105 endZ := startZ + size[2] 106 for z := startZ; z < endZ; z++ { 107 offset[2] = z 108 copy(volume[i:i+sliceBytes], makeSlice(offset, size2d)) 109 i += sliceBytes 110 } 111 return volume 112 } 113 114 func makeGrayscale(uuid dvid.UUID, t *testing.T, name string) *Data { 115 config := dvid.NewConfig() 116 dataservice, err := datastore.NewData(uuid, grayscaleT, dvid.InstanceName(name), config) 117 if err != nil { 118 t.Errorf("Unable to create grayscale instance %q: %v\n", name, err) 119 } 120 grayscale, ok := dataservice.(*Data) 121 if !ok { 122 t.Errorf("Can't cast uint8blk data service into Data\n") 123 } 124 return grayscale 125 } 126 127 func TestVoxelsInstanceCreation(t *testing.T) { 128 if err := server.OpenTest(); err != nil { 129 t.Fatalf("can't open test server: %v\n", err) 130 } 131 defer server.CloseTest() 132 133 uuid, _ := datastore.NewTestRepo() 134 135 // Create new voxels instance with optional parameters 136 name := "grayscale" 137 metadata := fmt.Sprintf(`{ 138 "typename": "uint8blk", 139 "dataname": %q, 140 "blocksize": "64,43,28", 141 "VoxelSize": "13.1, 14.2, 15.3", 142 "VoxelUnits": "picometers,nanometers,microns" 143 }`, name) 144 apiStr := fmt.Sprintf("%srepo/%s/instance", server.WebAPIPath, uuid) 145 server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString(metadata)) 146 147 // Get metadata and make sure optional settings have been set. 148 apiStr = fmt.Sprintf("%snode/%s/%s/info", server.WebAPIPath, uuid, name) 149 result := server.TestHTTP(t, "GET", apiStr, nil) 150 var parsed = struct { 151 Base struct { 152 TypeName, Name string 153 } 154 Extended struct { 155 BlockSize dvid.Point3d 156 VoxelSize dvid.NdFloat32 157 VoxelUnits dvid.NdString 158 } 159 }{} 160 if err := json.Unmarshal(result, &parsed); err != nil { 161 t.Fatalf("Error parsing JSON response of new instance metadata: %v\n", err) 162 } 163 if parsed.Base.Name != name { 164 t.Errorf("Parsed new instance has unexpected name: %s != %s (expected)\n", 165 parsed.Base.Name, name) 166 } 167 if parsed.Base.TypeName != "uint8blk" { 168 t.Errorf("Parsed new instance has unexpected type name: %s != uint8blk (expected)\n", 169 parsed.Base.TypeName) 170 } 171 if !parsed.Extended.BlockSize.Equals(dvid.Point3d{64, 43, 28}) { 172 t.Errorf("Bad block size in new uint8blk instance: %s\n", parsed.Extended.BlockSize) 173 } 174 if !parsed.Extended.VoxelSize.Equals(dvid.NdFloat32{13.1, 14.2, 15.3}) { 175 t.Errorf("Bad voxel size in new uint8blk instance: %s\n", parsed.Extended.VoxelSize) 176 } 177 if parsed.Extended.VoxelUnits[0] != "picometers" { 178 t.Errorf("Got %q for X voxel units, not picometers\n", parsed.Extended.VoxelUnits[0]) 179 } 180 if parsed.Extended.VoxelUnits[1] != "nanometers" { 181 t.Errorf("Got %q for X voxel units, not picometers\n", parsed.Extended.VoxelUnits[0]) 182 } 183 if parsed.Extended.VoxelUnits[2] != "microns" { 184 t.Errorf("Got %q for X voxel units, not picometers\n", parsed.Extended.VoxelUnits[0]) 185 } 186 } 187 188 func TestExtentsAndRes(t *testing.T) { 189 if err := server.OpenTest(); err != nil { 190 t.Fatalf("can't open test server: %v\n", err) 191 } 192 defer server.CloseTest() 193 194 uuid, _ := initTestRepo() 195 makeGrayscale(uuid, t, "grayscale") 196 197 extents := `{ 198 "MinPoint": [68, 127, 210], 199 "MaxPoint": [1023, 4811, 12187] 200 }` 201 apiStr := fmt.Sprintf("%snode/%s/grayscale/extents", server.WebAPIPath, uuid) 202 server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString(extents)) 203 204 resolution := `[4.0, 4.0, 4.0]` 205 apiStr = fmt.Sprintf("%snode/%s/grayscale/resolution", server.WebAPIPath, uuid) 206 server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString(resolution)) 207 208 apiStr = fmt.Sprintf("%snode/%s/grayscale/info", server.WebAPIPath, uuid) 209 result := server.TestHTTP(t, "GET", apiStr, nil) 210 var parsed = struct { 211 Base struct { 212 TypeName, Name string 213 } 214 Extended struct { 215 BlockSize dvid.Point3d 216 VoxelSize dvid.NdFloat32 217 VoxelUnits dvid.NdString 218 MinPoint dvid.Point3d 219 MaxPoint dvid.Point3d 220 MinIndex dvid.Point3d 221 MaxIndex dvid.Point3d 222 } 223 }{} 224 if err := json.Unmarshal(result, &parsed); err != nil { 225 t.Fatalf("Error parsing JSON response of new instance metadata: %v\n", err) 226 } 227 if !parsed.Extended.MinPoint.Equals(dvid.Point3d{68, 127, 210}) { 228 t.Errorf("Bad MinPoint in new uint8blk instance: %s\n", parsed.Extended.MinPoint) 229 } 230 if !parsed.Extended.MaxPoint.Equals(dvid.Point3d{1023, 4811, 12187}) { 231 t.Errorf("Bad MaxPoint in new uint8blk instance: %s\n", parsed.Extended.MaxPoint) 232 } 233 if !parsed.Extended.MinIndex.Equals(dvid.Point3d{2, 3, 6}) { 234 t.Errorf("Bad MinIndex in new uint8blk instance: %s\n", parsed.Extended.MinIndex) 235 } 236 if !parsed.Extended.MaxIndex.Equals(dvid.Point3d{31, 150, 380}) { 237 t.Errorf("Bad MaxIndex in new uint8blk instance: %s\n", parsed.Extended.MaxIndex) 238 } 239 if !parsed.Extended.VoxelSize.Equals(dvid.NdFloat32{4.0, 4.0, 4.0}) { 240 t.Errorf("Bad voxel size in new uint8blk instance: %s\n", parsed.Extended.VoxelSize) 241 } 242 } 243 244 func TestForegroundROI(t *testing.T) { 245 if err := server.OpenTest(); err != nil { 246 t.Fatalf("can't open test server: %v\n", err) 247 } 248 defer server.CloseTest() 249 250 uuid, _ := initTestRepo() 251 grayscale := makeGrayscale(uuid, t, "grayscale") 252 253 // Create a fake 128^3 volume with inner 64^3 foreground and 254 // outer background split between 0 and 255 at z = 63,64 255 data := make([]uint8, 128*128*128, 128*128*128) 256 for z := 64; z < 128; z++ { 257 for y := 0; y < 128; y++ { 258 for x := 0; x < 128; x++ { 259 data[z*128*128+y*128+x] = 255 260 } 261 } 262 } 263 for z := 32; z < 96; z++ { 264 for y := 32; y < 96; y++ { 265 for x := 32; x < 96; x++ { 266 data[z*128*128+y*128+x] = 128 267 } 268 } 269 } 270 271 // Put the volume so it's block-aligned 272 buf := bytes.NewBuffer(data) 273 putRequest := fmt.Sprintf("%snode/%s/grayscale/raw/0_1_2/128_128_128/160_64_128", server.WebAPIPath, uuid) 274 server.TestHTTP(t, "POST", putRequest, buf) 275 276 // Request a foreground ROI 277 var reply datastore.Response 278 cmd := dvid.Command{"node", string(uuid), "grayscale", "roi", "foreground", "0,255"} 279 if err := grayscale.DoRPC(datastore.Request{Command: cmd}, &reply); err != nil { 280 t.Fatalf("Error running foreground ROI command: %v\n", err) 281 } 282 283 if err := datastore.BlockOnUpdating(uuid, "foreground"); err != nil { 284 t.Fatalf("Error blocking on foreground roi updating: %v\n", err) 285 } 286 287 // Check results, making sure it's valid (200). 288 roiRequest := fmt.Sprintf("%snode/%s/foreground/roi", server.WebAPIPath, uuid) 289 resp := server.TestHTTP(t, "GET", roiRequest, nil) 290 291 // Check results 292 // We have block-aligned 2^3 block foreground 293 // from z = 32 -> 95, block coord 1 -> 2 + offset in z by 4 blocks = 5, 6 294 // from y = 32 -> 95, offset in y by 2 blocks = 3, 4 295 // from x = 32 -> 95, offset in x by 5 blocks = 6, 7 296 expected := "[[5,3,6,7],[5,4,6,7],[6,3,6,7],[6,4,6,7]]" 297 if string(resp) != expected { 298 t.Errorf("Expected the following foreground ROI:\n%s\nGot instead:\n%s\n", expected, string(resp)) 299 } 300 } 301 302 func TestDirectCalls(t *testing.T) { 303 if err := server.OpenTest(); err != nil { 304 t.Fatalf("can't open test server: %v\n", err) 305 } 306 defer server.CloseTest() 307 308 uuid, versionID := initTestRepo() 309 grayscale := makeGrayscale(uuid, t, "grayscale") 310 grayscaleCtx := datastore.NewVersionedCtx(grayscale, versionID) 311 312 // Create a block-aligned 8-bit grayscale image 313 offset := dvid.Point3d{512, 32, 1024} 314 size := dvid.Point3d{128, 96, 64} 315 subvol := dvid.NewSubvolume(offset, size) 316 data := makeVolume(offset, size) 317 origData := make([]byte, len(data)) 318 copy(origData, data) 319 320 // Store it into datastore at root 321 v, err := grayscale.NewVoxels(subvol, data) 322 if err != nil { 323 t.Fatalf("Unable to make new grayscale voxels: %v\n", err) 324 } 325 if err = grayscale.IngestVoxels(versionID, 1, v, ""); err != nil { 326 t.Errorf("Unable to put voxels for %s: %v\n", grayscaleCtx, err) 327 } 328 if v.NumVoxels() != int64(len(origData)) { 329 t.Errorf("# voxels (%d) after PutVoxels != # original voxels (%d)\n", 330 v.NumVoxels(), int64(len(origData))) 331 } 332 333 // Read the stored image 334 v2, err := grayscale.NewVoxels(subvol, nil) 335 if err != nil { 336 t.Errorf("Unable to make new grayscale ExtHandler: %v\n", err) 337 } 338 if err = grayscale.GetVoxels(versionID, v2, ""); err != nil { 339 t.Errorf("Unable to get voxels for %s: %v\n", grayscaleCtx, err) 340 } 341 342 // Make sure the retrieved image matches the original 343 if v.Stride() != v2.Stride() { 344 t.Errorf("Stride in retrieved subvol incorrect\n") 345 } 346 if v.Interpolable() != v2.Interpolable() { 347 t.Errorf("Interpolable bool in retrieved subvol incorrect\n") 348 } 349 if !reflect.DeepEqual(v.Size(), v2.Size()) { 350 t.Errorf("Size in retrieved subvol incorrect: %s vs expected %s\n", 351 v2.Size(), v.Size()) 352 } 353 if v.NumVoxels() != v2.NumVoxels() { 354 t.Errorf("# voxels in retrieved is different: %d vs expected %d\n", 355 v2.NumVoxels(), v.NumVoxels()) 356 } 357 data = v2.Data() 358 //dvid.PrintNonZero("original value", origData) 359 //dvid.PrintNonZero("returned value", data) 360 for i := int64(0); i < v2.NumVoxels(); i++ { 361 if data[i] != origData[i] { 362 t.Logf("Data returned != data stored for voxel %d\n", i) 363 t.Logf("Size of data: %d bytes from GET, %d bytes in PUT\n", len(data), len(origData)) 364 t.Fatalf("GET subvol (%d) != PUT subvol (%d) @ index %d", data[i], origData[i], i) 365 } 366 } 367 } 368 369 func TestBlockAPI(t *testing.T) { 370 if err := server.OpenTest(); err != nil { 371 t.Fatalf("can't open test server: %v\n", err) 372 } 373 defer server.CloseTest() 374 375 uuid, _ := initTestRepo() 376 grayscale := makeGrayscale(uuid, t, "grayscale") 377 378 // construct random blocks of data. 379 numBlockBytes := int32(grayscale.BlockSize().Prod()) 380 testBlocks := 1001 381 var blockData []byte 382 for i := 0; i < testBlocks; i++ { 383 blockData = append(blockData, dvid.RandomBytes(numBlockBytes)...) 384 } 385 386 // set start of span 387 x := rand.Int31() 388 y := rand.Int31() 389 z := rand.Int31() 390 391 // Test uncompressed roundtrip 392 393 // send the blocks 394 blockReq := fmt.Sprintf("%snode/%s/grayscale/blocks/%d_%d_%d/%d", server.WebAPIPath, uuid, x, y, z, testBlocks) 395 server.TestHTTP(t, "POST", blockReq, bytes.NewBuffer(blockData)) 396 397 // read same span of blocks 398 if err := datastore.BlockOnUpdating(uuid, "grayscale"); err != nil { 399 t.Fatalf("Error blocking on POST of grayscale: %v\n", err) 400 } 401 returnedData := server.TestHTTP(t, "GET", blockReq, nil) 402 403 // make sure blocks are same 404 totBytes := testBlocks * int(numBlockBytes) 405 if len(returnedData) != totBytes { 406 t.Errorf("Returned %d bytes, expected %d bytes", len(returnedData), totBytes) 407 } 408 if !reflect.DeepEqual(returnedData, blockData) { 409 t.Errorf("Returned block data != original block data\n") 410 } 411 412 // We should get blank blocks at different z 413 z += 1 414 blockReq = fmt.Sprintf("%snode/%s/grayscale/blocks/%d_%d_%d/%d", server.WebAPIPath, uuid, x, y, z, testBlocks) 415 returnedData = server.TestHTTP(t, "GET", blockReq, nil) 416 if len(returnedData) != totBytes { 417 t.Errorf("Returned %d bytes, expected %d bytes", len(returnedData), totBytes) 418 } 419 for i, b := range returnedData { 420 if b != 0 { 421 t.Fatalf("Expected 0 at returned byte %d, got %d instead.\n", i, b) 422 } 423 } 424 } 425 426 func TestGrayscaleRepoPersistence(t *testing.T) { 427 if err := server.OpenTest(); err != nil { 428 t.Fatalf("can't open test server: %v\n", err) 429 } 430 defer server.CloseTest() 431 432 uuid, _ := initTestRepo() 433 434 // Make grayscale and set various properties 435 config := dvid.NewConfig() 436 config.Set("BlockSize", "12,13,14") 437 config.Set("VoxelSize", "1.1,2.8,11") 438 config.Set("VoxelUnits", "microns,millimeters,nanometers") 439 dataservice, err := datastore.NewData(uuid, grayscaleT, "mygrayscale", config) 440 if err != nil { 441 t.Errorf("Unable to create grayscale instance: %s\n", err) 442 } 443 grayscale, ok := dataservice.(*Data) 444 if !ok { 445 t.Errorf("Can't cast uint8 data service into Data\n") 446 } 447 oldData := *grayscale 448 449 // Restart test datastore and see if datasets are still there. 450 if err = datastore.SaveDataByUUID(uuid, grayscale); err != nil { 451 t.Fatalf("Unable to save repo during grayscale persistence test: %v\n", err) 452 } 453 datastore.CloseReopenTest() 454 455 dataservice2, err := datastore.GetDataByUUIDName(uuid, "mygrayscale") 456 if err != nil { 457 t.Fatalf("Can't get grayscale instance from reloaded test db: %v\n", err) 458 } 459 grayscale2, ok := dataservice2.(*Data) 460 if !ok { 461 t.Errorf("Returned new data instance 2 is not imageblk.Data\n") 462 } 463 if !oldData.Equals(grayscale2) { 464 t.Errorf("Expected %v, got %v\n", oldData, *grayscale2) 465 } 466 }