github.com/janelia-flyem/dvid@v1.0.0/datatype/tarsupervoxels/tarsupervoxels_test.go (about)

     1  package tarsupervoxels
     2  
     3  import (
     4  	"archive/tar"
     5  	"bytes"
     6  	"encoding/binary"
     7  	"encoding/json"
     8  	"fmt"
     9  	"io"
    10  	"log"
    11  	"sync"
    12  	"testing"
    13  
    14  	"github.com/janelia-flyem/dvid/datastore"
    15  	"github.com/janelia-flyem/dvid/datatype/common/labels"
    16  	"github.com/janelia-flyem/dvid/dvid"
    17  	"github.com/janelia-flyem/dvid/server"
    18  	"github.com/janelia-flyem/dvid/storage"
    19  )
    20  
    21  var (
    22  	kvtype datastore.TypeService
    23  	testMu sync.Mutex
    24  )
    25  
    26  // Sets package-level testRepo and TestVersionID
    27  func initTestRepo() (dvid.UUID, dvid.VersionID) {
    28  	testMu.Lock()
    29  	defer testMu.Unlock()
    30  	if kvtype == nil {
    31  		var err error
    32  		kvtype, err = datastore.TypeServiceByName(TypeName)
    33  		if err != nil {
    34  			log.Fatalf("Can't get tarsupervoxels type: %s\n", err)
    35  		}
    36  	}
    37  	return datastore.NewTestRepo()
    38  }
    39  
    40  func testTarball(t *testing.T, storetype storage.Alias) {
    41  	testConfig := server.TestConfig{
    42  		KVStoresMap: storage.DataMap{"tarsupervoxels": storetype},
    43  	}
    44  	if err := server.OpenTest(testConfig); err != nil {
    45  		t.Fatalf("can't open test server: %v\n", err)
    46  	}
    47  	defer server.CloseTest()
    48  
    49  	uuid, _ := initTestRepo()
    50  	var config dvid.Config
    51  	labelname := "labels"
    52  	server.CreateTestInstance(t, uuid, "labelmap", labelname, config)
    53  	tarsvname := "blobs"
    54  	config.Set("Extension", "dat")
    55  	server.CreateTestInstance(t, uuid, "tarsupervoxels", tarsvname, config)
    56  	server.CreateTestSync(t, uuid, tarsvname, labelname)
    57  
    58  	if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil {
    59  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
    60  	}
    61  
    62  	// Put in supervoxels 1-64 at intersection of 8 default blocks.
    63  	n := 64
    64  	voxels := make([]byte, n*n*n*8)
    65  	for i := 0; i < n*n*n; i++ {
    66  		binary.LittleEndian.PutUint64(voxels[i*8:i*8+8], uint64((i%64)+1))
    67  	}
    68  	apiStr := fmt.Sprintf("%snode/%s/%s/raw/0_1_2/%d_%d_%d/64_64_64", server.WebAPIPath,
    69  		uuid, labelname, n, n, n)
    70  	server.TestHTTP(t, "POST", apiStr, bytes.NewBuffer(voxels))
    71  	if err := datastore.BlockOnUpdating(uuid, dvid.InstanceName(labelname)); err != nil {
    72  		t.Fatalf("Error blocking on sync of labels: %v\n", err)
    73  	}
    74  
    75  	// Merge a few supervoxels to create bodies to test.
    76  	apiStr = fmt.Sprintf("%snode/%s/labels/merge", server.WebAPIPath, uuid)
    77  	server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString("[30, 10, 15, 18, 19, 20, 21, 64]"))
    78  
    79  	// Add tarball data for first 63 supervoxels except for 15.
    80  	var buf bytes.Buffer
    81  	tw := tar.NewWriter(&buf)
    82  	for i := uint64(1); i <= 63; i++ {
    83  		if i == 15 {
    84  			continue
    85  		}
    86  		data := fmt.Sprintf("This is the data for supervoxel %d.", i)
    87  		hdr := &tar.Header{
    88  			Name: fmt.Sprintf("%d.dat", i),
    89  			Mode: 0755,
    90  			Size: int64(len(data)),
    91  		}
    92  		if err := tw.WriteHeader(hdr); err != nil {
    93  			t.Fatalf("unable to write tar file header for supervoxel %d: %v\n", i, err)
    94  		}
    95  		if _, err := tw.Write([]byte(data)); err != nil {
    96  			t.Fatalf("unable to write data for sueprvoxel %d: %v\n", i, err)
    97  		}
    98  	}
    99  	if err := tw.Close(); err != nil {
   100  		t.Fatalf("bad tar file close: %v\n", err)
   101  	}
   102  	apiStr = fmt.Sprintf("%snode/%s/%s/load", server.WebAPIPath, uuid, tarsvname)
   103  	server.TestHTTP(t, "POST", apiStr, &buf)
   104  
   105  	// Test single POST for last supervoxel
   106  	apiStr = fmt.Sprintf("%snode/%s/%s/supervoxel/64", server.WebAPIPath, uuid, tarsvname)
   107  	server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString("This is the data for supervoxel 64."))
   108  
   109  	// Check existence through HEAD
   110  	apiStr = fmt.Sprintf("%snode/%s/%s/tarfile/80", server.WebAPIPath, uuid, tarsvname)
   111  	server.TestBadHTTP(t, "HEAD", apiStr, nil)
   112  
   113  	apiStr = fmt.Sprintf("%snode/%s/%s/tarfile/10", server.WebAPIPath, uuid, tarsvname) // was merged
   114  	server.TestBadHTTP(t, "HEAD", apiStr, nil)
   115  
   116  	apiStr = fmt.Sprintf("%snode/%s/%s/tarfile/60", server.WebAPIPath, uuid, tarsvname)
   117  	server.TestHTTP(t, "HEAD", apiStr, nil)
   118  
   119  	apiStr = fmt.Sprintf("%snode/%s/%s/tarfile/30", server.WebAPIPath, uuid, tarsvname)
   120  	server.TestBadHTTP(t, "HEAD", apiStr, nil) // doesn't have supervoxel 15
   121  
   122  	// Check existence through /exists endpoint
   123  	apiStr = fmt.Sprintf("%snode/%s/%s/exists", server.WebAPIPath, uuid, tarsvname)
   124  	r := server.TestHTTP(t, "GET", apiStr, bytes.NewBufferString("[81, 10, 60]"))
   125  	var existences []bool
   126  	if err := json.Unmarshal(r, &existences); err != nil {
   127  		t.Fatalf("error trying to unmarshal existence list: %v\n", err)
   128  	}
   129  	expectedList := []bool{false, true, true}
   130  	if len(existences) != 3 {
   131  		t.Fatalf("expected 3 existences returned, got %d: %v\n", len(existences), existences)
   132  	}
   133  	for i, existence := range existences {
   134  		if expectedList[i] != existence {
   135  			t.Fatalf("expected existence %d to be %t, got %t\n", i, expectedList[i], existence)
   136  		}
   137  	}
   138  
   139  	// Check missing through /missing endpoint
   140  	apiStr = fmt.Sprintf("%snode/%s/%s/missing/30", server.WebAPIPath, uuid, tarsvname)
   141  	missingResp := server.TestHTTP(t, "GET", apiStr, nil)
   142  	if string(missingResp) != "[15]" {
   143  		t.Fatalf("For GET /missing/30, expected %q, got %q\n", "[15]", missingResp)
   144  	}
   145  
   146  	// Get tarball for body.
   147  	expected := labels.NewSet(30, 10, 15, 18, 19, 20, 21, 64)
   148  	apiStr = fmt.Sprintf("%snode/%s/%s/tarfile/30", server.WebAPIPath, uuid, tarsvname)
   149  	data := server.TestHTTP(t, "GET", apiStr, nil)
   150  	buf2 := bytes.NewBuffer(data)
   151  	tr := tar.NewReader(buf2)
   152  	var numFiles int
   153  	for {
   154  		hdr, err := tr.Next()
   155  		if err == io.EOF {
   156  			break
   157  		}
   158  		if err != nil {
   159  			t.Fatalf("error parsing tar: %v\n", err)
   160  		}
   161  		var svdata bytes.Buffer
   162  		if _, err := io.Copy(&svdata, tr); err != nil {
   163  			t.Fatalf("error reading tar data: %v\n", err)
   164  		}
   165  		var supervoxel uint64
   166  		var ext string
   167  		if _, err := fmt.Sscanf(hdr.Name, "%d.%s", &supervoxel, &ext); err != nil {
   168  			t.Fatalf("can't parse tar file name %q: %v\n", hdr.Name, err)
   169  		}
   170  		if supervoxel == 15 {
   171  			if ext != "missing" {
   172  				t.Fatalf("expected 15.missing but got %q\n", hdr.Name)
   173  			}
   174  		} else {
   175  			if ext != "dat" {
   176  				t.Fatalf("bad extension for tar file name %q\n", hdr.Name)
   177  			}
   178  			got := string(svdata.Bytes())
   179  			if got != fmt.Sprintf("This is the data for supervoxel %d.", supervoxel) {
   180  				t.Fatalf(`expected "This is the data for supervoxel %d.", got %q`, supervoxel, got)
   181  			}
   182  		}
   183  		if _, found := expected[supervoxel]; !found {
   184  			t.Fatalf("got back supervoxel %d in tarfile, which is not in set %s\n", supervoxel, expected)
   185  		}
   186  		numFiles++
   187  	}
   188  	if numFiles != 8 {
   189  		t.Fatalf("Only got %d files instead of expected 8\n", numFiles)
   190  	}
   191  
   192  	// Test single GET.
   193  	apiStr = fmt.Sprintf("%snode/%s/%s/supervoxel/17", server.WebAPIPath, uuid, tarsvname)
   194  	data = server.TestHTTP(t, "GET", apiStr, nil)
   195  	if string(data) != "This is the data for supervoxel 17." {
   196  		t.Fatalf("got bad supervoxel 17 data: %s\n", string(data))
   197  	}
   198  
   199  	buf.Reset()
   200  	tw = tar.NewWriter(&buf)
   201  	value15 := fmt.Sprintf("This is the data for supervoxel %d.", 15)
   202  	hdr := &tar.Header{
   203  		Name: fmt.Sprintf("%d.dat", 15),
   204  		Mode: 0755,
   205  		Size: int64(len(value15)),
   206  	}
   207  	if err := tw.WriteHeader(hdr); err != nil {
   208  		t.Fatalf("unable to write tar file header for supervoxel %d: %v\n", 15, err)
   209  	}
   210  	if _, err := tw.Write([]byte(value15)); err != nil {
   211  		t.Fatalf("unable to write data for sueprvoxel %d: %v\n", 15, err)
   212  	}
   213  	if err := tw.Close(); err != nil {
   214  		t.Fatalf("bad tar file close: %v\n", err)
   215  	}
   216  	apiStr = fmt.Sprintf("%snode/%s/%s/load", server.WebAPIPath, uuid, tarsvname)
   217  	server.TestHTTP(t, "POST", apiStr, &buf)
   218  
   219  	apiStr = fmt.Sprintf("%snode/%s/%s/tarfile/30", server.WebAPIPath, uuid, tarsvname)
   220  	server.TestHTTP(t, "HEAD", apiStr, nil) // now has every supervoxel including 15
   221  }