github.com/janelia-flyem/dvid@v1.0.0/datatype/tarsupervoxels/tarsupervoxels_test.go (about) 1 package tarsupervoxels 2 3 import ( 4 "archive/tar" 5 "bytes" 6 "encoding/binary" 7 "encoding/json" 8 "fmt" 9 "io" 10 "log" 11 "sync" 12 "testing" 13 14 "github.com/janelia-flyem/dvid/datastore" 15 "github.com/janelia-flyem/dvid/datatype/common/labels" 16 "github.com/janelia-flyem/dvid/dvid" 17 "github.com/janelia-flyem/dvid/server" 18 "github.com/janelia-flyem/dvid/storage" 19 ) 20 21 var ( 22 kvtype datastore.TypeService 23 testMu sync.Mutex 24 ) 25 26 // Sets package-level testRepo and TestVersionID 27 func initTestRepo() (dvid.UUID, dvid.VersionID) { 28 testMu.Lock() 29 defer testMu.Unlock() 30 if kvtype == nil { 31 var err error 32 kvtype, err = datastore.TypeServiceByName(TypeName) 33 if err != nil { 34 log.Fatalf("Can't get tarsupervoxels type: %s\n", err) 35 } 36 } 37 return datastore.NewTestRepo() 38 } 39 40 func testTarball(t *testing.T, storetype storage.Alias) { 41 testConfig := server.TestConfig{ 42 KVStoresMap: storage.DataMap{"tarsupervoxels": storetype}, 43 } 44 if err := server.OpenTest(testConfig); err != nil { 45 t.Fatalf("can't open test server: %v\n", err) 46 } 47 defer server.CloseTest() 48 49 uuid, _ := initTestRepo() 50 var config dvid.Config 51 labelname := "labels" 52 server.CreateTestInstance(t, uuid, "labelmap", labelname, config) 53 tarsvname := "blobs" 54 config.Set("Extension", "dat") 55 server.CreateTestInstance(t, uuid, "tarsupervoxels", tarsvname, config) 56 server.CreateTestSync(t, uuid, tarsvname, labelname) 57 58 if err := datastore.BlockOnUpdating(uuid, "labels"); err != nil { 59 t.Fatalf("Error blocking on sync of labels: %v\n", err) 60 } 61 62 // Put in supervoxels 1-64 at intersection of 8 default blocks. 63 n := 64 64 voxels := make([]byte, n*n*n*8) 65 for i := 0; i < n*n*n; i++ { 66 binary.LittleEndian.PutUint64(voxels[i*8:i*8+8], uint64((i%64)+1)) 67 } 68 apiStr := fmt.Sprintf("%snode/%s/%s/raw/0_1_2/%d_%d_%d/64_64_64", server.WebAPIPath, 69 uuid, labelname, n, n, n) 70 server.TestHTTP(t, "POST", apiStr, bytes.NewBuffer(voxels)) 71 if err := datastore.BlockOnUpdating(uuid, dvid.InstanceName(labelname)); err != nil { 72 t.Fatalf("Error blocking on sync of labels: %v\n", err) 73 } 74 75 // Merge a few supervoxels to create bodies to test. 76 apiStr = fmt.Sprintf("%snode/%s/labels/merge", server.WebAPIPath, uuid) 77 server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString("[30, 10, 15, 18, 19, 20, 21, 64]")) 78 79 // Add tarball data for first 63 supervoxels except for 15. 80 var buf bytes.Buffer 81 tw := tar.NewWriter(&buf) 82 for i := uint64(1); i <= 63; i++ { 83 if i == 15 { 84 continue 85 } 86 data := fmt.Sprintf("This is the data for supervoxel %d.", i) 87 hdr := &tar.Header{ 88 Name: fmt.Sprintf("%d.dat", i), 89 Mode: 0755, 90 Size: int64(len(data)), 91 } 92 if err := tw.WriteHeader(hdr); err != nil { 93 t.Fatalf("unable to write tar file header for supervoxel %d: %v\n", i, err) 94 } 95 if _, err := tw.Write([]byte(data)); err != nil { 96 t.Fatalf("unable to write data for sueprvoxel %d: %v\n", i, err) 97 } 98 } 99 if err := tw.Close(); err != nil { 100 t.Fatalf("bad tar file close: %v\n", err) 101 } 102 apiStr = fmt.Sprintf("%snode/%s/%s/load", server.WebAPIPath, uuid, tarsvname) 103 server.TestHTTP(t, "POST", apiStr, &buf) 104 105 // Test single POST for last supervoxel 106 apiStr = fmt.Sprintf("%snode/%s/%s/supervoxel/64", server.WebAPIPath, uuid, tarsvname) 107 server.TestHTTP(t, "POST", apiStr, bytes.NewBufferString("This is the data for supervoxel 64.")) 108 109 // Check existence through HEAD 110 apiStr = fmt.Sprintf("%snode/%s/%s/tarfile/80", server.WebAPIPath, uuid, tarsvname) 111 server.TestBadHTTP(t, "HEAD", apiStr, nil) 112 113 apiStr = fmt.Sprintf("%snode/%s/%s/tarfile/10", server.WebAPIPath, uuid, tarsvname) // was merged 114 server.TestBadHTTP(t, "HEAD", apiStr, nil) 115 116 apiStr = fmt.Sprintf("%snode/%s/%s/tarfile/60", server.WebAPIPath, uuid, tarsvname) 117 server.TestHTTP(t, "HEAD", apiStr, nil) 118 119 apiStr = fmt.Sprintf("%snode/%s/%s/tarfile/30", server.WebAPIPath, uuid, tarsvname) 120 server.TestBadHTTP(t, "HEAD", apiStr, nil) // doesn't have supervoxel 15 121 122 // Check existence through /exists endpoint 123 apiStr = fmt.Sprintf("%snode/%s/%s/exists", server.WebAPIPath, uuid, tarsvname) 124 r := server.TestHTTP(t, "GET", apiStr, bytes.NewBufferString("[81, 10, 60]")) 125 var existences []bool 126 if err := json.Unmarshal(r, &existences); err != nil { 127 t.Fatalf("error trying to unmarshal existence list: %v\n", err) 128 } 129 expectedList := []bool{false, true, true} 130 if len(existences) != 3 { 131 t.Fatalf("expected 3 existences returned, got %d: %v\n", len(existences), existences) 132 } 133 for i, existence := range existences { 134 if expectedList[i] != existence { 135 t.Fatalf("expected existence %d to be %t, got %t\n", i, expectedList[i], existence) 136 } 137 } 138 139 // Check missing through /missing endpoint 140 apiStr = fmt.Sprintf("%snode/%s/%s/missing/30", server.WebAPIPath, uuid, tarsvname) 141 missingResp := server.TestHTTP(t, "GET", apiStr, nil) 142 if string(missingResp) != "[15]" { 143 t.Fatalf("For GET /missing/30, expected %q, got %q\n", "[15]", missingResp) 144 } 145 146 // Get tarball for body. 147 expected := labels.NewSet(30, 10, 15, 18, 19, 20, 21, 64) 148 apiStr = fmt.Sprintf("%snode/%s/%s/tarfile/30", server.WebAPIPath, uuid, tarsvname) 149 data := server.TestHTTP(t, "GET", apiStr, nil) 150 buf2 := bytes.NewBuffer(data) 151 tr := tar.NewReader(buf2) 152 var numFiles int 153 for { 154 hdr, err := tr.Next() 155 if err == io.EOF { 156 break 157 } 158 if err != nil { 159 t.Fatalf("error parsing tar: %v\n", err) 160 } 161 var svdata bytes.Buffer 162 if _, err := io.Copy(&svdata, tr); err != nil { 163 t.Fatalf("error reading tar data: %v\n", err) 164 } 165 var supervoxel uint64 166 var ext string 167 if _, err := fmt.Sscanf(hdr.Name, "%d.%s", &supervoxel, &ext); err != nil { 168 t.Fatalf("can't parse tar file name %q: %v\n", hdr.Name, err) 169 } 170 if supervoxel == 15 { 171 if ext != "missing" { 172 t.Fatalf("expected 15.missing but got %q\n", hdr.Name) 173 } 174 } else { 175 if ext != "dat" { 176 t.Fatalf("bad extension for tar file name %q\n", hdr.Name) 177 } 178 got := string(svdata.Bytes()) 179 if got != fmt.Sprintf("This is the data for supervoxel %d.", supervoxel) { 180 t.Fatalf(`expected "This is the data for supervoxel %d.", got %q`, supervoxel, got) 181 } 182 } 183 if _, found := expected[supervoxel]; !found { 184 t.Fatalf("got back supervoxel %d in tarfile, which is not in set %s\n", supervoxel, expected) 185 } 186 numFiles++ 187 } 188 if numFiles != 8 { 189 t.Fatalf("Only got %d files instead of expected 8\n", numFiles) 190 } 191 192 // Test single GET. 193 apiStr = fmt.Sprintf("%snode/%s/%s/supervoxel/17", server.WebAPIPath, uuid, tarsvname) 194 data = server.TestHTTP(t, "GET", apiStr, nil) 195 if string(data) != "This is the data for supervoxel 17." { 196 t.Fatalf("got bad supervoxel 17 data: %s\n", string(data)) 197 } 198 199 buf.Reset() 200 tw = tar.NewWriter(&buf) 201 value15 := fmt.Sprintf("This is the data for supervoxel %d.", 15) 202 hdr := &tar.Header{ 203 Name: fmt.Sprintf("%d.dat", 15), 204 Mode: 0755, 205 Size: int64(len(value15)), 206 } 207 if err := tw.WriteHeader(hdr); err != nil { 208 t.Fatalf("unable to write tar file header for supervoxel %d: %v\n", 15, err) 209 } 210 if _, err := tw.Write([]byte(value15)); err != nil { 211 t.Fatalf("unable to write data for sueprvoxel %d: %v\n", 15, err) 212 } 213 if err := tw.Close(); err != nil { 214 t.Fatalf("bad tar file close: %v\n", err) 215 } 216 apiStr = fmt.Sprintf("%snode/%s/%s/load", server.WebAPIPath, uuid, tarsvname) 217 server.TestHTTP(t, "POST", apiStr, &buf) 218 219 apiStr = fmt.Sprintf("%snode/%s/%s/tarfile/30", server.WebAPIPath, uuid, tarsvname) 220 server.TestHTTP(t, "HEAD", apiStr, nil) // now has every supervoxel including 15 221 }