github.com/oskarth/go-ethereum@v1.6.8-0.20191013093314-dac24a9d3494/swarm/storage/chunker_test.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package storage 18 19 import ( 20 "bytes" 21 "context" 22 "crypto/rand" 23 "encoding/binary" 24 "fmt" 25 "io" 26 "testing" 27 28 "github.com/ethereum/go-ethereum/crypto/sha3" 29 ) 30 31 /* 32 Tests TreeChunker by splitting and joining a random byte slice 33 */ 34 35 type test interface { 36 Fatalf(string, ...interface{}) 37 Logf(string, ...interface{}) 38 } 39 40 type chunkerTester struct { 41 inputs map[uint64][]byte 42 t test 43 } 44 45 func newTestHasherStore(store ChunkStore, hash string) *hasherStore { 46 return NewHasherStore(store, MakeHashFunc(hash), false) 47 } 48 49 func testRandomBrokenData(n int, tester *chunkerTester) { 50 data := io.LimitReader(rand.Reader, int64(n)) 51 brokendata := brokenLimitReader(data, n, n/2) 52 53 buf := make([]byte, n) 54 _, err := brokendata.Read(buf) 55 if err == nil || err.Error() != "Broken reader" { 56 tester.t.Fatalf("Broken reader is not broken, hence broken. Returns: %v", err) 57 } 58 59 data = io.LimitReader(rand.Reader, int64(n)) 60 brokendata = brokenLimitReader(data, n, n/2) 61 62 putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) 63 64 expectedError := fmt.Errorf("Broken reader") 65 ctx := context.Background() 66 key, _, err := TreeSplit(ctx, brokendata, int64(n), putGetter) 67 if err == nil || err.Error() != expectedError.Error() { 68 tester.t.Fatalf("Not receiving the correct error! Expected %v, received %v", expectedError, err) 69 } 70 tester.t.Logf(" Address = %v\n", key) 71 } 72 73 func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) Address { 74 if tester.inputs == nil { 75 tester.inputs = make(map[uint64][]byte) 76 } 77 input, found := tester.inputs[uint64(n)] 78 var data io.Reader 79 if !found { 80 data, input = GenerateRandomData(n) 81 tester.inputs[uint64(n)] = input 82 } else { 83 data = io.LimitReader(bytes.NewReader(input), int64(n)) 84 } 85 86 putGetter := newTestHasherStore(NewMapChunkStore(), hash) 87 88 var addr Address 89 var wait func(context.Context) error 90 var err error 91 ctx := context.TODO() 92 if usePyramid { 93 addr, wait, err = PyramidSplit(ctx, data, putGetter, putGetter) 94 } else { 95 addr, wait, err = TreeSplit(ctx, data, int64(n), putGetter) 96 } 97 if err != nil { 98 tester.t.Fatalf(err.Error()) 99 } 100 tester.t.Logf(" Address = %v\n", addr) 101 err = wait(ctx) 102 if err != nil { 103 tester.t.Fatalf(err.Error()) 104 } 105 106 reader := TreeJoin(ctx, addr, putGetter, 0) 107 output := make([]byte, n) 108 r, err := reader.Read(output) 109 if r != n || err != io.EOF { 110 tester.t.Fatalf("read error read: %v n = %v err = %v\n", r, n, err) 111 } 112 if input != nil { 113 if !bytes.Equal(output, input) { 114 tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input, output) 115 } 116 } 117 118 // testing partial read 119 for i := 1; i < n; i += 10000 { 120 readableLength := n - i 121 output := make([]byte, readableLength) 122 r, err := reader.ReadAt(output, int64(i)) 123 if r != readableLength || err != io.EOF { 124 tester.t.Fatalf("readAt error with offset %v read: %v n = %v err = %v\n", i, r, readableLength, err) 125 } 126 if input != nil { 127 if !bytes.Equal(output, input[i:]) { 128 tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input[i:], output) 129 } 130 } 131 } 132 133 return addr 134 } 135 136 func TestSha3ForCorrectness(t *testing.T) { 137 tester := &chunkerTester{t: t} 138 139 size := 4096 140 input := make([]byte, size+8) 141 binary.LittleEndian.PutUint64(input[:8], uint64(size)) 142 143 io.LimitReader(bytes.NewReader(input[8:]), int64(size)) 144 145 rawSha3 := sha3.NewKeccak256() 146 rawSha3.Reset() 147 rawSha3.Write(input) 148 rawSha3Output := rawSha3.Sum(nil) 149 150 sha3FromMakeFunc := MakeHashFunc(SHA3Hash)() 151 sha3FromMakeFunc.ResetWithLength(input[:8]) 152 sha3FromMakeFunc.Write(input[8:]) 153 sha3FromMakeFuncOutput := sha3FromMakeFunc.Sum(nil) 154 155 if len(rawSha3Output) != len(sha3FromMakeFuncOutput) { 156 tester.t.Fatalf("Original SHA3 and abstracted Sha3 has different length %v:%v\n", len(rawSha3Output), len(sha3FromMakeFuncOutput)) 157 } 158 159 if !bytes.Equal(rawSha3Output, sha3FromMakeFuncOutput) { 160 tester.t.Fatalf("Original SHA3 and abstracted Sha3 mismatch %v:%v\n", rawSha3Output, sha3FromMakeFuncOutput) 161 } 162 163 } 164 165 func TestDataAppend(t *testing.T) { 166 sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678} 167 appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000} 168 169 tester := &chunkerTester{t: t} 170 for i := range sizes { 171 n := sizes[i] 172 m := appendSizes[i] 173 174 if tester.inputs == nil { 175 tester.inputs = make(map[uint64][]byte) 176 } 177 input, found := tester.inputs[uint64(n)] 178 var data io.Reader 179 if !found { 180 data, input = GenerateRandomData(n) 181 tester.inputs[uint64(n)] = input 182 } else { 183 data = io.LimitReader(bytes.NewReader(input), int64(n)) 184 } 185 186 store := NewMapChunkStore() 187 putGetter := newTestHasherStore(store, SHA3Hash) 188 189 ctx := context.TODO() 190 addr, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) 191 if err != nil { 192 tester.t.Fatalf(err.Error()) 193 } 194 err = wait(ctx) 195 if err != nil { 196 tester.t.Fatalf(err.Error()) 197 } 198 //create a append data stream 199 appendInput, found := tester.inputs[uint64(m)] 200 var appendData io.Reader 201 if !found { 202 appendData, appendInput = GenerateRandomData(m) 203 tester.inputs[uint64(m)] = appendInput 204 } else { 205 appendData = io.LimitReader(bytes.NewReader(appendInput), int64(m)) 206 } 207 208 putGetter = newTestHasherStore(store, SHA3Hash) 209 newAddr, wait, err := PyramidAppend(ctx, addr, appendData, putGetter, putGetter) 210 if err != nil { 211 tester.t.Fatalf(err.Error()) 212 } 213 err = wait(ctx) 214 if err != nil { 215 tester.t.Fatalf(err.Error()) 216 } 217 218 reader := TreeJoin(ctx, newAddr, putGetter, 0) 219 newOutput := make([]byte, n+m) 220 r, err := reader.Read(newOutput) 221 if r != (n + m) { 222 tester.t.Fatalf("read error read: %v n = %v m = %v err = %v\n", r, n, m, err) 223 } 224 225 newInput := append(input, appendInput...) 226 if !bytes.Equal(newOutput, newInput) { 227 tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", newInput, newOutput) 228 } 229 } 230 } 231 232 func TestRandomData(t *testing.T) { 233 // This test can validate files up to a relatively short length, as tree chunker slows down drastically. 234 // Validation of longer files is done by TestLocalStoreAndRetrieve in swarm package. 235 //sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 524288, 524288 + 1, 524288 + 4097, 7 * 524288, 7*524288 + 1, 7*524288 + 4097} 236 sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4097, 8191, 8192, 12288, 12289, 524288} 237 tester := &chunkerTester{t: t} 238 239 for _, s := range sizes { 240 treeChunkerAddress := testRandomData(false, SHA3Hash, s, tester) 241 pyramidChunkerAddress := testRandomData(true, SHA3Hash, s, tester) 242 if treeChunkerAddress.String() != pyramidChunkerAddress.String() { 243 tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerAddress.String(), pyramidChunkerAddress.String()) 244 } 245 } 246 247 for _, s := range sizes { 248 treeChunkerAddress := testRandomData(false, BMTHash, s, tester) 249 pyramidChunkerAddress := testRandomData(true, BMTHash, s, tester) 250 if treeChunkerAddress.String() != pyramidChunkerAddress.String() { 251 tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerAddress.String(), pyramidChunkerAddress.String()) 252 } 253 } 254 } 255 256 func TestRandomBrokenData(t *testing.T) { 257 sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678} 258 tester := &chunkerTester{t: t} 259 for _, s := range sizes { 260 testRandomBrokenData(s, tester) 261 } 262 } 263 264 func benchReadAll(reader LazySectionReader) { 265 size, _ := reader.Size(context.TODO(), nil) 266 output := make([]byte, 1000) 267 for pos := int64(0); pos < size; pos += 1000 { 268 reader.ReadAt(output, pos) 269 } 270 } 271 272 func benchmarkSplitJoin(n int, t *testing.B) { 273 t.ReportAllocs() 274 for i := 0; i < t.N; i++ { 275 data := testDataReader(n) 276 277 putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) 278 ctx := context.TODO() 279 key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) 280 if err != nil { 281 t.Fatalf(err.Error()) 282 } 283 err = wait(ctx) 284 if err != nil { 285 t.Fatalf(err.Error()) 286 } 287 reader := TreeJoin(ctx, key, putGetter, 0) 288 benchReadAll(reader) 289 } 290 } 291 292 func benchmarkSplitTreeSHA3(n int, t *testing.B) { 293 t.ReportAllocs() 294 for i := 0; i < t.N; i++ { 295 data := testDataReader(n) 296 putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash) 297 298 ctx := context.Background() 299 _, wait, err := TreeSplit(ctx, data, int64(n), putGetter) 300 if err != nil { 301 t.Fatalf(err.Error()) 302 } 303 err = wait(ctx) 304 if err != nil { 305 t.Fatalf(err.Error()) 306 } 307 308 } 309 } 310 311 func benchmarkSplitTreeBMT(n int, t *testing.B) { 312 t.ReportAllocs() 313 for i := 0; i < t.N; i++ { 314 data := testDataReader(n) 315 putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash) 316 317 ctx := context.Background() 318 _, wait, err := TreeSplit(ctx, data, int64(n), putGetter) 319 if err != nil { 320 t.Fatalf(err.Error()) 321 } 322 err = wait(ctx) 323 if err != nil { 324 t.Fatalf(err.Error()) 325 } 326 } 327 } 328 329 func benchmarkSplitPyramidBMT(n int, t *testing.B) { 330 t.ReportAllocs() 331 for i := 0; i < t.N; i++ { 332 data := testDataReader(n) 333 putGetter := newTestHasherStore(&FakeChunkStore{}, BMTHash) 334 335 ctx := context.Background() 336 _, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) 337 if err != nil { 338 t.Fatalf(err.Error()) 339 } 340 err = wait(ctx) 341 if err != nil { 342 t.Fatalf(err.Error()) 343 } 344 } 345 } 346 347 func benchmarkSplitPyramidSHA3(n int, t *testing.B) { 348 t.ReportAllocs() 349 for i := 0; i < t.N; i++ { 350 data := testDataReader(n) 351 putGetter := newTestHasherStore(&FakeChunkStore{}, SHA3Hash) 352 353 ctx := context.Background() 354 _, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) 355 if err != nil { 356 t.Fatalf(err.Error()) 357 } 358 err = wait(ctx) 359 if err != nil { 360 t.Fatalf(err.Error()) 361 } 362 } 363 } 364 365 func benchmarkSplitAppendPyramid(n, m int, t *testing.B) { 366 t.ReportAllocs() 367 for i := 0; i < t.N; i++ { 368 data := testDataReader(n) 369 data1 := testDataReader(m) 370 371 store := NewMapChunkStore() 372 putGetter := newTestHasherStore(store, SHA3Hash) 373 374 ctx := context.Background() 375 key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) 376 if err != nil { 377 t.Fatalf(err.Error()) 378 } 379 err = wait(ctx) 380 if err != nil { 381 t.Fatalf(err.Error()) 382 } 383 384 putGetter = newTestHasherStore(store, SHA3Hash) 385 _, wait, err = PyramidAppend(ctx, key, data1, putGetter, putGetter) 386 if err != nil { 387 t.Fatalf(err.Error()) 388 } 389 err = wait(ctx) 390 if err != nil { 391 t.Fatalf(err.Error()) 392 } 393 } 394 } 395 396 func BenchmarkSplitJoin_2(t *testing.B) { benchmarkSplitJoin(100, t) } 397 func BenchmarkSplitJoin_3(t *testing.B) { benchmarkSplitJoin(1000, t) } 398 func BenchmarkSplitJoin_4(t *testing.B) { benchmarkSplitJoin(10000, t) } 399 func BenchmarkSplitJoin_5(t *testing.B) { benchmarkSplitJoin(100000, t) } 400 func BenchmarkSplitJoin_6(t *testing.B) { benchmarkSplitJoin(1000000, t) } 401 func BenchmarkSplitJoin_7(t *testing.B) { benchmarkSplitJoin(10000000, t) } 402 403 // func BenchmarkSplitJoin_8(t *testing.B) { benchmarkJoin(100000000, t) } 404 405 func BenchmarkSplitTreeSHA3_2(t *testing.B) { benchmarkSplitTreeSHA3(100, t) } 406 func BenchmarkSplitTreeSHA3_2h(t *testing.B) { benchmarkSplitTreeSHA3(500, t) } 407 func BenchmarkSplitTreeSHA3_3(t *testing.B) { benchmarkSplitTreeSHA3(1000, t) } 408 func BenchmarkSplitTreeSHA3_3h(t *testing.B) { benchmarkSplitTreeSHA3(5000, t) } 409 func BenchmarkSplitTreeSHA3_4(t *testing.B) { benchmarkSplitTreeSHA3(10000, t) } 410 func BenchmarkSplitTreeSHA3_4h(t *testing.B) { benchmarkSplitTreeSHA3(50000, t) } 411 func BenchmarkSplitTreeSHA3_5(t *testing.B) { benchmarkSplitTreeSHA3(100000, t) } 412 func BenchmarkSplitTreeSHA3_6(t *testing.B) { benchmarkSplitTreeSHA3(1000000, t) } 413 func BenchmarkSplitTreeSHA3_7(t *testing.B) { benchmarkSplitTreeSHA3(10000000, t) } 414 415 // func BenchmarkSplitTreeSHA3_8(t *testing.B) { benchmarkSplitTreeSHA3(100000000, t) } 416 417 func BenchmarkSplitTreeBMT_2(t *testing.B) { benchmarkSplitTreeBMT(100, t) } 418 func BenchmarkSplitTreeBMT_2h(t *testing.B) { benchmarkSplitTreeBMT(500, t) } 419 func BenchmarkSplitTreeBMT_3(t *testing.B) { benchmarkSplitTreeBMT(1000, t) } 420 func BenchmarkSplitTreeBMT_3h(t *testing.B) { benchmarkSplitTreeBMT(5000, t) } 421 func BenchmarkSplitTreeBMT_4(t *testing.B) { benchmarkSplitTreeBMT(10000, t) } 422 func BenchmarkSplitTreeBMT_4h(t *testing.B) { benchmarkSplitTreeBMT(50000, t) } 423 func BenchmarkSplitTreeBMT_5(t *testing.B) { benchmarkSplitTreeBMT(100000, t) } 424 func BenchmarkSplitTreeBMT_6(t *testing.B) { benchmarkSplitTreeBMT(1000000, t) } 425 func BenchmarkSplitTreeBMT_7(t *testing.B) { benchmarkSplitTreeBMT(10000000, t) } 426 427 // func BenchmarkSplitTreeBMT_8(t *testing.B) { benchmarkSplitTreeBMT(100000000, t) } 428 429 func BenchmarkSplitPyramidSHA3_2(t *testing.B) { benchmarkSplitPyramidSHA3(100, t) } 430 func BenchmarkSplitPyramidSHA3_2h(t *testing.B) { benchmarkSplitPyramidSHA3(500, t) } 431 func BenchmarkSplitPyramidSHA3_3(t *testing.B) { benchmarkSplitPyramidSHA3(1000, t) } 432 func BenchmarkSplitPyramidSHA3_3h(t *testing.B) { benchmarkSplitPyramidSHA3(5000, t) } 433 func BenchmarkSplitPyramidSHA3_4(t *testing.B) { benchmarkSplitPyramidSHA3(10000, t) } 434 func BenchmarkSplitPyramidSHA3_4h(t *testing.B) { benchmarkSplitPyramidSHA3(50000, t) } 435 func BenchmarkSplitPyramidSHA3_5(t *testing.B) { benchmarkSplitPyramidSHA3(100000, t) } 436 func BenchmarkSplitPyramidSHA3_6(t *testing.B) { benchmarkSplitPyramidSHA3(1000000, t) } 437 func BenchmarkSplitPyramidSHA3_7(t *testing.B) { benchmarkSplitPyramidSHA3(10000000, t) } 438 439 // func BenchmarkSplitPyramidSHA3_8(t *testing.B) { benchmarkSplitPyramidSHA3(100000000, t) } 440 441 func BenchmarkSplitPyramidBMT_2(t *testing.B) { benchmarkSplitPyramidBMT(100, t) } 442 func BenchmarkSplitPyramidBMT_2h(t *testing.B) { benchmarkSplitPyramidBMT(500, t) } 443 func BenchmarkSplitPyramidBMT_3(t *testing.B) { benchmarkSplitPyramidBMT(1000, t) } 444 func BenchmarkSplitPyramidBMT_3h(t *testing.B) { benchmarkSplitPyramidBMT(5000, t) } 445 func BenchmarkSplitPyramidBMT_4(t *testing.B) { benchmarkSplitPyramidBMT(10000, t) } 446 func BenchmarkSplitPyramidBMT_4h(t *testing.B) { benchmarkSplitPyramidBMT(50000, t) } 447 func BenchmarkSplitPyramidBMT_5(t *testing.B) { benchmarkSplitPyramidBMT(100000, t) } 448 func BenchmarkSplitPyramidBMT_6(t *testing.B) { benchmarkSplitPyramidBMT(1000000, t) } 449 func BenchmarkSplitPyramidBMT_7(t *testing.B) { benchmarkSplitPyramidBMT(10000000, t) } 450 451 // func BenchmarkSplitPyramidBMT_8(t *testing.B) { benchmarkSplitPyramidBMT(100000000, t) } 452 453 func BenchmarkSplitAppendPyramid_2(t *testing.B) { benchmarkSplitAppendPyramid(100, 1000, t) } 454 func BenchmarkSplitAppendPyramid_2h(t *testing.B) { benchmarkSplitAppendPyramid(500, 1000, t) } 455 func BenchmarkSplitAppendPyramid_3(t *testing.B) { benchmarkSplitAppendPyramid(1000, 1000, t) } 456 func BenchmarkSplitAppendPyramid_4(t *testing.B) { benchmarkSplitAppendPyramid(10000, 1000, t) } 457 func BenchmarkSplitAppendPyramid_4h(t *testing.B) { benchmarkSplitAppendPyramid(50000, 1000, t) } 458 func BenchmarkSplitAppendPyramid_5(t *testing.B) { benchmarkSplitAppendPyramid(1000000, 1000, t) } 459 func BenchmarkSplitAppendPyramid_6(t *testing.B) { benchmarkSplitAppendPyramid(1000000, 1000, t) } 460 func BenchmarkSplitAppendPyramid_7(t *testing.B) { benchmarkSplitAppendPyramid(10000000, 1000, t) } 461 462 // func BenchmarkAppendPyramid_8(t *testing.B) { benchmarkAppendPyramid(100000000, 1000, t) } 463 464 // go test -timeout 20m -cpu 4 -bench=./swarm/storage -run no 465 // If you dont add the timeout argument above .. the benchmark will timeout and dump