github.com/linapex/ethereum-dpos-chinese@v0.0.0-20190316121959-b78b3a4a1ece/swarm/storage/chunker_test.go (about) 1 2 //<developer> 3 // <name>linapex 曹一峰</name> 4 // <email>linapex@163.com</email> 5 // <wx>superexc</wx> 6 // <qqgroup>128148617</qqgroup> 7 // <url>https://jsq.ink</url> 8 // <role>pku engineer</role> 9 // <date>2019-03-16 12:09:49</date> 10 //</624342680394010624> 11 12 // 13 // 14 // 15 // 16 // 17 // 18 // 19 // 20 // 21 // 22 // 23 // 24 // 25 // 26 // 27 28 package storage 29 30 import ( 31 "bytes" 32 "context" 33 "crypto/rand" 34 "encoding/binary" 35 "errors" 36 "fmt" 37 "io" 38 "testing" 39 40 "github.com/ethereum/go-ethereum/crypto/sha3" 41 ) 42 43 /* 44 45 */ 46 47 48 type test interface { 49 Fatalf(string, ...interface{}) 50 Logf(string, ...interface{}) 51 } 52 53 type chunkerTester struct { 54 inputs map[uint64][]byte 55 t test 56 } 57 58 // 59 // 60 // 61 type fakeChunkStore struct { 62 } 63 64 // 65 func (f *fakeChunkStore) Put(context.Context, *Chunk) { 66 } 67 68 // 69 func (f *fakeChunkStore) Get(context.Context, Address) (*Chunk, error) { 70 return nil, errors.New("FakeChunkStore doesn't support Get") 71 } 72 73 // 74 func (f *fakeChunkStore) Close() { 75 } 76 77 func newTestHasherStore(chunkStore ChunkStore, hash string) *hasherStore { 78 return NewHasherStore(chunkStore, MakeHashFunc(hash), false) 79 } 80 81 func testRandomBrokenData(n int, tester *chunkerTester) { 82 data := io.LimitReader(rand.Reader, int64(n)) 83 brokendata := brokenLimitReader(data, n, n/2) 84 85 buf := make([]byte, n) 86 _, err := brokendata.Read(buf) 87 if err == nil || err.Error() != "Broken reader" { 88 tester.t.Fatalf("Broken reader is not broken, hence broken. Returns: %v", err) 89 } 90 91 data = io.LimitReader(rand.Reader, int64(n)) 92 brokendata = brokenLimitReader(data, n, n/2) 93 94 putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) 95 96 expectedError := fmt.Errorf("Broken reader") 97 addr, _, err := TreeSplit(context.TODO(), brokendata, int64(n), putGetter) 98 if err == nil || err.Error() != expectedError.Error() { 99 tester.t.Fatalf("Not receiving the correct error! Expected %v, received %v", expectedError, err) 100 } 101 tester.t.Logf(" Key = %v\n", addr) 102 } 103 104 func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) Address { 105 if tester.inputs == nil { 106 tester.inputs = make(map[uint64][]byte) 107 } 108 input, found := tester.inputs[uint64(n)] 109 var data io.Reader 110 if !found { 111 data, input = generateRandomData(n) 112 tester.inputs[uint64(n)] = input 113 } else { 114 data = io.LimitReader(bytes.NewReader(input), int64(n)) 115 } 116 117 putGetter := newTestHasherStore(NewMapChunkStore(), hash) 118 119 var addr Address 120 var wait func(context.Context) error 121 var err error 122 ctx := context.TODO() 123 if usePyramid { 124 addr, wait, err = PyramidSplit(ctx, data, putGetter, putGetter) 125 } else { 126 addr, wait, err = TreeSplit(ctx, data, int64(n), putGetter) 127 } 128 if err != nil { 129 tester.t.Fatalf(err.Error()) 130 } 131 tester.t.Logf(" Key = %v\n", addr) 132 err = wait(ctx) 133 if err != nil { 134 tester.t.Fatalf(err.Error()) 135 } 136 137 reader := TreeJoin(context.TODO(), addr, putGetter, 0) 138 output := make([]byte, n) 139 r, err := reader.Read(output) 140 if r != n || err != io.EOF { 141 tester.t.Fatalf("read error read: %v n = %v err = %v\n", r, n, err) 142 } 143 if input != nil { 144 if !bytes.Equal(output, input) { 145 tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input, output) 146 } 147 } 148 149 // 150 for i := 1; i < n; i += 10000 { 151 readableLength := n - i 152 output := make([]byte, readableLength) 153 r, err := reader.ReadAt(output, int64(i)) 154 if r != readableLength || err != io.EOF { 155 tester.t.Fatalf("readAt error with offset %v read: %v n = %v err = %v\n", i, r, readableLength, err) 156 } 157 if input != nil { 158 if !bytes.Equal(output, input[i:]) { 159 tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input[i:], output) 160 } 161 } 162 } 163 164 return addr 165 } 166 167 func TestSha3ForCorrectness(t *testing.T) { 168 tester := &chunkerTester{t: t} 169 170 size := 4096 171 input := make([]byte, size+8) 172 binary.LittleEndian.PutUint64(input[:8], uint64(size)) 173 174 io.LimitReader(bytes.NewReader(input[8:]), int64(size)) 175 176 rawSha3 := sha3.NewKeccak256() 177 rawSha3.Reset() 178 rawSha3.Write(input) 179 rawSha3Output := rawSha3.Sum(nil) 180 181 sha3FromMakeFunc := MakeHashFunc(SHA3Hash)() 182 sha3FromMakeFunc.ResetWithLength(input[:8]) 183 sha3FromMakeFunc.Write(input[8:]) 184 sha3FromMakeFuncOutput := sha3FromMakeFunc.Sum(nil) 185 186 if len(rawSha3Output) != len(sha3FromMakeFuncOutput) { 187 tester.t.Fatalf("Original SHA3 and abstracted Sha3 has different length %v:%v\n", len(rawSha3Output), len(sha3FromMakeFuncOutput)) 188 } 189 190 if !bytes.Equal(rawSha3Output, sha3FromMakeFuncOutput) { 191 tester.t.Fatalf("Original SHA3 and abstracted Sha3 mismatch %v:%v\n", rawSha3Output, sha3FromMakeFuncOutput) 192 } 193 194 } 195 196 func TestDataAppend(t *testing.T) { 197 sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678} 198 appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000} 199 200 tester := &chunkerTester{t: t} 201 for i := range sizes { 202 n := sizes[i] 203 m := appendSizes[i] 204 205 if tester.inputs == nil { 206 tester.inputs = make(map[uint64][]byte) 207 } 208 input, found := tester.inputs[uint64(n)] 209 var data io.Reader 210 if !found { 211 data, input = generateRandomData(n) 212 tester.inputs[uint64(n)] = input 213 } else { 214 data = io.LimitReader(bytes.NewReader(input), int64(n)) 215 } 216 217 chunkStore := NewMapChunkStore() 218 putGetter := newTestHasherStore(chunkStore, SHA3Hash) 219 220 ctx := context.TODO() 221 addr, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) 222 if err != nil { 223 tester.t.Fatalf(err.Error()) 224 } 225 err = wait(ctx) 226 if err != nil { 227 tester.t.Fatalf(err.Error()) 228 } 229 230 // 231 appendInput, found := tester.inputs[uint64(m)] 232 var appendData io.Reader 233 if !found { 234 appendData, appendInput = generateRandomData(m) 235 tester.inputs[uint64(m)] = appendInput 236 } else { 237 appendData = io.LimitReader(bytes.NewReader(appendInput), int64(m)) 238 } 239 240 putGetter = newTestHasherStore(chunkStore, SHA3Hash) 241 newAddr, wait, err := PyramidAppend(ctx, addr, appendData, putGetter, putGetter) 242 if err != nil { 243 tester.t.Fatalf(err.Error()) 244 } 245 err = wait(ctx) 246 if err != nil { 247 tester.t.Fatalf(err.Error()) 248 } 249 250 reader := TreeJoin(ctx, newAddr, putGetter, 0) 251 newOutput := make([]byte, n+m) 252 r, err := reader.Read(newOutput) 253 if r != (n + m) { 254 tester.t.Fatalf("read error read: %v n = %v m = %v err = %v\n", r, n, m, err) 255 } 256 257 newInput := append(input, appendInput...) 258 if !bytes.Equal(newOutput, newInput) { 259 tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", newInput, newOutput) 260 } 261 } 262 } 263 264 func TestRandomData(t *testing.T) { 265 // 266 // 267 sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 524288, 524288 + 1, 524288 + 4097, 7 * 524288, 7*524288 + 1, 7*524288 + 4097} 268 tester := &chunkerTester{t: t} 269 270 for _, s := range sizes { 271 treeChunkerKey := testRandomData(false, SHA3Hash, s, tester) 272 pyramidChunkerKey := testRandomData(true, SHA3Hash, s, tester) 273 if treeChunkerKey.String() != pyramidChunkerKey.String() { 274 tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerKey.String(), pyramidChunkerKey.String()) 275 } 276 } 277 278 for _, s := range sizes { 279 treeChunkerKey := testRandomData(false, BMTHash, s, tester) 280 pyramidChunkerKey := testRandomData(true, BMTHash, s, tester) 281 if treeChunkerKey.String() != pyramidChunkerKey.String() { 282 tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerKey.String(), pyramidChunkerKey.String()) 283 } 284 } 285 } 286 287 func TestRandomBrokenData(t *testing.T) { 288 sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678} 289 tester := &chunkerTester{t: t} 290 for _, s := range sizes { 291 testRandomBrokenData(s, tester) 292 } 293 } 294 295 func benchReadAll(reader LazySectionReader) { 296 size, _ := reader.Size(context.TODO(), nil) 297 output := make([]byte, 1000) 298 for pos := int64(0); pos < size; pos += 1000 { 299 reader.ReadAt(output, pos) 300 } 301 } 302 303 func benchmarkSplitJoin(n int, t *testing.B) { 304 t.ReportAllocs() 305 for i := 0; i < t.N; i++ { 306 data := testDataReader(n) 307 308 putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash) 309 ctx := context.TODO() 310 key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) 311 if err != nil { 312 t.Fatalf(err.Error()) 313 } 314 err = wait(ctx) 315 if err != nil { 316 t.Fatalf(err.Error()) 317 } 318 reader := TreeJoin(ctx, key, putGetter, 0) 319 benchReadAll(reader) 320 } 321 } 322 323 func benchmarkSplitTreeSHA3(n int, t *testing.B) { 324 t.ReportAllocs() 325 for i := 0; i < t.N; i++ { 326 data := testDataReader(n) 327 putGetter := newTestHasherStore(&fakeChunkStore{}, SHA3Hash) 328 329 _, _, err := TreeSplit(context.TODO(), data, int64(n), putGetter) 330 if err != nil { 331 t.Fatalf(err.Error()) 332 } 333 } 334 } 335 336 func benchmarkSplitTreeBMT(n int, t *testing.B) { 337 t.ReportAllocs() 338 for i := 0; i < t.N; i++ { 339 data := testDataReader(n) 340 putGetter := newTestHasherStore(&fakeChunkStore{}, BMTHash) 341 342 _, _, err := TreeSplit(context.TODO(), data, int64(n), putGetter) 343 if err != nil { 344 t.Fatalf(err.Error()) 345 } 346 } 347 } 348 349 func benchmarkSplitPyramidSHA3(n int, t *testing.B) { 350 t.ReportAllocs() 351 for i := 0; i < t.N; i++ { 352 data := testDataReader(n) 353 putGetter := newTestHasherStore(&fakeChunkStore{}, SHA3Hash) 354 355 _, _, err := PyramidSplit(context.TODO(), data, putGetter, putGetter) 356 if err != nil { 357 t.Fatalf(err.Error()) 358 } 359 360 } 361 } 362 363 func benchmarkSplitPyramidBMT(n int, t *testing.B) { 364 t.ReportAllocs() 365 for i := 0; i < t.N; i++ { 366 data := testDataReader(n) 367 putGetter := newTestHasherStore(&fakeChunkStore{}, BMTHash) 368 369 _, _, err := PyramidSplit(context.TODO(), data, putGetter, putGetter) 370 if err != nil { 371 t.Fatalf(err.Error()) 372 } 373 } 374 } 375 376 func benchmarkSplitAppendPyramid(n, m int, t *testing.B) { 377 t.ReportAllocs() 378 for i := 0; i < t.N; i++ { 379 data := testDataReader(n) 380 data1 := testDataReader(m) 381 382 chunkStore := NewMapChunkStore() 383 putGetter := newTestHasherStore(chunkStore, SHA3Hash) 384 385 ctx := context.TODO() 386 key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter) 387 if err != nil { 388 t.Fatalf(err.Error()) 389 } 390 err = wait(ctx) 391 if err != nil { 392 t.Fatalf(err.Error()) 393 } 394 395 putGetter = newTestHasherStore(chunkStore, SHA3Hash) 396 _, wait, err = PyramidAppend(ctx, key, data1, putGetter, putGetter) 397 if err != nil { 398 t.Fatalf(err.Error()) 399 } 400 err = wait(ctx) 401 if err != nil { 402 t.Fatalf(err.Error()) 403 } 404 } 405 } 406 407 func BenchmarkSplitJoin_2(t *testing.B) { benchmarkSplitJoin(100, t) } 408 func BenchmarkSplitJoin_3(t *testing.B) { benchmarkSplitJoin(1000, t) } 409 func BenchmarkSplitJoin_4(t *testing.B) { benchmarkSplitJoin(10000, t) } 410 func BenchmarkSplitJoin_5(t *testing.B) { benchmarkSplitJoin(100000, t) } 411 func BenchmarkSplitJoin_6(t *testing.B) { benchmarkSplitJoin(1000000, t) } 412 func BenchmarkSplitJoin_7(t *testing.B) { benchmarkSplitJoin(10000000, t) } 413 414 // 415 416 func BenchmarkSplitTreeSHA3_2(t *testing.B) { benchmarkSplitTreeSHA3(100, t) } 417 func BenchmarkSplitTreeSHA3_2h(t *testing.B) { benchmarkSplitTreeSHA3(500, t) } 418 func BenchmarkSplitTreeSHA3_3(t *testing.B) { benchmarkSplitTreeSHA3(1000, t) } 419 func BenchmarkSplitTreeSHA3_3h(t *testing.B) { benchmarkSplitTreeSHA3(5000, t) } 420 func BenchmarkSplitTreeSHA3_4(t *testing.B) { benchmarkSplitTreeSHA3(10000, t) } 421 func BenchmarkSplitTreeSHA3_4h(t *testing.B) { benchmarkSplitTreeSHA3(50000, t) } 422 func BenchmarkSplitTreeSHA3_5(t *testing.B) { benchmarkSplitTreeSHA3(100000, t) } 423 func BenchmarkSplitTreeSHA3_6(t *testing.B) { benchmarkSplitTreeSHA3(1000000, t) } 424 func BenchmarkSplitTreeSHA3_7(t *testing.B) { benchmarkSplitTreeSHA3(10000000, t) } 425 426 // 427 428 func BenchmarkSplitTreeBMT_2(t *testing.B) { benchmarkSplitTreeBMT(100, t) } 429 func BenchmarkSplitTreeBMT_2h(t *testing.B) { benchmarkSplitTreeBMT(500, t) } 430 func BenchmarkSplitTreeBMT_3(t *testing.B) { benchmarkSplitTreeBMT(1000, t) } 431 func BenchmarkSplitTreeBMT_3h(t *testing.B) { benchmarkSplitTreeBMT(5000, t) } 432 func BenchmarkSplitTreeBMT_4(t *testing.B) { benchmarkSplitTreeBMT(10000, t) } 433 func BenchmarkSplitTreeBMT_4h(t *testing.B) { benchmarkSplitTreeBMT(50000, t) } 434 func BenchmarkSplitTreeBMT_5(t *testing.B) { benchmarkSplitTreeBMT(100000, t) } 435 func BenchmarkSplitTreeBMT_6(t *testing.B) { benchmarkSplitTreeBMT(1000000, t) } 436 func BenchmarkSplitTreeBMT_7(t *testing.B) { benchmarkSplitTreeBMT(10000000, t) } 437 438 // 439 440 func BenchmarkSplitPyramidSHA3_2(t *testing.B) { benchmarkSplitPyramidSHA3(100, t) } 441 func BenchmarkSplitPyramidSHA3_2h(t *testing.B) { benchmarkSplitPyramidSHA3(500, t) } 442 func BenchmarkSplitPyramidSHA3_3(t *testing.B) { benchmarkSplitPyramidSHA3(1000, t) } 443 func BenchmarkSplitPyramidSHA3_3h(t *testing.B) { benchmarkSplitPyramidSHA3(5000, t) } 444 func BenchmarkSplitPyramidSHA3_4(t *testing.B) { benchmarkSplitPyramidSHA3(10000, t) } 445 func BenchmarkSplitPyramidSHA3_4h(t *testing.B) { benchmarkSplitPyramidSHA3(50000, t) } 446 func BenchmarkSplitPyramidSHA3_5(t *testing.B) { benchmarkSplitPyramidSHA3(100000, t) } 447 func BenchmarkSplitPyramidSHA3_6(t *testing.B) { benchmarkSplitPyramidSHA3(1000000, t) } 448 func BenchmarkSplitPyramidSHA3_7(t *testing.B) { benchmarkSplitPyramidSHA3(10000000, t) } 449 450 // 451 452 func BenchmarkSplitPyramidBMT_2(t *testing.B) { benchmarkSplitPyramidBMT(100, t) } 453 func BenchmarkSplitPyramidBMT_2h(t *testing.B) { benchmarkSplitPyramidBMT(500, t) } 454 func BenchmarkSplitPyramidBMT_3(t *testing.B) { benchmarkSplitPyramidBMT(1000, t) } 455 func BenchmarkSplitPyramidBMT_3h(t *testing.B) { benchmarkSplitPyramidBMT(5000, t) } 456 func BenchmarkSplitPyramidBMT_4(t *testing.B) { benchmarkSplitPyramidBMT(10000, t) } 457 func BenchmarkSplitPyramidBMT_4h(t *testing.B) { benchmarkSplitPyramidBMT(50000, t) } 458 func BenchmarkSplitPyramidBMT_5(t *testing.B) { benchmarkSplitPyramidBMT(100000, t) } 459 func BenchmarkSplitPyramidBMT_6(t *testing.B) { benchmarkSplitPyramidBMT(1000000, t) } 460 func BenchmarkSplitPyramidBMT_7(t *testing.B) { benchmarkSplitPyramidBMT(10000000, t) } 461 462 // 463 464 func BenchmarkSplitAppendPyramid_2(t *testing.B) { benchmarkSplitAppendPyramid(100, 1000, t) } 465 func BenchmarkSplitAppendPyramid_2h(t *testing.B) { benchmarkSplitAppendPyramid(500, 1000, t) } 466 func BenchmarkSplitAppendPyramid_3(t *testing.B) { benchmarkSplitAppendPyramid(1000, 1000, t) } 467 func BenchmarkSplitAppendPyramid_4(t *testing.B) { benchmarkSplitAppendPyramid(10000, 1000, t) } 468 func BenchmarkSplitAppendPyramid_4h(t *testing.B) { benchmarkSplitAppendPyramid(50000, 1000, t) } 469 func BenchmarkSplitAppendPyramid_5(t *testing.B) { benchmarkSplitAppendPyramid(1000000, 1000, t) } 470 func BenchmarkSplitAppendPyramid_6(t *testing.B) { benchmarkSplitAppendPyramid(1000000, 1000, t) } 471 func BenchmarkSplitAppendPyramid_7(t *testing.B) { benchmarkSplitAppendPyramid(10000000, 1000, t) } 472 473 // 474 475 // 476 // 477