github.com/m3shine/gochain@v2.2.26+incompatible/swarm/storage/chunker_test.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package storage 18 19 import ( 20 "bytes" 21 "crypto/rand" 22 "encoding/binary" 23 "errors" 24 "fmt" 25 "io" 26 "sync" 27 "testing" 28 "time" 29 30 "github.com/gochain-io/gochain/crypto/sha3" 31 ) 32 33 /* 34 Tests TreeChunker by splitting and joining a random byte slice 35 */ 36 37 type test interface { 38 Fatalf(string, ...interface{}) 39 Logf(string, ...interface{}) 40 } 41 42 type chunkerTester struct { 43 inputs map[uint64][]byte 44 chunksMu sync.RWMutex 45 chunks map[string]*Chunk 46 t test 47 } 48 49 func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) { 50 // reset 51 self.chunksMu.Lock() 52 self.chunks = make(map[string]*Chunk) 53 self.chunksMu.Unlock() 54 55 if self.inputs == nil { 56 self.inputs = make(map[uint64][]byte) 57 } 58 59 quitC := make(chan bool) 60 defer close(quitC) 61 if chunkC != nil { 62 go func() { 63 for { 64 select { 65 case <-quitC: 66 return 67 case chunk := <-chunkC: 68 self.chunksMu.Lock() 69 self.chunks[chunk.Key.String()] = chunk 70 self.chunksMu.Unlock() 71 chunk.Done() 72 } 73 74 } 75 }() 76 } 77 78 key, err = chunker.Split(data, size, chunkC, swg) 79 if err != nil && expectedError == nil { 80 err = fmt.Errorf("Split error: %v", err) 81 } 82 return key, err 83 } 84 85 func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) { 86 quitC := make(chan bool) 87 timeout := time.After(60 * time.Second) 88 if chunkC != nil { 89 go func() error { 90 for { 91 select { 92 case <-timeout: 93 return errors.New("Append timeout error") 94 case <-quitC: 95 return nil 96 case chunk := <-chunkC: 97 if chunk != nil { 98 self.chunksMu.RLock() 99 stored, success := self.chunks[chunk.Key.String()] 100 self.chunksMu.RUnlock() 101 if !success { 102 // Requesting data 103 self.chunksMu.Lock() 104 self.chunks[chunk.Key.String()] = chunk 105 self.chunksMu.Unlock() 106 chunk.Done() 107 } else { 108 // getting data 109 chunk.SData = stored.SData 110 chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) 111 close(chunk.C) 112 } 113 } 114 } 115 } 116 }() 117 } 118 119 key, err = chunker.Append(rootKey, data, chunkC, swg) 120 if err != nil && expectedError == nil { 121 err = fmt.Errorf("Append error: %v", err) 122 } 123 124 if chunkC != nil { 125 if swg != nil { 126 swg.Wait() 127 } 128 close(quitC) 129 } 130 return key, err 131 } 132 133 func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader { 134 // reset but not the chunks 135 136 reader := chunker.Join(key, chunkC) 137 138 timeout := time.After(600 * time.Second) 139 i := 0 140 go func() error { 141 for { 142 select { 143 case <-timeout: 144 return errors.New("Join timeout error") 145 case chunk, ok := <-chunkC: 146 if !ok { 147 close(quitC) 148 return nil 149 } 150 // this just mocks the behaviour of a chunk store retrieval 151 self.chunksMu.RLock() 152 stored, success := self.chunks[chunk.Key.String()] 153 self.chunksMu.RUnlock() 154 if !success { 155 return errors.New("Not found") 156 } 157 chunk.SData = stored.SData 158 chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) 159 close(chunk.C) 160 i++ 161 } 162 } 163 }() 164 return reader 165 } 166 167 func testRandomBrokenData(splitter Splitter, n int, tester *chunkerTester) { 168 data := io.LimitReader(rand.Reader, int64(n)) 169 brokendata := brokenLimitReader(data, n, n/2) 170 171 buf := make([]byte, n) 172 _, err := brokendata.Read(buf) 173 if err == nil || err.Error() != "Broken reader" { 174 tester.t.Fatalf("Broken reader is not broken, hence broken. Returns: %v", err) 175 } 176 177 data = io.LimitReader(rand.Reader, int64(n)) 178 brokendata = brokenLimitReader(data, n, n/2) 179 180 chunkC := make(chan *Chunk, 1000) 181 swg := &sync.WaitGroup{} 182 183 expectedError := fmt.Errorf("Broken reader") 184 key, err := tester.Split(splitter, brokendata, int64(n), chunkC, swg, expectedError) 185 if err == nil || err.Error() != expectedError.Error() { 186 tester.t.Fatalf("Not receiving the correct error! Expected %v, received %v", expectedError, err) 187 } 188 tester.t.Logf(" Key = %v\n", key) 189 } 190 191 func testRandomData(splitter Splitter, n int, tester *chunkerTester) Key { 192 if tester.inputs == nil { 193 tester.inputs = make(map[uint64][]byte) 194 } 195 input, found := tester.inputs[uint64(n)] 196 var data io.Reader 197 if !found { 198 data, input = testDataReaderAndSlice(n) 199 tester.inputs[uint64(n)] = input 200 } else { 201 data = io.LimitReader(bytes.NewReader(input), int64(n)) 202 } 203 204 chunkC := make(chan *Chunk, 1000) 205 206 key, err := tester.Split(splitter, data, int64(n), chunkC, &sync.WaitGroup{}, nil) 207 if err != nil { 208 tester.t.Fatalf(err.Error()) 209 } 210 tester.t.Logf(" Key = %v\n", key) 211 212 chunkC = make(chan *Chunk, 1000) 213 quitC := make(chan bool) 214 215 chunker := NewTreeChunker(NewChunkerParams()) 216 reader := tester.Join(chunker, key, 0, chunkC, quitC) 217 output := make([]byte, n) 218 r, err := reader.Read(output) 219 if r != n || err != io.EOF { 220 tester.t.Fatalf("read error read: %v n = %v err = %v\n", r, n, err) 221 } 222 if input != nil { 223 if !bytes.Equal(output, input) { 224 tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input, output) 225 } 226 } 227 close(chunkC) 228 <-quitC 229 230 return key 231 } 232 233 func testRandomDataAppend(splitter Splitter, n, m int, tester *chunkerTester) { 234 if tester.inputs == nil { 235 tester.inputs = make(map[uint64][]byte) 236 } 237 input, found := tester.inputs[uint64(n)] 238 var data io.Reader 239 if !found { 240 data, input = testDataReaderAndSlice(n) 241 tester.inputs[uint64(n)] = input 242 } else { 243 data = io.LimitReader(bytes.NewReader(input), int64(n)) 244 } 245 246 chunkC := make(chan *Chunk, 1000) 247 248 key, err := tester.Split(splitter, data, int64(n), chunkC, &sync.WaitGroup{}, nil) 249 if err != nil { 250 tester.t.Fatalf(err.Error()) 251 } 252 tester.t.Logf(" Key = %v\n", key) 253 254 //create a append data stream 255 appendInput, found := tester.inputs[uint64(m)] 256 var appendData io.Reader 257 if !found { 258 appendData, appendInput = testDataReaderAndSlice(m) 259 tester.inputs[uint64(m)] = appendInput 260 } else { 261 appendData = io.LimitReader(bytes.NewReader(appendInput), int64(m)) 262 } 263 264 chunkC = make(chan *Chunk, 1000) 265 266 newKey, err := tester.Append(splitter, key, appendData, chunkC, &sync.WaitGroup{}, nil) 267 if err != nil { 268 tester.t.Fatalf(err.Error()) 269 } 270 tester.t.Logf(" NewKey = %v\n", newKey) 271 272 chunkC = make(chan *Chunk, 1000) 273 quitC := make(chan bool) 274 275 chunker := NewTreeChunker(NewChunkerParams()) 276 reader := tester.Join(chunker, newKey, 0, chunkC, quitC) 277 newOutput := make([]byte, n+m) 278 r, err := reader.Read(newOutput) 279 if r != (n + m) { 280 tester.t.Fatalf("read error read: %v n = %v err = %v\n", r, n, err) 281 } 282 283 newInput := append(input, appendInput...) 284 if !bytes.Equal(newOutput, newInput) { 285 tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", newInput, newOutput) 286 } 287 288 close(chunkC) 289 } 290 291 func TestSha3ForCorrectness(t *testing.T) { 292 tester := &chunkerTester{t: t} 293 294 size := 4096 295 input := make([]byte, size+8) 296 binary.LittleEndian.PutUint64(input[:8], uint64(size)) 297 298 io.LimitReader(bytes.NewReader(input[8:]), int64(size)) 299 300 rawSha3 := sha3.NewKeccak256() 301 rawSha3.Reset() 302 rawSha3.Write(input) 303 rawSha3Output := rawSha3.Sum(nil) 304 305 sha3FromMakeFunc := MakeHashFunc(SHA3Hash)() 306 sha3FromMakeFunc.ResetWithLength(input[:8]) 307 sha3FromMakeFunc.Write(input[8:]) 308 sha3FromMakeFuncOutput := sha3FromMakeFunc.Sum(nil) 309 310 if len(rawSha3Output) != len(sha3FromMakeFuncOutput) { 311 tester.t.Fatalf("Original SHA3 and abstracted Sha3 has different length %v:%v\n", len(rawSha3Output), len(sha3FromMakeFuncOutput)) 312 } 313 314 if !bytes.Equal(rawSha3Output, sha3FromMakeFuncOutput) { 315 tester.t.Fatalf("Original SHA3 and abstracted Sha3 mismatch %v:%v\n", rawSha3Output, sha3FromMakeFuncOutput) 316 } 317 318 } 319 320 func TestDataAppend(t *testing.T) { 321 sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678} 322 appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000} 323 324 tester := &chunkerTester{t: t} 325 chunker := NewPyramidChunker(NewChunkerParams()) 326 for i, s := range sizes { 327 testRandomDataAppend(chunker, s, appendSizes[i], tester) 328 329 } 330 } 331 332 func TestRandomData(t *testing.T) { 333 sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678} 334 tester := &chunkerTester{t: t} 335 336 chunker := NewTreeChunker(NewChunkerParams()) 337 pyramid := NewPyramidChunker(NewChunkerParams()) 338 for _, s := range sizes { 339 treeChunkerKey := testRandomData(chunker, s, tester) 340 pyramidChunkerKey := testRandomData(pyramid, s, tester) 341 if treeChunkerKey.String() != pyramidChunkerKey.String() { 342 tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerKey.String(), pyramidChunkerKey.String()) 343 } 344 } 345 346 cp := NewChunkerParams() 347 cp.Hash = BMTHash 348 chunker = NewTreeChunker(cp) 349 pyramid = NewPyramidChunker(cp) 350 for _, s := range sizes { 351 treeChunkerKey := testRandomData(chunker, s, tester) 352 pyramidChunkerKey := testRandomData(pyramid, s, tester) 353 if treeChunkerKey.String() != pyramidChunkerKey.String() { 354 tester.t.Fatalf("tree chunker BMT and pyramid chunker BMT key mismatch for size %v \n TC: %v\n PC: %v\n", s, treeChunkerKey.String(), pyramidChunkerKey.String()) 355 } 356 } 357 358 } 359 360 func TestRandomBrokenData(t *testing.T) { 361 sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678} 362 tester := &chunkerTester{t: t} 363 chunker := NewTreeChunker(NewChunkerParams()) 364 for _, s := range sizes { 365 testRandomBrokenData(chunker, s, tester) 366 } 367 } 368 369 func benchReadAll(reader LazySectionReader) { 370 size, _ := reader.Size(nil) 371 output := make([]byte, 1000) 372 for pos := int64(0); pos < size; pos += 1000 { 373 reader.ReadAt(output, pos) 374 } 375 } 376 377 func benchmarkJoin(n int, t *testing.B) { 378 t.ReportAllocs() 379 for i := 0; i < t.N; i++ { 380 chunker := NewTreeChunker(NewChunkerParams()) 381 tester := &chunkerTester{t: t} 382 data := testDataReader(n) 383 384 chunkC := make(chan *Chunk, 1000) 385 386 key, err := tester.Split(chunker, data, int64(n), chunkC, &sync.WaitGroup{}, nil) 387 if err != nil { 388 tester.t.Fatalf(err.Error()) 389 } 390 chunkC = make(chan *Chunk, 1000) 391 quitC := make(chan bool) 392 reader := tester.Join(chunker, key, i, chunkC, quitC) 393 benchReadAll(reader) 394 close(chunkC) 395 <-quitC 396 } 397 } 398 399 func benchmarkSplitTreeSHA3(n int, t *testing.B) { 400 t.ReportAllocs() 401 for i := 0; i < t.N; i++ { 402 chunker := NewTreeChunker(NewChunkerParams()) 403 tester := &chunkerTester{t: t} 404 data := testDataReader(n) 405 _, err := tester.Split(chunker, data, int64(n), nil, nil, nil) 406 if err != nil { 407 tester.t.Fatalf(err.Error()) 408 } 409 } 410 } 411 412 func benchmarkSplitTreeBMT(n int, t *testing.B) { 413 t.ReportAllocs() 414 for i := 0; i < t.N; i++ { 415 cp := NewChunkerParams() 416 cp.Hash = BMTHash 417 chunker := NewTreeChunker(cp) 418 tester := &chunkerTester{t: t} 419 data := testDataReader(n) 420 _, err := tester.Split(chunker, data, int64(n), nil, nil, nil) 421 if err != nil { 422 tester.t.Fatalf(err.Error()) 423 } 424 } 425 } 426 427 func benchmarkSplitPyramidSHA3(n int, t *testing.B) { 428 t.ReportAllocs() 429 for i := 0; i < t.N; i++ { 430 splitter := NewPyramidChunker(NewChunkerParams()) 431 tester := &chunkerTester{t: t} 432 data := testDataReader(n) 433 _, err := tester.Split(splitter, data, int64(n), nil, nil, nil) 434 if err != nil { 435 tester.t.Fatalf(err.Error()) 436 } 437 } 438 } 439 440 func benchmarkSplitPyramidBMT(n int, t *testing.B) { 441 t.ReportAllocs() 442 for i := 0; i < t.N; i++ { 443 cp := NewChunkerParams() 444 cp.Hash = BMTHash 445 splitter := NewPyramidChunker(cp) 446 tester := &chunkerTester{t: t} 447 data := testDataReader(n) 448 _, err := tester.Split(splitter, data, int64(n), nil, nil, nil) 449 if err != nil { 450 tester.t.Fatalf(err.Error()) 451 } 452 } 453 } 454 455 func benchmarkAppendPyramid(n, m int, t *testing.B) { 456 t.ReportAllocs() 457 for i := 0; i < t.N; i++ { 458 chunker := NewPyramidChunker(NewChunkerParams()) 459 tester := &chunkerTester{t: t} 460 data := testDataReader(n) 461 data1 := testDataReader(m) 462 463 chunkC := make(chan *Chunk, 1000) 464 key, err := tester.Split(chunker, data, int64(n), chunkC, &sync.WaitGroup{}, nil) 465 if err != nil { 466 tester.t.Fatalf(err.Error()) 467 } 468 469 chunkC = make(chan *Chunk, 1000) 470 471 _, err = tester.Append(chunker, key, data1, chunkC, &sync.WaitGroup{}, nil) 472 if err != nil { 473 tester.t.Fatalf(err.Error()) 474 } 475 476 close(chunkC) 477 } 478 } 479 480 func BenchmarkJoin_2(t *testing.B) { benchmarkJoin(100, t) } 481 func BenchmarkJoin_3(t *testing.B) { benchmarkJoin(1000, t) } 482 func BenchmarkJoin_4(t *testing.B) { benchmarkJoin(10000, t) } 483 func BenchmarkJoin_5(t *testing.B) { benchmarkJoin(100000, t) } 484 func BenchmarkJoin_6(t *testing.B) { benchmarkJoin(1000000, t) } 485 func BenchmarkJoin_7(t *testing.B) { benchmarkJoin(10000000, t) } 486 func BenchmarkJoin_8(t *testing.B) { benchmarkJoin(100000000, t) } 487 488 func BenchmarkSplitTreeSHA3_2(t *testing.B) { benchmarkSplitTreeSHA3(100, t) } 489 func BenchmarkSplitTreeSHA3_2h(t *testing.B) { benchmarkSplitTreeSHA3(500, t) } 490 func BenchmarkSplitTreeSHA3_3(t *testing.B) { benchmarkSplitTreeSHA3(1000, t) } 491 func BenchmarkSplitTreeSHA3_3h(t *testing.B) { benchmarkSplitTreeSHA3(5000, t) } 492 func BenchmarkSplitTreeSHA3_4(t *testing.B) { benchmarkSplitTreeSHA3(10000, t) } 493 func BenchmarkSplitTreeSHA3_4h(t *testing.B) { benchmarkSplitTreeSHA3(50000, t) } 494 func BenchmarkSplitTreeSHA3_5(t *testing.B) { benchmarkSplitTreeSHA3(100000, t) } 495 func BenchmarkSplitTreeSHA3_6(t *testing.B) { benchmarkSplitTreeSHA3(1000000, t) } 496 func BenchmarkSplitTreeSHA3_7(t *testing.B) { benchmarkSplitTreeSHA3(10000000, t) } 497 func BenchmarkSplitTreeSHA3_8(t *testing.B) { benchmarkSplitTreeSHA3(100000000, t) } 498 499 func BenchmarkSplitTreeBMT_2(t *testing.B) { benchmarkSplitTreeBMT(100, t) } 500 func BenchmarkSplitTreeBMT_2h(t *testing.B) { benchmarkSplitTreeBMT(500, t) } 501 func BenchmarkSplitTreeBMT_3(t *testing.B) { benchmarkSplitTreeBMT(1000, t) } 502 func BenchmarkSplitTreeBMT_3h(t *testing.B) { benchmarkSplitTreeBMT(5000, t) } 503 func BenchmarkSplitTreeBMT_4(t *testing.B) { benchmarkSplitTreeBMT(10000, t) } 504 func BenchmarkSplitTreeBMT_4h(t *testing.B) { benchmarkSplitTreeBMT(50000, t) } 505 func BenchmarkSplitTreeBMT_5(t *testing.B) { benchmarkSplitTreeBMT(100000, t) } 506 func BenchmarkSplitTreeBMT_6(t *testing.B) { benchmarkSplitTreeBMT(1000000, t) } 507 func BenchmarkSplitTreeBMT_7(t *testing.B) { benchmarkSplitTreeBMT(10000000, t) } 508 func BenchmarkSplitTreeBMT_8(t *testing.B) { benchmarkSplitTreeBMT(100000000, t) } 509 510 func BenchmarkSplitPyramidSHA3_2(t *testing.B) { benchmarkSplitPyramidSHA3(100, t) } 511 func BenchmarkSplitPyramidSHA3_2h(t *testing.B) { benchmarkSplitPyramidSHA3(500, t) } 512 func BenchmarkSplitPyramidSHA3_3(t *testing.B) { benchmarkSplitPyramidSHA3(1000, t) } 513 func BenchmarkSplitPyramidSHA3_3h(t *testing.B) { benchmarkSplitPyramidSHA3(5000, t) } 514 func BenchmarkSplitPyramidSHA3_4(t *testing.B) { benchmarkSplitPyramidSHA3(10000, t) } 515 func BenchmarkSplitPyramidSHA3_4h(t *testing.B) { benchmarkSplitPyramidSHA3(50000, t) } 516 func BenchmarkSplitPyramidSHA3_5(t *testing.B) { benchmarkSplitPyramidSHA3(100000, t) } 517 func BenchmarkSplitPyramidSHA3_6(t *testing.B) { benchmarkSplitPyramidSHA3(1000000, t) } 518 func BenchmarkSplitPyramidSHA3_7(t *testing.B) { benchmarkSplitPyramidSHA3(10000000, t) } 519 func BenchmarkSplitPyramidSHA3_8(t *testing.B) { benchmarkSplitPyramidSHA3(100000000, t) } 520 521 func BenchmarkSplitPyramidBMT_2(t *testing.B) { benchmarkSplitPyramidBMT(100, t) } 522 func BenchmarkSplitPyramidBMT_2h(t *testing.B) { benchmarkSplitPyramidBMT(500, t) } 523 func BenchmarkSplitPyramidBMT_3(t *testing.B) { benchmarkSplitPyramidBMT(1000, t) } 524 func BenchmarkSplitPyramidBMT_3h(t *testing.B) { benchmarkSplitPyramidBMT(5000, t) } 525 func BenchmarkSplitPyramidBMT_4(t *testing.B) { benchmarkSplitPyramidBMT(10000, t) } 526 func BenchmarkSplitPyramidBMT_4h(t *testing.B) { benchmarkSplitPyramidBMT(50000, t) } 527 func BenchmarkSplitPyramidBMT_5(t *testing.B) { benchmarkSplitPyramidBMT(100000, t) } 528 func BenchmarkSplitPyramidBMT_6(t *testing.B) { benchmarkSplitPyramidBMT(1000000, t) } 529 func BenchmarkSplitPyramidBMT_7(t *testing.B) { benchmarkSplitPyramidBMT(10000000, t) } 530 func BenchmarkSplitPyramidBMT_8(t *testing.B) { benchmarkSplitPyramidBMT(100000000, t) } 531 532 func BenchmarkAppendPyramid_2(t *testing.B) { benchmarkAppendPyramid(100, 1000, t) } 533 func BenchmarkAppendPyramid_2h(t *testing.B) { benchmarkAppendPyramid(500, 1000, t) } 534 func BenchmarkAppendPyramid_3(t *testing.B) { benchmarkAppendPyramid(1000, 1000, t) } 535 func BenchmarkAppendPyramid_4(t *testing.B) { benchmarkAppendPyramid(10000, 1000, t) } 536 func BenchmarkAppendPyramid_4h(t *testing.B) { benchmarkAppendPyramid(50000, 1000, t) } 537 func BenchmarkAppendPyramid_5(t *testing.B) { benchmarkAppendPyramid(1000000, 1000, t) } 538 func BenchmarkAppendPyramid_6(t *testing.B) { benchmarkAppendPyramid(1000000, 1000, t) } 539 func BenchmarkAppendPyramid_7(t *testing.B) { benchmarkAppendPyramid(10000000, 1000, t) } 540 func BenchmarkAppendPyramid_8(t *testing.B) { benchmarkAppendPyramid(100000000, 1000, t) } 541 542 // go test -timeout 20m -cpu 4 -bench=./swarm/storage -run no 543 // If you dont add the timeout argument above .. the benchmark will timeout and dump