github.com/sberex/go-sberex@v1.8.2-0.20181113200658-ed96ac38f7d7/swarm/storage/chunker_test.go (about) 1 // This file is part of the go-sberex library. The go-sberex library is 2 // free software: you can redistribute it and/or modify it under the terms 3 // of the GNU Lesser General Public License as published by the Free 4 // Software Foundation, either version 3 of the License, or (at your option) 5 // any later version. 6 // 7 // The go-sberex library is distributed in the hope that it will be useful, 8 // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser 10 // General Public License <http://www.gnu.org/licenses/> for more details. 11 12 package storage 13 14 import ( 15 "bytes" 16 "crypto/rand" 17 "encoding/binary" 18 "errors" 19 "fmt" 20 "io" 21 "sync" 22 "testing" 23 "time" 24 25 "github.com/Sberex/go-sberex/crypto/sha3" 26 ) 27 28 /* 29 Tests TreeChunker by splitting and joining a random byte slice 30 */ 31 32 type test interface { 33 Fatalf(string, ...interface{}) 34 Logf(string, ...interface{}) 35 } 36 37 type chunkerTester struct { 38 inputs map[uint64][]byte 39 chunks map[string]*Chunk 40 t test 41 } 42 43 func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) { 44 // reset 45 self.chunks = make(map[string]*Chunk) 46 47 if self.inputs == nil { 48 self.inputs = make(map[uint64][]byte) 49 } 50 51 quitC := make(chan bool) 52 timeout := time.After(600 * time.Second) 53 if chunkC != nil { 54 go func() error { 55 for { 56 select { 57 case <-timeout: 58 return errors.New("Split timeout error") 59 case <-quitC: 60 return nil 61 case chunk := <-chunkC: 62 // self.chunks = append(self.chunks, chunk) 63 self.chunks[chunk.Key.String()] = chunk 64 if chunk.wg != nil { 65 chunk.wg.Done() 66 } 67 } 68 69 } 70 }() 71 } 72 73 key, err = chunker.Split(data, size, chunkC, swg, nil) 74 if err != nil && expectedError == nil { 75 err = fmt.Errorf("Split error: %v", err) 76 } 77 78 if chunkC != nil { 79 if swg != nil { 80 swg.Wait() 81 } 82 close(quitC) 83 } 84 return key, err 85 } 86 87 func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key, err error) { 88 quitC := make(chan bool) 89 timeout := time.After(60 * time.Second) 90 if chunkC != nil { 91 go func() error { 92 for { 93 select { 94 case <-timeout: 95 return errors.New("Append timeout error") 96 case <-quitC: 97 return nil 98 case chunk := <-chunkC: 99 if chunk != nil { 100 stored, success := self.chunks[chunk.Key.String()] 101 if !success { 102 // Requesting data 103 self.chunks[chunk.Key.String()] = chunk 104 if chunk.wg != nil { 105 chunk.wg.Done() 106 } 107 } else { 108 // getting data 109 chunk.SData = stored.SData 110 chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) 111 close(chunk.C) 112 } 113 } 114 } 115 } 116 }() 117 } 118 119 key, err = chunker.Append(rootKey, data, chunkC, swg, nil) 120 if err != nil && expectedError == nil { 121 err = fmt.Errorf("Append error: %v", err) 122 } 123 124 if chunkC != nil { 125 if swg != nil { 126 swg.Wait() 127 } 128 close(quitC) 129 } 130 return key, err 131 } 132 133 func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader { 134 // reset but not the chunks 135 136 reader := chunker.Join(key, chunkC) 137 138 timeout := time.After(600 * time.Second) 139 i := 0 140 go func() error { 141 for { 142 select { 143 case <-timeout: 144 return errors.New("Join timeout error") 145 case chunk, ok := <-chunkC: 146 if !ok { 147 close(quitC) 148 return nil 149 } 150 // this just mocks the behaviour of a chunk store retrieval 151 stored, success := self.chunks[chunk.Key.String()] 152 if !success { 153 return errors.New("Not found") 154 } 155 chunk.SData = stored.SData 156 chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) 157 close(chunk.C) 158 i++ 159 } 160 } 161 }() 162 return reader 163 } 164 165 func testRandomBrokenData(splitter Splitter, n int, tester *chunkerTester) { 166 data := io.LimitReader(rand.Reader, int64(n)) 167 brokendata := brokenLimitReader(data, n, n/2) 168 169 buf := make([]byte, n) 170 _, err := brokendata.Read(buf) 171 if err == nil || err.Error() != "Broken reader" { 172 tester.t.Fatalf("Broken reader is not broken, hence broken. Returns: %v", err) 173 } 174 175 data = io.LimitReader(rand.Reader, int64(n)) 176 brokendata = brokenLimitReader(data, n, n/2) 177 178 chunkC := make(chan *Chunk, 1000) 179 swg := &sync.WaitGroup{} 180 181 expectedError := fmt.Errorf("Broken reader") 182 key, err := tester.Split(splitter, brokendata, int64(n), chunkC, swg, expectedError) 183 if err == nil || err.Error() != expectedError.Error() { 184 tester.t.Fatalf("Not receiving the correct error! Expected %v, received %v", expectedError, err) 185 } 186 tester.t.Logf(" Key = %v\n", key) 187 } 188 189 func testRandomData(splitter Splitter, n int, tester *chunkerTester) Key { 190 if tester.inputs == nil { 191 tester.inputs = make(map[uint64][]byte) 192 } 193 input, found := tester.inputs[uint64(n)] 194 var data io.Reader 195 if !found { 196 data, input = testDataReaderAndSlice(n) 197 tester.inputs[uint64(n)] = input 198 } else { 199 data = io.LimitReader(bytes.NewReader(input), int64(n)) 200 } 201 202 chunkC := make(chan *Chunk, 1000) 203 swg := &sync.WaitGroup{} 204 205 key, err := tester.Split(splitter, data, int64(n), chunkC, swg, nil) 206 if err != nil { 207 tester.t.Fatalf(err.Error()) 208 } 209 tester.t.Logf(" Key = %v\n", key) 210 211 chunkC = make(chan *Chunk, 1000) 212 quitC := make(chan bool) 213 214 chunker := NewTreeChunker(NewChunkerParams()) 215 reader := tester.Join(chunker, key, 0, chunkC, quitC) 216 output := make([]byte, n) 217 r, err := reader.Read(output) 218 if r != n || err != io.EOF { 219 tester.t.Fatalf("read error read: %v n = %v err = %v\n", r, n, err) 220 } 221 if input != nil { 222 if !bytes.Equal(output, input) { 223 tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input, output) 224 } 225 } 226 close(chunkC) 227 <-quitC 228 229 return key 230 } 231 232 func testRandomDataAppend(splitter Splitter, n, m int, tester *chunkerTester) { 233 if tester.inputs == nil { 234 tester.inputs = make(map[uint64][]byte) 235 } 236 input, found := tester.inputs[uint64(n)] 237 var data io.Reader 238 if !found { 239 data, input = testDataReaderAndSlice(n) 240 tester.inputs[uint64(n)] = input 241 } else { 242 data = io.LimitReader(bytes.NewReader(input), int64(n)) 243 } 244 245 chunkC := make(chan *Chunk, 1000) 246 swg := &sync.WaitGroup{} 247 248 key, err := tester.Split(splitter, data, int64(n), chunkC, swg, nil) 249 if err != nil { 250 tester.t.Fatalf(err.Error()) 251 } 252 tester.t.Logf(" Key = %v\n", key) 253 254 //create a append data stream 255 appendInput, found := tester.inputs[uint64(m)] 256 var appendData io.Reader 257 if !found { 258 appendData, appendInput = testDataReaderAndSlice(m) 259 tester.inputs[uint64(m)] = appendInput 260 } else { 261 appendData = io.LimitReader(bytes.NewReader(appendInput), int64(m)) 262 } 263 264 chunkC = make(chan *Chunk, 1000) 265 swg = &sync.WaitGroup{} 266 267 newKey, err := tester.Append(splitter, key, appendData, chunkC, swg, nil) 268 if err != nil { 269 tester.t.Fatalf(err.Error()) 270 } 271 tester.t.Logf(" NewKey = %v\n", newKey) 272 273 chunkC = make(chan *Chunk, 1000) 274 quitC := make(chan bool) 275 276 chunker := NewTreeChunker(NewChunkerParams()) 277 reader := tester.Join(chunker, newKey, 0, chunkC, quitC) 278 newOutput := make([]byte, n+m) 279 r, err := reader.Read(newOutput) 280 if r != (n + m) { 281 tester.t.Fatalf("read error read: %v n = %v err = %v\n", r, n, err) 282 } 283 284 newInput := append(input, appendInput...) 285 if !bytes.Equal(newOutput, newInput) { 286 tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", newInput, newOutput) 287 } 288 289 close(chunkC) 290 } 291 292 func TestSha3ForCorrectness(t *testing.T) { 293 tester := &chunkerTester{t: t} 294 295 size := 4096 296 input := make([]byte, size+8) 297 binary.LittleEndian.PutUint64(input[:8], uint64(size)) 298 299 io.LimitReader(bytes.NewReader(input[8:]), int64(size)) 300 301 rawSha3 := sha3.NewKeccak256() 302 rawSha3.Reset() 303 rawSha3.Write(input) 304 rawSha3Output := rawSha3.Sum(nil) 305 306 sha3FromMakeFunc := MakeHashFunc(SHA3Hash)() 307 sha3FromMakeFunc.ResetWithLength(input[:8]) 308 sha3FromMakeFunc.Write(input[8:]) 309 sha3FromMakeFuncOutput := sha3FromMakeFunc.Sum(nil) 310 311 if len(rawSha3Output) != len(sha3FromMakeFuncOutput) { 312 tester.t.Fatalf("Original SHA3 and abstracted Sha3 has different length %v:%v\n", len(rawSha3Output), len(sha3FromMakeFuncOutput)) 313 } 314 315 if !bytes.Equal(rawSha3Output, sha3FromMakeFuncOutput) { 316 tester.t.Fatalf("Original SHA3 and abstracted Sha3 mismatch %v:%v\n", rawSha3Output, sha3FromMakeFuncOutput) 317 } 318 319 } 320 321 func TestDataAppend(t *testing.T) { 322 sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678} 323 appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000} 324 325 tester := &chunkerTester{t: t} 326 chunker := NewPyramidChunker(NewChunkerParams()) 327 for i, s := range sizes { 328 testRandomDataAppend(chunker, s, appendSizes[i], tester) 329 330 } 331 } 332 333 func TestRandomData(t *testing.T) { 334 sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678} 335 tester := &chunkerTester{t: t} 336 337 chunker := NewTreeChunker(NewChunkerParams()) 338 pyramid := NewPyramidChunker(NewChunkerParams()) 339 for _, s := range sizes { 340 treeChunkerKey := testRandomData(chunker, s, tester) 341 pyramidChunkerKey := testRandomData(pyramid, s, tester) 342 if treeChunkerKey.String() != pyramidChunkerKey.String() { 343 tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerKey.String(), pyramidChunkerKey.String()) 344 } 345 } 346 347 cp := NewChunkerParams() 348 cp.Hash = BMTHash 349 chunker = NewTreeChunker(cp) 350 pyramid = NewPyramidChunker(cp) 351 for _, s := range sizes { 352 treeChunkerKey := testRandomData(chunker, s, tester) 353 pyramidChunkerKey := testRandomData(pyramid, s, tester) 354 if treeChunkerKey.String() != pyramidChunkerKey.String() { 355 tester.t.Fatalf("tree chunker BMT and pyramid chunker BMT key mismatch for size %v \n TC: %v\n PC: %v\n", s, treeChunkerKey.String(), pyramidChunkerKey.String()) 356 } 357 } 358 359 } 360 361 func TestRandomBrokenData(t *testing.T) { 362 sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678} 363 tester := &chunkerTester{t: t} 364 chunker := NewTreeChunker(NewChunkerParams()) 365 for _, s := range sizes { 366 testRandomBrokenData(chunker, s, tester) 367 } 368 } 369 370 func benchReadAll(reader LazySectionReader) { 371 size, _ := reader.Size(nil) 372 output := make([]byte, 1000) 373 for pos := int64(0); pos < size; pos += 1000 { 374 reader.ReadAt(output, pos) 375 } 376 } 377 378 func benchmarkJoin(n int, t *testing.B) { 379 t.ReportAllocs() 380 for i := 0; i < t.N; i++ { 381 chunker := NewTreeChunker(NewChunkerParams()) 382 tester := &chunkerTester{t: t} 383 data := testDataReader(n) 384 385 chunkC := make(chan *Chunk, 1000) 386 swg := &sync.WaitGroup{} 387 388 key, err := tester.Split(chunker, data, int64(n), chunkC, swg, nil) 389 if err != nil { 390 tester.t.Fatalf(err.Error()) 391 } 392 chunkC = make(chan *Chunk, 1000) 393 quitC := make(chan bool) 394 reader := tester.Join(chunker, key, i, chunkC, quitC) 395 benchReadAll(reader) 396 close(chunkC) 397 <-quitC 398 } 399 } 400 401 func benchmarkSplitTreeSHA3(n int, t *testing.B) { 402 t.ReportAllocs() 403 for i := 0; i < t.N; i++ { 404 chunker := NewTreeChunker(NewChunkerParams()) 405 tester := &chunkerTester{t: t} 406 data := testDataReader(n) 407 _, err := tester.Split(chunker, data, int64(n), nil, nil, nil) 408 if err != nil { 409 tester.t.Fatalf(err.Error()) 410 } 411 } 412 } 413 414 func benchmarkSplitTreeBMT(n int, t *testing.B) { 415 t.ReportAllocs() 416 for i := 0; i < t.N; i++ { 417 cp := NewChunkerParams() 418 cp.Hash = BMTHash 419 chunker := NewTreeChunker(cp) 420 tester := &chunkerTester{t: t} 421 data := testDataReader(n) 422 _, err := tester.Split(chunker, data, int64(n), nil, nil, nil) 423 if err != nil { 424 tester.t.Fatalf(err.Error()) 425 } 426 } 427 } 428 429 func benchmarkSplitPyramidSHA3(n int, t *testing.B) { 430 t.ReportAllocs() 431 for i := 0; i < t.N; i++ { 432 splitter := NewPyramidChunker(NewChunkerParams()) 433 tester := &chunkerTester{t: t} 434 data := testDataReader(n) 435 _, err := tester.Split(splitter, data, int64(n), nil, nil, nil) 436 if err != nil { 437 tester.t.Fatalf(err.Error()) 438 } 439 } 440 } 441 442 func benchmarkSplitPyramidBMT(n int, t *testing.B) { 443 t.ReportAllocs() 444 for i := 0; i < t.N; i++ { 445 cp := NewChunkerParams() 446 cp.Hash = BMTHash 447 splitter := NewPyramidChunker(cp) 448 tester := &chunkerTester{t: t} 449 data := testDataReader(n) 450 _, err := tester.Split(splitter, data, int64(n), nil, nil, nil) 451 if err != nil { 452 tester.t.Fatalf(err.Error()) 453 } 454 } 455 } 456 457 func benchmarkAppendPyramid(n, m int, t *testing.B) { 458 t.ReportAllocs() 459 for i := 0; i < t.N; i++ { 460 chunker := NewPyramidChunker(NewChunkerParams()) 461 tester := &chunkerTester{t: t} 462 data := testDataReader(n) 463 data1 := testDataReader(m) 464 465 chunkC := make(chan *Chunk, 1000) 466 swg := &sync.WaitGroup{} 467 key, err := tester.Split(chunker, data, int64(n), chunkC, swg, nil) 468 if err != nil { 469 tester.t.Fatalf(err.Error()) 470 } 471 472 chunkC = make(chan *Chunk, 1000) 473 swg = &sync.WaitGroup{} 474 475 _, err = tester.Append(chunker, key, data1, chunkC, swg, nil) 476 if err != nil { 477 tester.t.Fatalf(err.Error()) 478 } 479 480 close(chunkC) 481 } 482 } 483 484 func BenchmarkJoin_2(t *testing.B) { benchmarkJoin(100, t) } 485 func BenchmarkJoin_3(t *testing.B) { benchmarkJoin(1000, t) } 486 func BenchmarkJoin_4(t *testing.B) { benchmarkJoin(10000, t) } 487 func BenchmarkJoin_5(t *testing.B) { benchmarkJoin(100000, t) } 488 func BenchmarkJoin_6(t *testing.B) { benchmarkJoin(1000000, t) } 489 func BenchmarkJoin_7(t *testing.B) { benchmarkJoin(10000000, t) } 490 func BenchmarkJoin_8(t *testing.B) { benchmarkJoin(100000000, t) } 491 492 func BenchmarkSplitTreeSHA3_2(t *testing.B) { benchmarkSplitTreeSHA3(100, t) } 493 func BenchmarkSplitTreeSHA3_2h(t *testing.B) { benchmarkSplitTreeSHA3(500, t) } 494 func BenchmarkSplitTreeSHA3_3(t *testing.B) { benchmarkSplitTreeSHA3(1000, t) } 495 func BenchmarkSplitTreeSHA3_3h(t *testing.B) { benchmarkSplitTreeSHA3(5000, t) } 496 func BenchmarkSplitTreeSHA3_4(t *testing.B) { benchmarkSplitTreeSHA3(10000, t) } 497 func BenchmarkSplitTreeSHA3_4h(t *testing.B) { benchmarkSplitTreeSHA3(50000, t) } 498 func BenchmarkSplitTreeSHA3_5(t *testing.B) { benchmarkSplitTreeSHA3(100000, t) } 499 func BenchmarkSplitTreeSHA3_6(t *testing.B) { benchmarkSplitTreeSHA3(1000000, t) } 500 func BenchmarkSplitTreeSHA3_7(t *testing.B) { benchmarkSplitTreeSHA3(10000000, t) } 501 func BenchmarkSplitTreeSHA3_8(t *testing.B) { benchmarkSplitTreeSHA3(100000000, t) } 502 503 func BenchmarkSplitTreeBMT_2(t *testing.B) { benchmarkSplitTreeBMT(100, t) } 504 func BenchmarkSplitTreeBMT_2h(t *testing.B) { benchmarkSplitTreeBMT(500, t) } 505 func BenchmarkSplitTreeBMT_3(t *testing.B) { benchmarkSplitTreeBMT(1000, t) } 506 func BenchmarkSplitTreeBMT_3h(t *testing.B) { benchmarkSplitTreeBMT(5000, t) } 507 func BenchmarkSplitTreeBMT_4(t *testing.B) { benchmarkSplitTreeBMT(10000, t) } 508 func BenchmarkSplitTreeBMT_4h(t *testing.B) { benchmarkSplitTreeBMT(50000, t) } 509 func BenchmarkSplitTreeBMT_5(t *testing.B) { benchmarkSplitTreeBMT(100000, t) } 510 func BenchmarkSplitTreeBMT_6(t *testing.B) { benchmarkSplitTreeBMT(1000000, t) } 511 func BenchmarkSplitTreeBMT_7(t *testing.B) { benchmarkSplitTreeBMT(10000000, t) } 512 func BenchmarkSplitTreeBMT_8(t *testing.B) { benchmarkSplitTreeBMT(100000000, t) } 513 514 func BenchmarkSplitPyramidSHA3_2(t *testing.B) { benchmarkSplitPyramidSHA3(100, t) } 515 func BenchmarkSplitPyramidSHA3_2h(t *testing.B) { benchmarkSplitPyramidSHA3(500, t) } 516 func BenchmarkSplitPyramidSHA3_3(t *testing.B) { benchmarkSplitPyramidSHA3(1000, t) } 517 func BenchmarkSplitPyramidSHA3_3h(t *testing.B) { benchmarkSplitPyramidSHA3(5000, t) } 518 func BenchmarkSplitPyramidSHA3_4(t *testing.B) { benchmarkSplitPyramidSHA3(10000, t) } 519 func BenchmarkSplitPyramidSHA3_4h(t *testing.B) { benchmarkSplitPyramidSHA3(50000, t) } 520 func BenchmarkSplitPyramidSHA3_5(t *testing.B) { benchmarkSplitPyramidSHA3(100000, t) } 521 func BenchmarkSplitPyramidSHA3_6(t *testing.B) { benchmarkSplitPyramidSHA3(1000000, t) } 522 func BenchmarkSplitPyramidSHA3_7(t *testing.B) { benchmarkSplitPyramidSHA3(10000000, t) } 523 func BenchmarkSplitPyramidSHA3_8(t *testing.B) { benchmarkSplitPyramidSHA3(100000000, t) } 524 525 func BenchmarkSplitPyramidBMT_2(t *testing.B) { benchmarkSplitPyramidBMT(100, t) } 526 func BenchmarkSplitPyramidBMT_2h(t *testing.B) { benchmarkSplitPyramidBMT(500, t) } 527 func BenchmarkSplitPyramidBMT_3(t *testing.B) { benchmarkSplitPyramidBMT(1000, t) } 528 func BenchmarkSplitPyramidBMT_3h(t *testing.B) { benchmarkSplitPyramidBMT(5000, t) } 529 func BenchmarkSplitPyramidBMT_4(t *testing.B) { benchmarkSplitPyramidBMT(10000, t) } 530 func BenchmarkSplitPyramidBMT_4h(t *testing.B) { benchmarkSplitPyramidBMT(50000, t) } 531 func BenchmarkSplitPyramidBMT_5(t *testing.B) { benchmarkSplitPyramidBMT(100000, t) } 532 func BenchmarkSplitPyramidBMT_6(t *testing.B) { benchmarkSplitPyramidBMT(1000000, t) } 533 func BenchmarkSplitPyramidBMT_7(t *testing.B) { benchmarkSplitPyramidBMT(10000000, t) } 534 func BenchmarkSplitPyramidBMT_8(t *testing.B) { benchmarkSplitPyramidBMT(100000000, t) } 535 536 func BenchmarkAppendPyramid_2(t *testing.B) { benchmarkAppendPyramid(100, 1000, t) } 537 func BenchmarkAppendPyramid_2h(t *testing.B) { benchmarkAppendPyramid(500, 1000, t) } 538 func BenchmarkAppendPyramid_3(t *testing.B) { benchmarkAppendPyramid(1000, 1000, t) } 539 func BenchmarkAppendPyramid_4(t *testing.B) { benchmarkAppendPyramid(10000, 1000, t) } 540 func BenchmarkAppendPyramid_4h(t *testing.B) { benchmarkAppendPyramid(50000, 1000, t) } 541 func BenchmarkAppendPyramid_5(t *testing.B) { benchmarkAppendPyramid(1000000, 1000, t) } 542 func BenchmarkAppendPyramid_6(t *testing.B) { benchmarkAppendPyramid(1000000, 1000, t) } 543 func BenchmarkAppendPyramid_7(t *testing.B) { benchmarkAppendPyramid(10000000, 1000, t) } 544 func BenchmarkAppendPyramid_8(t *testing.B) { benchmarkAppendPyramid(100000000, 1000, t) } 545 546 // go test -timeout 20m -cpu 4 -bench=./swarm/storage -run no 547 // If you dont add the timeout argument above .. the benchmark will timeout and dump