github.com/etherbanking/go-etherbanking@v1.7.1-0.20181009210156-cf649bca5aba/swarm/storage/chunker_test.go (about)

     1  // Copyright 2016 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package storage
    18  
    19  import (
    20  	"bytes"
    21  	"crypto/rand"
    22  	"encoding/binary"
    23  	"fmt"
    24  	"io"
    25  	"runtime"
    26  	"sync"
    27  	"testing"
    28  	"time"
    29  )
    30  
    31  /*
    32  Tests TreeChunker by splitting and joining a random byte slice
    33  */
    34  
    35  type test interface {
    36  	Fatalf(string, ...interface{})
    37  	Logf(string, ...interface{})
    38  }
    39  
    40  type chunkerTester struct {
    41  	inputs map[uint64][]byte
    42  	chunks map[string]*Chunk
    43  	t      test
    44  }
    45  
    46  func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key) {
    47  	// reset
    48  	self.chunks = make(map[string]*Chunk)
    49  
    50  	if self.inputs == nil {
    51  		self.inputs = make(map[uint64][]byte)
    52  	}
    53  
    54  	quitC := make(chan bool)
    55  	timeout := time.After(600 * time.Second)
    56  	if chunkC != nil {
    57  		go func() {
    58  			for {
    59  				select {
    60  				case <-timeout:
    61  					self.t.Fatalf("Join timeout error")
    62  				case <-quitC:
    63  					return
    64  				case chunk := <-chunkC:
    65  					// self.chunks = append(self.chunks, chunk)
    66  					self.chunks[chunk.Key.String()] = chunk
    67  					if chunk.wg != nil {
    68  						chunk.wg.Done()
    69  					}
    70  				}
    71  			}
    72  		}()
    73  	}
    74  	key, err := chunker.Split(data, size, chunkC, swg, nil)
    75  	if err != nil && expectedError == nil {
    76  		self.t.Fatalf("Split error: %v", err)
    77  	} else if expectedError != nil && (err == nil || err.Error() != expectedError.Error()) {
    78  		self.t.Fatalf("Not receiving the correct error! Expected %v, received %v", expectedError, err)
    79  	}
    80  	if chunkC != nil {
    81  		if swg != nil {
    82  			swg.Wait()
    83  		}
    84  		close(quitC)
    85  	}
    86  	return
    87  }
    88  
    89  func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader {
    90  	// reset but not the chunks
    91  
    92  	reader := chunker.Join(key, chunkC)
    93  
    94  	timeout := time.After(600 * time.Second)
    95  	i := 0
    96  	go func() {
    97  		for {
    98  			select {
    99  			case <-timeout:
   100  				self.t.Fatalf("Join timeout error")
   101  
   102  			case chunk, ok := <-chunkC:
   103  				if !ok {
   104  					close(quitC)
   105  					return
   106  				}
   107  				// this just mocks the behaviour of a chunk store retrieval
   108  				stored, success := self.chunks[chunk.Key.String()]
   109  				if !success {
   110  					self.t.Fatalf("not found")
   111  					return
   112  				}
   113  				chunk.SData = stored.SData
   114  				chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8]))
   115  				close(chunk.C)
   116  				i++
   117  			}
   118  		}
   119  	}()
   120  	return reader
   121  }
   122  
   123  func testRandomBrokenData(splitter Splitter, n int, tester *chunkerTester) {
   124  	data := io.LimitReader(rand.Reader, int64(n))
   125  	brokendata := brokenLimitReader(data, n, n/2)
   126  
   127  	buf := make([]byte, n)
   128  	_, err := brokendata.Read(buf)
   129  	if err == nil || err.Error() != "Broken reader" {
   130  		tester.t.Fatalf("Broken reader is not broken, hence broken. Returns: %v", err)
   131  	}
   132  
   133  	data = io.LimitReader(rand.Reader, int64(n))
   134  	brokendata = brokenLimitReader(data, n, n/2)
   135  
   136  	chunkC := make(chan *Chunk, 1000)
   137  	swg := &sync.WaitGroup{}
   138  
   139  	key := tester.Split(splitter, brokendata, int64(n), chunkC, swg, fmt.Errorf("Broken reader"))
   140  	tester.t.Logf(" Key = %v\n", key)
   141  }
   142  
   143  func testRandomData(splitter Splitter, n int, tester *chunkerTester) {
   144  	if tester.inputs == nil {
   145  		tester.inputs = make(map[uint64][]byte)
   146  	}
   147  	input, found := tester.inputs[uint64(n)]
   148  	var data io.Reader
   149  	if !found {
   150  		data, input = testDataReaderAndSlice(n)
   151  		tester.inputs[uint64(n)] = input
   152  	} else {
   153  		data = io.LimitReader(bytes.NewReader(input), int64(n))
   154  	}
   155  
   156  	chunkC := make(chan *Chunk, 1000)
   157  	swg := &sync.WaitGroup{}
   158  
   159  	key := tester.Split(splitter, data, int64(n), chunkC, swg, nil)
   160  	tester.t.Logf(" Key = %v\n", key)
   161  
   162  	chunkC = make(chan *Chunk, 1000)
   163  	quitC := make(chan bool)
   164  
   165  	chunker := NewTreeChunker(NewChunkerParams())
   166  	reader := tester.Join(chunker, key, 0, chunkC, quitC)
   167  	output := make([]byte, n)
   168  	r, err := reader.Read(output)
   169  	if r != n || err != io.EOF {
   170  		tester.t.Fatalf("read error  read: %v  n = %v  err = %v\n", r, n, err)
   171  	}
   172  	if input != nil {
   173  		if !bytes.Equal(output, input) {
   174  			tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input, output)
   175  		}
   176  	}
   177  	close(chunkC)
   178  	<-quitC
   179  }
   180  
   181  func TestRandomData(t *testing.T) {
   182  	// sizes := []int{123456}
   183  	sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 123456, 2345678}
   184  	tester := &chunkerTester{t: t}
   185  	chunker := NewTreeChunker(NewChunkerParams())
   186  	for _, s := range sizes {
   187  		testRandomData(chunker, s, tester)
   188  	}
   189  	pyramid := NewPyramidChunker(NewChunkerParams())
   190  	for _, s := range sizes {
   191  		testRandomData(pyramid, s, tester)
   192  	}
   193  }
   194  
   195  func TestRandomBrokenData(t *testing.T) {
   196  	sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 123456, 2345678}
   197  	tester := &chunkerTester{t: t}
   198  	chunker := NewTreeChunker(NewChunkerParams())
   199  	for _, s := range sizes {
   200  		testRandomBrokenData(chunker, s, tester)
   201  		t.Logf("done size: %v", s)
   202  	}
   203  }
   204  
   205  func benchReadAll(reader LazySectionReader) {
   206  	size, _ := reader.Size(nil)
   207  	output := make([]byte, 1000)
   208  	for pos := int64(0); pos < size; pos += 1000 {
   209  		reader.ReadAt(output, pos)
   210  	}
   211  }
   212  
   213  func benchmarkJoin(n int, t *testing.B) {
   214  	t.ReportAllocs()
   215  	for i := 0; i < t.N; i++ {
   216  		chunker := NewTreeChunker(NewChunkerParams())
   217  		tester := &chunkerTester{t: t}
   218  		data := testDataReader(n)
   219  
   220  		chunkC := make(chan *Chunk, 1000)
   221  		swg := &sync.WaitGroup{}
   222  
   223  		key := tester.Split(chunker, data, int64(n), chunkC, swg, nil)
   224  		// t.StartTimer()
   225  		chunkC = make(chan *Chunk, 1000)
   226  		quitC := make(chan bool)
   227  		reader := tester.Join(chunker, key, i, chunkC, quitC)
   228  		benchReadAll(reader)
   229  		close(chunkC)
   230  		<-quitC
   231  		// t.StopTimer()
   232  	}
   233  	stats := new(runtime.MemStats)
   234  	runtime.ReadMemStats(stats)
   235  	fmt.Println(stats.Sys)
   236  }
   237  
   238  func benchmarkSplitTree(n int, t *testing.B) {
   239  	t.ReportAllocs()
   240  	for i := 0; i < t.N; i++ {
   241  		chunker := NewTreeChunker(NewChunkerParams())
   242  		tester := &chunkerTester{t: t}
   243  		data := testDataReader(n)
   244  		tester.Split(chunker, data, int64(n), nil, nil, nil)
   245  	}
   246  	stats := new(runtime.MemStats)
   247  	runtime.ReadMemStats(stats)
   248  	fmt.Println(stats.Sys)
   249  }
   250  
   251  func benchmarkSplitPyramid(n int, t *testing.B) {
   252  	t.ReportAllocs()
   253  	for i := 0; i < t.N; i++ {
   254  		splitter := NewPyramidChunker(NewChunkerParams())
   255  		tester := &chunkerTester{t: t}
   256  		data := testDataReader(n)
   257  		tester.Split(splitter, data, int64(n), nil, nil, nil)
   258  	}
   259  	stats := new(runtime.MemStats)
   260  	runtime.ReadMemStats(stats)
   261  	fmt.Println(stats.Sys)
   262  }
   263  
   264  func BenchmarkJoin_2(t *testing.B) { benchmarkJoin(100, t) }
   265  func BenchmarkJoin_3(t *testing.B) { benchmarkJoin(1000, t) }
   266  func BenchmarkJoin_4(t *testing.B) { benchmarkJoin(10000, t) }
   267  func BenchmarkJoin_5(t *testing.B) { benchmarkJoin(100000, t) }
   268  func BenchmarkJoin_6(t *testing.B) { benchmarkJoin(1000000, t) }
   269  func BenchmarkJoin_7(t *testing.B) { benchmarkJoin(10000000, t) }
   270  func BenchmarkJoin_8(t *testing.B) { benchmarkJoin(100000000, t) }
   271  
   272  func BenchmarkSplitTree_2(t *testing.B)  { benchmarkSplitTree(100, t) }
   273  func BenchmarkSplitTree_2h(t *testing.B) { benchmarkSplitTree(500, t) }
   274  func BenchmarkSplitTree_3(t *testing.B)  { benchmarkSplitTree(1000, t) }
   275  func BenchmarkSplitTree_3h(t *testing.B) { benchmarkSplitTree(5000, t) }
   276  func BenchmarkSplitTree_4(t *testing.B)  { benchmarkSplitTree(10000, t) }
   277  func BenchmarkSplitTree_4h(t *testing.B) { benchmarkSplitTree(50000, t) }
   278  func BenchmarkSplitTree_5(t *testing.B)  { benchmarkSplitTree(100000, t) }
   279  func BenchmarkSplitTree_6(t *testing.B)  { benchmarkSplitTree(1000000, t) }
   280  func BenchmarkSplitTree_7(t *testing.B)  { benchmarkSplitTree(10000000, t) }
   281  func BenchmarkSplitTree_8(t *testing.B)  { benchmarkSplitTree(100000000, t) }
   282  
   283  func BenchmarkSplitPyramid_2(t *testing.B)  { benchmarkSplitPyramid(100, t) }
   284  func BenchmarkSplitPyramid_2h(t *testing.B) { benchmarkSplitPyramid(500, t) }
   285  func BenchmarkSplitPyramid_3(t *testing.B)  { benchmarkSplitPyramid(1000, t) }
   286  func BenchmarkSplitPyramid_3h(t *testing.B) { benchmarkSplitPyramid(5000, t) }
   287  func BenchmarkSplitPyramid_4(t *testing.B)  { benchmarkSplitPyramid(10000, t) }
   288  func BenchmarkSplitPyramid_4h(t *testing.B) { benchmarkSplitPyramid(50000, t) }
   289  func BenchmarkSplitPyramid_5(t *testing.B)  { benchmarkSplitPyramid(100000, t) }
   290  func BenchmarkSplitPyramid_6(t *testing.B)  { benchmarkSplitPyramid(1000000, t) }
   291  func BenchmarkSplitPyramid_7(t *testing.B)  { benchmarkSplitPyramid(10000000, t) }
   292  func BenchmarkSplitPyramid_8(t *testing.B)  { benchmarkSplitPyramid(100000000, t) }
   293  
   294  // godep go test -bench ./swarm/storage -cpuprofile cpu.out -memprofile mem.out