github.com/etherbanking/go-etherbanking@v1.7.1-0.20181009210156-cf649bca5aba/swarm/storage/chunker_test.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package storage 18 19 import ( 20 "bytes" 21 "crypto/rand" 22 "encoding/binary" 23 "fmt" 24 "io" 25 "runtime" 26 "sync" 27 "testing" 28 "time" 29 ) 30 31 /* 32 Tests TreeChunker by splitting and joining a random byte slice 33 */ 34 35 type test interface { 36 Fatalf(string, ...interface{}) 37 Logf(string, ...interface{}) 38 } 39 40 type chunkerTester struct { 41 inputs map[uint64][]byte 42 chunks map[string]*Chunk 43 t test 44 } 45 46 func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, chunkC chan *Chunk, swg *sync.WaitGroup, expectedError error) (key Key) { 47 // reset 48 self.chunks = make(map[string]*Chunk) 49 50 if self.inputs == nil { 51 self.inputs = make(map[uint64][]byte) 52 } 53 54 quitC := make(chan bool) 55 timeout := time.After(600 * time.Second) 56 if chunkC != nil { 57 go func() { 58 for { 59 select { 60 case <-timeout: 61 self.t.Fatalf("Join timeout error") 62 case <-quitC: 63 return 64 case chunk := <-chunkC: 65 // self.chunks = append(self.chunks, chunk) 66 self.chunks[chunk.Key.String()] = chunk 67 if chunk.wg != nil { 68 chunk.wg.Done() 69 } 70 } 71 } 72 }() 73 } 74 key, err := chunker.Split(data, size, chunkC, swg, nil) 75 if err != nil && expectedError == nil { 76 self.t.Fatalf("Split error: %v", err) 77 } else if expectedError != nil && (err == nil || err.Error() != expectedError.Error()) { 78 self.t.Fatalf("Not receiving the correct error! Expected %v, received %v", expectedError, err) 79 } 80 if chunkC != nil { 81 if swg != nil { 82 swg.Wait() 83 } 84 close(quitC) 85 } 86 return 87 } 88 89 func (self *chunkerTester) Join(chunker Chunker, key Key, c int, chunkC chan *Chunk, quitC chan bool) LazySectionReader { 90 // reset but not the chunks 91 92 reader := chunker.Join(key, chunkC) 93 94 timeout := time.After(600 * time.Second) 95 i := 0 96 go func() { 97 for { 98 select { 99 case <-timeout: 100 self.t.Fatalf("Join timeout error") 101 102 case chunk, ok := <-chunkC: 103 if !ok { 104 close(quitC) 105 return 106 } 107 // this just mocks the behaviour of a chunk store retrieval 108 stored, success := self.chunks[chunk.Key.String()] 109 if !success { 110 self.t.Fatalf("not found") 111 return 112 } 113 chunk.SData = stored.SData 114 chunk.Size = int64(binary.LittleEndian.Uint64(chunk.SData[0:8])) 115 close(chunk.C) 116 i++ 117 } 118 } 119 }() 120 return reader 121 } 122 123 func testRandomBrokenData(splitter Splitter, n int, tester *chunkerTester) { 124 data := io.LimitReader(rand.Reader, int64(n)) 125 brokendata := brokenLimitReader(data, n, n/2) 126 127 buf := make([]byte, n) 128 _, err := brokendata.Read(buf) 129 if err == nil || err.Error() != "Broken reader" { 130 tester.t.Fatalf("Broken reader is not broken, hence broken. Returns: %v", err) 131 } 132 133 data = io.LimitReader(rand.Reader, int64(n)) 134 brokendata = brokenLimitReader(data, n, n/2) 135 136 chunkC := make(chan *Chunk, 1000) 137 swg := &sync.WaitGroup{} 138 139 key := tester.Split(splitter, brokendata, int64(n), chunkC, swg, fmt.Errorf("Broken reader")) 140 tester.t.Logf(" Key = %v\n", key) 141 } 142 143 func testRandomData(splitter Splitter, n int, tester *chunkerTester) { 144 if tester.inputs == nil { 145 tester.inputs = make(map[uint64][]byte) 146 } 147 input, found := tester.inputs[uint64(n)] 148 var data io.Reader 149 if !found { 150 data, input = testDataReaderAndSlice(n) 151 tester.inputs[uint64(n)] = input 152 } else { 153 data = io.LimitReader(bytes.NewReader(input), int64(n)) 154 } 155 156 chunkC := make(chan *Chunk, 1000) 157 swg := &sync.WaitGroup{} 158 159 key := tester.Split(splitter, data, int64(n), chunkC, swg, nil) 160 tester.t.Logf(" Key = %v\n", key) 161 162 chunkC = make(chan *Chunk, 1000) 163 quitC := make(chan bool) 164 165 chunker := NewTreeChunker(NewChunkerParams()) 166 reader := tester.Join(chunker, key, 0, chunkC, quitC) 167 output := make([]byte, n) 168 r, err := reader.Read(output) 169 if r != n || err != io.EOF { 170 tester.t.Fatalf("read error read: %v n = %v err = %v\n", r, n, err) 171 } 172 if input != nil { 173 if !bytes.Equal(output, input) { 174 tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input, output) 175 } 176 } 177 close(chunkC) 178 <-quitC 179 } 180 181 func TestRandomData(t *testing.T) { 182 // sizes := []int{123456} 183 sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 123456, 2345678} 184 tester := &chunkerTester{t: t} 185 chunker := NewTreeChunker(NewChunkerParams()) 186 for _, s := range sizes { 187 testRandomData(chunker, s, tester) 188 } 189 pyramid := NewPyramidChunker(NewChunkerParams()) 190 for _, s := range sizes { 191 testRandomData(pyramid, s, tester) 192 } 193 } 194 195 func TestRandomBrokenData(t *testing.T) { 196 sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 123456, 2345678} 197 tester := &chunkerTester{t: t} 198 chunker := NewTreeChunker(NewChunkerParams()) 199 for _, s := range sizes { 200 testRandomBrokenData(chunker, s, tester) 201 t.Logf("done size: %v", s) 202 } 203 } 204 205 func benchReadAll(reader LazySectionReader) { 206 size, _ := reader.Size(nil) 207 output := make([]byte, 1000) 208 for pos := int64(0); pos < size; pos += 1000 { 209 reader.ReadAt(output, pos) 210 } 211 } 212 213 func benchmarkJoin(n int, t *testing.B) { 214 t.ReportAllocs() 215 for i := 0; i < t.N; i++ { 216 chunker := NewTreeChunker(NewChunkerParams()) 217 tester := &chunkerTester{t: t} 218 data := testDataReader(n) 219 220 chunkC := make(chan *Chunk, 1000) 221 swg := &sync.WaitGroup{} 222 223 key := tester.Split(chunker, data, int64(n), chunkC, swg, nil) 224 // t.StartTimer() 225 chunkC = make(chan *Chunk, 1000) 226 quitC := make(chan bool) 227 reader := tester.Join(chunker, key, i, chunkC, quitC) 228 benchReadAll(reader) 229 close(chunkC) 230 <-quitC 231 // t.StopTimer() 232 } 233 stats := new(runtime.MemStats) 234 runtime.ReadMemStats(stats) 235 fmt.Println(stats.Sys) 236 } 237 238 func benchmarkSplitTree(n int, t *testing.B) { 239 t.ReportAllocs() 240 for i := 0; i < t.N; i++ { 241 chunker := NewTreeChunker(NewChunkerParams()) 242 tester := &chunkerTester{t: t} 243 data := testDataReader(n) 244 tester.Split(chunker, data, int64(n), nil, nil, nil) 245 } 246 stats := new(runtime.MemStats) 247 runtime.ReadMemStats(stats) 248 fmt.Println(stats.Sys) 249 } 250 251 func benchmarkSplitPyramid(n int, t *testing.B) { 252 t.ReportAllocs() 253 for i := 0; i < t.N; i++ { 254 splitter := NewPyramidChunker(NewChunkerParams()) 255 tester := &chunkerTester{t: t} 256 data := testDataReader(n) 257 tester.Split(splitter, data, int64(n), nil, nil, nil) 258 } 259 stats := new(runtime.MemStats) 260 runtime.ReadMemStats(stats) 261 fmt.Println(stats.Sys) 262 } 263 264 func BenchmarkJoin_2(t *testing.B) { benchmarkJoin(100, t) } 265 func BenchmarkJoin_3(t *testing.B) { benchmarkJoin(1000, t) } 266 func BenchmarkJoin_4(t *testing.B) { benchmarkJoin(10000, t) } 267 func BenchmarkJoin_5(t *testing.B) { benchmarkJoin(100000, t) } 268 func BenchmarkJoin_6(t *testing.B) { benchmarkJoin(1000000, t) } 269 func BenchmarkJoin_7(t *testing.B) { benchmarkJoin(10000000, t) } 270 func BenchmarkJoin_8(t *testing.B) { benchmarkJoin(100000000, t) } 271 272 func BenchmarkSplitTree_2(t *testing.B) { benchmarkSplitTree(100, t) } 273 func BenchmarkSplitTree_2h(t *testing.B) { benchmarkSplitTree(500, t) } 274 func BenchmarkSplitTree_3(t *testing.B) { benchmarkSplitTree(1000, t) } 275 func BenchmarkSplitTree_3h(t *testing.B) { benchmarkSplitTree(5000, t) } 276 func BenchmarkSplitTree_4(t *testing.B) { benchmarkSplitTree(10000, t) } 277 func BenchmarkSplitTree_4h(t *testing.B) { benchmarkSplitTree(50000, t) } 278 func BenchmarkSplitTree_5(t *testing.B) { benchmarkSplitTree(100000, t) } 279 func BenchmarkSplitTree_6(t *testing.B) { benchmarkSplitTree(1000000, t) } 280 func BenchmarkSplitTree_7(t *testing.B) { benchmarkSplitTree(10000000, t) } 281 func BenchmarkSplitTree_8(t *testing.B) { benchmarkSplitTree(100000000, t) } 282 283 func BenchmarkSplitPyramid_2(t *testing.B) { benchmarkSplitPyramid(100, t) } 284 func BenchmarkSplitPyramid_2h(t *testing.B) { benchmarkSplitPyramid(500, t) } 285 func BenchmarkSplitPyramid_3(t *testing.B) { benchmarkSplitPyramid(1000, t) } 286 func BenchmarkSplitPyramid_3h(t *testing.B) { benchmarkSplitPyramid(5000, t) } 287 func BenchmarkSplitPyramid_4(t *testing.B) { benchmarkSplitPyramid(10000, t) } 288 func BenchmarkSplitPyramid_4h(t *testing.B) { benchmarkSplitPyramid(50000, t) } 289 func BenchmarkSplitPyramid_5(t *testing.B) { benchmarkSplitPyramid(100000, t) } 290 func BenchmarkSplitPyramid_6(t *testing.B) { benchmarkSplitPyramid(1000000, t) } 291 func BenchmarkSplitPyramid_7(t *testing.B) { benchmarkSplitPyramid(10000000, t) } 292 func BenchmarkSplitPyramid_8(t *testing.B) { benchmarkSplitPyramid(100000000, t) } 293 294 // godep go test -bench ./swarm/storage -cpuprofile cpu.out -memprofile mem.out