github.com/linapex/ethereum-dpos-chinese@v0.0.0-20190316121959-b78b3a4a1ece/swarm/storage/chunker_test.go (about)

     1  
     2  //<developer>
     3  //    <name>linapex 曹一峰</name>
     4  //    <email>linapex@163.com</email>
     5  //    <wx>superexc</wx>
     6  //    <qqgroup>128148617</qqgroup>
     7  //    <url>https://jsq.ink</url>
     8  //    <role>pku engineer</role>
     9  //    <date>2019-03-16 12:09:49</date>
    10  //</624342680394010624>
    11  
    12  //
    13  //
    14  //
    15  //
    16  //
    17  //
    18  //
    19  //
    20  //
    21  //
    22  //
    23  //
    24  //
    25  //
    26  //
    27  
    28  package storage
    29  
    30  import (
    31  	"bytes"
    32  	"context"
    33  	"crypto/rand"
    34  	"encoding/binary"
    35  	"errors"
    36  	"fmt"
    37  	"io"
    38  	"testing"
    39  
    40  	"github.com/ethereum/go-ethereum/crypto/sha3"
    41  )
    42  
    43  /*
    44  
    45  */
    46  
    47  
    48  type test interface {
    49  	Fatalf(string, ...interface{})
    50  	Logf(string, ...interface{})
    51  }
    52  
    53  type chunkerTester struct {
    54  	inputs map[uint64][]byte
    55  	t      test
    56  }
    57  
    58  //
    59  //
    60  //
    61  type fakeChunkStore struct {
    62  }
    63  
    64  //
    65  func (f *fakeChunkStore) Put(context.Context, *Chunk) {
    66  }
    67  
    68  //
    69  func (f *fakeChunkStore) Get(context.Context, Address) (*Chunk, error) {
    70  	return nil, errors.New("FakeChunkStore doesn't support Get")
    71  }
    72  
    73  //
    74  func (f *fakeChunkStore) Close() {
    75  }
    76  
    77  func newTestHasherStore(chunkStore ChunkStore, hash string) *hasherStore {
    78  	return NewHasherStore(chunkStore, MakeHashFunc(hash), false)
    79  }
    80  
    81  func testRandomBrokenData(n int, tester *chunkerTester) {
    82  	data := io.LimitReader(rand.Reader, int64(n))
    83  	brokendata := brokenLimitReader(data, n, n/2)
    84  
    85  	buf := make([]byte, n)
    86  	_, err := brokendata.Read(buf)
    87  	if err == nil || err.Error() != "Broken reader" {
    88  		tester.t.Fatalf("Broken reader is not broken, hence broken. Returns: %v", err)
    89  	}
    90  
    91  	data = io.LimitReader(rand.Reader, int64(n))
    92  	brokendata = brokenLimitReader(data, n, n/2)
    93  
    94  	putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash)
    95  
    96  	expectedError := fmt.Errorf("Broken reader")
    97  	addr, _, err := TreeSplit(context.TODO(), brokendata, int64(n), putGetter)
    98  	if err == nil || err.Error() != expectedError.Error() {
    99  		tester.t.Fatalf("Not receiving the correct error! Expected %v, received %v", expectedError, err)
   100  	}
   101  	tester.t.Logf(" Key = %v\n", addr)
   102  }
   103  
   104  func testRandomData(usePyramid bool, hash string, n int, tester *chunkerTester) Address {
   105  	if tester.inputs == nil {
   106  		tester.inputs = make(map[uint64][]byte)
   107  	}
   108  	input, found := tester.inputs[uint64(n)]
   109  	var data io.Reader
   110  	if !found {
   111  		data, input = generateRandomData(n)
   112  		tester.inputs[uint64(n)] = input
   113  	} else {
   114  		data = io.LimitReader(bytes.NewReader(input), int64(n))
   115  	}
   116  
   117  	putGetter := newTestHasherStore(NewMapChunkStore(), hash)
   118  
   119  	var addr Address
   120  	var wait func(context.Context) error
   121  	var err error
   122  	ctx := context.TODO()
   123  	if usePyramid {
   124  		addr, wait, err = PyramidSplit(ctx, data, putGetter, putGetter)
   125  	} else {
   126  		addr, wait, err = TreeSplit(ctx, data, int64(n), putGetter)
   127  	}
   128  	if err != nil {
   129  		tester.t.Fatalf(err.Error())
   130  	}
   131  	tester.t.Logf(" Key = %v\n", addr)
   132  	err = wait(ctx)
   133  	if err != nil {
   134  		tester.t.Fatalf(err.Error())
   135  	}
   136  
   137  	reader := TreeJoin(context.TODO(), addr, putGetter, 0)
   138  	output := make([]byte, n)
   139  	r, err := reader.Read(output)
   140  	if r != n || err != io.EOF {
   141  		tester.t.Fatalf("read error  read: %v  n = %v  err = %v\n", r, n, err)
   142  	}
   143  	if input != nil {
   144  		if !bytes.Equal(output, input) {
   145  			tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input, output)
   146  		}
   147  	}
   148  
   149  //
   150  	for i := 1; i < n; i += 10000 {
   151  		readableLength := n - i
   152  		output := make([]byte, readableLength)
   153  		r, err := reader.ReadAt(output, int64(i))
   154  		if r != readableLength || err != io.EOF {
   155  			tester.t.Fatalf("readAt error with offset %v read: %v  n = %v  err = %v\n", i, r, readableLength, err)
   156  		}
   157  		if input != nil {
   158  			if !bytes.Equal(output, input[i:]) {
   159  				tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", input[i:], output)
   160  			}
   161  		}
   162  	}
   163  
   164  	return addr
   165  }
   166  
   167  func TestSha3ForCorrectness(t *testing.T) {
   168  	tester := &chunkerTester{t: t}
   169  
   170  	size := 4096
   171  	input := make([]byte, size+8)
   172  	binary.LittleEndian.PutUint64(input[:8], uint64(size))
   173  
   174  	io.LimitReader(bytes.NewReader(input[8:]), int64(size))
   175  
   176  	rawSha3 := sha3.NewKeccak256()
   177  	rawSha3.Reset()
   178  	rawSha3.Write(input)
   179  	rawSha3Output := rawSha3.Sum(nil)
   180  
   181  	sha3FromMakeFunc := MakeHashFunc(SHA3Hash)()
   182  	sha3FromMakeFunc.ResetWithLength(input[:8])
   183  	sha3FromMakeFunc.Write(input[8:])
   184  	sha3FromMakeFuncOutput := sha3FromMakeFunc.Sum(nil)
   185  
   186  	if len(rawSha3Output) != len(sha3FromMakeFuncOutput) {
   187  		tester.t.Fatalf("Original SHA3 and abstracted Sha3 has different length %v:%v\n", len(rawSha3Output), len(sha3FromMakeFuncOutput))
   188  	}
   189  
   190  	if !bytes.Equal(rawSha3Output, sha3FromMakeFuncOutput) {
   191  		tester.t.Fatalf("Original SHA3 and abstracted Sha3 mismatch %v:%v\n", rawSha3Output, sha3FromMakeFuncOutput)
   192  	}
   193  
   194  }
   195  
   196  func TestDataAppend(t *testing.T) {
   197  	sizes := []int{1, 1, 1, 4095, 4096, 4097, 1, 1, 1, 123456, 2345678, 2345678}
   198  	appendSizes := []int{4095, 4096, 4097, 1, 1, 1, 8191, 8192, 8193, 9000, 3000, 5000}
   199  
   200  	tester := &chunkerTester{t: t}
   201  	for i := range sizes {
   202  		n := sizes[i]
   203  		m := appendSizes[i]
   204  
   205  		if tester.inputs == nil {
   206  			tester.inputs = make(map[uint64][]byte)
   207  		}
   208  		input, found := tester.inputs[uint64(n)]
   209  		var data io.Reader
   210  		if !found {
   211  			data, input = generateRandomData(n)
   212  			tester.inputs[uint64(n)] = input
   213  		} else {
   214  			data = io.LimitReader(bytes.NewReader(input), int64(n))
   215  		}
   216  
   217  		chunkStore := NewMapChunkStore()
   218  		putGetter := newTestHasherStore(chunkStore, SHA3Hash)
   219  
   220  		ctx := context.TODO()
   221  		addr, wait, err := PyramidSplit(ctx, data, putGetter, putGetter)
   222  		if err != nil {
   223  			tester.t.Fatalf(err.Error())
   224  		}
   225  		err = wait(ctx)
   226  		if err != nil {
   227  			tester.t.Fatalf(err.Error())
   228  		}
   229  
   230  //
   231  		appendInput, found := tester.inputs[uint64(m)]
   232  		var appendData io.Reader
   233  		if !found {
   234  			appendData, appendInput = generateRandomData(m)
   235  			tester.inputs[uint64(m)] = appendInput
   236  		} else {
   237  			appendData = io.LimitReader(bytes.NewReader(appendInput), int64(m))
   238  		}
   239  
   240  		putGetter = newTestHasherStore(chunkStore, SHA3Hash)
   241  		newAddr, wait, err := PyramidAppend(ctx, addr, appendData, putGetter, putGetter)
   242  		if err != nil {
   243  			tester.t.Fatalf(err.Error())
   244  		}
   245  		err = wait(ctx)
   246  		if err != nil {
   247  			tester.t.Fatalf(err.Error())
   248  		}
   249  
   250  		reader := TreeJoin(ctx, newAddr, putGetter, 0)
   251  		newOutput := make([]byte, n+m)
   252  		r, err := reader.Read(newOutput)
   253  		if r != (n + m) {
   254  			tester.t.Fatalf("read error  read: %v  n = %v  m = %v  err = %v\n", r, n, m, err)
   255  		}
   256  
   257  		newInput := append(input, appendInput...)
   258  		if !bytes.Equal(newOutput, newInput) {
   259  			tester.t.Fatalf("input and output mismatch\n IN: %v\nOUT: %v\n", newInput, newOutput)
   260  		}
   261  	}
   262  }
   263  
   264  func TestRandomData(t *testing.T) {
   265  //
   266  //
   267  	sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 524288, 524288 + 1, 524288 + 4097, 7 * 524288, 7*524288 + 1, 7*524288 + 4097}
   268  	tester := &chunkerTester{t: t}
   269  
   270  	for _, s := range sizes {
   271  		treeChunkerKey := testRandomData(false, SHA3Hash, s, tester)
   272  		pyramidChunkerKey := testRandomData(true, SHA3Hash, s, tester)
   273  		if treeChunkerKey.String() != pyramidChunkerKey.String() {
   274  			tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerKey.String(), pyramidChunkerKey.String())
   275  		}
   276  	}
   277  
   278  	for _, s := range sizes {
   279  		treeChunkerKey := testRandomData(false, BMTHash, s, tester)
   280  		pyramidChunkerKey := testRandomData(true, BMTHash, s, tester)
   281  		if treeChunkerKey.String() != pyramidChunkerKey.String() {
   282  			tester.t.Fatalf("tree chunker and pyramid chunker key mismatch for size %v\n TC: %v\n PC: %v\n", s, treeChunkerKey.String(), pyramidChunkerKey.String())
   283  		}
   284  	}
   285  }
   286  
   287  func TestRandomBrokenData(t *testing.T) {
   288  	sizes := []int{1, 60, 83, 179, 253, 1024, 4095, 4096, 4097, 8191, 8192, 8193, 12287, 12288, 12289, 123456, 2345678}
   289  	tester := &chunkerTester{t: t}
   290  	for _, s := range sizes {
   291  		testRandomBrokenData(s, tester)
   292  	}
   293  }
   294  
   295  func benchReadAll(reader LazySectionReader) {
   296  	size, _ := reader.Size(context.TODO(), nil)
   297  	output := make([]byte, 1000)
   298  	for pos := int64(0); pos < size; pos += 1000 {
   299  		reader.ReadAt(output, pos)
   300  	}
   301  }
   302  
   303  func benchmarkSplitJoin(n int, t *testing.B) {
   304  	t.ReportAllocs()
   305  	for i := 0; i < t.N; i++ {
   306  		data := testDataReader(n)
   307  
   308  		putGetter := newTestHasherStore(NewMapChunkStore(), SHA3Hash)
   309  		ctx := context.TODO()
   310  		key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter)
   311  		if err != nil {
   312  			t.Fatalf(err.Error())
   313  		}
   314  		err = wait(ctx)
   315  		if err != nil {
   316  			t.Fatalf(err.Error())
   317  		}
   318  		reader := TreeJoin(ctx, key, putGetter, 0)
   319  		benchReadAll(reader)
   320  	}
   321  }
   322  
   323  func benchmarkSplitTreeSHA3(n int, t *testing.B) {
   324  	t.ReportAllocs()
   325  	for i := 0; i < t.N; i++ {
   326  		data := testDataReader(n)
   327  		putGetter := newTestHasherStore(&fakeChunkStore{}, SHA3Hash)
   328  
   329  		_, _, err := TreeSplit(context.TODO(), data, int64(n), putGetter)
   330  		if err != nil {
   331  			t.Fatalf(err.Error())
   332  		}
   333  	}
   334  }
   335  
   336  func benchmarkSplitTreeBMT(n int, t *testing.B) {
   337  	t.ReportAllocs()
   338  	for i := 0; i < t.N; i++ {
   339  		data := testDataReader(n)
   340  		putGetter := newTestHasherStore(&fakeChunkStore{}, BMTHash)
   341  
   342  		_, _, err := TreeSplit(context.TODO(), data, int64(n), putGetter)
   343  		if err != nil {
   344  			t.Fatalf(err.Error())
   345  		}
   346  	}
   347  }
   348  
   349  func benchmarkSplitPyramidSHA3(n int, t *testing.B) {
   350  	t.ReportAllocs()
   351  	for i := 0; i < t.N; i++ {
   352  		data := testDataReader(n)
   353  		putGetter := newTestHasherStore(&fakeChunkStore{}, SHA3Hash)
   354  
   355  		_, _, err := PyramidSplit(context.TODO(), data, putGetter, putGetter)
   356  		if err != nil {
   357  			t.Fatalf(err.Error())
   358  		}
   359  
   360  	}
   361  }
   362  
   363  func benchmarkSplitPyramidBMT(n int, t *testing.B) {
   364  	t.ReportAllocs()
   365  	for i := 0; i < t.N; i++ {
   366  		data := testDataReader(n)
   367  		putGetter := newTestHasherStore(&fakeChunkStore{}, BMTHash)
   368  
   369  		_, _, err := PyramidSplit(context.TODO(), data, putGetter, putGetter)
   370  		if err != nil {
   371  			t.Fatalf(err.Error())
   372  		}
   373  	}
   374  }
   375  
   376  func benchmarkSplitAppendPyramid(n, m int, t *testing.B) {
   377  	t.ReportAllocs()
   378  	for i := 0; i < t.N; i++ {
   379  		data := testDataReader(n)
   380  		data1 := testDataReader(m)
   381  
   382  		chunkStore := NewMapChunkStore()
   383  		putGetter := newTestHasherStore(chunkStore, SHA3Hash)
   384  
   385  		ctx := context.TODO()
   386  		key, wait, err := PyramidSplit(ctx, data, putGetter, putGetter)
   387  		if err != nil {
   388  			t.Fatalf(err.Error())
   389  		}
   390  		err = wait(ctx)
   391  		if err != nil {
   392  			t.Fatalf(err.Error())
   393  		}
   394  
   395  		putGetter = newTestHasherStore(chunkStore, SHA3Hash)
   396  		_, wait, err = PyramidAppend(ctx, key, data1, putGetter, putGetter)
   397  		if err != nil {
   398  			t.Fatalf(err.Error())
   399  		}
   400  		err = wait(ctx)
   401  		if err != nil {
   402  			t.Fatalf(err.Error())
   403  		}
   404  	}
   405  }
   406  
   407  func BenchmarkSplitJoin_2(t *testing.B) { benchmarkSplitJoin(100, t) }
   408  func BenchmarkSplitJoin_3(t *testing.B) { benchmarkSplitJoin(1000, t) }
   409  func BenchmarkSplitJoin_4(t *testing.B) { benchmarkSplitJoin(10000, t) }
   410  func BenchmarkSplitJoin_5(t *testing.B) { benchmarkSplitJoin(100000, t) }
   411  func BenchmarkSplitJoin_6(t *testing.B) { benchmarkSplitJoin(1000000, t) }
   412  func BenchmarkSplitJoin_7(t *testing.B) { benchmarkSplitJoin(10000000, t) }
   413  
   414  //
   415  
   416  func BenchmarkSplitTreeSHA3_2(t *testing.B)  { benchmarkSplitTreeSHA3(100, t) }
   417  func BenchmarkSplitTreeSHA3_2h(t *testing.B) { benchmarkSplitTreeSHA3(500, t) }
   418  func BenchmarkSplitTreeSHA3_3(t *testing.B)  { benchmarkSplitTreeSHA3(1000, t) }
   419  func BenchmarkSplitTreeSHA3_3h(t *testing.B) { benchmarkSplitTreeSHA3(5000, t) }
   420  func BenchmarkSplitTreeSHA3_4(t *testing.B)  { benchmarkSplitTreeSHA3(10000, t) }
   421  func BenchmarkSplitTreeSHA3_4h(t *testing.B) { benchmarkSplitTreeSHA3(50000, t) }
   422  func BenchmarkSplitTreeSHA3_5(t *testing.B)  { benchmarkSplitTreeSHA3(100000, t) }
   423  func BenchmarkSplitTreeSHA3_6(t *testing.B)  { benchmarkSplitTreeSHA3(1000000, t) }
   424  func BenchmarkSplitTreeSHA3_7(t *testing.B)  { benchmarkSplitTreeSHA3(10000000, t) }
   425  
   426  //
   427  
   428  func BenchmarkSplitTreeBMT_2(t *testing.B)  { benchmarkSplitTreeBMT(100, t) }
   429  func BenchmarkSplitTreeBMT_2h(t *testing.B) { benchmarkSplitTreeBMT(500, t) }
   430  func BenchmarkSplitTreeBMT_3(t *testing.B)  { benchmarkSplitTreeBMT(1000, t) }
   431  func BenchmarkSplitTreeBMT_3h(t *testing.B) { benchmarkSplitTreeBMT(5000, t) }
   432  func BenchmarkSplitTreeBMT_4(t *testing.B)  { benchmarkSplitTreeBMT(10000, t) }
   433  func BenchmarkSplitTreeBMT_4h(t *testing.B) { benchmarkSplitTreeBMT(50000, t) }
   434  func BenchmarkSplitTreeBMT_5(t *testing.B)  { benchmarkSplitTreeBMT(100000, t) }
   435  func BenchmarkSplitTreeBMT_6(t *testing.B)  { benchmarkSplitTreeBMT(1000000, t) }
   436  func BenchmarkSplitTreeBMT_7(t *testing.B)  { benchmarkSplitTreeBMT(10000000, t) }
   437  
   438  //
   439  
   440  func BenchmarkSplitPyramidSHA3_2(t *testing.B)  { benchmarkSplitPyramidSHA3(100, t) }
   441  func BenchmarkSplitPyramidSHA3_2h(t *testing.B) { benchmarkSplitPyramidSHA3(500, t) }
   442  func BenchmarkSplitPyramidSHA3_3(t *testing.B)  { benchmarkSplitPyramidSHA3(1000, t) }
   443  func BenchmarkSplitPyramidSHA3_3h(t *testing.B) { benchmarkSplitPyramidSHA3(5000, t) }
   444  func BenchmarkSplitPyramidSHA3_4(t *testing.B)  { benchmarkSplitPyramidSHA3(10000, t) }
   445  func BenchmarkSplitPyramidSHA3_4h(t *testing.B) { benchmarkSplitPyramidSHA3(50000, t) }
   446  func BenchmarkSplitPyramidSHA3_5(t *testing.B)  { benchmarkSplitPyramidSHA3(100000, t) }
   447  func BenchmarkSplitPyramidSHA3_6(t *testing.B)  { benchmarkSplitPyramidSHA3(1000000, t) }
   448  func BenchmarkSplitPyramidSHA3_7(t *testing.B)  { benchmarkSplitPyramidSHA3(10000000, t) }
   449  
   450  //
   451  
   452  func BenchmarkSplitPyramidBMT_2(t *testing.B)  { benchmarkSplitPyramidBMT(100, t) }
   453  func BenchmarkSplitPyramidBMT_2h(t *testing.B) { benchmarkSplitPyramidBMT(500, t) }
   454  func BenchmarkSplitPyramidBMT_3(t *testing.B)  { benchmarkSplitPyramidBMT(1000, t) }
   455  func BenchmarkSplitPyramidBMT_3h(t *testing.B) { benchmarkSplitPyramidBMT(5000, t) }
   456  func BenchmarkSplitPyramidBMT_4(t *testing.B)  { benchmarkSplitPyramidBMT(10000, t) }
   457  func BenchmarkSplitPyramidBMT_4h(t *testing.B) { benchmarkSplitPyramidBMT(50000, t) }
   458  func BenchmarkSplitPyramidBMT_5(t *testing.B)  { benchmarkSplitPyramidBMT(100000, t) }
   459  func BenchmarkSplitPyramidBMT_6(t *testing.B)  { benchmarkSplitPyramidBMT(1000000, t) }
   460  func BenchmarkSplitPyramidBMT_7(t *testing.B)  { benchmarkSplitPyramidBMT(10000000, t) }
   461  
   462  //
   463  
   464  func BenchmarkSplitAppendPyramid_2(t *testing.B)  { benchmarkSplitAppendPyramid(100, 1000, t) }
   465  func BenchmarkSplitAppendPyramid_2h(t *testing.B) { benchmarkSplitAppendPyramid(500, 1000, t) }
   466  func BenchmarkSplitAppendPyramid_3(t *testing.B)  { benchmarkSplitAppendPyramid(1000, 1000, t) }
   467  func BenchmarkSplitAppendPyramid_4(t *testing.B)  { benchmarkSplitAppendPyramid(10000, 1000, t) }
   468  func BenchmarkSplitAppendPyramid_4h(t *testing.B) { benchmarkSplitAppendPyramid(50000, 1000, t) }
   469  func BenchmarkSplitAppendPyramid_5(t *testing.B)  { benchmarkSplitAppendPyramid(1000000, 1000, t) }
   470  func BenchmarkSplitAppendPyramid_6(t *testing.B)  { benchmarkSplitAppendPyramid(1000000, 1000, t) }
   471  func BenchmarkSplitAppendPyramid_7(t *testing.B)  { benchmarkSplitAppendPyramid(10000000, 1000, t) }
   472  
   473  //
   474  
   475  //
   476  //
   477