github.com/sabhiram/gsync@v0.0.0-20180214150410-b9089a1b7f13/gsync_test.go (about)

     1  // This Source Code Form is subject to the terms of the Mozilla Public
     2  // License, version 2.0. If a copy of the MPL was not distributed with this
     3  // file, You can obtain one at http://mozilla.org/MPL/2.0/.
     4  
     5  package gsync
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"crypto/md5"
    11  	"fmt"
    12  	"hash"
    13  	"io"
    14  	"io/ioutil"
    15  	"math/rand"
    16  	"testing"
    17  	"time"
    18  
    19  	"github.com/hooklift/assert"
    20  	"github.com/pkg/profile"
    21  )
    22  
    23  // TestRollingHash tests that incrementally calculated signatures arrive to the same
    24  // value as the full block signature.
    25  func TestRollingHash(t *testing.T) {
    26  	_, _, target := rollingHash([]byte("abcd"))          // file's content in server
    27  	reader := bytes.NewReader([]byte("aaabcdbbabcdddf")) // new file's content in client
    28  
    29  	var (
    30  		r1, r2, r, old uint32
    31  		offset         int64
    32  		rolling        bool
    33  	)
    34  
    35  	delta := make([]byte, 0)
    36  	for {
    37  		buffer := make([]byte, 4) // block size of 4
    38  		n, err := reader.ReadAt(buffer, offset)
    39  
    40  		block := buffer[:n]
    41  		if rolling {
    42  			new := uint32(block[n-1])
    43  			r1, r2, r = rollingHash2(uint32(n), r1, r2, old, new)
    44  		} else {
    45  			r1, r2, r = rollingHash(block)
    46  		}
    47  
    48  		if r == target {
    49  			if err == io.EOF {
    50  				break
    51  			}
    52  
    53  			rolling = false
    54  			old, r, r1, r2 = 0, 0, 0, 0
    55  			offset += int64(n)
    56  		} else {
    57  			if err == io.EOF {
    58  				// If EOF is reached and not matched data is found, we need to add trailing data
    59  				// to delta array.
    60  				delta = append(delta, block...)
    61  				break
    62  			}
    63  
    64  			rolling = true
    65  			old = uint32(block[0])
    66  			delta = append(delta, block[0])
    67  			offset++
    68  		}
    69  
    70  		assert.Ok(t, err)
    71  	}
    72  
    73  	assert.Equals(t, []byte("aabbddf"), delta)
    74  }
    75  
    76  var alpha = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789\n"
    77  
    78  // srand generates a random string of fixed size.
    79  func srand(seed int64, size int) []byte {
    80  	buf := make([]byte, size)
    81  	rand.Seed(seed)
    82  	for i := 0; i < size; i++ {
    83  		buf[i] = alpha[rand.Intn(len(alpha))]
    84  	}
    85  	return buf
    86  }
    87  
    88  func TestSync(t *testing.T) {
    89  	defer profile.Start().Stop()
    90  	tests := []struct {
    91  		desc   string
    92  		source []byte
    93  		cache  []byte
    94  		h      hash.Hash
    95  	}{
    96  		{
    97  			"full sync, no cache, 2mb file",
    98  			srand(10, (2*1024)*1024),
    99  			nil,
   100  			md5.New(),
   101  		},
   102  		{
   103  			"partial sync, 1mb cached, 2mb file",
   104  			srand(20, (2*1024)*1024),
   105  			srand(20, (1*1024)*1024),
   106  			md5.New(),
   107  		},
   108  	}
   109  
   110  	for _, tt := range tests {
   111  		t.Run(tt.desc, func(t *testing.T) {
   112  			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
   113  			defer cancel()
   114  
   115  			if len(tt.cache) > 0 {
   116  				assert.Equals(t, tt.source[:len(tt.cache)], tt.cache)
   117  			}
   118  
   119  			fmt.Print("Signatures... ")
   120  			sigsCh, err := Signatures(ctx, bytes.NewReader(tt.cache), tt.h)
   121  			assert.Ok(t, err)
   122  			fmt.Println("done")
   123  
   124  			fmt.Print("LookUpTable... ")
   125  			cacheSigs, err := LookUpTable(ctx, sigsCh)
   126  			assert.Ok(t, err)
   127  			fmt.Printf("%d blocks found in cache. done\n", len(cacheSigs))
   128  
   129  			fmt.Print("Sync... ")
   130  			opsCh, err := Sync(ctx, bytes.NewReader(tt.source), tt.h, cacheSigs)
   131  			assert.Ok(t, err)
   132  			fmt.Println("done")
   133  
   134  			fmt.Print("Apply... ")
   135  			target := new(bytes.Buffer)
   136  			err = Apply(ctx, target, bytes.NewReader(tt.cache), opsCh)
   137  			assert.Ok(t, err)
   138  			fmt.Println("done")
   139  
   140  			assert.Cond(t, target.Len() != 0, "target file should not be empty")
   141  			if !bytes.Equal(tt.source, target.Bytes()) {
   142  				ioutil.WriteFile("source.txt", tt.source, 0640)
   143  				ioutil.WriteFile("cache.txt", tt.cache, 0640)
   144  				ioutil.WriteFile("target.txt", target.Bytes(), 0640)
   145  			}
   146  			assert.Cond(t, bytes.Equal(tt.source, target.Bytes()), "source and target files are different")
   147  		})
   148  	}
   149  }
   150  
   151  func Benchmark6kbBlockSize(b *testing.B)    {}
   152  func Benchmark128kbBlockSize(b *testing.B)  {}
   153  func Benchmark512kbBlockSize(b *testing.B)  {}
   154  func Benchmark1024kbBlockSize(b *testing.B) {}
   155  
   156  func BenchmarkMD5(b *testing.B)     {}
   157  func BenchmarkSHA256(b *testing.B)  {}
   158  func BenchmarkSHA512(b *testing.B)  {}
   159  func BenchmarkMurmur3(b *testing.B) {}
   160  func BenchmarkXXHash(b *testing.B)  {}