github.com/keltia/go-ipfs@v0.3.8-0.20150909044612-210793031c63/importer/chunk/rabin_test.go (about)

     1  package chunk
     2  
     3  import (
     4  	"bytes"
     5  	"fmt"
     6  	"github.com/ipfs/go-ipfs/blocks"
     7  	"github.com/ipfs/go-ipfs/blocks/key"
     8  	"github.com/ipfs/go-ipfs/util"
     9  	"io"
    10  	"testing"
    11  )
    12  
    13  func TestRabinChunking(t *testing.T) {
    14  	data := make([]byte, 1024*1024*16)
    15  	util.NewTimeSeededRand().Read(data)
    16  
    17  	r := NewRabin(bytes.NewReader(data), 1024*256)
    18  
    19  	var chunks [][]byte
    20  
    21  	for {
    22  		chunk, err := r.NextBytes()
    23  		if err != nil {
    24  			if err == io.EOF {
    25  				break
    26  			}
    27  			t.Fatal(err)
    28  		}
    29  
    30  		chunks = append(chunks, chunk)
    31  	}
    32  
    33  	fmt.Printf("average block size: %d\n", len(data)/len(chunks))
    34  
    35  	unchunked := bytes.Join(chunks, nil)
    36  	if !bytes.Equal(unchunked, data) {
    37  		fmt.Printf("%d %d\n", len(unchunked), len(data))
    38  		t.Fatal("data was chunked incorrectly")
    39  	}
    40  }
    41  
    42  func chunkData(t *testing.T, data []byte) map[key.Key]*blocks.Block {
    43  	r := NewRabin(bytes.NewReader(data), 1024*256)
    44  
    45  	blkmap := make(map[key.Key]*blocks.Block)
    46  
    47  	for {
    48  		blk, err := r.NextBytes()
    49  		if err != nil {
    50  			if err == io.EOF {
    51  				break
    52  			}
    53  			t.Fatal(err)
    54  		}
    55  
    56  		b := blocks.NewBlock(blk)
    57  		blkmap[b.Key()] = b
    58  	}
    59  
    60  	return blkmap
    61  }
    62  
    63  func TestRabinChunkReuse(t *testing.T) {
    64  	data := make([]byte, 1024*1024*16)
    65  	util.NewTimeSeededRand().Read(data)
    66  
    67  	ch1 := chunkData(t, data[1000:])
    68  	ch2 := chunkData(t, data)
    69  
    70  	var extra int
    71  	for k, _ := range ch2 {
    72  		_, ok := ch1[k]
    73  		if !ok {
    74  			extra++
    75  		}
    76  	}
    77  
    78  	if extra > 2 {
    79  		t.Log("too many spare chunks made")
    80  	}
    81  }