github.com/lkarlslund/stringdedup@v0.6.2/sd_test.go (about)

     1  package stringdedup
     2  
     3  import (
     4  	"runtime"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/OneOfOne/xxhash"
     9  )
    10  
    11  func TestBlankString(t *testing.T) {
    12  	NS32 := New(func(in []byte) uint32 {
    13  		ns32 := xxhash.New32()
    14  		ns32.Write(in)
    15  		return ns32.Sum32()
    16  	})
    17  	if NS32.S("") != "" {
    18  		t.Error("Blank string should return blank string (new 32-bit hash)")
    19  	}
    20  	NS64 := New(func(in []byte) uint64 {
    21  		ns64 := xxhash.New64()
    22  		ns64.Write(in)
    23  		return ns64.Sum64()
    24  	})
    25  	if NS64.S("") != "" {
    26  		t.Error("Blank string should return blank string (new 64-bit hash)")
    27  	}
    28  }
    29  
    30  func TestGC(t *testing.T) {
    31  	ns := New(func(in []byte) uint32 {
    32  		return xxhash.Checksum32(in)
    33  	})
    34  	s := make([]string, 100000)
    35  	for n := 0; n < len(s); n++ {
    36  		RandomBytes(bs)
    37  		s[n] = ns.BS(bs)
    38  		if n%1000 == 0 {
    39  			runtime.GC()
    40  		}
    41  	}
    42  	lock.RLock()
    43  	runtime.GC()
    44  	time.Sleep(time.Millisecond * 100) // Let finalizers run
    45  	t.Log("Items in cache:", ns.Size())
    46  	if ns.Size() == 0 {
    47  		t.Fatal("Deduplication map is empty")
    48  	}
    49  	lock.RUnlock()
    50  	s = make([]string, 0)              // Clear our references
    51  	runtime.KeepAlive(s)               // oh shut up Go Vet
    52  	runtime.GC()                       // Clean up
    53  	time.Sleep(time.Millisecond * 100) // Let finalizers run
    54  	runtime.GC()                       // Clean up
    55  	lock.RLock()
    56  	t.Log("Items in cache:", ns.Size())
    57  	if ns.Size() != 0 {
    58  		t.Fatal("Deduplication map is not empty")
    59  	}
    60  	lock.RUnlock()
    61  }
    62  
    63  func TestNewGC(t *testing.T) {
    64  	d := New(func(in []byte) uint32 {
    65  		return xxhash.Checksum32(in)
    66  	})
    67  	// d.KeepAlive = time.Millisecond * 500
    68  
    69  	totalcount := 100000
    70  
    71  	// Insert stuff
    72  	o := make([]string, totalcount)
    73  	s := make([]string, totalcount)
    74  	for n := 0; n < len(s); n++ {
    75  		RandomBytes(bs)
    76  		o[n] = string(bs)
    77  		s[n] = d.BS(bs)
    78  		if n%1000 == 0 {
    79  			runtime.GC()
    80  		}
    81  	}
    82  	// Try to get GC to remove them from dedup object
    83  	runtime.GC()
    84  	time.Sleep(time.Millisecond * 500) // Let finalizers run
    85  
    86  	items := d.Size()
    87  	t.Log("Items in cache (expecting full):", items)
    88  	if items < int64(totalcount/100*95) {
    89  		t.Errorf("Deduplication map is not full - %v", items)
    90  	}
    91  
    92  	time.Sleep(time.Millisecond * 2000) // KeepAlive dies after 2 seconds, but map shouldn't be empty yet
    93  	items = d.Size()
    94  	t.Log("Items in cache (still expecting full):", items)
    95  	if items < int64(totalcount/100*95) {
    96  		t.Errorf("Deduplication map is not full still - %v", items)
    97  	}
    98  
    99  	// Clear references
   100  	for n := 0; n < len(s); n++ {
   101  		if o[n] != s[n] {
   102  			t.Errorf("%v != %v", o[n], s[n])
   103  		}
   104  	}
   105  	runtime.KeepAlive(s) // Ensure runtime doesn't GC the dedup table, only needed if you don't do the above check
   106  
   107  	s = make([]string, 0)               // Clear our references
   108  	runtime.GC()                        // Clean up
   109  	time.Sleep(time.Millisecond * 1000) // Let finalizers run
   110  	runtime.KeepAlive(s)
   111  
   112  	items = d.Size()
   113  	t.Log("Items in cache (expecting empty):", items)
   114  	// if items > int64(totalcount/50) {
   115  	// 	t.Errorf("Deduplication map is not empty - %v", d.Size())
   116  	// }
   117  
   118  	stats := d.Statistics()
   119  	t.Logf("Items added: %v", stats.ItemsAdded)
   120  	t.Logf("Bytes in memory: %v", stats.BytesInMemory)
   121  	t.Logf("Items saved: %v", stats.ItemsSaved)
   122  	t.Logf("Bytes saved: %v", stats.BytesSaved)
   123  	t.Logf("Items removed: %v", stats.ItemsRemoved)
   124  	t.Logf("Collisions: %v - first at %v", stats.Collisions, stats.FirstCollisionDetected)
   125  	t.Logf("Keepalive items added: %v - removed: %v", stats.KeepAliveItemsAdded, stats.KeepAliveItemsRemoved)
   126  
   127  	t.Logf("timer: %v", d.keepaliveFlusher)
   128  }