github.com/andy2046/gopie@v0.7.0/pkg/bloom/bloomscale.go (about)

     1  package bloom
     2  
     3  import (
     4  	"math"
     5  
     6  	"github.com/andy2046/bitmap"
     7  )
     8  
     9  type (
    10  	scalableBloomFilter struct {
    11  		filterz   []*bloomFilterBit // bloom filters list
    12  		count     uint64            // number of elements in the bloom filter
    13  		n         uint64            // estimated number of elements
    14  		p         float64           // target False Positive rate
    15  		r         float64           // optimal tightening ratio
    16  		fillRatio float64           // fill ratio
    17  	}
    18  )
    19  
    20  const (
    21  	rDefault  float64 = 0.8
    22  	fillRatio float64 = 0.5
    23  )
    24  
    25  // NewS creates scalable bloom filter based on the provided fpRate.
    26  // fpRate is the target False Positive probability.
    27  func NewS(fpRate float64) Bloom {
    28  	return NewSGuess(10000, fpRate, rDefault)
    29  }
    30  
    31  // NewSGuess estimates m/k based on the provided n/p then creates scalable bloom filter.
    32  // n is the estimated number of elements in the bloom filter.
    33  // p is the false positive probability.
    34  // r is the optimal tightening ratio.
    35  func NewSGuess(n uint64, p, r float64) Bloom {
    36  	m, k := Guess(n, p)
    37  	mm, exponent := adjustM(m)
    38  
    39  	sBF := scalableBloomFilter{
    40  		filterz:   make([]*bloomFilterBit, 0, 1),
    41  		r:         r,
    42  		fillRatio: fillRatio,
    43  		p:         p,
    44  		n:         n,
    45  	}
    46  
    47  	sBF.filterz = append(sBF.filterz, &bloomFilterBit{
    48  		bitmap: bitmap.New(mm),
    49  		m:      mm - 1, // x % 2^i = x & (2^i - 1)
    50  		k:      k,
    51  		shift:  64 - exponent,
    52  	})
    53  	return &sBF
    54  }
    55  
    56  func (bf *scalableBloomFilter) Add(entry []byte) {
    57  	idx := len(bf.filterz) - 1
    58  	if bf.filterz[idx].estimatedFillRatio() >= bf.fillRatio {
    59  		fp := bf.p * math.Pow(bf.r, float64(len(bf.filterz)))
    60  		m, k := Guess(bf.n, fp)
    61  		mm, exponent := adjustM(m)
    62  		bf.filterz = append(bf.filterz, &bloomFilterBit{
    63  			bitmap: bitmap.New(mm),
    64  			m:      mm - 1, // x % 2^i = x & (2^i - 1)
    65  			k:      k,
    66  			shift:  64 - exponent,
    67  		})
    68  		idx++
    69  	}
    70  	bf.filterz[idx].Add(entry)
    71  	bf.count++
    72  }
    73  
    74  func (bf *scalableBloomFilter) AddString(entry string) {
    75  	bf.Add([]byte(entry))
    76  }
    77  
    78  func (bf *scalableBloomFilter) Exist(entry []byte) bool {
    79  	for _, f := range bf.filterz {
    80  		if f.Exist(entry) {
    81  			return true
    82  		}
    83  	}
    84  	return false
    85  }
    86  
    87  func (bf *scalableBloomFilter) ExistString(entry string) bool {
    88  	return bf.Exist([]byte(entry))
    89  }
    90  
    91  func (bf *scalableBloomFilter) FalsePositive() float64 {
    92  	rez := 1.0
    93  	for _, f := range bf.filterz {
    94  		rez *= (1.0 - f.FalsePositive())
    95  	}
    96  	return 1.0 - rez
    97  }
    98  
    99  func (bf *scalableBloomFilter) GuessFalsePositive(n uint64) float64 {
   100  	rez := 1.0
   101  	for _, f := range bf.filterz {
   102  		rez *= (1.0 - f.GuessFalsePositive(n))
   103  	}
   104  	return 1.0 - rez
   105  }
   106  
   107  func (bf *scalableBloomFilter) M() uint64 {
   108  	m := uint64(0)
   109  	for _, f := range bf.filterz {
   110  		m += f.M()
   111  	}
   112  	return m
   113  }
   114  
   115  func (bf *scalableBloomFilter) K() uint64 {
   116  	return bf.filterz[0].K()
   117  }
   118  
   119  func (bf *scalableBloomFilter) N() uint64 {
   120  	return bf.count
   121  }
   122  
   123  func (bf *scalableBloomFilter) Clear() {
   124  	for i := range bf.filterz {
   125  		bf.filterz[i] = nil
   126  	}
   127  	bf.filterz = make([]*bloomFilterBit, 0, 1)
   128  	m, k := Guess(bf.n, bf.p)
   129  	mm, exponent := adjustM(m)
   130  	bf.filterz = append(bf.filterz, &bloomFilterBit{
   131  		bitmap: bitmap.New(mm),
   132  		m:      mm - 1, // x % 2^i = x & (2^i - 1)
   133  		k:      k,
   134  		shift:  64 - exponent,
   135  	})
   136  	bf.count = 0
   137  }