github.com/coyove/sdss@v0.0.0-20231129015646-c2ec58cca6a2/contrib/bitmap/xorfilter.go (about)

     1  package bitmap
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/binary"
     6  	"unsafe"
     7  
     8  	"github.com/FastFilter/xorfilter"
     9  )
    10  
    11  func xfNew(data []uint64) []byte {
    12  	if len(data) == 0 {
    13  		panic("empty data")
    14  	}
    15  	p := &bytes.Buffer{}
    16  	if len(data) <= 12 {
    17  		binary.Write(p, binary.BigEndian, uint32(0))
    18  		tmp := make([]uint32, len(data))
    19  		for i := range tmp {
    20  			tmp[i] = uint32(data[i])
    21  		}
    22  
    23  		var buf []byte
    24  		*(*[3]int)(unsafe.Pointer(&buf)) = [3]int{
    25  			*(*int)(unsafe.Pointer(&tmp)),
    26  			len(tmp) * 4,
    27  			len(tmp) * 4,
    28  		}
    29  		p.Write(buf)
    30  		return p.Bytes()
    31  	}
    32  
    33  	data = append(data, data...)
    34  	for i, half := 0, len(data)/2; i < half; i++ {
    35  		data[half+i] = ^data[i]
    36  	}
    37  	x, err := xorfilter.Populate(data)
    38  	if err != nil {
    39  		panic(err)
    40  	}
    41  	binary.Write(p, binary.BigEndian, x.BlockLength) // 4b
    42  	binary.Write(p, binary.BigEndian, x.Seed)        // 8b
    43  	p.Write(x.Fingerprints)                          // bytes
    44  	return p.Bytes()
    45  }
    46  
    47  // Validness of 'data' is not checked.
    48  func xfBuild(data []byte) (xorfilter.Xor8, []uint32) {
    49  	x := xorfilter.Xor8{}
    50  	x.BlockLength = binary.BigEndian.Uint32(data[:4])
    51  	if x.BlockLength == 0 {
    52  		var values []uint32
    53  		l := len(data) - 4
    54  		*(*[3]int)(unsafe.Pointer(&values)) = [3]int{
    55  			int(uintptr(unsafe.Pointer(&data[4]))),
    56  			l / 4,
    57  			l / 4,
    58  		}
    59  		return x, values
    60  	}
    61  	x.Seed = binary.BigEndian.Uint64(data[4:12])
    62  	x.Fingerprints = data[12:]
    63  	return x, nil
    64  }
    65  
    66  func xfContains(x xorfilter.Xor8, vs []uint32, v uint64) bool {
    67  	if len(vs) == 0 {
    68  		return x.Contains(v) && x.Contains(^v)
    69  	}
    70  	for _, v0 := range vs {
    71  		if v0 == uint32(v) {
    72  			return true
    73  		}
    74  	}
    75  	return false
    76  }