github.com/benoitkugler/goacve@v0.0.0-20201217100549-151ce6e55dc8/server/core/datamodel/base_hash.go

package datamodel

import (
	"bytes"
	"compress/gzip"
	"encoding/binary"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"math"
)

// BaseHash stores, for each table (links included), a hash of each item (identified by id)
type BaseHash map[string]map[int64]uint32

// NewBaseHash décompresse les hashs, reçus au format renvoyé
// par `Compress`.
func NewBaseHash(headerJSON []byte, compressed []byte) (BaseHash, error) {
	gr, err := gzip.NewReader(bytes.NewReader(compressed))
	if err != nil {
		return nil, err
	}
	decompressed, err := ioutil.ReadAll(gr)
	if err != nil {
		return nil, err
	}

	var header map[string][2]int

	err = json.Unmarshal(headerJSON, &header)
	if err != nil {
		return nil, err
	}

	out := make(BaseHash, len(header))
	for name, offsets := range header {
		tmp, err := decodeCompressHashes(decompressed[offsets[0] : offsets[0]+offsets[1]])
		if err != nil {
			return nil, err
		}
		out[name] = tmp.decompress()
		if err != nil {
			return nil, err
		}
	}
	return out, nil
}

// HashDiff stocke les éléments qui ont changé sur le serveur:
//	- les éléments nouveaux ou modifiés
//	- les éléments supprimés
type HashDiff struct {
	Modified map[string][]int64
	Deleted  map[string][]int64
}

// Total renvoie le nombre de modifications et de suppressions, cumulé sur
// toutes les tables.
func (h HashDiff) Total() (int, int) {
	m, d := 0, 0
	for _, table := range h.Modified {
		m += len(table)
	}
	for _, table := range h.Deleted {
		d += len(table)
	}
	return m, d
}

// Compare compare le hash envoyé par le serveur à celui présent en local.
func (current BaseHash) Compare(incomming BaseHash) HashDiff {
	out := HashDiff{
		Modified: make(map[string][]int64, len(current)),
		Deleted:  make(map[string][]int64, len(current)),
	}

	for name := range incomming {
		currentTableHash, incomingTableHash := current[name], incomming[name]
		var diff []int64
		for id, h := range incomingTableHash {
			currentHash, has := currentTableHash[id]
			if !has { // l'item est nouveau
				diff = append(diff, id)
			} else if h != currentHash { // l'item a changé
				diff = append(diff, id)
			}
		}
		out.Modified[name] = diff
	}

	for name := range current {
		currentTableHash, incomingTableHash := current[name], incomming[name]
		var toSupp []int64
		for id := range currentTableHash {
			if _, in := incomingTableHash[id]; !in { // l'item n'existe plus, il pourra être supprimé
				toSupp = append(toSupp, id)
			}
		}
		out.Deleted[name] = toSupp
	}
	return out
}

// Compress écrit les hash sous forme binaire, compact. Le header renvoyé
// est nécessaire pour décrypter les données binaires.
func (b BaseHash) Compress() (headerJSON []byte, bin []byte, err error) {
	header, compact := b.binary()

	var out bytes.Buffer
	gzOut := gzip.NewWriter(&out)
	if err != nil {
		return nil, nil, err
	}
	_, err = io.Copy(gzOut, bytes.NewReader(compact))
	if err != nil {
		return nil, nil, err
	}
	err = gzOut.Close()
	if err != nil {
		return nil, nil, err
	}

	headerJSON, err = json.Marshal(header)
	return headerJSON, out.Bytes(), err
}

type compressHashes struct {
	First   int64
	Hashes  []uint32
	Encoded []byte
}

func decodeCompressHashes(buf []byte) (compressHashes, error) {
	if len(buf) < 8 {
		return compressHashes{}, errors.New("version binaire des hash invalide")
	}
	first := int64(binary.BigEndian.Uint64(buf))
	buf = buf[8:]
	if len(buf)%5 != 0 {
		return compressHashes{}, errors.New("version binaire des hash invalide")
	}
	L := len(buf) / 5
	out := compressHashes{
		First:   first,
		Hashes:  make([]uint32, L),
		Encoded: make([]byte, L),
	}
	for i := range out.Hashes {
		out.Encoded[i] = buf[5*i]
		out.Hashes[i] = binary.BigEndian.Uint32(buf[5*i+1:])
	}
	return out, nil
}

func newCompressHashes(hashes map[int64]uint32) compressHashes {
	if len(hashes) == 0 {
		return compressHashes{}
	}
	max, first := int64(0), int64(math.MaxInt64)
	for id := range hashes {
		if id > max {
			max = id
		}
		if id < first {
			first = id
		}
	}
	out := compressHashes{
		First:   first,
		Hashes:  make([]uint32, (max - first + 1)),
		Encoded: make([]byte, (max - first + 1)),
	}

	for id, hash := range hashes {
		out.Hashes[id-first] = hash
		out.Encoded[id-first] = 1
	}
	return out
}

// suppose the len(c.Encoded) == len(c.Hashes)
func (c compressHashes) decompress() map[int64]uint32 {
	out := make(map[int64]uint32)
	for i, e := range c.Encoded {
		if e == 1 {
			out[c.First+int64(i)] = c.Hashes[i]
		}
	}
	return out
}

func (c compressHashes) binary() []byte {
	out := make([]byte, 5*len(c.Encoded)+8)
	binary.BigEndian.PutUint64(out, uint64(c.First))
	data := out[8:]
	for i, e := range c.Encoded {
		data[5*i] = e
		binary.BigEndian.PutUint32(data[5*i+1:], c.Hashes[i])
	}
	return out
}

func (b BaseHash) binary() (map[string][2]int, []byte) {
	var (
		header = make(map[string][2]int)
		out    []byte
	)
	currentOffset := 0
	for name, m := range b {
		data := newCompressHashes(m).binary()
		out = append(out, data...)
		header[name] = [2]int{currentOffset, len(data)}
		currentOffset += len(data)
	}
	return header, out
}