github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/types/codec.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  // This file incorporates work covered by the following copyright and
    16  // permission notice:
    17  //
    18  // Copyright 2016 Attic Labs, Inc. All rights reserved.
    19  // Licensed under the Apache License, version 2.0:
    20  // http://www.apache.org/licenses/LICENSE-2.0
    21  
    22  package types
    23  
    24  import (
    25  	"encoding/binary"
    26  	"math"
    27  	"time"
    28  	"unsafe"
    29  
    30  	"github.com/google/uuid"
    31  	"github.com/shopspring/decimal"
    32  
    33  	"github.com/dolthub/dolt/go/store/chunks"
    34  	"github.com/dolthub/dolt/go/store/d"
    35  	"github.com/dolthub/dolt/go/store/hash"
    36  )
    37  
    38  const initialBufferSize = 2048
    39  
    40  func EncodeValue(v Value, nbf *NomsBinFormat) (chunks.Chunk, error) {
    41  	if v.Kind() == UnknownKind {
    42  		return chunks.EmptyChunk, ErrUnknownType
    43  	}
    44  
    45  	w := binaryNomsWriter{make([]byte, 4), 0}
    46  	err := v.writeTo(&w, nbf)
    47  
    48  	if err != nil {
    49  		return chunks.EmptyChunk, err
    50  	}
    51  
    52  	return chunks.NewChunk(w.data()), nil
    53  }
    54  
    55  func decodeFromBytes(data []byte, vrw ValueReadWriter) (Value, error) {
    56  	dec := newValueDecoder(data, vrw)
    57  	v, err := dec.readValue(vrw.Format())
    58  
    59  	if err != nil {
    60  		return nil, err
    61  	}
    62  
    63  	d.PanicIfFalse(dec.pos() == uint32(len(data)))
    64  	return v, nil
    65  }
    66  
    67  func decodeFromBytesWithValidation(data []byte, vrw ValueReadWriter) (Value, error) {
    68  	r := binaryNomsReader{data, 0}
    69  	dec := newValueDecoderWithValidation(r, vrw)
    70  	v, err := dec.readValue(vrw.Format())
    71  
    72  	if err != nil {
    73  		return nil, err
    74  	}
    75  
    76  	d.PanicIfFalse(dec.pos() == uint32(len(data)))
    77  	return v, nil
    78  }
    79  
    80  // DecodeValue decodes a value from a chunk source. It is an error to provide an empty chunk.
    81  func DecodeValue(c chunks.Chunk, vrw ValueReadWriter) (Value, error) {
    82  	d.PanicIfTrue(c.IsEmpty())
    83  	return decodeFromBytes(c.Data(), vrw)
    84  }
    85  
    86  type nomsWriter interface {
    87  	writeBool(b bool)
    88  	writeCount(count uint64)
    89  	writeHash(h hash.Hash)
    90  	writeFloat(v Float, nbf *NomsBinFormat)
    91  	writeInt(v Int)
    92  	writeUint(v Uint)
    93  	writeString(v string)
    94  	writeUint8(v uint8)
    95  	writeUint16(v uint16)
    96  	writeRaw(buff []byte)
    97  }
    98  
    99  type binaryNomsReader struct {
   100  	buff   []byte
   101  	offset uint32
   102  }
   103  
   104  func (b *binaryNomsReader) readBytes(count uint32) []byte {
   105  	v := b.buff[b.offset : b.offset+count]
   106  	b.offset += count
   107  	return v
   108  }
   109  
   110  func (b *binaryNomsReader) readCopyOfBytes(count uint32) []byte {
   111  	v := make([]byte, count)
   112  	copy(v, b.buff[b.offset:b.offset+count])
   113  	b.offset += count
   114  	return v
   115  }
   116  
   117  func (b *binaryNomsReader) skipBytes(count uint32) {
   118  	b.offset += count
   119  }
   120  
   121  func (b *binaryNomsReader) pos() uint32 {
   122  	return b.offset
   123  }
   124  
   125  func (b *binaryNomsReader) readUint8() uint8 {
   126  	v := uint8(b.buff[b.offset])
   127  	b.offset++
   128  	return v
   129  }
   130  
   131  func (b *binaryNomsReader) peekUint8() uint8 {
   132  	return uint8(b.buff[b.offset])
   133  }
   134  
   135  func (b *binaryNomsReader) skipUint8() {
   136  	b.offset++
   137  }
   138  
   139  func (b *binaryNomsReader) readUint16() uint16 {
   140  	v := binary.BigEndian.Uint16(b.buff[b.offset:])
   141  	b.offset += 2
   142  	return v
   143  }
   144  
   145  func (b *binaryNomsReader) PeekKind() NomsKind {
   146  	return NomsKind(b.peekUint8())
   147  }
   148  
   149  func (b *binaryNomsReader) ReadKind() NomsKind {
   150  	return NomsKind(b.readUint8())
   151  }
   152  
   153  func (b *binaryNomsReader) skipKind() {
   154  	b.skipUint8()
   155  }
   156  
   157  func (b *binaryNomsReader) readCount() uint64 {
   158  	return b.ReadUint()
   159  }
   160  
   161  func (b *binaryNomsReader) skipCount() {
   162  	b.skipUint()
   163  }
   164  
   165  func (b *binaryNomsReader) ReadFloat(nbf *NomsBinFormat) float64 {
   166  	if isFormat_7_18(nbf) {
   167  		i := b.ReadInt()
   168  		exp := b.ReadInt()
   169  		return fracExpToFloat(i, int(exp))
   170  	} else {
   171  		floatbits := binary.BigEndian.Uint64(b.readBytes(8))
   172  		return math.Float64frombits(floatbits)
   173  	}
   174  }
   175  
   176  func (b *binaryNomsReader) ReadDecimal() (decimal.Decimal, error) {
   177  	size := uint32(b.readUint16())
   178  	db := b.readBytes(size)
   179  
   180  	var dec decimal.Decimal
   181  	err := dec.GobDecode(db)
   182  	return dec, err
   183  }
   184  
   185  func (b *binaryNomsReader) ReadTimestamp() (time.Time, error) {
   186  	data := b.readBytes(timestampNumBytes)
   187  
   188  	var t time.Time
   189  	err := t.UnmarshalBinary(data)
   190  	return t, err
   191  }
   192  
   193  func (b *binaryNomsReader) skipFloat(nbf *NomsBinFormat) {
   194  	if isFormat_7_18(nbf) {
   195  		b.skipInt()
   196  		b.skipInt()
   197  	} else {
   198  		b.skipBytes(8)
   199  	}
   200  }
   201  
   202  func (b *binaryNomsReader) skipInt() {
   203  	maxOffset := b.offset + 10
   204  	for ; b.offset < maxOffset; b.offset++ {
   205  		if b.buff[b.offset]&0x80 == 0 {
   206  			b.offset++
   207  			return
   208  		}
   209  	}
   210  }
   211  
   212  func (b *binaryNomsReader) ReadInt() int64 {
   213  	v, count := unrolledDecodeVarint(b.buff[b.offset:])
   214  	b.offset += uint32(count)
   215  	return v
   216  }
   217  
   218  func (b *binaryNomsReader) ReadUint() uint64 {
   219  	v, count := unrolledDecodeUVarint(b.buff[b.offset:])
   220  	b.offset += uint32(count)
   221  	return v
   222  }
   223  
   224  func unrolledDecodeUVarint(buf []byte) (uint64, int) {
   225  	b := uint64(buf[0])
   226  	if b < 0x80 {
   227  		return b, 1
   228  	}
   229  
   230  	x := b & 0x7f
   231  	b = uint64(buf[1])
   232  	if b < 0x80 {
   233  		return x | (b << 7), 2
   234  	}
   235  
   236  	x |= (b & 0x7f) << 7
   237  	b = uint64(buf[2])
   238  	if b < 0x80 {
   239  		return x | (b << 14), 3
   240  	}
   241  
   242  	x |= (b & 0x7f) << 14
   243  	b = uint64(buf[3])
   244  	if b < 0x80 {
   245  		return x | (b << 21), 4
   246  	}
   247  
   248  	x |= (b & 0x7f) << 21
   249  	b = uint64(buf[4])
   250  	if b < 0x80 {
   251  		return x | (b << 28), 5
   252  	}
   253  
   254  	x |= (b & 0x7f) << 28
   255  	b = uint64(buf[5])
   256  	if b < 0x80 {
   257  		return x | (b << 35), 6
   258  	}
   259  
   260  	x |= (b & 0x7f) << 35
   261  	b = uint64(buf[6])
   262  	if b < 0x80 {
   263  		return x | (b << 42), 7
   264  	}
   265  
   266  	x |= (b & 0x7f) << 42
   267  	b = uint64(buf[7])
   268  	if b < 0x80 {
   269  		return x | (b << 49), 8
   270  	}
   271  
   272  	x |= (b & 0x7f) << 49
   273  	b = uint64(buf[8])
   274  	if b < 0x80 {
   275  		return x | (b << 56), 9
   276  	}
   277  
   278  	x |= (b & 0x7f) << 56
   279  	b = uint64(buf[9])
   280  	if b == 1 {
   281  		return x | (1 << 63), 10
   282  	}
   283  
   284  	return 0, -10
   285  }
   286  
   287  func unrolledDecodeVarint(buf []byte) (int64, int) {
   288  	ux, n := unrolledDecodeUVarint(buf) // ok to continue in presence of error
   289  	x := int64(ux >> 1)
   290  	if ux&1 != 0 {
   291  		x = ^x
   292  	}
   293  	return x, n
   294  }
   295  
   296  func (b *binaryNomsReader) skipUint() {
   297  	maxOffset := b.offset + 10
   298  	for ; b.offset < maxOffset; b.offset++ {
   299  		if b.buff[b.offset]&0x80 == 0 {
   300  			b.offset++
   301  			return
   302  		}
   303  	}
   304  }
   305  
   306  func (b *binaryNomsReader) ReadBool() bool {
   307  	return b.readUint8() == 1
   308  }
   309  
   310  func (b *binaryNomsReader) skipBool() {
   311  	b.skipUint8()
   312  }
   313  
   314  func (b *binaryNomsReader) ReadString() string {
   315  	size := uint32(b.readCount())
   316  	strBytes := b.buff[b.offset : b.offset+size]
   317  	b.offset += size
   318  	return *(*string)(unsafe.Pointer(&strBytes))
   319  }
   320  
   321  func (b *binaryNomsReader) ReadInlineBlob() []byte {
   322  	size := uint32(b.readUint16())
   323  	bytes := b.buff[b.offset : b.offset+size]
   324  	b.offset += size
   325  	return bytes
   326  }
   327  
   328  func (b *binaryNomsReader) ReadUUID() uuid.UUID {
   329  	id := uuid.UUID{}
   330  	copy(id[:uuidNumBytes], b.readBytes(uuidNumBytes))
   331  	return id
   332  }
   333  
   334  func (b *binaryNomsReader) skipString() {
   335  	size := uint32(b.readCount())
   336  	b.offset += size
   337  }
   338  
   339  func (b *binaryNomsReader) readHash() hash.Hash {
   340  	h := hash.Hash{}
   341  	copy(h[:], b.buff[b.offset:b.offset+hash.ByteLen])
   342  	b.offset += hash.ByteLen
   343  	return h
   344  }
   345  
   346  func (b *binaryNomsReader) skipHash() {
   347  	b.offset += hash.ByteLen
   348  }
   349  
   350  func (b *binaryNomsReader) byteSlice(start, end uint32) []byte {
   351  	return b.buff[start:end]
   352  }
   353  
   354  type binaryNomsWriter struct {
   355  	buff   []byte
   356  	offset uint32
   357  }
   358  
   359  func newBinaryNomsWriterWithSizeHint(sizeHint uint64) binaryNomsWriter {
   360  	size := uint32(initialBufferSize)
   361  	if sizeHint >= math.MaxUint32 {
   362  		size = math.MaxUint32
   363  	} else if sizeHint > uint64(size) {
   364  		size = uint32(sizeHint)
   365  	}
   366  
   367  	return binaryNomsWriter{make([]byte, size), 0}
   368  }
   369  
   370  func newBinaryNomsWriter() binaryNomsWriter {
   371  	return binaryNomsWriter{make([]byte, initialBufferSize), 0}
   372  }
   373  
   374  func (b *binaryNomsWriter) data() []byte {
   375  	return b.buff[0:b.offset]
   376  }
   377  
   378  func (b *binaryNomsWriter) reset() {
   379  	b.offset = 0
   380  }
   381  
   382  const (
   383  	GigsHalf = 1 << 29
   384  	Gigs2    = 1 << 31
   385  )
   386  
   387  func (b *binaryNomsWriter) ensureCapacity(n uint32) {
   388  	length := uint64(len(b.buff))
   389  	minLength := uint64(b.offset) + uint64(n)
   390  	if length >= minLength {
   391  		return
   392  	}
   393  
   394  	old := b.buff
   395  
   396  	if minLength > math.MaxUint32 {
   397  		panic("overflow")
   398  	}
   399  
   400  	for minLength > length {
   401  		length = length * 2
   402  
   403  		if length >= Gigs2 {
   404  			length = Gigs2
   405  			break
   406  		}
   407  	}
   408  
   409  	for minLength > length {
   410  		length += GigsHalf
   411  
   412  		if length >= math.MaxUint32 {
   413  			length = math.MaxUint32
   414  			break
   415  		}
   416  	}
   417  
   418  	b.buff = make([]byte, length)
   419  	copy(b.buff, old)
   420  }
   421  
   422  func (b *binaryNomsWriter) writeUint8(v uint8) {
   423  	b.ensureCapacity(1)
   424  	b.buff[b.offset] = byte(v)
   425  	b.offset++
   426  }
   427  
   428  func (b *binaryNomsWriter) writeCount(v uint64) {
   429  	b.ensureCapacity(binary.MaxVarintLen64)
   430  	count := binary.PutUvarint(b.buff[b.offset:], v)
   431  	b.offset += uint32(count)
   432  }
   433  
   434  func (b *binaryNomsWriter) writeInt(v Int) {
   435  	b.ensureCapacity(binary.MaxVarintLen64)
   436  	count := binary.PutVarint(b.buff[b.offset:], int64(v))
   437  	b.offset += uint32(count)
   438  }
   439  
   440  func (b *binaryNomsWriter) writeUint(v Uint) {
   441  	b.ensureCapacity(binary.MaxVarintLen64)
   442  	count := binary.PutUvarint(b.buff[b.offset:], uint64(v))
   443  	b.offset += uint32(count)
   444  }
   445  
   446  func (b *binaryNomsWriter) writeUint16(v uint16) {
   447  	b.ensureCapacity(2)
   448  	binary.BigEndian.PutUint16(b.buff[b.offset:], v)
   449  	b.offset += 2
   450  }
   451  
   452  func (b *binaryNomsWriter) writeFloat(v Float, nbf *NomsBinFormat) {
   453  	if isFormat_7_18(nbf) {
   454  		b.ensureCapacity(binary.MaxVarintLen64 * 2)
   455  		i, exp := float64ToIntExp(float64(v))
   456  		count := binary.PutVarint(b.buff[b.offset:], i)
   457  		b.offset += uint32(count)
   458  		count = binary.PutVarint(b.buff[b.offset:], int64(exp))
   459  		b.offset += uint32(count)
   460  	} else {
   461  		b.ensureCapacity(8)
   462  		binary.BigEndian.PutUint64(b.buff[b.offset:], math.Float64bits(float64(v)))
   463  		b.offset += 8
   464  	}
   465  }
   466  
   467  func (b *binaryNomsWriter) writeBool(v bool) {
   468  	if v {
   469  		b.writeUint8(uint8(1))
   470  	} else {
   471  		b.writeUint8(uint8(0))
   472  	}
   473  }
   474  
   475  func (b *binaryNomsWriter) writeString(v string) {
   476  	size := uint32(len(v))
   477  	b.writeCount(uint64(size))
   478  
   479  	b.ensureCapacity(size)
   480  	copy(b.buff[b.offset:], v)
   481  	b.offset += size
   482  }
   483  
   484  func (b *binaryNomsWriter) writeHash(h hash.Hash) {
   485  	b.ensureCapacity(hash.ByteLen)
   486  	copy(b.buff[b.offset:], h[:])
   487  	b.offset += hash.ByteLen
   488  }
   489  
   490  func (b *binaryNomsWriter) writeRaw(buff []byte) {
   491  	size := uint32(len(buff))
   492  	b.ensureCapacity(size)
   493  	copy(b.buff[b.offset:], buff)
   494  	b.offset += size
   495  }