github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/types/codec.go (about) 1 // Copyright 2019 Dolthub, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 // 15 // This file incorporates work covered by the following copyright and 16 // permission notice: 17 // 18 // Copyright 2016 Attic Labs, Inc. All rights reserved. 19 // Licensed under the Apache License, version 2.0: 20 // http://www.apache.org/licenses/LICENSE-2.0 21 22 package types 23 24 import ( 25 "encoding/binary" 26 "math" 27 "time" 28 "unsafe" 29 30 "github.com/google/uuid" 31 "github.com/shopspring/decimal" 32 33 "github.com/dolthub/dolt/go/store/chunks" 34 "github.com/dolthub/dolt/go/store/d" 35 "github.com/dolthub/dolt/go/store/hash" 36 ) 37 38 const initialBufferSize = 2048 39 40 func EncodeValue(v Value, nbf *NomsBinFormat) (chunks.Chunk, error) { 41 if v.Kind() == UnknownKind { 42 return chunks.EmptyChunk, ErrUnknownType 43 } 44 45 w := binaryNomsWriter{make([]byte, 4), 0} 46 err := v.writeTo(&w, nbf) 47 48 if err != nil { 49 return chunks.EmptyChunk, err 50 } 51 52 return chunks.NewChunk(w.data()), nil 53 } 54 55 func decodeFromBytes(data []byte, vrw ValueReadWriter) (Value, error) { 56 dec := newValueDecoder(data, vrw) 57 v, err := dec.readValue(vrw.Format()) 58 59 if err != nil { 60 return nil, err 61 } 62 63 d.PanicIfFalse(dec.pos() == uint32(len(data))) 64 return v, nil 65 } 66 67 func decodeFromBytesWithValidation(data []byte, vrw ValueReadWriter) (Value, error) { 68 r := binaryNomsReader{data, 0} 69 dec := newValueDecoderWithValidation(r, vrw) 70 v, err := dec.readValue(vrw.Format()) 71 72 if err != nil { 73 return nil, err 74 } 75 76 d.PanicIfFalse(dec.pos() == uint32(len(data))) 77 return v, nil 78 } 79 80 // DecodeValue decodes a value from a chunk source. It is an error to provide an empty chunk. 81 func DecodeValue(c chunks.Chunk, vrw ValueReadWriter) (Value, error) { 82 d.PanicIfTrue(c.IsEmpty()) 83 return decodeFromBytes(c.Data(), vrw) 84 } 85 86 type nomsWriter interface { 87 writeBool(b bool) 88 writeCount(count uint64) 89 writeHash(h hash.Hash) 90 writeFloat(v Float, nbf *NomsBinFormat) 91 writeInt(v Int) 92 writeUint(v Uint) 93 writeString(v string) 94 writeUint8(v uint8) 95 writeUint16(v uint16) 96 writeRaw(buff []byte) 97 } 98 99 type binaryNomsReader struct { 100 buff []byte 101 offset uint32 102 } 103 104 func (b *binaryNomsReader) readBytes(count uint32) []byte { 105 v := b.buff[b.offset : b.offset+count] 106 b.offset += count 107 return v 108 } 109 110 func (b *binaryNomsReader) readCopyOfBytes(count uint32) []byte { 111 v := make([]byte, count) 112 copy(v, b.buff[b.offset:b.offset+count]) 113 b.offset += count 114 return v 115 } 116 117 func (b *binaryNomsReader) skipBytes(count uint32) { 118 b.offset += count 119 } 120 121 func (b *binaryNomsReader) pos() uint32 { 122 return b.offset 123 } 124 125 func (b *binaryNomsReader) readUint8() uint8 { 126 v := uint8(b.buff[b.offset]) 127 b.offset++ 128 return v 129 } 130 131 func (b *binaryNomsReader) peekUint8() uint8 { 132 return uint8(b.buff[b.offset]) 133 } 134 135 func (b *binaryNomsReader) skipUint8() { 136 b.offset++ 137 } 138 139 func (b *binaryNomsReader) readUint16() uint16 { 140 v := binary.BigEndian.Uint16(b.buff[b.offset:]) 141 b.offset += 2 142 return v 143 } 144 145 func (b *binaryNomsReader) PeekKind() NomsKind { 146 return NomsKind(b.peekUint8()) 147 } 148 149 func (b *binaryNomsReader) ReadKind() NomsKind { 150 return NomsKind(b.readUint8()) 151 } 152 153 func (b *binaryNomsReader) skipKind() { 154 b.skipUint8() 155 } 156 157 func (b *binaryNomsReader) readCount() uint64 { 158 return b.ReadUint() 159 } 160 161 func (b *binaryNomsReader) skipCount() { 162 b.skipUint() 163 } 164 165 func (b *binaryNomsReader) ReadFloat(nbf *NomsBinFormat) float64 { 166 if isFormat_7_18(nbf) { 167 i := b.ReadInt() 168 exp := b.ReadInt() 169 return fracExpToFloat(i, int(exp)) 170 } else { 171 floatbits := binary.BigEndian.Uint64(b.readBytes(8)) 172 return math.Float64frombits(floatbits) 173 } 174 } 175 176 func (b *binaryNomsReader) ReadDecimal() (decimal.Decimal, error) { 177 size := uint32(b.readUint16()) 178 db := b.readBytes(size) 179 180 var dec decimal.Decimal 181 err := dec.GobDecode(db) 182 return dec, err 183 } 184 185 func (b *binaryNomsReader) ReadTimestamp() (time.Time, error) { 186 data := b.readBytes(timestampNumBytes) 187 188 var t time.Time 189 err := t.UnmarshalBinary(data) 190 return t, err 191 } 192 193 func (b *binaryNomsReader) skipFloat(nbf *NomsBinFormat) { 194 if isFormat_7_18(nbf) { 195 b.skipInt() 196 b.skipInt() 197 } else { 198 b.skipBytes(8) 199 } 200 } 201 202 func (b *binaryNomsReader) skipInt() { 203 maxOffset := b.offset + 10 204 for ; b.offset < maxOffset; b.offset++ { 205 if b.buff[b.offset]&0x80 == 0 { 206 b.offset++ 207 return 208 } 209 } 210 } 211 212 func (b *binaryNomsReader) ReadInt() int64 { 213 v, count := unrolledDecodeVarint(b.buff[b.offset:]) 214 b.offset += uint32(count) 215 return v 216 } 217 218 func (b *binaryNomsReader) ReadUint() uint64 { 219 v, count := unrolledDecodeUVarint(b.buff[b.offset:]) 220 b.offset += uint32(count) 221 return v 222 } 223 224 func unrolledDecodeUVarint(buf []byte) (uint64, int) { 225 b := uint64(buf[0]) 226 if b < 0x80 { 227 return b, 1 228 } 229 230 x := b & 0x7f 231 b = uint64(buf[1]) 232 if b < 0x80 { 233 return x | (b << 7), 2 234 } 235 236 x |= (b & 0x7f) << 7 237 b = uint64(buf[2]) 238 if b < 0x80 { 239 return x | (b << 14), 3 240 } 241 242 x |= (b & 0x7f) << 14 243 b = uint64(buf[3]) 244 if b < 0x80 { 245 return x | (b << 21), 4 246 } 247 248 x |= (b & 0x7f) << 21 249 b = uint64(buf[4]) 250 if b < 0x80 { 251 return x | (b << 28), 5 252 } 253 254 x |= (b & 0x7f) << 28 255 b = uint64(buf[5]) 256 if b < 0x80 { 257 return x | (b << 35), 6 258 } 259 260 x |= (b & 0x7f) << 35 261 b = uint64(buf[6]) 262 if b < 0x80 { 263 return x | (b << 42), 7 264 } 265 266 x |= (b & 0x7f) << 42 267 b = uint64(buf[7]) 268 if b < 0x80 { 269 return x | (b << 49), 8 270 } 271 272 x |= (b & 0x7f) << 49 273 b = uint64(buf[8]) 274 if b < 0x80 { 275 return x | (b << 56), 9 276 } 277 278 x |= (b & 0x7f) << 56 279 b = uint64(buf[9]) 280 if b == 1 { 281 return x | (1 << 63), 10 282 } 283 284 return 0, -10 285 } 286 287 func unrolledDecodeVarint(buf []byte) (int64, int) { 288 ux, n := unrolledDecodeUVarint(buf) // ok to continue in presence of error 289 x := int64(ux >> 1) 290 if ux&1 != 0 { 291 x = ^x 292 } 293 return x, n 294 } 295 296 func (b *binaryNomsReader) skipUint() { 297 maxOffset := b.offset + 10 298 for ; b.offset < maxOffset; b.offset++ { 299 if b.buff[b.offset]&0x80 == 0 { 300 b.offset++ 301 return 302 } 303 } 304 } 305 306 func (b *binaryNomsReader) ReadBool() bool { 307 return b.readUint8() == 1 308 } 309 310 func (b *binaryNomsReader) skipBool() { 311 b.skipUint8() 312 } 313 314 func (b *binaryNomsReader) ReadString() string { 315 size := uint32(b.readCount()) 316 strBytes := b.buff[b.offset : b.offset+size] 317 b.offset += size 318 return *(*string)(unsafe.Pointer(&strBytes)) 319 } 320 321 func (b *binaryNomsReader) ReadInlineBlob() []byte { 322 size := uint32(b.readUint16()) 323 bytes := b.buff[b.offset : b.offset+size] 324 b.offset += size 325 return bytes 326 } 327 328 func (b *binaryNomsReader) ReadUUID() uuid.UUID { 329 id := uuid.UUID{} 330 copy(id[:uuidNumBytes], b.readBytes(uuidNumBytes)) 331 return id 332 } 333 334 func (b *binaryNomsReader) skipString() { 335 size := uint32(b.readCount()) 336 b.offset += size 337 } 338 339 func (b *binaryNomsReader) readHash() hash.Hash { 340 h := hash.Hash{} 341 copy(h[:], b.buff[b.offset:b.offset+hash.ByteLen]) 342 b.offset += hash.ByteLen 343 return h 344 } 345 346 func (b *binaryNomsReader) skipHash() { 347 b.offset += hash.ByteLen 348 } 349 350 func (b *binaryNomsReader) byteSlice(start, end uint32) []byte { 351 return b.buff[start:end] 352 } 353 354 type binaryNomsWriter struct { 355 buff []byte 356 offset uint32 357 } 358 359 func newBinaryNomsWriterWithSizeHint(sizeHint uint64) binaryNomsWriter { 360 size := uint32(initialBufferSize) 361 if sizeHint >= math.MaxUint32 { 362 size = math.MaxUint32 363 } else if sizeHint > uint64(size) { 364 size = uint32(sizeHint) 365 } 366 367 return binaryNomsWriter{make([]byte, size), 0} 368 } 369 370 func newBinaryNomsWriter() binaryNomsWriter { 371 return binaryNomsWriter{make([]byte, initialBufferSize), 0} 372 } 373 374 func (b *binaryNomsWriter) data() []byte { 375 return b.buff[0:b.offset] 376 } 377 378 func (b *binaryNomsWriter) reset() { 379 b.offset = 0 380 } 381 382 const ( 383 GigsHalf = 1 << 29 384 Gigs2 = 1 << 31 385 ) 386 387 func (b *binaryNomsWriter) ensureCapacity(n uint32) { 388 length := uint64(len(b.buff)) 389 minLength := uint64(b.offset) + uint64(n) 390 if length >= minLength { 391 return 392 } 393 394 old := b.buff 395 396 if minLength > math.MaxUint32 { 397 panic("overflow") 398 } 399 400 for minLength > length { 401 length = length * 2 402 403 if length >= Gigs2 { 404 length = Gigs2 405 break 406 } 407 } 408 409 for minLength > length { 410 length += GigsHalf 411 412 if length >= math.MaxUint32 { 413 length = math.MaxUint32 414 break 415 } 416 } 417 418 b.buff = make([]byte, length) 419 copy(b.buff, old) 420 } 421 422 func (b *binaryNomsWriter) writeUint8(v uint8) { 423 b.ensureCapacity(1) 424 b.buff[b.offset] = byte(v) 425 b.offset++ 426 } 427 428 func (b *binaryNomsWriter) writeCount(v uint64) { 429 b.ensureCapacity(binary.MaxVarintLen64) 430 count := binary.PutUvarint(b.buff[b.offset:], v) 431 b.offset += uint32(count) 432 } 433 434 func (b *binaryNomsWriter) writeInt(v Int) { 435 b.ensureCapacity(binary.MaxVarintLen64) 436 count := binary.PutVarint(b.buff[b.offset:], int64(v)) 437 b.offset += uint32(count) 438 } 439 440 func (b *binaryNomsWriter) writeUint(v Uint) { 441 b.ensureCapacity(binary.MaxVarintLen64) 442 count := binary.PutUvarint(b.buff[b.offset:], uint64(v)) 443 b.offset += uint32(count) 444 } 445 446 func (b *binaryNomsWriter) writeUint16(v uint16) { 447 b.ensureCapacity(2) 448 binary.BigEndian.PutUint16(b.buff[b.offset:], v) 449 b.offset += 2 450 } 451 452 func (b *binaryNomsWriter) writeFloat(v Float, nbf *NomsBinFormat) { 453 if isFormat_7_18(nbf) { 454 b.ensureCapacity(binary.MaxVarintLen64 * 2) 455 i, exp := float64ToIntExp(float64(v)) 456 count := binary.PutVarint(b.buff[b.offset:], i) 457 b.offset += uint32(count) 458 count = binary.PutVarint(b.buff[b.offset:], int64(exp)) 459 b.offset += uint32(count) 460 } else { 461 b.ensureCapacity(8) 462 binary.BigEndian.PutUint64(b.buff[b.offset:], math.Float64bits(float64(v))) 463 b.offset += 8 464 } 465 } 466 467 func (b *binaryNomsWriter) writeBool(v bool) { 468 if v { 469 b.writeUint8(uint8(1)) 470 } else { 471 b.writeUint8(uint8(0)) 472 } 473 } 474 475 func (b *binaryNomsWriter) writeString(v string) { 476 size := uint32(len(v)) 477 b.writeCount(uint64(size)) 478 479 b.ensureCapacity(size) 480 copy(b.buff[b.offset:], v) 481 b.offset += size 482 } 483 484 func (b *binaryNomsWriter) writeHash(h hash.Hash) { 485 b.ensureCapacity(hash.ByteLen) 486 copy(b.buff[b.offset:], h[:]) 487 b.offset += hash.ByteLen 488 } 489 490 func (b *binaryNomsWriter) writeRaw(buff []byte) { 491 size := uint32(len(buff)) 492 b.ensureCapacity(size) 493 copy(b.buff[b.offset:], buff) 494 b.offset += size 495 }