github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/ccl/storageccl/engineccl/rocksdb.go (about)

     1  // Copyright 2017 The Cockroach Authors.
     2  //
     3  // Licensed as a CockroachDB Enterprise file under the Cockroach Community
     4  // License (the "License"); you may not use this file except in compliance with
     5  // the License. You may obtain a copy of the License at
     6  //
     7  //     https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt
     8  
     9  package engineccl
    10  
    11  import (
    12  	"unsafe"
    13  
    14  	"github.com/cockroachdb/cockroach/pkg/roachpb"
    15  	"github.com/cockroachdb/cockroach/pkg/storage"
    16  	"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
    17  	"github.com/cockroachdb/errors"
    18  )
    19  
    20  // TODO(tamird): why does rocksdb not link jemalloc,snappy statically?
    21  
    22  // #cgo CPPFLAGS: -I../../../../c-deps/libroach/include -I../../../../c-deps/libroach/ccl/include
    23  // #cgo LDFLAGS: -lroachccl
    24  // #cgo LDFLAGS: -lroach
    25  // #cgo LDFLAGS: -lprotobuf
    26  // #cgo LDFLAGS: -lrocksdb
    27  // #cgo LDFLAGS: -lsnappy
    28  // #cgo LDFLAGS: -lcryptopp
    29  // #cgo linux LDFLAGS: -lrt -lpthread
    30  // #cgo windows LDFLAGS: -lshlwapi -lrpcrt4
    31  //
    32  // #include <stdlib.h>
    33  // #include <libroachccl.h>
    34  import "C"
    35  
    36  func init() {
    37  	storage.SetRocksDBOpenHook(C.DBOpenHookCCL)
    38  }
    39  
    40  // VerifyBatchRepr asserts that all keys in a BatchRepr are between the specified
    41  // start and end keys and computes the enginepb.MVCCStats for it.
    42  func VerifyBatchRepr(
    43  	repr []byte, start, end storage.MVCCKey, nowNanos int64,
    44  ) (enginepb.MVCCStats, error) {
    45  	// We store a 4 byte checksum of each key/value entry in the value. Make
    46  	// sure the all ones in this BatchRepr validate.
    47  	//
    48  	// TODO(dan): After a number of hours of trying, I was unable to get a
    49  	// performant c++ implementation of crc32 ieee, so this verifies the
    50  	// checksums in go and constructs the MVCCStats in c++. It'd be nice to move
    51  	// one or the other.
    52  	{
    53  		r, err := storage.NewRocksDBBatchReader(repr)
    54  		if err != nil {
    55  			return enginepb.MVCCStats{}, errors.Wrapf(err, "verifying key/value checksums")
    56  		}
    57  		for r.Next() {
    58  			switch r.BatchType() {
    59  			case storage.BatchTypeValue:
    60  				mvccKey, err := r.MVCCKey()
    61  				if err != nil {
    62  					return enginepb.MVCCStats{}, errors.Wrapf(err, "verifying key/value checksums")
    63  				}
    64  				v := roachpb.Value{RawBytes: r.Value()}
    65  				if err := v.Verify(mvccKey.Key); err != nil {
    66  					return enginepb.MVCCStats{}, err
    67  				}
    68  			default:
    69  				return enginepb.MVCCStats{}, errors.Errorf(
    70  					"unexpected entry type in batch: %d", r.BatchType())
    71  			}
    72  		}
    73  		if err := r.Error(); err != nil {
    74  			return enginepb.MVCCStats{}, errors.Wrapf(err, "verifying key/value checksums")
    75  		}
    76  	}
    77  
    78  	var stats C.MVCCStatsResult
    79  	if err := statusToError(C.DBBatchReprVerify(
    80  		goToCSlice(repr), goToCKey(start), goToCKey(end), C.int64_t(nowNanos), &stats,
    81  	)); err != nil {
    82  		return enginepb.MVCCStats{}, err
    83  	}
    84  	return cStatsToGoStats(stats, nowNanos)
    85  }
    86  
    87  // TODO(dan): The following are all duplicated from storage/engine/rocksdb.go,
    88  // but if you export the ones there and reuse them here, it doesn't work.
    89  //
    90  // `cannot use engine.GoToCSlice(repr) (type engine.C.struct___0) as type C.struct___0 in argument to _Cfunc_DBBatchReprVerify`
    91  //
    92  // The issue is tracked upstream at https://github.com/golang/go/issues/13467.
    93  // At worst, we could split these out into a separate file in storage/engine and
    94  // use `go generate` to copy it here so they stay in sync.
    95  
    96  // goToCSlice converts a go byte slice to a DBSlice. Note that this is
    97  // potentially dangerous as the DBSlice holds a reference to the go
    98  // byte slice memory that the Go GC does not know about. This method
    99  // is only intended for use in converting arguments to C
   100  // functions. The C function must copy any data that it wishes to
   101  // retain once the function returns.
   102  func goToCSlice(b []byte) C.DBSlice {
   103  	if len(b) == 0 {
   104  		return C.DBSlice{data: nil, len: 0}
   105  	}
   106  	return C.DBSlice{
   107  		data: (*C.char)(unsafe.Pointer(&b[0])),
   108  		len:  C.size_t(len(b)),
   109  	}
   110  }
   111  
   112  func goToCKey(key storage.MVCCKey) C.DBKey {
   113  	return C.DBKey{
   114  		key:       goToCSlice(key.Key),
   115  		wall_time: C.int64_t(key.Timestamp.WallTime),
   116  		logical:   C.int32_t(key.Timestamp.Logical),
   117  	}
   118  }
   119  
   120  func cSliceToUnsafeGoBytes(s C.DBSlice) []byte {
   121  	if s.data == nil {
   122  		return nil
   123  	}
   124  	// Interpret the C pointer as a pointer to a Go array, then slice.
   125  	return (*[storage.MaxArrayLen]byte)(unsafe.Pointer(s.data))[:s.len:s.len]
   126  }
   127  
   128  func cStringToGoString(s C.DBString) string {
   129  	if s.data == nil {
   130  		return ""
   131  	}
   132  	// Reinterpret the string as a slice, then cast to string which does a copy.
   133  	result := string(cSliceToUnsafeGoBytes(C.DBSlice{s.data, s.len}))
   134  	C.free(unsafe.Pointer(s.data))
   135  	return result
   136  }
   137  
   138  func statusToError(s C.DBStatus) error {
   139  	if s.data == nil {
   140  		return nil
   141  	}
   142  	return errors.Newf("%s", cStringToGoString(s))
   143  }
   144  
   145  func cStatsToGoStats(stats C.MVCCStatsResult, nowNanos int64) (enginepb.MVCCStats, error) {
   146  	ms := enginepb.MVCCStats{}
   147  	if err := statusToError(stats.status); err != nil {
   148  		return ms, err
   149  	}
   150  	ms.ContainsEstimates = 0
   151  	ms.LiveBytes = int64(stats.live_bytes)
   152  	ms.KeyBytes = int64(stats.key_bytes)
   153  	ms.ValBytes = int64(stats.val_bytes)
   154  	ms.IntentBytes = int64(stats.intent_bytes)
   155  	ms.LiveCount = int64(stats.live_count)
   156  	ms.KeyCount = int64(stats.key_count)
   157  	ms.ValCount = int64(stats.val_count)
   158  	ms.IntentCount = int64(stats.intent_count)
   159  	ms.IntentAge = int64(stats.intent_age)
   160  	ms.GCBytesAge = int64(stats.gc_bytes_age)
   161  	ms.SysBytes = int64(stats.sys_bytes)
   162  	ms.SysCount = int64(stats.sys_count)
   163  	ms.LastUpdateNanos = nowNanos
   164  	return ms, nil
   165  }