github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/ccl/storageccl/engineccl/rocksdb.go (about) 1 // Copyright 2017 The Cockroach Authors. 2 // 3 // Licensed as a CockroachDB Enterprise file under the Cockroach Community 4 // License (the "License"); you may not use this file except in compliance with 5 // the License. You may obtain a copy of the License at 6 // 7 // https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt 8 9 package engineccl 10 11 import ( 12 "unsafe" 13 14 "github.com/cockroachdb/cockroach/pkg/roachpb" 15 "github.com/cockroachdb/cockroach/pkg/storage" 16 "github.com/cockroachdb/cockroach/pkg/storage/enginepb" 17 "github.com/cockroachdb/errors" 18 ) 19 20 // TODO(tamird): why does rocksdb not link jemalloc,snappy statically? 21 22 // #cgo CPPFLAGS: -I../../../../c-deps/libroach/include -I../../../../c-deps/libroach/ccl/include 23 // #cgo LDFLAGS: -lroachccl 24 // #cgo LDFLAGS: -lroach 25 // #cgo LDFLAGS: -lprotobuf 26 // #cgo LDFLAGS: -lrocksdb 27 // #cgo LDFLAGS: -lsnappy 28 // #cgo LDFLAGS: -lcryptopp 29 // #cgo linux LDFLAGS: -lrt -lpthread 30 // #cgo windows LDFLAGS: -lshlwapi -lrpcrt4 31 // 32 // #include <stdlib.h> 33 // #include <libroachccl.h> 34 import "C" 35 36 func init() { 37 storage.SetRocksDBOpenHook(C.DBOpenHookCCL) 38 } 39 40 // VerifyBatchRepr asserts that all keys in a BatchRepr are between the specified 41 // start and end keys and computes the enginepb.MVCCStats for it. 42 func VerifyBatchRepr( 43 repr []byte, start, end storage.MVCCKey, nowNanos int64, 44 ) (enginepb.MVCCStats, error) { 45 // We store a 4 byte checksum of each key/value entry in the value. Make 46 // sure the all ones in this BatchRepr validate. 47 // 48 // TODO(dan): After a number of hours of trying, I was unable to get a 49 // performant c++ implementation of crc32 ieee, so this verifies the 50 // checksums in go and constructs the MVCCStats in c++. It'd be nice to move 51 // one or the other. 52 { 53 r, err := storage.NewRocksDBBatchReader(repr) 54 if err != nil { 55 return enginepb.MVCCStats{}, errors.Wrapf(err, "verifying key/value checksums") 56 } 57 for r.Next() { 58 switch r.BatchType() { 59 case storage.BatchTypeValue: 60 mvccKey, err := r.MVCCKey() 61 if err != nil { 62 return enginepb.MVCCStats{}, errors.Wrapf(err, "verifying key/value checksums") 63 } 64 v := roachpb.Value{RawBytes: r.Value()} 65 if err := v.Verify(mvccKey.Key); err != nil { 66 return enginepb.MVCCStats{}, err 67 } 68 default: 69 return enginepb.MVCCStats{}, errors.Errorf( 70 "unexpected entry type in batch: %d", r.BatchType()) 71 } 72 } 73 if err := r.Error(); err != nil { 74 return enginepb.MVCCStats{}, errors.Wrapf(err, "verifying key/value checksums") 75 } 76 } 77 78 var stats C.MVCCStatsResult 79 if err := statusToError(C.DBBatchReprVerify( 80 goToCSlice(repr), goToCKey(start), goToCKey(end), C.int64_t(nowNanos), &stats, 81 )); err != nil { 82 return enginepb.MVCCStats{}, err 83 } 84 return cStatsToGoStats(stats, nowNanos) 85 } 86 87 // TODO(dan): The following are all duplicated from storage/engine/rocksdb.go, 88 // but if you export the ones there and reuse them here, it doesn't work. 89 // 90 // `cannot use engine.GoToCSlice(repr) (type engine.C.struct___0) as type C.struct___0 in argument to _Cfunc_DBBatchReprVerify` 91 // 92 // The issue is tracked upstream at https://github.com/golang/go/issues/13467. 93 // At worst, we could split these out into a separate file in storage/engine and 94 // use `go generate` to copy it here so they stay in sync. 95 96 // goToCSlice converts a go byte slice to a DBSlice. Note that this is 97 // potentially dangerous as the DBSlice holds a reference to the go 98 // byte slice memory that the Go GC does not know about. This method 99 // is only intended for use in converting arguments to C 100 // functions. The C function must copy any data that it wishes to 101 // retain once the function returns. 102 func goToCSlice(b []byte) C.DBSlice { 103 if len(b) == 0 { 104 return C.DBSlice{data: nil, len: 0} 105 } 106 return C.DBSlice{ 107 data: (*C.char)(unsafe.Pointer(&b[0])), 108 len: C.size_t(len(b)), 109 } 110 } 111 112 func goToCKey(key storage.MVCCKey) C.DBKey { 113 return C.DBKey{ 114 key: goToCSlice(key.Key), 115 wall_time: C.int64_t(key.Timestamp.WallTime), 116 logical: C.int32_t(key.Timestamp.Logical), 117 } 118 } 119 120 func cSliceToUnsafeGoBytes(s C.DBSlice) []byte { 121 if s.data == nil { 122 return nil 123 } 124 // Interpret the C pointer as a pointer to a Go array, then slice. 125 return (*[storage.MaxArrayLen]byte)(unsafe.Pointer(s.data))[:s.len:s.len] 126 } 127 128 func cStringToGoString(s C.DBString) string { 129 if s.data == nil { 130 return "" 131 } 132 // Reinterpret the string as a slice, then cast to string which does a copy. 133 result := string(cSliceToUnsafeGoBytes(C.DBSlice{s.data, s.len})) 134 C.free(unsafe.Pointer(s.data)) 135 return result 136 } 137 138 func statusToError(s C.DBStatus) error { 139 if s.data == nil { 140 return nil 141 } 142 return errors.Newf("%s", cStringToGoString(s)) 143 } 144 145 func cStatsToGoStats(stats C.MVCCStatsResult, nowNanos int64) (enginepb.MVCCStats, error) { 146 ms := enginepb.MVCCStats{} 147 if err := statusToError(stats.status); err != nil { 148 return ms, err 149 } 150 ms.ContainsEstimates = 0 151 ms.LiveBytes = int64(stats.live_bytes) 152 ms.KeyBytes = int64(stats.key_bytes) 153 ms.ValBytes = int64(stats.val_bytes) 154 ms.IntentBytes = int64(stats.intent_bytes) 155 ms.LiveCount = int64(stats.live_count) 156 ms.KeyCount = int64(stats.key_count) 157 ms.ValCount = int64(stats.val_count) 158 ms.IntentCount = int64(stats.intent_count) 159 ms.IntentAge = int64(stats.intent_age) 160 ms.GCBytesAge = int64(stats.gc_bytes_age) 161 ms.SysBytes = int64(stats.sys_bytes) 162 ms.SysCount = int64(stats.sys_count) 163 ms.LastUpdateNanos = nowNanos 164 return ms, nil 165 }