github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/ccl/storageccl/engineccl/encrypted_fs.go (about) 1 // Copyright 2019 The Cockroach Authors. 2 // 3 // Licensed as a CockroachDB Enterprise file under the Cockroach Community 4 // License (the "License"); you may not use this file except in compliance with 5 // the License. You may obtain a copy of the License at 6 // 7 // https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt 8 9 package engineccl 10 11 import ( 12 "context" 13 "fmt" 14 15 "github.com/cockroachdb/cockroach/pkg/ccl/baseccl" 16 "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl" 17 "github.com/cockroachdb/cockroach/pkg/storage" 18 "github.com/cockroachdb/cockroach/pkg/storage/enginepb" 19 "github.com/cockroachdb/cockroach/pkg/util/protoutil" 20 "github.com/cockroachdb/cockroach/pkg/util/syncutil" 21 "github.com/cockroachdb/pebble/vfs" 22 ) 23 24 // High-level code structure. 25 // 26 // A pebble instance can operate in two modes: 27 // - With a single FS, used for all files. 28 // - With three FSs, when encryption-at-rest is on. We refer to these FSs as the base-FS, 29 // store-FS and data-FS. The base-FS contains unencrypted files, the store-FS contains 30 // files encrypted using the user-specified store keys, and the data-FS contains files 31 // encrypted using generated keys. 32 // 33 // Both the data-FS and store-FS are wrappers around the base-FS, i.e., they encrypt and 34 // decrypt data being written to and read from files in the base-FS. The environment in which 35 // these exist knows which file to open in which FS. Specifically, 36 // - The file registry uses the base-FS to store the FileRegistry proto. This registry provides 37 // information about files in the store-FS and data-FS (the EnvType::Store and EnvType::Data 38 // respectively). This information includes which FS the file belongs to (for consistency 39 // checking that a file created using one FS is not being read in another) and information 40 // about encryption settings used for the file, including the key id. 41 // - The StoreKeyManager uses the base-FS to read the user-specified store keys at startup. 42 // These are in two key files: the active key file and the old key file, which contain the 43 // key id and the key. 44 // - The store-FS is used only for storing the key file for the generated keys. It is used by 45 // the DataKeyManager. These keys are rotated periodically in a simple manner -- a new 46 // active key is generated for future file writes. Existing files are not affected. 47 // 48 // The data-FS and store-FS both use a common implementation. They consume: 49 // - the FS they are wrapping: it is always the base-FS in our case, but it does not matter. 50 // - The single file registry they share to record information about their files. 51 // - A KeyManager interface wrapped in a FileStreamCreator, that provides a simple 52 // interface for encrypting and decrypting data in a file at arbitrary byte offsets. 53 // 54 // Both data-FS and store-FS can be operating in plaintext mode if the user-specified store 55 // keys are "plain". 56 // 57 // For query execution spilling to disk: we want to use encryption, but the registries do not need 58 // to be on disk since on restart the query path will wipe all the existing files it has written. 59 // The setup would include a memFS and there would logically be four FSs: base-FS (unencrypted 60 // disk based FS), mem-FS (unencrypted memory based FS), store-FS (encrypted memory based FS), 61 // data-FS (encrypted disk based FS). 62 // - The file registry uses mem-FS to store the FileRegistry proto. 63 // - The StoreKeyManager uses the base-FS to read the user-specified store keys. 64 // - The store-FS wraps a mem-FS for reading/writing data keys. 65 // - The DataKeyManager uses the store-FS. 66 // - The data-FS wraps a base-FS for reading/writing data. 67 68 // encryptedFile implements vfs.File. 69 type encryptedFile struct { 70 vfs.File 71 mu struct { 72 syncutil.Mutex 73 rOffset int64 74 wOffset int64 75 } 76 stream FileStream 77 } 78 79 // Write implements io.Writer. 80 func (f *encryptedFile) Write(p []byte) (n int, err error) { 81 f.mu.Lock() 82 defer f.mu.Unlock() 83 f.stream.Encrypt(f.mu.wOffset, p) 84 n, err = f.File.Write(p) 85 f.mu.wOffset += int64(n) 86 return n, err 87 } 88 89 // Read implements io.Reader. 90 func (f *encryptedFile) Read(p []byte) (n int, err error) { 91 f.mu.Lock() 92 defer f.mu.Unlock() 93 n, err = f.ReadAt(p, f.mu.rOffset) 94 f.mu.rOffset += int64(n) 95 return n, err 96 } 97 98 // ReadAt implements io.ReaderAt 99 func (f *encryptedFile) ReadAt(p []byte, off int64) (n int, err error) { 100 n, err = f.File.ReadAt(p, off) 101 if n > 0 { 102 f.stream.Decrypt(off, p[:n]) 103 } 104 return n, err 105 } 106 107 // encryptedFS implements vfs.FS. 108 type encryptedFS struct { 109 vfs.FS 110 fileRegistry *storage.PebbleFileRegistry 111 streamCreator *FileCipherStreamCreator 112 } 113 114 // Create implements vfs.FS.Create. 115 func (fs *encryptedFS) Create(name string) (vfs.File, error) { 116 f, err := fs.FS.Create(name) 117 if err != nil { 118 return f, err 119 } 120 settings, stream, err := fs.streamCreator.CreateNew(context.TODO()) 121 if err != nil { 122 f.Close() 123 return nil, err 124 } 125 fproto := &enginepb.FileEntry{} 126 fproto.EnvType = fs.streamCreator.envType 127 if fproto.EncryptionSettings, err = protoutil.Marshal(settings); err != nil { 128 f.Close() 129 return nil, err 130 } 131 if err = fs.fileRegistry.SetFileEntry(name, fproto); err != nil { 132 f.Close() 133 return nil, err 134 } 135 return &encryptedFile{File: f, stream: stream}, nil 136 } 137 138 // Link implements vfs.FS.Link. 139 func (fs *encryptedFS) Link(oldname, newname string) error { 140 if err := fs.FS.Link(oldname, newname); err != nil { 141 return err 142 } 143 return fs.fileRegistry.MaybeLinkEntry(oldname, newname) 144 } 145 146 // Open implements vfs.FS.Open. 147 func (fs *encryptedFS) Open(name string, opts ...vfs.OpenOption) (vfs.File, error) { 148 f, err := fs.FS.Open(name, opts...) 149 if err != nil { 150 return f, err 151 } 152 fileEntry := fs.fileRegistry.GetFileEntry(name) 153 var settings *enginepbccl.EncryptionSettings 154 if fileEntry != nil { 155 if fileEntry.EnvType != fs.streamCreator.envType { 156 f.Close() 157 return nil, fmt.Errorf("filename: %s has env %d not equal to FS env %d", 158 name, fileEntry.EnvType, fs.streamCreator.envType) 159 } 160 settings = &enginepbccl.EncryptionSettings{} 161 if err := protoutil.Unmarshal(fileEntry.EncryptionSettings, settings); err != nil { 162 f.Close() 163 return nil, err 164 } 165 } 166 stream, err := fs.streamCreator.CreateExisting(settings) 167 if err != nil { 168 f.Close() 169 return nil, err 170 } 171 return &encryptedFile{File: f, stream: stream}, nil 172 } 173 174 // Remove implements vfs.FS.Remove. 175 func (fs *encryptedFS) Remove(name string) error { 176 if err := fs.FS.Remove(name); err != nil { 177 return err 178 } 179 return fs.fileRegistry.MaybeDeleteEntry(name) 180 } 181 182 // Rename implements vfs.FS.Rename. 183 func (fs *encryptedFS) Rename(oldname, newname string) error { 184 if err := fs.FS.Rename(oldname, newname); err != nil { 185 return err 186 } 187 return fs.fileRegistry.MaybeRenameEntry(oldname, newname) 188 } 189 190 // ReuseForWrite implements vfs.FS.ReuseForWrite. 191 // 192 // We cannot change any of the key/iv/nonce and reuse the same file since RocksDB does not 193 // like non-empty WAL files with zero readable entries. There is a todo in env_encryption.cc 194 // to change this RocksDB behavior. We need to handle a user switching from Pebble to RocksDB, 195 // so cannot generate WAL files that RocksDB will complain about. 196 func (fs *encryptedFS) ReuseForWrite(oldname, newname string) (vfs.File, error) { 197 // This is slower than simply calling Create(newname) since the Remove() and Create() 198 // will write and sync the file registry file twice. We can optimize this if needed. 199 if err := fs.Remove(oldname); err != nil { 200 return nil, err 201 } 202 return fs.Create(newname) 203 } 204 205 type encryptionStatsHandler struct { 206 storeKM *StoreKeyManager 207 dataKM *DataKeyManager 208 } 209 210 func (e *encryptionStatsHandler) GetEncryptionStatus() ([]byte, error) { 211 var s enginepbccl.EncryptionStatus 212 if e.storeKM.activeKey != nil { 213 s.ActiveStoreKey = e.storeKM.activeKey.Info 214 } 215 k, err := e.dataKM.ActiveKey(context.TODO()) 216 if err != nil { 217 return nil, err 218 } 219 if k != nil { 220 s.ActiveDataKey = k.Info 221 } 222 return protoutil.Marshal(&s) 223 } 224 225 func (e *encryptionStatsHandler) GetDataKeysRegistry() ([]byte, error) { 226 r := e.dataKM.getScrubbedRegistry() 227 return protoutil.Marshal(r) 228 } 229 230 func (e *encryptionStatsHandler) GetActiveDataKeyID() (string, error) { 231 k, err := e.dataKM.ActiveKey(context.TODO()) 232 if err != nil { 233 return "", err 234 } 235 if k != nil { 236 return k.Info.KeyId, nil 237 } 238 return "plain", nil 239 } 240 241 func (e *encryptionStatsHandler) GetActiveStoreKeyType() int32 { 242 if e.storeKM.activeKey != nil { 243 return int32(e.storeKM.activeKey.Info.EncryptionType) 244 } 245 return int32(enginepbccl.EncryptionType_Plaintext) 246 } 247 248 func (e *encryptionStatsHandler) GetKeyIDFromSettings(settings []byte) (string, error) { 249 var s enginepbccl.EncryptionSettings 250 if err := protoutil.Unmarshal(settings, &s); err != nil { 251 return "", err 252 } 253 return s.KeyId, nil 254 } 255 256 // Init initializes engine.NewEncryptedEncFunc. 257 func init() { 258 storage.NewEncryptedEnvFunc = newEncryptedEnv 259 } 260 261 // newEncryptedEnv creates an encrypted environment and returns the vfs.FS to use for reading and 262 // writing data. The optionBytes is a binary serialized baseccl.EncryptionOptions, so that non-CCL 263 // code does not depend on CCL code. 264 // 265 // See the comment at the top of this file for the structure of this environment. 266 func newEncryptedEnv( 267 fs vfs.FS, fr *storage.PebbleFileRegistry, dbDir string, readOnly bool, optionBytes []byte, 268 ) (vfs.FS, storage.EncryptionStatsHandler, error) { 269 options := &baseccl.EncryptionOptions{} 270 if err := protoutil.Unmarshal(optionBytes, options); err != nil { 271 return nil, nil, err 272 } 273 if options.KeySource != baseccl.EncryptionKeySource_KeyFiles { 274 return nil, nil, fmt.Errorf("unknown encryption key source: %d", options.KeySource) 275 } 276 storeKeyManager := &StoreKeyManager{ 277 fs: fs, 278 activeKeyFilename: options.KeyFiles.CurrentKey, 279 oldKeyFilename: options.KeyFiles.OldKey, 280 } 281 if err := storeKeyManager.Load(context.TODO()); err != nil { 282 return nil, nil, err 283 } 284 storeFS := &encryptedFS{ 285 FS: fs, 286 fileRegistry: fr, 287 streamCreator: &FileCipherStreamCreator{ 288 envType: enginepb.EnvType_Store, 289 keyManager: storeKeyManager, 290 }, 291 } 292 dataKeyManager := &DataKeyManager{ 293 fs: storeFS, 294 dbDir: dbDir, 295 rotationPeriod: options.DataKeyRotationPeriod, 296 } 297 if err := dataKeyManager.Load(context.TODO()); err != nil { 298 return nil, nil, err 299 } 300 dataFS := &encryptedFS{ 301 FS: fs, 302 fileRegistry: fr, 303 streamCreator: &FileCipherStreamCreator{ 304 envType: enginepb.EnvType_Data, 305 keyManager: dataKeyManager, 306 }, 307 } 308 309 if !readOnly { 310 key, err := storeKeyManager.ActiveKey(context.TODO()) 311 if err != nil { 312 return nil, nil, err 313 } 314 if err := dataKeyManager.SetActiveStoreKeyInfo(context.TODO(), key.Info); err != nil { 315 return nil, nil, err 316 } 317 } 318 return dataFS, &encryptionStatsHandler{storeKM: storeKeyManager, dataKM: dataKeyManager}, nil 319 }