github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/ccl/storageccl/engineccl/encrypted_fs.go (about)

     1  // Copyright 2019 The Cockroach Authors.
     2  //
     3  // Licensed as a CockroachDB Enterprise file under the Cockroach Community
     4  // License (the "License"); you may not use this file except in compliance with
     5  // the License. You may obtain a copy of the License at
     6  //
     7  //     https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt
     8  
     9  package engineccl
    10  
    11  import (
    12  	"context"
    13  	"fmt"
    14  
    15  	"github.com/cockroachdb/cockroach/pkg/ccl/baseccl"
    16  	"github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl"
    17  	"github.com/cockroachdb/cockroach/pkg/storage"
    18  	"github.com/cockroachdb/cockroach/pkg/storage/enginepb"
    19  	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
    20  	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
    21  	"github.com/cockroachdb/pebble/vfs"
    22  )
    23  
    24  // High-level code structure.
    25  //
    26  // A pebble instance can operate in two modes:
    27  // - With a single FS, used for all files.
    28  // - With three FSs, when encryption-at-rest is on. We refer to these FSs as the base-FS,
    29  //   store-FS and data-FS. The base-FS contains unencrypted files, the store-FS contains
    30  //   files encrypted using the user-specified store keys, and the data-FS contains files
    31  //   encrypted using generated keys.
    32  //
    33  // Both the data-FS and store-FS are wrappers around the base-FS, i.e., they encrypt and
    34  // decrypt data being written to and read from files in the base-FS. The environment in which
    35  // these exist knows which file to open in which FS. Specifically,
    36  // - The file registry uses the base-FS to store the FileRegistry proto. This registry provides
    37  //   information about files in the store-FS and data-FS (the EnvType::Store and EnvType::Data
    38  //   respectively). This information includes which FS the file belongs to (for consistency
    39  //   checking that a file created using one FS is not being read in another) and information
    40  //   about encryption settings used for the file, including the key id.
    41  // - The StoreKeyManager uses the base-FS to read the user-specified store keys at startup.
    42  //   These are in two key files: the active key file and the old key file, which contain the
    43  //   key id and the key.
    44  // - The store-FS is used only for storing the key file for the generated keys. It is used by
    45  //   the DataKeyManager. These keys are rotated periodically in a simple manner -- a new
    46  //   active key is generated for future file writes. Existing files are not affected.
    47  //
    48  // The data-FS and store-FS both use a common implementation. They consume:
    49  // - the FS they are wrapping: it is always the base-FS in our case, but it does not matter.
    50  // - The single file registry they share to record information about their files.
    51  // - A KeyManager interface wrapped in a FileStreamCreator, that provides a simple
    52  //   interface for encrypting and decrypting data in a file at arbitrary byte offsets.
    53  //
    54  // Both data-FS and store-FS can be operating in plaintext mode if the user-specified store
    55  // keys are "plain".
    56  //
    57  // For query execution spilling to disk: we want to use encryption, but the registries do not need
    58  // to be on disk since on restart the query path will wipe all the existing files it has written.
    59  // The setup would include a memFS and there would logically be four FSs: base-FS (unencrypted
    60  // disk based FS), mem-FS (unencrypted memory based FS), store-FS (encrypted memory based FS),
    61  // data-FS (encrypted disk based FS).
    62  // - The file registry uses mem-FS to store the FileRegistry proto.
    63  // - The StoreKeyManager uses the base-FS to read the user-specified store keys.
    64  // - The store-FS wraps a mem-FS for reading/writing data keys.
    65  // - The DataKeyManager uses the store-FS.
    66  // - The data-FS wraps a base-FS for reading/writing data.
    67  
    68  // encryptedFile implements vfs.File.
    69  type encryptedFile struct {
    70  	vfs.File
    71  	mu struct {
    72  		syncutil.Mutex
    73  		rOffset int64
    74  		wOffset int64
    75  	}
    76  	stream FileStream
    77  }
    78  
    79  // Write implements io.Writer.
    80  func (f *encryptedFile) Write(p []byte) (n int, err error) {
    81  	f.mu.Lock()
    82  	defer f.mu.Unlock()
    83  	f.stream.Encrypt(f.mu.wOffset, p)
    84  	n, err = f.File.Write(p)
    85  	f.mu.wOffset += int64(n)
    86  	return n, err
    87  }
    88  
    89  // Read implements io.Reader.
    90  func (f *encryptedFile) Read(p []byte) (n int, err error) {
    91  	f.mu.Lock()
    92  	defer f.mu.Unlock()
    93  	n, err = f.ReadAt(p, f.mu.rOffset)
    94  	f.mu.rOffset += int64(n)
    95  	return n, err
    96  }
    97  
    98  // ReadAt implements io.ReaderAt
    99  func (f *encryptedFile) ReadAt(p []byte, off int64) (n int, err error) {
   100  	n, err = f.File.ReadAt(p, off)
   101  	if n > 0 {
   102  		f.stream.Decrypt(off, p[:n])
   103  	}
   104  	return n, err
   105  }
   106  
   107  // encryptedFS implements vfs.FS.
   108  type encryptedFS struct {
   109  	vfs.FS
   110  	fileRegistry  *storage.PebbleFileRegistry
   111  	streamCreator *FileCipherStreamCreator
   112  }
   113  
   114  // Create implements vfs.FS.Create.
   115  func (fs *encryptedFS) Create(name string) (vfs.File, error) {
   116  	f, err := fs.FS.Create(name)
   117  	if err != nil {
   118  		return f, err
   119  	}
   120  	settings, stream, err := fs.streamCreator.CreateNew(context.TODO())
   121  	if err != nil {
   122  		f.Close()
   123  		return nil, err
   124  	}
   125  	fproto := &enginepb.FileEntry{}
   126  	fproto.EnvType = fs.streamCreator.envType
   127  	if fproto.EncryptionSettings, err = protoutil.Marshal(settings); err != nil {
   128  		f.Close()
   129  		return nil, err
   130  	}
   131  	if err = fs.fileRegistry.SetFileEntry(name, fproto); err != nil {
   132  		f.Close()
   133  		return nil, err
   134  	}
   135  	return &encryptedFile{File: f, stream: stream}, nil
   136  }
   137  
   138  // Link implements vfs.FS.Link.
   139  func (fs *encryptedFS) Link(oldname, newname string) error {
   140  	if err := fs.FS.Link(oldname, newname); err != nil {
   141  		return err
   142  	}
   143  	return fs.fileRegistry.MaybeLinkEntry(oldname, newname)
   144  }
   145  
   146  // Open implements vfs.FS.Open.
   147  func (fs *encryptedFS) Open(name string, opts ...vfs.OpenOption) (vfs.File, error) {
   148  	f, err := fs.FS.Open(name, opts...)
   149  	if err != nil {
   150  		return f, err
   151  	}
   152  	fileEntry := fs.fileRegistry.GetFileEntry(name)
   153  	var settings *enginepbccl.EncryptionSettings
   154  	if fileEntry != nil {
   155  		if fileEntry.EnvType != fs.streamCreator.envType {
   156  			f.Close()
   157  			return nil, fmt.Errorf("filename: %s has env %d not equal to FS env %d",
   158  				name, fileEntry.EnvType, fs.streamCreator.envType)
   159  		}
   160  		settings = &enginepbccl.EncryptionSettings{}
   161  		if err := protoutil.Unmarshal(fileEntry.EncryptionSettings, settings); err != nil {
   162  			f.Close()
   163  			return nil, err
   164  		}
   165  	}
   166  	stream, err := fs.streamCreator.CreateExisting(settings)
   167  	if err != nil {
   168  		f.Close()
   169  		return nil, err
   170  	}
   171  	return &encryptedFile{File: f, stream: stream}, nil
   172  }
   173  
   174  // Remove implements vfs.FS.Remove.
   175  func (fs *encryptedFS) Remove(name string) error {
   176  	if err := fs.FS.Remove(name); err != nil {
   177  		return err
   178  	}
   179  	return fs.fileRegistry.MaybeDeleteEntry(name)
   180  }
   181  
   182  // Rename implements vfs.FS.Rename.
   183  func (fs *encryptedFS) Rename(oldname, newname string) error {
   184  	if err := fs.FS.Rename(oldname, newname); err != nil {
   185  		return err
   186  	}
   187  	return fs.fileRegistry.MaybeRenameEntry(oldname, newname)
   188  }
   189  
   190  // ReuseForWrite implements vfs.FS.ReuseForWrite.
   191  //
   192  // We cannot change any of the key/iv/nonce and reuse the same file since RocksDB does not
   193  // like non-empty WAL files with zero readable entries. There is a todo in env_encryption.cc
   194  // to change this RocksDB behavior. We need to handle a user switching from Pebble to RocksDB,
   195  // so cannot generate WAL files that RocksDB will complain about.
   196  func (fs *encryptedFS) ReuseForWrite(oldname, newname string) (vfs.File, error) {
   197  	// This is slower than simply calling Create(newname) since the Remove() and Create()
   198  	// will write and sync the file registry file twice. We can optimize this if needed.
   199  	if err := fs.Remove(oldname); err != nil {
   200  		return nil, err
   201  	}
   202  	return fs.Create(newname)
   203  }
   204  
   205  type encryptionStatsHandler struct {
   206  	storeKM *StoreKeyManager
   207  	dataKM  *DataKeyManager
   208  }
   209  
   210  func (e *encryptionStatsHandler) GetEncryptionStatus() ([]byte, error) {
   211  	var s enginepbccl.EncryptionStatus
   212  	if e.storeKM.activeKey != nil {
   213  		s.ActiveStoreKey = e.storeKM.activeKey.Info
   214  	}
   215  	k, err := e.dataKM.ActiveKey(context.TODO())
   216  	if err != nil {
   217  		return nil, err
   218  	}
   219  	if k != nil {
   220  		s.ActiveDataKey = k.Info
   221  	}
   222  	return protoutil.Marshal(&s)
   223  }
   224  
   225  func (e *encryptionStatsHandler) GetDataKeysRegistry() ([]byte, error) {
   226  	r := e.dataKM.getScrubbedRegistry()
   227  	return protoutil.Marshal(r)
   228  }
   229  
   230  func (e *encryptionStatsHandler) GetActiveDataKeyID() (string, error) {
   231  	k, err := e.dataKM.ActiveKey(context.TODO())
   232  	if err != nil {
   233  		return "", err
   234  	}
   235  	if k != nil {
   236  		return k.Info.KeyId, nil
   237  	}
   238  	return "plain", nil
   239  }
   240  
   241  func (e *encryptionStatsHandler) GetActiveStoreKeyType() int32 {
   242  	if e.storeKM.activeKey != nil {
   243  		return int32(e.storeKM.activeKey.Info.EncryptionType)
   244  	}
   245  	return int32(enginepbccl.EncryptionType_Plaintext)
   246  }
   247  
   248  func (e *encryptionStatsHandler) GetKeyIDFromSettings(settings []byte) (string, error) {
   249  	var s enginepbccl.EncryptionSettings
   250  	if err := protoutil.Unmarshal(settings, &s); err != nil {
   251  		return "", err
   252  	}
   253  	return s.KeyId, nil
   254  }
   255  
   256  // Init initializes engine.NewEncryptedEncFunc.
   257  func init() {
   258  	storage.NewEncryptedEnvFunc = newEncryptedEnv
   259  }
   260  
   261  // newEncryptedEnv creates an encrypted environment and returns the vfs.FS to use for reading and
   262  // writing data. The optionBytes is a binary serialized baseccl.EncryptionOptions, so that non-CCL
   263  // code does not depend on CCL code.
   264  //
   265  // See the comment at the top of this file for the structure of this environment.
   266  func newEncryptedEnv(
   267  	fs vfs.FS, fr *storage.PebbleFileRegistry, dbDir string, readOnly bool, optionBytes []byte,
   268  ) (vfs.FS, storage.EncryptionStatsHandler, error) {
   269  	options := &baseccl.EncryptionOptions{}
   270  	if err := protoutil.Unmarshal(optionBytes, options); err != nil {
   271  		return nil, nil, err
   272  	}
   273  	if options.KeySource != baseccl.EncryptionKeySource_KeyFiles {
   274  		return nil, nil, fmt.Errorf("unknown encryption key source: %d", options.KeySource)
   275  	}
   276  	storeKeyManager := &StoreKeyManager{
   277  		fs:                fs,
   278  		activeKeyFilename: options.KeyFiles.CurrentKey,
   279  		oldKeyFilename:    options.KeyFiles.OldKey,
   280  	}
   281  	if err := storeKeyManager.Load(context.TODO()); err != nil {
   282  		return nil, nil, err
   283  	}
   284  	storeFS := &encryptedFS{
   285  		FS:           fs,
   286  		fileRegistry: fr,
   287  		streamCreator: &FileCipherStreamCreator{
   288  			envType:    enginepb.EnvType_Store,
   289  			keyManager: storeKeyManager,
   290  		},
   291  	}
   292  	dataKeyManager := &DataKeyManager{
   293  		fs:             storeFS,
   294  		dbDir:          dbDir,
   295  		rotationPeriod: options.DataKeyRotationPeriod,
   296  	}
   297  	if err := dataKeyManager.Load(context.TODO()); err != nil {
   298  		return nil, nil, err
   299  	}
   300  	dataFS := &encryptedFS{
   301  		FS:           fs,
   302  		fileRegistry: fr,
   303  		streamCreator: &FileCipherStreamCreator{
   304  			envType:    enginepb.EnvType_Data,
   305  			keyManager: dataKeyManager,
   306  		},
   307  	}
   308  
   309  	if !readOnly {
   310  		key, err := storeKeyManager.ActiveKey(context.TODO())
   311  		if err != nil {
   312  			return nil, nil, err
   313  		}
   314  		if err := dataKeyManager.SetActiveStoreKeyInfo(context.TODO(), key.Info); err != nil {
   315  			return nil, nil, err
   316  		}
   317  	}
   318  	return dataFS, &encryptionStatsHandler{storeKM: storeKeyManager, dataKM: dataKeyManager}, nil
   319  }