github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/ccl/storageccl/engineccl/pebble_key_manager.go (about)

     1  // Copyright 2019 The Cockroach Authors.
     2  //
     3  // Licensed as a CockroachDB Enterprise file under the Cockroach Community
     4  // License (the "License"); you may not use this file except in compliance with
     5  // the License. You may obtain a copy of the License at
     6  //
     7  //     https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt
     8  
     9  package engineccl
    10  
    11  import (
    12  	"context"
    13  	"crypto/rand"
    14  	"encoding/hex"
    15  	"fmt"
    16  	"io/ioutil"
    17  	"os"
    18  	"time"
    19  
    20  	"github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl"
    21  	"github.com/cockroachdb/cockroach/pkg/storage"
    22  	"github.com/cockroachdb/cockroach/pkg/util/log"
    23  	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
    24  	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
    25  	"github.com/cockroachdb/pebble/vfs"
    26  	"github.com/gogo/protobuf/proto"
    27  )
    28  
    29  const (
    30  	// The key filename for the StoreKeyManager when using plaintext.
    31  	storeFileNamePlain = "plain"
    32  	// The key id used in KeyInfo for a plaintext "key".
    33  	plainKeyID = "plain"
    34  	// The length of a real key id.
    35  	keyIDLength = 32
    36  	// The filename used for writing the data keys by the DataKeyManager.
    37  	keyRegistryFilename = "COCKROACHDB_DATA_KEYS"
    38  )
    39  
    40  // PebbleKeyManager manages encryption keys. There are two implementations. See encrypted_fs.go for
    41  // high-level context.
    42  type PebbleKeyManager interface {
    43  	// ActiveKey returns the currently active key. If plaintext should be used it can return nil or
    44  	// a key with encryption_type = Plaintext.
    45  	ActiveKey(ctx context.Context) (*enginepbccl.SecretKey, error)
    46  
    47  	// GetKey gets the key for the given id. Returns an error if the key was not found.
    48  	GetKey(id string) (*enginepbccl.SecretKey, error)
    49  }
    50  
    51  var _ PebbleKeyManager = &StoreKeyManager{}
    52  var _ PebbleKeyManager = &DataKeyManager{}
    53  
    54  // Overridden for testing.
    55  var kmTimeNow = time.Now
    56  
    57  // StoreKeyManager manages the user-provided keys. Implements PebbleKeyManager.
    58  type StoreKeyManager struct {
    59  	// Initialize the following before calling Load().
    60  	fs                vfs.FS
    61  	activeKeyFilename string
    62  	oldKeyFilename    string
    63  
    64  	// Implementation. Both are not nil after a successful call to Load().
    65  	activeKey *enginepbccl.SecretKey
    66  	oldKey    *enginepbccl.SecretKey
    67  }
    68  
    69  // Load must be called before calling other functions.
    70  func (m *StoreKeyManager) Load(ctx context.Context) error {
    71  	var err error
    72  	m.activeKey, err = loadKeyFromFile(m.fs, m.activeKeyFilename)
    73  	if err != nil {
    74  		return err
    75  	}
    76  	m.oldKey, err = loadKeyFromFile(m.fs, m.oldKeyFilename)
    77  	if err != nil {
    78  		return err
    79  	}
    80  	log.Infof(ctx, "loaded active store key: %s, old store key: %s",
    81  		proto.CompactTextString(m.activeKey.Info), proto.CompactTextString(m.oldKey.Info))
    82  	return nil
    83  }
    84  
    85  // ActiveKey implements PebbleKeyManager.ActiveKey.
    86  func (m *StoreKeyManager) ActiveKey(ctx context.Context) (*enginepbccl.SecretKey, error) {
    87  	return m.activeKey, nil
    88  }
    89  
    90  // GetKey implements PebbleKeyManager.GetKey.
    91  func (m *StoreKeyManager) GetKey(id string) (*enginepbccl.SecretKey, error) {
    92  	if m.activeKey.Info.KeyId == id {
    93  		return m.activeKey, nil
    94  	}
    95  	if m.oldKey.Info.KeyId == id {
    96  		return m.oldKey, nil
    97  	}
    98  	return nil, fmt.Errorf("store key ID %s was not found", id)
    99  }
   100  
   101  func loadKeyFromFile(fs vfs.FS, filename string) (*enginepbccl.SecretKey, error) {
   102  	now := kmTimeNow().Unix()
   103  	key := &enginepbccl.SecretKey{}
   104  	key.Info = &enginepbccl.KeyInfo{}
   105  	if filename == storeFileNamePlain {
   106  		key.Info.EncryptionType = enginepbccl.EncryptionType_Plaintext
   107  		key.Info.KeyId = plainKeyID
   108  		key.Info.CreationTime = now
   109  		key.Info.Source = storeFileNamePlain
   110  		return key, nil
   111  	}
   112  
   113  	f, err := fs.Open(filename)
   114  	if err != nil {
   115  		return nil, err
   116  	}
   117  	defer f.Close()
   118  	b, err := ioutil.ReadAll(f)
   119  	if err != nil {
   120  		return nil, err
   121  	}
   122  	// keyIDLength bytes for the ID, followed by the key.
   123  	keyLength := len(b) - keyIDLength
   124  	switch keyLength {
   125  	case 16:
   126  		key.Info.EncryptionType = enginepbccl.EncryptionType_AES128_CTR
   127  	case 24:
   128  		key.Info.EncryptionType = enginepbccl.EncryptionType_AES192_CTR
   129  	case 32:
   130  		key.Info.EncryptionType = enginepbccl.EncryptionType_AES256_CTR
   131  	default:
   132  		return nil, fmt.Errorf("store key of unsupported length: %d", keyLength)
   133  	}
   134  	key.Key = b[keyIDLength:]
   135  	// Hex encoding to make it human readable.
   136  	key.Info.KeyId = hex.EncodeToString(b[:keyIDLength])
   137  	key.Info.CreationTime = now
   138  	key.Info.Source = filename
   139  
   140  	return key, nil
   141  }
   142  
   143  // DataKeyManager manages data keys. Implements PebbleKeyManager. Key rotation does not begin until
   144  // SetActiveStoreKeyInfo() is called.
   145  type DataKeyManager struct {
   146  	// Initialize the following before calling Load().
   147  	fs             vfs.FS
   148  	dbDir          string
   149  	rotationPeriod int64 // seconds
   150  
   151  	// Implementation.
   152  
   153  	// Initialized in Load()
   154  	registryFilename string
   155  
   156  	mu struct {
   157  		syncutil.Mutex
   158  		// Non-nil after Load()
   159  		keyRegistry *enginepbccl.DataKeysRegistry
   160  		// rotationEnabled => non-nil
   161  		activeKey *enginepbccl.SecretKey
   162  		// Transitions to true when SetActiveStoreKeyInfo() is called for the
   163  		// first time.
   164  		rotationEnabled bool
   165  	}
   166  }
   167  
   168  func makeRegistryProto() *enginepbccl.DataKeysRegistry {
   169  	return &enginepbccl.DataKeysRegistry{
   170  		StoreKeys: make(map[string]*enginepbccl.KeyInfo),
   171  		DataKeys:  make(map[string]*enginepbccl.SecretKey),
   172  	}
   173  }
   174  
   175  // Load must be called before calling other methods.
   176  func (m *DataKeyManager) Load(ctx context.Context) error {
   177  	m.registryFilename = m.fs.PathJoin(m.dbDir, keyRegistryFilename)
   178  	_, err := m.fs.Stat(m.registryFilename)
   179  	m.mu.Lock()
   180  	defer m.mu.Unlock()
   181  	if os.IsNotExist(err) {
   182  		// First run.
   183  		m.mu.keyRegistry = makeRegistryProto()
   184  		return nil
   185  	}
   186  	if err != nil {
   187  		return err
   188  	}
   189  
   190  	f, err := m.fs.Open(m.registryFilename)
   191  	if err != nil {
   192  		return err
   193  	}
   194  	defer f.Close()
   195  	b, err := ioutil.ReadAll(f)
   196  	if err != nil {
   197  		return err
   198  	}
   199  	m.mu.keyRegistry = makeRegistryProto()
   200  	if err = protoutil.Unmarshal(b, m.mu.keyRegistry); err != nil {
   201  		return err
   202  	}
   203  	if err = validateRegistry(m.mu.keyRegistry); err != nil {
   204  		return err
   205  	}
   206  	if m.mu.keyRegistry.ActiveDataKeyId != "" {
   207  		key, found := m.mu.keyRegistry.DataKeys[m.mu.keyRegistry.ActiveDataKeyId]
   208  		if !found {
   209  			// This should have resulted in an error in validateRegistry()
   210  			panic("unexpected inconsistent DataKeysRegistry")
   211  		}
   212  		m.mu.activeKey = key
   213  		log.Infof(ctx, "loaded active data key: %s", m.mu.activeKey.Info.String())
   214  	} else {
   215  		log.Infof(ctx, "no active data key yet")
   216  	}
   217  	return nil
   218  }
   219  
   220  // ActiveKey implements PebbleKeyManager.ActiveKey.
   221  //
   222  // TODO(sbhola): do rotation via a background activity instead of in this function so that we don't
   223  // slow down creation of files.
   224  func (m *DataKeyManager) ActiveKey(ctx context.Context) (*enginepbccl.SecretKey, error) {
   225  	m.mu.Lock()
   226  	defer m.mu.Unlock()
   227  	if m.mu.rotationEnabled {
   228  		now := kmTimeNow().Unix()
   229  		if now-m.mu.activeKey.Info.CreationTime > m.rotationPeriod {
   230  			keyRegistry := makeRegistryProto()
   231  			proto.Merge(keyRegistry, m.mu.keyRegistry)
   232  			if err := m.rotateDataKeyAndWrite(ctx, keyRegistry); err != nil {
   233  				return nil, err
   234  			}
   235  		}
   236  	}
   237  	return m.mu.activeKey, nil
   238  }
   239  
   240  // GetKey implements PebbleKeyManager.GetKey.
   241  func (m *DataKeyManager) GetKey(id string) (*enginepbccl.SecretKey, error) {
   242  	m.mu.Lock()
   243  	defer m.mu.Unlock()
   244  	key, found := m.mu.keyRegistry.DataKeys[id]
   245  	if !found {
   246  		return nil, fmt.Errorf("key %s is not found", id)
   247  	}
   248  	return key, nil
   249  }
   250  
   251  // SetActiveStoreKeyInfo sets the current active store key. Even though there may be a valid
   252  // ActiveStoreKeyId in the DataKeysRegistry loaded from file, key rotation does not start until
   253  // the first call to the following function. Each call to this function will rotate the active
   254  // data key under the following conditions:
   255  // - there is no active data key.
   256  // - the active store key has changed.
   257  //
   258  // This function should not be called for a read only store.
   259  func (m *DataKeyManager) SetActiveStoreKeyInfo(
   260  	ctx context.Context, storeKeyInfo *enginepbccl.KeyInfo,
   261  ) error {
   262  	m.mu.Lock()
   263  	defer m.mu.Unlock()
   264  	prevActiveStoreKey, found := m.mu.keyRegistry.StoreKeys[m.mu.keyRegistry.ActiveStoreKeyId]
   265  	if found && prevActiveStoreKey.KeyId == storeKeyInfo.KeyId && m.mu.activeKey != nil {
   266  		// The active store key has not changed and we already have an active data key,
   267  		// so no need to do anything.
   268  		return nil
   269  	}
   270  	// For keys other than plaintext, make sure the user is not reusing inactive keys.
   271  	if storeKeyInfo.EncryptionType != enginepbccl.EncryptionType_Plaintext {
   272  		if _, found := m.mu.keyRegistry.StoreKeys[storeKeyInfo.KeyId]; found {
   273  			return fmt.Errorf("new active store key ID %s already exists as an inactive key -- this"+
   274  				"is really dangerous", storeKeyInfo.KeyId)
   275  		}
   276  	}
   277  
   278  	// The keyRegistry proto that will replace the current one.
   279  	keyRegistry := makeRegistryProto()
   280  	proto.Merge(keyRegistry, m.mu.keyRegistry)
   281  	keyRegistry.StoreKeys[storeKeyInfo.KeyId] = storeKeyInfo
   282  	keyRegistry.ActiveStoreKeyId = storeKeyInfo.KeyId
   283  	if storeKeyInfo.EncryptionType == enginepbccl.EncryptionType_Plaintext {
   284  		// Mark all data keys as exposed.
   285  		for _, key := range keyRegistry.DataKeys {
   286  			key.Info.WasExposed = true
   287  		}
   288  	}
   289  	if err := m.rotateDataKeyAndWrite(ctx, keyRegistry); err != nil {
   290  		return err
   291  	}
   292  	m.mu.rotationEnabled = true
   293  	return nil
   294  }
   295  
   296  func (m *DataKeyManager) getScrubbedRegistry() *enginepbccl.DataKeysRegistry {
   297  	m.mu.Lock()
   298  	defer m.mu.Unlock()
   299  	r := makeRegistryProto()
   300  	proto.Merge(r, m.mu.keyRegistry)
   301  	for _, v := range r.DataKeys {
   302  		v.Key = nil
   303  	}
   304  	return r
   305  }
   306  
   307  func validateRegistry(keyRegistry *enginepbccl.DataKeysRegistry) error {
   308  	if keyRegistry.ActiveStoreKeyId != "" && keyRegistry.StoreKeys[keyRegistry.ActiveStoreKeyId] == nil {
   309  		return fmt.Errorf("active store key %s not found", keyRegistry.ActiveStoreKeyId)
   310  	}
   311  	if keyRegistry.ActiveDataKeyId != "" && keyRegistry.DataKeys[keyRegistry.ActiveDataKeyId] == nil {
   312  		return fmt.Errorf("active data key %s not found", keyRegistry.ActiveDataKeyId)
   313  	}
   314  	return nil
   315  }
   316  
   317  // Generates a new data key and adds it to the keyRegistry proto and sets it as the active key.
   318  func generateAndSetNewDataKey(
   319  	ctx context.Context, keyRegistry *enginepbccl.DataKeysRegistry,
   320  ) (*enginepbccl.SecretKey, error) {
   321  	activeStoreKey := keyRegistry.StoreKeys[keyRegistry.ActiveStoreKeyId]
   322  	if activeStoreKey == nil {
   323  		panic("expected registry with active store key")
   324  	}
   325  	key := &enginepbccl.SecretKey{}
   326  	key.Info = &enginepbccl.KeyInfo{}
   327  	key.Info.EncryptionType = activeStoreKey.EncryptionType
   328  	key.Info.CreationTime = kmTimeNow().Unix()
   329  	key.Info.Source = "data key manager"
   330  	key.Info.ParentKeyId = activeStoreKey.KeyId
   331  
   332  	if activeStoreKey.EncryptionType == enginepbccl.EncryptionType_Plaintext {
   333  		key.Info.KeyId = plainKeyID
   334  		key.Info.WasExposed = true
   335  	} else {
   336  		var keyLength int
   337  		switch activeStoreKey.EncryptionType {
   338  		case enginepbccl.EncryptionType_AES128_CTR:
   339  			keyLength = 16
   340  		case enginepbccl.EncryptionType_AES192_CTR:
   341  			keyLength = 24
   342  		case enginepbccl.EncryptionType_AES256_CTR:
   343  			keyLength = 32
   344  		default:
   345  			return nil, fmt.Errorf("unknown encryption type %d for key ID %s",
   346  				activeStoreKey.EncryptionType, activeStoreKey.KeyId)
   347  		}
   348  		key.Key = make([]byte, keyLength)
   349  		n, err := rand.Read(key.Key)
   350  		if err != nil {
   351  			return nil, err
   352  		}
   353  		if n != keyLength {
   354  			log.Fatalf(ctx, "rand.Read returned no error but fewer bytes %d than promised %d", n, keyLength)
   355  		}
   356  		keyID := make([]byte, keyIDLength)
   357  		if n, err = rand.Read(keyID); err != nil {
   358  			return nil, err
   359  		}
   360  		if n != keyIDLength {
   361  			log.Fatalf(ctx, "rand.Read returned no error but fewer bytes %d than promised %d", n, keyIDLength)
   362  		}
   363  		// Hex encoding to make it human readable.
   364  		key.Info.KeyId = hex.EncodeToString(keyID)
   365  		key.Info.WasExposed = false
   366  	}
   367  	keyRegistry.DataKeys[key.Info.KeyId] = key
   368  	keyRegistry.ActiveDataKeyId = key.Info.KeyId
   369  	return key, nil
   370  }
   371  
   372  func (m *DataKeyManager) rotateDataKeyAndWrite(
   373  	ctx context.Context, keyRegistry *enginepbccl.DataKeysRegistry,
   374  ) (err error) {
   375  	defer func() {
   376  		if err != nil {
   377  			log.Infof(ctx, "error while attempting to rotate data key: %s", err)
   378  		} else {
   379  			log.Infof(ctx, "rotated to new active data key: %s", proto.CompactTextString(m.mu.activeKey.Info))
   380  		}
   381  	}()
   382  
   383  	var newKey *enginepbccl.SecretKey
   384  	if newKey, err = generateAndSetNewDataKey(ctx, keyRegistry); err != nil {
   385  		return
   386  	}
   387  	if err = validateRegistry(keyRegistry); err != nil {
   388  		return
   389  	}
   390  	bytes, err := protoutil.Marshal(keyRegistry)
   391  	if err != nil {
   392  		return
   393  	}
   394  	if err = storage.SafeWriteToFile(m.fs, m.dbDir, m.registryFilename, bytes); err != nil {
   395  		return
   396  	}
   397  	m.mu.keyRegistry = keyRegistry
   398  	m.mu.activeKey = newKey
   399  	return
   400  }