github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/ccl/storageccl/engineccl/pebble_key_manager.go (about) 1 // Copyright 2019 The Cockroach Authors. 2 // 3 // Licensed as a CockroachDB Enterprise file under the Cockroach Community 4 // License (the "License"); you may not use this file except in compliance with 5 // the License. You may obtain a copy of the License at 6 // 7 // https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt 8 9 package engineccl 10 11 import ( 12 "context" 13 "crypto/rand" 14 "encoding/hex" 15 "fmt" 16 "io/ioutil" 17 "os" 18 "time" 19 20 "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl" 21 "github.com/cockroachdb/cockroach/pkg/storage" 22 "github.com/cockroachdb/cockroach/pkg/util/log" 23 "github.com/cockroachdb/cockroach/pkg/util/protoutil" 24 "github.com/cockroachdb/cockroach/pkg/util/syncutil" 25 "github.com/cockroachdb/pebble/vfs" 26 "github.com/gogo/protobuf/proto" 27 ) 28 29 const ( 30 // The key filename for the StoreKeyManager when using plaintext. 31 storeFileNamePlain = "plain" 32 // The key id used in KeyInfo for a plaintext "key". 33 plainKeyID = "plain" 34 // The length of a real key id. 35 keyIDLength = 32 36 // The filename used for writing the data keys by the DataKeyManager. 37 keyRegistryFilename = "COCKROACHDB_DATA_KEYS" 38 ) 39 40 // PebbleKeyManager manages encryption keys. There are two implementations. See encrypted_fs.go for 41 // high-level context. 42 type PebbleKeyManager interface { 43 // ActiveKey returns the currently active key. If plaintext should be used it can return nil or 44 // a key with encryption_type = Plaintext. 45 ActiveKey(ctx context.Context) (*enginepbccl.SecretKey, error) 46 47 // GetKey gets the key for the given id. Returns an error if the key was not found. 48 GetKey(id string) (*enginepbccl.SecretKey, error) 49 } 50 51 var _ PebbleKeyManager = &StoreKeyManager{} 52 var _ PebbleKeyManager = &DataKeyManager{} 53 54 // Overridden for testing. 55 var kmTimeNow = time.Now 56 57 // StoreKeyManager manages the user-provided keys. Implements PebbleKeyManager. 58 type StoreKeyManager struct { 59 // Initialize the following before calling Load(). 60 fs vfs.FS 61 activeKeyFilename string 62 oldKeyFilename string 63 64 // Implementation. Both are not nil after a successful call to Load(). 65 activeKey *enginepbccl.SecretKey 66 oldKey *enginepbccl.SecretKey 67 } 68 69 // Load must be called before calling other functions. 70 func (m *StoreKeyManager) Load(ctx context.Context) error { 71 var err error 72 m.activeKey, err = loadKeyFromFile(m.fs, m.activeKeyFilename) 73 if err != nil { 74 return err 75 } 76 m.oldKey, err = loadKeyFromFile(m.fs, m.oldKeyFilename) 77 if err != nil { 78 return err 79 } 80 log.Infof(ctx, "loaded active store key: %s, old store key: %s", 81 proto.CompactTextString(m.activeKey.Info), proto.CompactTextString(m.oldKey.Info)) 82 return nil 83 } 84 85 // ActiveKey implements PebbleKeyManager.ActiveKey. 86 func (m *StoreKeyManager) ActiveKey(ctx context.Context) (*enginepbccl.SecretKey, error) { 87 return m.activeKey, nil 88 } 89 90 // GetKey implements PebbleKeyManager.GetKey. 91 func (m *StoreKeyManager) GetKey(id string) (*enginepbccl.SecretKey, error) { 92 if m.activeKey.Info.KeyId == id { 93 return m.activeKey, nil 94 } 95 if m.oldKey.Info.KeyId == id { 96 return m.oldKey, nil 97 } 98 return nil, fmt.Errorf("store key ID %s was not found", id) 99 } 100 101 func loadKeyFromFile(fs vfs.FS, filename string) (*enginepbccl.SecretKey, error) { 102 now := kmTimeNow().Unix() 103 key := &enginepbccl.SecretKey{} 104 key.Info = &enginepbccl.KeyInfo{} 105 if filename == storeFileNamePlain { 106 key.Info.EncryptionType = enginepbccl.EncryptionType_Plaintext 107 key.Info.KeyId = plainKeyID 108 key.Info.CreationTime = now 109 key.Info.Source = storeFileNamePlain 110 return key, nil 111 } 112 113 f, err := fs.Open(filename) 114 if err != nil { 115 return nil, err 116 } 117 defer f.Close() 118 b, err := ioutil.ReadAll(f) 119 if err != nil { 120 return nil, err 121 } 122 // keyIDLength bytes for the ID, followed by the key. 123 keyLength := len(b) - keyIDLength 124 switch keyLength { 125 case 16: 126 key.Info.EncryptionType = enginepbccl.EncryptionType_AES128_CTR 127 case 24: 128 key.Info.EncryptionType = enginepbccl.EncryptionType_AES192_CTR 129 case 32: 130 key.Info.EncryptionType = enginepbccl.EncryptionType_AES256_CTR 131 default: 132 return nil, fmt.Errorf("store key of unsupported length: %d", keyLength) 133 } 134 key.Key = b[keyIDLength:] 135 // Hex encoding to make it human readable. 136 key.Info.KeyId = hex.EncodeToString(b[:keyIDLength]) 137 key.Info.CreationTime = now 138 key.Info.Source = filename 139 140 return key, nil 141 } 142 143 // DataKeyManager manages data keys. Implements PebbleKeyManager. Key rotation does not begin until 144 // SetActiveStoreKeyInfo() is called. 145 type DataKeyManager struct { 146 // Initialize the following before calling Load(). 147 fs vfs.FS 148 dbDir string 149 rotationPeriod int64 // seconds 150 151 // Implementation. 152 153 // Initialized in Load() 154 registryFilename string 155 156 mu struct { 157 syncutil.Mutex 158 // Non-nil after Load() 159 keyRegistry *enginepbccl.DataKeysRegistry 160 // rotationEnabled => non-nil 161 activeKey *enginepbccl.SecretKey 162 // Transitions to true when SetActiveStoreKeyInfo() is called for the 163 // first time. 164 rotationEnabled bool 165 } 166 } 167 168 func makeRegistryProto() *enginepbccl.DataKeysRegistry { 169 return &enginepbccl.DataKeysRegistry{ 170 StoreKeys: make(map[string]*enginepbccl.KeyInfo), 171 DataKeys: make(map[string]*enginepbccl.SecretKey), 172 } 173 } 174 175 // Load must be called before calling other methods. 176 func (m *DataKeyManager) Load(ctx context.Context) error { 177 m.registryFilename = m.fs.PathJoin(m.dbDir, keyRegistryFilename) 178 _, err := m.fs.Stat(m.registryFilename) 179 m.mu.Lock() 180 defer m.mu.Unlock() 181 if os.IsNotExist(err) { 182 // First run. 183 m.mu.keyRegistry = makeRegistryProto() 184 return nil 185 } 186 if err != nil { 187 return err 188 } 189 190 f, err := m.fs.Open(m.registryFilename) 191 if err != nil { 192 return err 193 } 194 defer f.Close() 195 b, err := ioutil.ReadAll(f) 196 if err != nil { 197 return err 198 } 199 m.mu.keyRegistry = makeRegistryProto() 200 if err = protoutil.Unmarshal(b, m.mu.keyRegistry); err != nil { 201 return err 202 } 203 if err = validateRegistry(m.mu.keyRegistry); err != nil { 204 return err 205 } 206 if m.mu.keyRegistry.ActiveDataKeyId != "" { 207 key, found := m.mu.keyRegistry.DataKeys[m.mu.keyRegistry.ActiveDataKeyId] 208 if !found { 209 // This should have resulted in an error in validateRegistry() 210 panic("unexpected inconsistent DataKeysRegistry") 211 } 212 m.mu.activeKey = key 213 log.Infof(ctx, "loaded active data key: %s", m.mu.activeKey.Info.String()) 214 } else { 215 log.Infof(ctx, "no active data key yet") 216 } 217 return nil 218 } 219 220 // ActiveKey implements PebbleKeyManager.ActiveKey. 221 // 222 // TODO(sbhola): do rotation via a background activity instead of in this function so that we don't 223 // slow down creation of files. 224 func (m *DataKeyManager) ActiveKey(ctx context.Context) (*enginepbccl.SecretKey, error) { 225 m.mu.Lock() 226 defer m.mu.Unlock() 227 if m.mu.rotationEnabled { 228 now := kmTimeNow().Unix() 229 if now-m.mu.activeKey.Info.CreationTime > m.rotationPeriod { 230 keyRegistry := makeRegistryProto() 231 proto.Merge(keyRegistry, m.mu.keyRegistry) 232 if err := m.rotateDataKeyAndWrite(ctx, keyRegistry); err != nil { 233 return nil, err 234 } 235 } 236 } 237 return m.mu.activeKey, nil 238 } 239 240 // GetKey implements PebbleKeyManager.GetKey. 241 func (m *DataKeyManager) GetKey(id string) (*enginepbccl.SecretKey, error) { 242 m.mu.Lock() 243 defer m.mu.Unlock() 244 key, found := m.mu.keyRegistry.DataKeys[id] 245 if !found { 246 return nil, fmt.Errorf("key %s is not found", id) 247 } 248 return key, nil 249 } 250 251 // SetActiveStoreKeyInfo sets the current active store key. Even though there may be a valid 252 // ActiveStoreKeyId in the DataKeysRegistry loaded from file, key rotation does not start until 253 // the first call to the following function. Each call to this function will rotate the active 254 // data key under the following conditions: 255 // - there is no active data key. 256 // - the active store key has changed. 257 // 258 // This function should not be called for a read only store. 259 func (m *DataKeyManager) SetActiveStoreKeyInfo( 260 ctx context.Context, storeKeyInfo *enginepbccl.KeyInfo, 261 ) error { 262 m.mu.Lock() 263 defer m.mu.Unlock() 264 prevActiveStoreKey, found := m.mu.keyRegistry.StoreKeys[m.mu.keyRegistry.ActiveStoreKeyId] 265 if found && prevActiveStoreKey.KeyId == storeKeyInfo.KeyId && m.mu.activeKey != nil { 266 // The active store key has not changed and we already have an active data key, 267 // so no need to do anything. 268 return nil 269 } 270 // For keys other than plaintext, make sure the user is not reusing inactive keys. 271 if storeKeyInfo.EncryptionType != enginepbccl.EncryptionType_Plaintext { 272 if _, found := m.mu.keyRegistry.StoreKeys[storeKeyInfo.KeyId]; found { 273 return fmt.Errorf("new active store key ID %s already exists as an inactive key -- this"+ 274 "is really dangerous", storeKeyInfo.KeyId) 275 } 276 } 277 278 // The keyRegistry proto that will replace the current one. 279 keyRegistry := makeRegistryProto() 280 proto.Merge(keyRegistry, m.mu.keyRegistry) 281 keyRegistry.StoreKeys[storeKeyInfo.KeyId] = storeKeyInfo 282 keyRegistry.ActiveStoreKeyId = storeKeyInfo.KeyId 283 if storeKeyInfo.EncryptionType == enginepbccl.EncryptionType_Plaintext { 284 // Mark all data keys as exposed. 285 for _, key := range keyRegistry.DataKeys { 286 key.Info.WasExposed = true 287 } 288 } 289 if err := m.rotateDataKeyAndWrite(ctx, keyRegistry); err != nil { 290 return err 291 } 292 m.mu.rotationEnabled = true 293 return nil 294 } 295 296 func (m *DataKeyManager) getScrubbedRegistry() *enginepbccl.DataKeysRegistry { 297 m.mu.Lock() 298 defer m.mu.Unlock() 299 r := makeRegistryProto() 300 proto.Merge(r, m.mu.keyRegistry) 301 for _, v := range r.DataKeys { 302 v.Key = nil 303 } 304 return r 305 } 306 307 func validateRegistry(keyRegistry *enginepbccl.DataKeysRegistry) error { 308 if keyRegistry.ActiveStoreKeyId != "" && keyRegistry.StoreKeys[keyRegistry.ActiveStoreKeyId] == nil { 309 return fmt.Errorf("active store key %s not found", keyRegistry.ActiveStoreKeyId) 310 } 311 if keyRegistry.ActiveDataKeyId != "" && keyRegistry.DataKeys[keyRegistry.ActiveDataKeyId] == nil { 312 return fmt.Errorf("active data key %s not found", keyRegistry.ActiveDataKeyId) 313 } 314 return nil 315 } 316 317 // Generates a new data key and adds it to the keyRegistry proto and sets it as the active key. 318 func generateAndSetNewDataKey( 319 ctx context.Context, keyRegistry *enginepbccl.DataKeysRegistry, 320 ) (*enginepbccl.SecretKey, error) { 321 activeStoreKey := keyRegistry.StoreKeys[keyRegistry.ActiveStoreKeyId] 322 if activeStoreKey == nil { 323 panic("expected registry with active store key") 324 } 325 key := &enginepbccl.SecretKey{} 326 key.Info = &enginepbccl.KeyInfo{} 327 key.Info.EncryptionType = activeStoreKey.EncryptionType 328 key.Info.CreationTime = kmTimeNow().Unix() 329 key.Info.Source = "data key manager" 330 key.Info.ParentKeyId = activeStoreKey.KeyId 331 332 if activeStoreKey.EncryptionType == enginepbccl.EncryptionType_Plaintext { 333 key.Info.KeyId = plainKeyID 334 key.Info.WasExposed = true 335 } else { 336 var keyLength int 337 switch activeStoreKey.EncryptionType { 338 case enginepbccl.EncryptionType_AES128_CTR: 339 keyLength = 16 340 case enginepbccl.EncryptionType_AES192_CTR: 341 keyLength = 24 342 case enginepbccl.EncryptionType_AES256_CTR: 343 keyLength = 32 344 default: 345 return nil, fmt.Errorf("unknown encryption type %d for key ID %s", 346 activeStoreKey.EncryptionType, activeStoreKey.KeyId) 347 } 348 key.Key = make([]byte, keyLength) 349 n, err := rand.Read(key.Key) 350 if err != nil { 351 return nil, err 352 } 353 if n != keyLength { 354 log.Fatalf(ctx, "rand.Read returned no error but fewer bytes %d than promised %d", n, keyLength) 355 } 356 keyID := make([]byte, keyIDLength) 357 if n, err = rand.Read(keyID); err != nil { 358 return nil, err 359 } 360 if n != keyIDLength { 361 log.Fatalf(ctx, "rand.Read returned no error but fewer bytes %d than promised %d", n, keyIDLength) 362 } 363 // Hex encoding to make it human readable. 364 key.Info.KeyId = hex.EncodeToString(keyID) 365 key.Info.WasExposed = false 366 } 367 keyRegistry.DataKeys[key.Info.KeyId] = key 368 keyRegistry.ActiveDataKeyId = key.Info.KeyId 369 return key, nil 370 } 371 372 func (m *DataKeyManager) rotateDataKeyAndWrite( 373 ctx context.Context, keyRegistry *enginepbccl.DataKeysRegistry, 374 ) (err error) { 375 defer func() { 376 if err != nil { 377 log.Infof(ctx, "error while attempting to rotate data key: %s", err) 378 } else { 379 log.Infof(ctx, "rotated to new active data key: %s", proto.CompactTextString(m.mu.activeKey.Info)) 380 } 381 }() 382 383 var newKey *enginepbccl.SecretKey 384 if newKey, err = generateAndSetNewDataKey(ctx, keyRegistry); err != nil { 385 return 386 } 387 if err = validateRegistry(keyRegistry); err != nil { 388 return 389 } 390 bytes, err := protoutil.Marshal(keyRegistry) 391 if err != nil { 392 return 393 } 394 if err = storage.SafeWriteToFile(m.fs, m.dbDir, m.registryFilename, bytes); err != nil { 395 return 396 } 397 m.mu.keyRegistry = keyRegistry 398 m.mu.activeKey = newKey 399 return 400 }