github.com/zhuohuang-hust/src-cbuild@v0.0.0-20230105071821-c7aab3e7c840/mergeCode/libnetwork/datastore/datastore.go (about) 1 package datastore 2 3 import ( 4 "fmt" 5 "log" 6 "reflect" 7 "strings" 8 "sync" 9 10 "github.com/docker/libkv" 11 "github.com/docker/libkv/store" 12 "github.com/docker/libnetwork/discoverapi" 13 "github.com/docker/libnetwork/types" 14 ) 15 16 //DataStore exported 17 type DataStore interface { 18 // GetObject gets data from datastore and unmarshals to the specified object 19 GetObject(key string, o KVObject) error 20 // PutObject adds a new Record based on an object into the datastore 21 PutObject(kvObject KVObject) error 22 // PutObjectAtomic provides an atomic add and update operation for a Record 23 PutObjectAtomic(kvObject KVObject) error 24 // DeleteObject deletes a record 25 DeleteObject(kvObject KVObject) error 26 // DeleteObjectAtomic performs an atomic delete operation 27 DeleteObjectAtomic(kvObject KVObject) error 28 // DeleteTree deletes a record 29 DeleteTree(kvObject KVObject) error 30 // Watchable returns whether the store is watchable or not 31 Watchable() bool 32 // Watch for changes on a KVObject 33 Watch(kvObject KVObject, stopCh <-chan struct{}) (<-chan KVObject, error) 34 // RestartWatch retriggers stopped Watches 35 RestartWatch() 36 // Active returns if the store is active 37 Active() bool 38 // List returns of a list of KVObjects belonging to the parent 39 // key. The caller must pass a KVObject of the same type as 40 // the objects that need to be listed 41 List(string, KVObject) ([]KVObject, error) 42 // Scope returns the scope of the store 43 Scope() string 44 // KVStore returns access to the KV Store 45 KVStore() store.Store 46 // Close closes the data store 47 Close() 48 } 49 50 // ErrKeyModified is raised for an atomic update when the update is working on a stale state 51 var ( 52 ErrKeyModified = store.ErrKeyModified 53 ErrKeyNotFound = store.ErrKeyNotFound 54 ) 55 56 type datastore struct { 57 scope string 58 store store.Store 59 cache *cache 60 watchCh chan struct{} 61 active bool 62 sequential bool 63 sync.Mutex 64 } 65 66 // KVObject is Key/Value interface used by objects to be part of the DataStore 67 type KVObject interface { 68 // Key method lets an object provide the Key to be used in KV Store 69 Key() []string 70 // KeyPrefix method lets an object return immediate parent key that can be used for tree walk 71 KeyPrefix() []string 72 // Value method lets an object marshal its content to be stored in the KV store 73 Value() []byte 74 // SetValue is used by the datastore to set the object's value when loaded from the data store. 75 SetValue([]byte) error 76 // Index method returns the latest DB Index as seen by the object 77 Index() uint64 78 // SetIndex method allows the datastore to store the latest DB Index into the object 79 SetIndex(uint64) 80 // True if the object exists in the datastore, false if it hasn't been stored yet. 81 // When SetIndex() is called, the object has been stored. 82 Exists() bool 83 // DataScope indicates the storage scope of the KV object 84 DataScope() string 85 // Skip provides a way for a KV Object to avoid persisting it in the KV Store 86 Skip() bool 87 } 88 89 // KVConstructor interface defines methods which can construct a KVObject from another. 90 type KVConstructor interface { 91 // New returns a new object which is created based on the 92 // source object 93 New() KVObject 94 // CopyTo deep copies the contents of the implementing object 95 // to the passed destination object 96 CopyTo(KVObject) error 97 } 98 99 // ScopeCfg represents Datastore configuration. 100 type ScopeCfg struct { 101 Client ScopeClientCfg 102 } 103 104 // ScopeClientCfg represents Datastore Client-only mode configuration 105 type ScopeClientCfg struct { 106 Provider string 107 Address string 108 Config *store.Config 109 } 110 111 const ( 112 // LocalScope indicates to store the KV object in local datastore such as boltdb 113 LocalScope = "local" 114 // GlobalScope indicates to store the KV object in global datastore such as consul/etcd/zookeeper 115 GlobalScope = "global" 116 defaultPrefix = "/var/lib/docker/network/files" 117 ) 118 119 const ( 120 // NetworkKeyPrefix is the prefix for network key in the kv store 121 NetworkKeyPrefix = "network" 122 // EndpointKeyPrefix is the prefix for endpoint key in the kv store 123 EndpointKeyPrefix = "endpoint" 124 ) 125 126 var ( 127 defaultScopes = makeDefaultScopes() 128 ) 129 130 func makeDefaultScopes() map[string]*ScopeCfg { 131 def := make(map[string]*ScopeCfg) 132 def[LocalScope] = &ScopeCfg{ 133 Client: ScopeClientCfg{ 134 Provider: string(store.BOLTDB), 135 Address: defaultPrefix + "/local-kv.db", 136 Config: &store.Config{ 137 Bucket: "libnetwork", 138 }, 139 }, 140 } 141 142 return def 143 } 144 145 var defaultRootChain = []string{"docker", "network", "v1.0"} 146 var rootChain = defaultRootChain 147 148 // DefaultScopes returns a map of default scopes and it's config for clients to use. 149 func DefaultScopes(dataDir string) map[string]*ScopeCfg { 150 if dataDir != "" { 151 defaultScopes[LocalScope].Client.Address = dataDir + "/network/files/local-kv.db" 152 return defaultScopes 153 } 154 155 defaultScopes[LocalScope].Client.Address = defaultPrefix + "/local-kv.db" 156 return defaultScopes 157 } 158 159 // IsValid checks if the scope config has valid configuration. 160 func (cfg *ScopeCfg) IsValid() bool { 161 if cfg == nil || 162 strings.TrimSpace(cfg.Client.Provider) == "" || 163 strings.TrimSpace(cfg.Client.Address) == "" { 164 return false 165 } 166 167 return true 168 } 169 170 //Key provides convenient method to create a Key 171 func Key(key ...string) string { 172 keychain := append(rootChain, key...) 173 str := strings.Join(keychain, "/") 174 return str + "/" 175 } 176 177 //ParseKey provides convenient method to unpack the key to complement the Key function 178 func ParseKey(key string) ([]string, error) { 179 chain := strings.Split(strings.Trim(key, "/"), "/") 180 181 // The key must atleast be equal to the rootChain in order to be considered as valid 182 if len(chain) <= len(rootChain) || !reflect.DeepEqual(chain[0:len(rootChain)], rootChain) { 183 return nil, types.BadRequestErrorf("invalid Key : %s", key) 184 } 185 return chain[len(rootChain):], nil 186 } 187 188 // newClient used to connect to KV Store 189 func newClient(scope string, kv string, addr string, config *store.Config, cached bool) (DataStore, error) { 190 191 if cached && scope != LocalScope { 192 return nil, fmt.Errorf("caching supported only for scope %s", LocalScope) 193 } 194 sequential := false 195 if scope == LocalScope { 196 sequential = true 197 } 198 199 if config == nil { 200 config = &store.Config{} 201 } 202 203 var addrs []string 204 205 if kv == string(store.BOLTDB) { 206 // Parse file path 207 addrs = strings.Split(addr, ",") 208 } else { 209 // Parse URI 210 parts := strings.SplitN(addr, "/", 2) 211 addrs = strings.Split(parts[0], ",") 212 213 // Add the custom prefix to the root chain 214 if len(parts) == 2 { 215 rootChain = append([]string{parts[1]}, defaultRootChain...) 216 } 217 } 218 219 store, err := libkv.NewStore(store.Backend(kv), addrs, config) 220 if err != nil { 221 return nil, err 222 } 223 224 ds := &datastore{scope: scope, store: store, active: true, watchCh: make(chan struct{}), sequential: sequential} 225 if cached { 226 ds.cache = newCache(ds) 227 } 228 229 return ds, nil 230 } 231 232 // NewDataStore creates a new instance of LibKV data store 233 func NewDataStore(scope string, cfg *ScopeCfg) (DataStore, error) { 234 if cfg == nil || cfg.Client.Provider == "" || cfg.Client.Address == "" { 235 c, ok := defaultScopes[scope] 236 if !ok || c.Client.Provider == "" || c.Client.Address == "" { 237 return nil, fmt.Errorf("unexpected scope %s without configuration passed", scope) 238 } 239 240 cfg = c 241 } 242 243 var cached bool 244 if scope == LocalScope { 245 cached = true 246 } 247 248 return newClient(scope, cfg.Client.Provider, cfg.Client.Address, cfg.Client.Config, cached) 249 } 250 251 // NewDataStoreFromConfig creates a new instance of LibKV data store starting from the datastore config data 252 func NewDataStoreFromConfig(dsc discoverapi.DatastoreConfigData) (DataStore, error) { 253 var ( 254 ok bool 255 sCfgP *store.Config 256 ) 257 258 sCfgP, ok = dsc.Config.(*store.Config) 259 if !ok && dsc.Config != nil { 260 return nil, fmt.Errorf("cannot parse store configuration: %v", dsc.Config) 261 } 262 263 scopeCfg := &ScopeCfg{ 264 Client: ScopeClientCfg{ 265 Address: dsc.Address, 266 Provider: dsc.Provider, 267 Config: sCfgP, 268 }, 269 } 270 271 ds, err := NewDataStore(dsc.Scope, scopeCfg) 272 if err != nil { 273 return nil, fmt.Errorf("failed to construct datastore client from datastore configuration %v: %v", dsc, err) 274 } 275 276 return ds, err 277 } 278 279 func (ds *datastore) Close() { 280 ds.store.Close() 281 } 282 283 func (ds *datastore) Scope() string { 284 return ds.scope 285 } 286 287 func (ds *datastore) Active() bool { 288 return ds.active 289 } 290 291 func (ds *datastore) Watchable() bool { 292 return ds.scope != LocalScope 293 } 294 295 func (ds *datastore) Watch(kvObject KVObject, stopCh <-chan struct{}) (<-chan KVObject, error) { 296 sCh := make(chan struct{}) 297 298 ctor, ok := kvObject.(KVConstructor) 299 if !ok { 300 return nil, fmt.Errorf("error watching object type %T, object does not implement KVConstructor interface", kvObject) 301 } 302 303 kvpCh, err := ds.store.Watch(Key(kvObject.Key()...), sCh) 304 if err != nil { 305 return nil, err 306 } 307 308 kvoCh := make(chan KVObject) 309 310 go func() { 311 retry_watch: 312 var err error 313 314 // Make sure to get a new instance of watch channel 315 ds.Lock() 316 watchCh := ds.watchCh 317 ds.Unlock() 318 319 loop: 320 for { 321 select { 322 case <-stopCh: 323 close(sCh) 324 return 325 case kvPair := <-kvpCh: 326 // If the backend KV store gets reset libkv's go routine 327 // for the watch can exit resulting in a nil value in 328 // channel. 329 if kvPair == nil { 330 ds.Lock() 331 ds.active = false 332 ds.Unlock() 333 break loop 334 } 335 336 dstO := ctor.New() 337 338 if err = dstO.SetValue(kvPair.Value); err != nil { 339 log.Printf("Could not unmarshal kvpair value = %s", string(kvPair.Value)) 340 break 341 } 342 343 dstO.SetIndex(kvPair.LastIndex) 344 kvoCh <- dstO 345 } 346 } 347 348 // Wait on watch channel for a re-trigger when datastore becomes active 349 <-watchCh 350 351 kvpCh, err = ds.store.Watch(Key(kvObject.Key()...), sCh) 352 if err != nil { 353 log.Printf("Could not watch the key %s in store: %v", Key(kvObject.Key()...), err) 354 } 355 356 goto retry_watch 357 }() 358 359 return kvoCh, nil 360 } 361 362 func (ds *datastore) RestartWatch() { 363 ds.Lock() 364 defer ds.Unlock() 365 366 ds.active = true 367 watchCh := ds.watchCh 368 ds.watchCh = make(chan struct{}) 369 close(watchCh) 370 } 371 372 func (ds *datastore) KVStore() store.Store { 373 return ds.store 374 } 375 376 // PutObjectAtomic adds a new Record based on an object into the datastore 377 func (ds *datastore) PutObjectAtomic(kvObject KVObject) error { 378 var ( 379 previous *store.KVPair 380 pair *store.KVPair 381 err error 382 ) 383 if ds.sequential { 384 ds.Lock() 385 defer ds.Unlock() 386 } 387 388 if kvObject == nil { 389 return types.BadRequestErrorf("invalid KV Object : nil") 390 } 391 392 kvObjValue := kvObject.Value() 393 394 if kvObjValue == nil { 395 return types.BadRequestErrorf("invalid KV Object with a nil Value for key %s", Key(kvObject.Key()...)) 396 } 397 398 if kvObject.Skip() { 399 goto add_cache 400 } 401 402 if kvObject.Exists() { 403 previous = &store.KVPair{Key: Key(kvObject.Key()...), LastIndex: kvObject.Index()} 404 } else { 405 previous = nil 406 } 407 408 _, pair, err = ds.store.AtomicPut(Key(kvObject.Key()...), kvObjValue, previous, nil) 409 if err != nil { 410 if err == store.ErrKeyExists { 411 return ErrKeyModified 412 } 413 return err 414 } 415 416 kvObject.SetIndex(pair.LastIndex) 417 418 add_cache: 419 if ds.cache != nil { 420 // If persistent store is skipped, sequencing needs to 421 // happen in cache. 422 return ds.cache.add(kvObject, kvObject.Skip()) 423 } 424 425 return nil 426 } 427 428 // PutObject adds a new Record based on an object into the datastore 429 func (ds *datastore) PutObject(kvObject KVObject) error { 430 if ds.sequential { 431 ds.Lock() 432 defer ds.Unlock() 433 } 434 435 if kvObject == nil { 436 return types.BadRequestErrorf("invalid KV Object : nil") 437 } 438 439 if kvObject.Skip() { 440 goto add_cache 441 } 442 443 if err := ds.putObjectWithKey(kvObject, kvObject.Key()...); err != nil { 444 return err 445 } 446 447 add_cache: 448 if ds.cache != nil { 449 // If persistent store is skipped, sequencing needs to 450 // happen in cache. 451 return ds.cache.add(kvObject, kvObject.Skip()) 452 } 453 454 return nil 455 } 456 457 func (ds *datastore) putObjectWithKey(kvObject KVObject, key ...string) error { 458 kvObjValue := kvObject.Value() 459 460 if kvObjValue == nil { 461 return types.BadRequestErrorf("invalid KV Object with a nil Value for key %s", Key(kvObject.Key()...)) 462 } 463 return ds.store.Put(Key(key...), kvObjValue, nil) 464 } 465 466 // GetObject returns a record matching the key 467 func (ds *datastore) GetObject(key string, o KVObject) error { 468 if ds.sequential { 469 ds.Lock() 470 defer ds.Unlock() 471 } 472 473 if ds.cache != nil { 474 return ds.cache.get(key, o) 475 } 476 477 kvPair, err := ds.store.Get(key) 478 if err != nil { 479 return err 480 } 481 482 if err := o.SetValue(kvPair.Value); err != nil { 483 return err 484 } 485 486 // Make sure the object has a correct view of the DB index in 487 // case we need to modify it and update the DB. 488 o.SetIndex(kvPair.LastIndex) 489 return nil 490 } 491 492 func (ds *datastore) ensureParent(parent string) error { 493 exists, err := ds.store.Exists(parent) 494 if err != nil { 495 return err 496 } 497 if exists { 498 return nil 499 } 500 return ds.store.Put(parent, []byte{}, &store.WriteOptions{IsDir: true}) 501 } 502 503 func (ds *datastore) List(key string, kvObject KVObject) ([]KVObject, error) { 504 if ds.sequential { 505 ds.Lock() 506 defer ds.Unlock() 507 } 508 509 if ds.cache != nil { 510 return ds.cache.list(kvObject) 511 } 512 513 // Bail out right away if the kvObject does not implement KVConstructor 514 ctor, ok := kvObject.(KVConstructor) 515 if !ok { 516 return nil, fmt.Errorf("error listing objects, object does not implement KVConstructor interface") 517 } 518 519 // Make sure the parent key exists 520 if err := ds.ensureParent(key); err != nil { 521 return nil, err 522 } 523 524 kvList, err := ds.store.List(key) 525 if err != nil { 526 return nil, err 527 } 528 529 var kvol []KVObject 530 for _, kvPair := range kvList { 531 if len(kvPair.Value) == 0 { 532 continue 533 } 534 535 dstO := ctor.New() 536 if err := dstO.SetValue(kvPair.Value); err != nil { 537 return nil, err 538 } 539 540 // Make sure the object has a correct view of the DB index in 541 // case we need to modify it and update the DB. 542 dstO.SetIndex(kvPair.LastIndex) 543 544 kvol = append(kvol, dstO) 545 } 546 547 return kvol, nil 548 } 549 550 // DeleteObject unconditionally deletes a record from the store 551 func (ds *datastore) DeleteObject(kvObject KVObject) error { 552 if ds.sequential { 553 ds.Lock() 554 defer ds.Unlock() 555 } 556 557 // cleaup the cache first 558 if ds.cache != nil { 559 // If persistent store is skipped, sequencing needs to 560 // happen in cache. 561 ds.cache.del(kvObject, kvObject.Skip()) 562 } 563 564 if kvObject.Skip() { 565 return nil 566 } 567 568 return ds.store.Delete(Key(kvObject.Key()...)) 569 } 570 571 // DeleteObjectAtomic performs atomic delete on a record 572 func (ds *datastore) DeleteObjectAtomic(kvObject KVObject) error { 573 if ds.sequential { 574 ds.Lock() 575 defer ds.Unlock() 576 } 577 578 if kvObject == nil { 579 return types.BadRequestErrorf("invalid KV Object : nil") 580 } 581 582 previous := &store.KVPair{Key: Key(kvObject.Key()...), LastIndex: kvObject.Index()} 583 584 if kvObject.Skip() { 585 goto del_cache 586 } 587 588 if _, err := ds.store.AtomicDelete(Key(kvObject.Key()...), previous); err != nil { 589 if err == store.ErrKeyExists { 590 return ErrKeyModified 591 } 592 return err 593 } 594 595 del_cache: 596 // cleanup the cache only if AtomicDelete went through successfully 597 if ds.cache != nil { 598 // If persistent store is skipped, sequencing needs to 599 // happen in cache. 600 return ds.cache.del(kvObject, kvObject.Skip()) 601 } 602 603 return nil 604 } 605 606 // DeleteTree unconditionally deletes a record from the store 607 func (ds *datastore) DeleteTree(kvObject KVObject) error { 608 if ds.sequential { 609 ds.Lock() 610 defer ds.Unlock() 611 } 612 613 // cleaup the cache first 614 if ds.cache != nil { 615 // If persistent store is skipped, sequencing needs to 616 // happen in cache. 617 ds.cache.del(kvObject, kvObject.Skip()) 618 } 619 620 if kvObject.Skip() { 621 return nil 622 } 623 624 return ds.store.DeleteTree(Key(kvObject.KeyPrefix()...)) 625 }