github.com/zhuohuang-hust/src-cbuild@v0.0.0-20230105071821-c7aab3e7c840/mergeCode/libnetwork/datastore/datastore.go (about)

     1  package datastore
     2  
     3  import (
     4  	"fmt"
     5  	"log"
     6  	"reflect"
     7  	"strings"
     8  	"sync"
     9  
    10  	"github.com/docker/libkv"
    11  	"github.com/docker/libkv/store"
    12  	"github.com/docker/libnetwork/discoverapi"
    13  	"github.com/docker/libnetwork/types"
    14  )
    15  
    16  //DataStore exported
    17  type DataStore interface {
    18  	// GetObject gets data from datastore and unmarshals to the specified object
    19  	GetObject(key string, o KVObject) error
    20  	// PutObject adds a new Record based on an object into the datastore
    21  	PutObject(kvObject KVObject) error
    22  	// PutObjectAtomic provides an atomic add and update operation for a Record
    23  	PutObjectAtomic(kvObject KVObject) error
    24  	// DeleteObject deletes a record
    25  	DeleteObject(kvObject KVObject) error
    26  	// DeleteObjectAtomic performs an atomic delete operation
    27  	DeleteObjectAtomic(kvObject KVObject) error
    28  	// DeleteTree deletes a record
    29  	DeleteTree(kvObject KVObject) error
    30  	// Watchable returns whether the store is watchable or not
    31  	Watchable() bool
    32  	// Watch for changes on a KVObject
    33  	Watch(kvObject KVObject, stopCh <-chan struct{}) (<-chan KVObject, error)
    34  	// RestartWatch retriggers stopped Watches
    35  	RestartWatch()
    36  	// Active returns if the store is active
    37  	Active() bool
    38  	// List returns of a list of KVObjects belonging to the parent
    39  	// key. The caller must pass a KVObject of the same type as
    40  	// the objects that need to be listed
    41  	List(string, KVObject) ([]KVObject, error)
    42  	// Scope returns the scope of the store
    43  	Scope() string
    44  	// KVStore returns access to the KV Store
    45  	KVStore() store.Store
    46  	// Close closes the data store
    47  	Close()
    48  }
    49  
    50  // ErrKeyModified is raised for an atomic update when the update is working on a stale state
    51  var (
    52  	ErrKeyModified = store.ErrKeyModified
    53  	ErrKeyNotFound = store.ErrKeyNotFound
    54  )
    55  
    56  type datastore struct {
    57  	scope      string
    58  	store      store.Store
    59  	cache      *cache
    60  	watchCh    chan struct{}
    61  	active     bool
    62  	sequential bool
    63  	sync.Mutex
    64  }
    65  
    66  // KVObject is Key/Value interface used by objects to be part of the DataStore
    67  type KVObject interface {
    68  	// Key method lets an object provide the Key to be used in KV Store
    69  	Key() []string
    70  	// KeyPrefix method lets an object return immediate parent key that can be used for tree walk
    71  	KeyPrefix() []string
    72  	// Value method lets an object marshal its content to be stored in the KV store
    73  	Value() []byte
    74  	// SetValue is used by the datastore to set the object's value when loaded from the data store.
    75  	SetValue([]byte) error
    76  	// Index method returns the latest DB Index as seen by the object
    77  	Index() uint64
    78  	// SetIndex method allows the datastore to store the latest DB Index into the object
    79  	SetIndex(uint64)
    80  	// True if the object exists in the datastore, false if it hasn't been stored yet.
    81  	// When SetIndex() is called, the object has been stored.
    82  	Exists() bool
    83  	// DataScope indicates the storage scope of the KV object
    84  	DataScope() string
    85  	// Skip provides a way for a KV Object to avoid persisting it in the KV Store
    86  	Skip() bool
    87  }
    88  
    89  // KVConstructor interface defines methods which can construct a KVObject from another.
    90  type KVConstructor interface {
    91  	// New returns a new object which is created based on the
    92  	// source object
    93  	New() KVObject
    94  	// CopyTo deep copies the contents of the implementing object
    95  	// to the passed destination object
    96  	CopyTo(KVObject) error
    97  }
    98  
    99  // ScopeCfg represents Datastore configuration.
   100  type ScopeCfg struct {
   101  	Client ScopeClientCfg
   102  }
   103  
   104  // ScopeClientCfg represents Datastore Client-only mode configuration
   105  type ScopeClientCfg struct {
   106  	Provider string
   107  	Address  string
   108  	Config   *store.Config
   109  }
   110  
   111  const (
   112  	// LocalScope indicates to store the KV object in local datastore such as boltdb
   113  	LocalScope = "local"
   114  	// GlobalScope indicates to store the KV object in global datastore such as consul/etcd/zookeeper
   115  	GlobalScope   = "global"
   116  	defaultPrefix = "/var/lib/docker/network/files"
   117  )
   118  
   119  const (
   120  	// NetworkKeyPrefix is the prefix for network key in the kv store
   121  	NetworkKeyPrefix = "network"
   122  	// EndpointKeyPrefix is the prefix for endpoint key in the kv store
   123  	EndpointKeyPrefix = "endpoint"
   124  )
   125  
   126  var (
   127  	defaultScopes = makeDefaultScopes()
   128  )
   129  
   130  func makeDefaultScopes() map[string]*ScopeCfg {
   131  	def := make(map[string]*ScopeCfg)
   132  	def[LocalScope] = &ScopeCfg{
   133  		Client: ScopeClientCfg{
   134  			Provider: string(store.BOLTDB),
   135  			Address:  defaultPrefix + "/local-kv.db",
   136  			Config: &store.Config{
   137  				Bucket: "libnetwork",
   138  			},
   139  		},
   140  	}
   141  
   142  	return def
   143  }
   144  
   145  var defaultRootChain = []string{"docker", "network", "v1.0"}
   146  var rootChain = defaultRootChain
   147  
   148  // DefaultScopes returns a map of default scopes and it's config for clients to use.
   149  func DefaultScopes(dataDir string) map[string]*ScopeCfg {
   150  	if dataDir != "" {
   151  		defaultScopes[LocalScope].Client.Address = dataDir + "/network/files/local-kv.db"
   152  		return defaultScopes
   153  	}
   154  
   155  	defaultScopes[LocalScope].Client.Address = defaultPrefix + "/local-kv.db"
   156  	return defaultScopes
   157  }
   158  
   159  // IsValid checks if the scope config has valid configuration.
   160  func (cfg *ScopeCfg) IsValid() bool {
   161  	if cfg == nil ||
   162  		strings.TrimSpace(cfg.Client.Provider) == "" ||
   163  		strings.TrimSpace(cfg.Client.Address) == "" {
   164  		return false
   165  	}
   166  
   167  	return true
   168  }
   169  
   170  //Key provides convenient method to create a Key
   171  func Key(key ...string) string {
   172  	keychain := append(rootChain, key...)
   173  	str := strings.Join(keychain, "/")
   174  	return str + "/"
   175  }
   176  
   177  //ParseKey provides convenient method to unpack the key to complement the Key function
   178  func ParseKey(key string) ([]string, error) {
   179  	chain := strings.Split(strings.Trim(key, "/"), "/")
   180  
   181  	// The key must atleast be equal to the rootChain in order to be considered as valid
   182  	if len(chain) <= len(rootChain) || !reflect.DeepEqual(chain[0:len(rootChain)], rootChain) {
   183  		return nil, types.BadRequestErrorf("invalid Key : %s", key)
   184  	}
   185  	return chain[len(rootChain):], nil
   186  }
   187  
   188  // newClient used to connect to KV Store
   189  func newClient(scope string, kv string, addr string, config *store.Config, cached bool) (DataStore, error) {
   190  
   191  	if cached && scope != LocalScope {
   192  		return nil, fmt.Errorf("caching supported only for scope %s", LocalScope)
   193  	}
   194  	sequential := false
   195  	if scope == LocalScope {
   196  		sequential = true
   197  	}
   198  
   199  	if config == nil {
   200  		config = &store.Config{}
   201  	}
   202  
   203  	var addrs []string
   204  
   205  	if kv == string(store.BOLTDB) {
   206  		// Parse file path
   207  		addrs = strings.Split(addr, ",")
   208  	} else {
   209  		// Parse URI
   210  		parts := strings.SplitN(addr, "/", 2)
   211  		addrs = strings.Split(parts[0], ",")
   212  
   213  		// Add the custom prefix to the root chain
   214  		if len(parts) == 2 {
   215  			rootChain = append([]string{parts[1]}, defaultRootChain...)
   216  		}
   217  	}
   218  
   219  	store, err := libkv.NewStore(store.Backend(kv), addrs, config)
   220  	if err != nil {
   221  		return nil, err
   222  	}
   223  
   224  	ds := &datastore{scope: scope, store: store, active: true, watchCh: make(chan struct{}), sequential: sequential}
   225  	if cached {
   226  		ds.cache = newCache(ds)
   227  	}
   228  
   229  	return ds, nil
   230  }
   231  
   232  // NewDataStore creates a new instance of LibKV data store
   233  func NewDataStore(scope string, cfg *ScopeCfg) (DataStore, error) {
   234  	if cfg == nil || cfg.Client.Provider == "" || cfg.Client.Address == "" {
   235  		c, ok := defaultScopes[scope]
   236  		if !ok || c.Client.Provider == "" || c.Client.Address == "" {
   237  			return nil, fmt.Errorf("unexpected scope %s without configuration passed", scope)
   238  		}
   239  
   240  		cfg = c
   241  	}
   242  
   243  	var cached bool
   244  	if scope == LocalScope {
   245  		cached = true
   246  	}
   247  
   248  	return newClient(scope, cfg.Client.Provider, cfg.Client.Address, cfg.Client.Config, cached)
   249  }
   250  
   251  // NewDataStoreFromConfig creates a new instance of LibKV data store starting from the datastore config data
   252  func NewDataStoreFromConfig(dsc discoverapi.DatastoreConfigData) (DataStore, error) {
   253  	var (
   254  		ok    bool
   255  		sCfgP *store.Config
   256  	)
   257  
   258  	sCfgP, ok = dsc.Config.(*store.Config)
   259  	if !ok && dsc.Config != nil {
   260  		return nil, fmt.Errorf("cannot parse store configuration: %v", dsc.Config)
   261  	}
   262  
   263  	scopeCfg := &ScopeCfg{
   264  		Client: ScopeClientCfg{
   265  			Address:  dsc.Address,
   266  			Provider: dsc.Provider,
   267  			Config:   sCfgP,
   268  		},
   269  	}
   270  
   271  	ds, err := NewDataStore(dsc.Scope, scopeCfg)
   272  	if err != nil {
   273  		return nil, fmt.Errorf("failed to construct datastore client from datastore configuration %v: %v", dsc, err)
   274  	}
   275  
   276  	return ds, err
   277  }
   278  
   279  func (ds *datastore) Close() {
   280  	ds.store.Close()
   281  }
   282  
   283  func (ds *datastore) Scope() string {
   284  	return ds.scope
   285  }
   286  
   287  func (ds *datastore) Active() bool {
   288  	return ds.active
   289  }
   290  
   291  func (ds *datastore) Watchable() bool {
   292  	return ds.scope != LocalScope
   293  }
   294  
   295  func (ds *datastore) Watch(kvObject KVObject, stopCh <-chan struct{}) (<-chan KVObject, error) {
   296  	sCh := make(chan struct{})
   297  
   298  	ctor, ok := kvObject.(KVConstructor)
   299  	if !ok {
   300  		return nil, fmt.Errorf("error watching object type %T, object does not implement KVConstructor interface", kvObject)
   301  	}
   302  
   303  	kvpCh, err := ds.store.Watch(Key(kvObject.Key()...), sCh)
   304  	if err != nil {
   305  		return nil, err
   306  	}
   307  
   308  	kvoCh := make(chan KVObject)
   309  
   310  	go func() {
   311  	retry_watch:
   312  		var err error
   313  
   314  		// Make sure to get a new instance of watch channel
   315  		ds.Lock()
   316  		watchCh := ds.watchCh
   317  		ds.Unlock()
   318  
   319  	loop:
   320  		for {
   321  			select {
   322  			case <-stopCh:
   323  				close(sCh)
   324  				return
   325  			case kvPair := <-kvpCh:
   326  				// If the backend KV store gets reset libkv's go routine
   327  				// for the watch can exit resulting in a nil value in
   328  				// channel.
   329  				if kvPair == nil {
   330  					ds.Lock()
   331  					ds.active = false
   332  					ds.Unlock()
   333  					break loop
   334  				}
   335  
   336  				dstO := ctor.New()
   337  
   338  				if err = dstO.SetValue(kvPair.Value); err != nil {
   339  					log.Printf("Could not unmarshal kvpair value = %s", string(kvPair.Value))
   340  					break
   341  				}
   342  
   343  				dstO.SetIndex(kvPair.LastIndex)
   344  				kvoCh <- dstO
   345  			}
   346  		}
   347  
   348  		// Wait on watch channel for a re-trigger when datastore becomes active
   349  		<-watchCh
   350  
   351  		kvpCh, err = ds.store.Watch(Key(kvObject.Key()...), sCh)
   352  		if err != nil {
   353  			log.Printf("Could not watch the key %s in store: %v", Key(kvObject.Key()...), err)
   354  		}
   355  
   356  		goto retry_watch
   357  	}()
   358  
   359  	return kvoCh, nil
   360  }
   361  
   362  func (ds *datastore) RestartWatch() {
   363  	ds.Lock()
   364  	defer ds.Unlock()
   365  
   366  	ds.active = true
   367  	watchCh := ds.watchCh
   368  	ds.watchCh = make(chan struct{})
   369  	close(watchCh)
   370  }
   371  
   372  func (ds *datastore) KVStore() store.Store {
   373  	return ds.store
   374  }
   375  
   376  // PutObjectAtomic adds a new Record based on an object into the datastore
   377  func (ds *datastore) PutObjectAtomic(kvObject KVObject) error {
   378  	var (
   379  		previous *store.KVPair
   380  		pair     *store.KVPair
   381  		err      error
   382  	)
   383  	if ds.sequential {
   384  		ds.Lock()
   385  		defer ds.Unlock()
   386  	}
   387  
   388  	if kvObject == nil {
   389  		return types.BadRequestErrorf("invalid KV Object : nil")
   390  	}
   391  
   392  	kvObjValue := kvObject.Value()
   393  
   394  	if kvObjValue == nil {
   395  		return types.BadRequestErrorf("invalid KV Object with a nil Value for key %s", Key(kvObject.Key()...))
   396  	}
   397  
   398  	if kvObject.Skip() {
   399  		goto add_cache
   400  	}
   401  
   402  	if kvObject.Exists() {
   403  		previous = &store.KVPair{Key: Key(kvObject.Key()...), LastIndex: kvObject.Index()}
   404  	} else {
   405  		previous = nil
   406  	}
   407  
   408  	_, pair, err = ds.store.AtomicPut(Key(kvObject.Key()...), kvObjValue, previous, nil)
   409  	if err != nil {
   410  		if err == store.ErrKeyExists {
   411  			return ErrKeyModified
   412  		}
   413  		return err
   414  	}
   415  
   416  	kvObject.SetIndex(pair.LastIndex)
   417  
   418  add_cache:
   419  	if ds.cache != nil {
   420  		// If persistent store is skipped, sequencing needs to
   421  		// happen in cache.
   422  		return ds.cache.add(kvObject, kvObject.Skip())
   423  	}
   424  
   425  	return nil
   426  }
   427  
   428  // PutObject adds a new Record based on an object into the datastore
   429  func (ds *datastore) PutObject(kvObject KVObject) error {
   430  	if ds.sequential {
   431  		ds.Lock()
   432  		defer ds.Unlock()
   433  	}
   434  
   435  	if kvObject == nil {
   436  		return types.BadRequestErrorf("invalid KV Object : nil")
   437  	}
   438  
   439  	if kvObject.Skip() {
   440  		goto add_cache
   441  	}
   442  
   443  	if err := ds.putObjectWithKey(kvObject, kvObject.Key()...); err != nil {
   444  		return err
   445  	}
   446  
   447  add_cache:
   448  	if ds.cache != nil {
   449  		// If persistent store is skipped, sequencing needs to
   450  		// happen in cache.
   451  		return ds.cache.add(kvObject, kvObject.Skip())
   452  	}
   453  
   454  	return nil
   455  }
   456  
   457  func (ds *datastore) putObjectWithKey(kvObject KVObject, key ...string) error {
   458  	kvObjValue := kvObject.Value()
   459  
   460  	if kvObjValue == nil {
   461  		return types.BadRequestErrorf("invalid KV Object with a nil Value for key %s", Key(kvObject.Key()...))
   462  	}
   463  	return ds.store.Put(Key(key...), kvObjValue, nil)
   464  }
   465  
   466  // GetObject returns a record matching the key
   467  func (ds *datastore) GetObject(key string, o KVObject) error {
   468  	if ds.sequential {
   469  		ds.Lock()
   470  		defer ds.Unlock()
   471  	}
   472  
   473  	if ds.cache != nil {
   474  		return ds.cache.get(key, o)
   475  	}
   476  
   477  	kvPair, err := ds.store.Get(key)
   478  	if err != nil {
   479  		return err
   480  	}
   481  
   482  	if err := o.SetValue(kvPair.Value); err != nil {
   483  		return err
   484  	}
   485  
   486  	// Make sure the object has a correct view of the DB index in
   487  	// case we need to modify it and update the DB.
   488  	o.SetIndex(kvPair.LastIndex)
   489  	return nil
   490  }
   491  
   492  func (ds *datastore) ensureParent(parent string) error {
   493  	exists, err := ds.store.Exists(parent)
   494  	if err != nil {
   495  		return err
   496  	}
   497  	if exists {
   498  		return nil
   499  	}
   500  	return ds.store.Put(parent, []byte{}, &store.WriteOptions{IsDir: true})
   501  }
   502  
   503  func (ds *datastore) List(key string, kvObject KVObject) ([]KVObject, error) {
   504  	if ds.sequential {
   505  		ds.Lock()
   506  		defer ds.Unlock()
   507  	}
   508  
   509  	if ds.cache != nil {
   510  		return ds.cache.list(kvObject)
   511  	}
   512  
   513  	// Bail out right away if the kvObject does not implement KVConstructor
   514  	ctor, ok := kvObject.(KVConstructor)
   515  	if !ok {
   516  		return nil, fmt.Errorf("error listing objects, object does not implement KVConstructor interface")
   517  	}
   518  
   519  	// Make sure the parent key exists
   520  	if err := ds.ensureParent(key); err != nil {
   521  		return nil, err
   522  	}
   523  
   524  	kvList, err := ds.store.List(key)
   525  	if err != nil {
   526  		return nil, err
   527  	}
   528  
   529  	var kvol []KVObject
   530  	for _, kvPair := range kvList {
   531  		if len(kvPair.Value) == 0 {
   532  			continue
   533  		}
   534  
   535  		dstO := ctor.New()
   536  		if err := dstO.SetValue(kvPair.Value); err != nil {
   537  			return nil, err
   538  		}
   539  
   540  		// Make sure the object has a correct view of the DB index in
   541  		// case we need to modify it and update the DB.
   542  		dstO.SetIndex(kvPair.LastIndex)
   543  
   544  		kvol = append(kvol, dstO)
   545  	}
   546  
   547  	return kvol, nil
   548  }
   549  
   550  // DeleteObject unconditionally deletes a record from the store
   551  func (ds *datastore) DeleteObject(kvObject KVObject) error {
   552  	if ds.sequential {
   553  		ds.Lock()
   554  		defer ds.Unlock()
   555  	}
   556  
   557  	// cleaup the cache first
   558  	if ds.cache != nil {
   559  		// If persistent store is skipped, sequencing needs to
   560  		// happen in cache.
   561  		ds.cache.del(kvObject, kvObject.Skip())
   562  	}
   563  
   564  	if kvObject.Skip() {
   565  		return nil
   566  	}
   567  
   568  	return ds.store.Delete(Key(kvObject.Key()...))
   569  }
   570  
   571  // DeleteObjectAtomic performs atomic delete on a record
   572  func (ds *datastore) DeleteObjectAtomic(kvObject KVObject) error {
   573  	if ds.sequential {
   574  		ds.Lock()
   575  		defer ds.Unlock()
   576  	}
   577  
   578  	if kvObject == nil {
   579  		return types.BadRequestErrorf("invalid KV Object : nil")
   580  	}
   581  
   582  	previous := &store.KVPair{Key: Key(kvObject.Key()...), LastIndex: kvObject.Index()}
   583  
   584  	if kvObject.Skip() {
   585  		goto del_cache
   586  	}
   587  
   588  	if _, err := ds.store.AtomicDelete(Key(kvObject.Key()...), previous); err != nil {
   589  		if err == store.ErrKeyExists {
   590  			return ErrKeyModified
   591  		}
   592  		return err
   593  	}
   594  
   595  del_cache:
   596  	// cleanup the cache only if AtomicDelete went through successfully
   597  	if ds.cache != nil {
   598  		// If persistent store is skipped, sequencing needs to
   599  		// happen in cache.
   600  		return ds.cache.del(kvObject, kvObject.Skip())
   601  	}
   602  
   603  	return nil
   604  }
   605  
   606  // DeleteTree unconditionally deletes a record from the store
   607  func (ds *datastore) DeleteTree(kvObject KVObject) error {
   608  	if ds.sequential {
   609  		ds.Lock()
   610  		defer ds.Unlock()
   611  	}
   612  
   613  	// cleaup the cache first
   614  	if ds.cache != nil {
   615  		// If persistent store is skipped, sequencing needs to
   616  		// happen in cache.
   617  		ds.cache.del(kvObject, kvObject.Skip())
   618  	}
   619  
   620  	if kvObject.Skip() {
   621  		return nil
   622  	}
   623  
   624  	return ds.store.DeleteTree(Key(kvObject.KeyPrefix()...))
   625  }