github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/kv/kvserver/closedts/container/container.go (about)

     1  // Copyright 2018 The Cockroach Authors.
     2  //
     3  // Use of this software is governed by the Business Source License
     4  // included in the file licenses/BSL.txt.
     5  //
     6  // As of the Change Date specified in that file, in accordance with
     7  // the Business Source License, use of this software will be governed
     8  // by the Apache License, Version 2.0, included in the file
     9  // licenses/APL.txt.
    10  
    11  package container
    12  
    13  import (
    14  	"sync/atomic"
    15  	"time"
    16  
    17  	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts"
    18  	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/ctpb"
    19  	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/minprop"
    20  	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/provider"
    21  	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/storage"
    22  	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/closedts/transport"
    23  	"github.com/cockroachdb/cockroach/pkg/roachpb"
    24  	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
    25  	"github.com/cockroachdb/cockroach/pkg/util/stop"
    26  	"github.com/cockroachdb/errors"
    27  	"google.golang.org/grpc"
    28  )
    29  
    30  // Config is a container that holds references to all of the components required
    31  // to set up a full closed timestamp subsystem.
    32  type Config struct {
    33  	Settings *cluster.Settings
    34  	Stopper  *stop.Stopper
    35  	Clock    closedts.LiveClockFn
    36  	Refresh  closedts.RefreshFn
    37  	Dialer   closedts.Dialer
    38  }
    39  
    40  // A Container is a full closed timestamp subsystem along with the Config it was
    41  // created from.
    42  type Container struct {
    43  	Config
    44  	// Initialized on Start().
    45  	Tracker  closedts.TrackerI
    46  	Storage  closedts.Storage
    47  	Provider closedts.Provider
    48  	Server   ctpb.Server
    49  	Clients  closedts.ClientRegistry
    50  
    51  	nodeID        roachpb.NodeID
    52  	delayedServer *delayedServer
    53  	noop          bool // if true, is NoopContainer
    54  }
    55  
    56  const (
    57  	// For each node, keep two historical buckets (i.e. one recent one, and one that
    58  	// lagging followers can still satisfy some reads from).
    59  	storageBucketNum = 2
    60  	// StorageBucketScale determines the (exponential) spacing of storage buckets.
    61  	// For example, a scale of 5s means that the second bucket will attempt to hold
    62  	// a closed timestamp 5s in the past from the first, and the third 5*5=25s from
    63  	// the first, etc.
    64  	//
    65  	// TODO(tschottdorf): it's straightforward to make this dynamic. It should track
    66  	// the interval at which timestamps are closed out, ideally being a little shorter.
    67  	// The effect of that would be that the most recent closed timestamp and the previous
    68  	// one can be queried against separately.
    69  	StorageBucketScale = 10 * time.Second
    70  )
    71  
    72  // NewContainer initializes a Container from the given Config. The Container
    73  // will need to be started separately, and will only be populated during Start().
    74  //
    75  // However, its RegisterClosedTimestampServer method can only be called before
    76  // the Container is started.
    77  func NewContainer(cfg Config) *Container {
    78  	return &Container{
    79  		Config: cfg,
    80  	}
    81  }
    82  
    83  type delayedServer struct {
    84  	active int32 // atomic
    85  	s      ctpb.Server
    86  }
    87  
    88  func (s *delayedServer) Start() {
    89  	atomic.StoreInt32(&s.active, 1)
    90  }
    91  
    92  func (s delayedServer) Get(client ctpb.ClosedTimestamp_GetServer) error {
    93  	if atomic.LoadInt32(&s.active) == 0 {
    94  		return errors.New("not available yet")
    95  	}
    96  	return s.s.Get(client)
    97  }
    98  
    99  // RegisterClosedTimestampServer registers the Server contained in the container
   100  // with gRPC.
   101  func (c *Container) RegisterClosedTimestampServer(s *grpc.Server) {
   102  	c.delayedServer = &delayedServer{}
   103  	ctpb.RegisterClosedTimestampServer(s, c.delayedServer)
   104  }
   105  
   106  // Start starts the Container. The Stopper used to create the Container is in
   107  // charge of stopping it.
   108  func (c *Container) Start(nodeID roachpb.NodeID) {
   109  	cfg := c.Config
   110  
   111  	if c.noop {
   112  		return
   113  	}
   114  
   115  	storage := storage.NewMultiStorage(func() storage.SingleStorage {
   116  		return storage.NewMemStorage(StorageBucketScale, storageBucketNum)
   117  	})
   118  
   119  	tracker := minprop.NewTracker()
   120  
   121  	pConf := provider.Config{
   122  		NodeID:   nodeID,
   123  		Settings: cfg.Settings,
   124  		Stopper:  cfg.Stopper,
   125  		Storage:  storage,
   126  		Clock:    cfg.Clock,
   127  		Close:    closedts.AsCloseFn(tracker),
   128  	}
   129  
   130  	provider := provider.NewProvider(&pConf)
   131  
   132  	server := transport.NewServer(cfg.Stopper, provider, cfg.Refresh)
   133  
   134  	rConf := transport.Config{
   135  		NodeID:   nodeID,
   136  		Settings: cfg.Settings,
   137  		Stopper:  cfg.Stopper,
   138  		Dialer:   cfg.Dialer,
   139  		Sink:     provider,
   140  	}
   141  
   142  	c.nodeID = nodeID
   143  	c.Storage = storage
   144  	c.Tracker = tracker
   145  	c.Server = server
   146  	c.Clients = transport.NewClients(rConf)
   147  	c.Provider = provider
   148  	c.Provider.Start()
   149  	if c.delayedServer != nil {
   150  		c.delayedServer.s = server
   151  		c.delayedServer.Start()
   152  	}
   153  }