github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/roachpb/metadata.proto

// Copyright 2014 The Cockroach Authors.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.

syntax = "proto2";
package cockroach.roachpb;
option go_package = "roachpb";

import "util/unresolved_addr.proto";
import "util/hlc/timestamp.proto";
import "gogoproto/gogo.proto";

// Attributes specifies a list of arbitrary strings describing
// node topology, store type, and machine capabilities.
message Attributes {
  option (gogoproto.goproto_stringer) = false;

  repeated string attrs = 1 [(gogoproto.moretags) = "yaml:\"attrs,flow\""];
}

// ReplicationTarget identifies a node/store pair.
message ReplicationTarget {
  option (gogoproto.goproto_stringer) = false;
  option (gogoproto.equal) = true;

  optional int32 node_id = 1 [(gogoproto.nullable) = false,
      (gogoproto.customname) = "NodeID", (gogoproto.casttype) = "NodeID"];
  optional int32 store_id = 2 [(gogoproto.nullable) = false,
      (gogoproto.customname) = "StoreID", (gogoproto.casttype) = "StoreID"];
}

// ReplicaType identifies which raft activities a replica participates in. In
// normal operation, VOTER_FULL and LEARNER are the only used states. However,
// atomic replication changes require a transition through a "joint config"; in
// this joint config, the VOTER_DEMOTING and VOTER_INCOMING types are used as
// well to denote voters which are being downgraded to learners and newly added
// by the change, respectively. A demoting voter is turning into a learner,
// which we prefer over a direct removal, which was used prior to v20.1 and
// uses the VOTER_OUTGOING type instead (see VersionChangeReplicasDemotion for
// details on why we're not doing that any more).
//
// All voter types indicate a replica that participates in all raft activities,
// including voting for leadership and committing entries. Typically, this
// requires a majority of voters to reach a decision. In a joint config, two
// separate majorities are required: one from the set of replicas that have
// either type VOTER or VOTER_OUTOING or VOTER_DEMOTING, as well as that of the
// set of types VOTER and VOTER_INCOMING . For example, when type VOTER_FULL is
// assigned to replicas 1 and 2, while 3 is VOTER_OUTGOING and 4 is
// VOTER_INCOMING, then the two sets over which quorums need to be achieved are
// {1,2,3} and {1,2,4}. Thus, {1,2} is a quorum of both, {1,3} is a quorum of
// the first but not the second, {1,4} is a quorum of the second but not the
// first, and {3,4} is a quorum of neither.
enum ReplicaType {
  option (gogoproto.goproto_enum_prefix) = false;

  // VOTER_FULL indicates a replica that is a voter both in the
  // incoming and outgoing set.
  VOTER_FULL = 0;
  // VOTER_INCOMING indicates a voting replica that will be a
  // VOTER_FULL once the ongoing atomic replication change is finalized; that is,
  // it is in the process of being added. In practice, this replica type should
  // be treated like a VOTER_FULL.
  VOTER_INCOMING = 2;
  // VOTER_OUTGOING indicates a voting replica that will not be part
  // of the descriptor once the ongoing atomic replication change is finalized;
  // that is, it is in the process of being removed. In practice, a replica of
  // this type should be treated accordingly and no work should be assigned to
  // it.
  VOTER_OUTGOING = 3;
  // VOTER_DEMOTING indicates a voting replica that will become a learner once
  // the ongoing atomic replication change is finalized; that is, it is in the
  // process of being demoted. Since learners are currently short-lived, this
  // replica is really being removed, with an intermediate step, and no work
  // should be assigned to it.
  VOTER_DEMOTING = 4;
  // LEARNER indicates a replica that applies committed entries, but does not
  // count towards the quorum(s). Candidates will not ask for (or take into
  // account) votes of (peers they consider) LEARNERs for leadership nor do
  // their acknowledged log entries get taken into account for determining the
  // committed index. At the time of writing, learners in CockroachDB are a
  // short-term transient state: a replica being added and on its way to being a
  // VOTER_{FULL,INCOMING}, or a VOTER_DEMOTING being removed.
  LEARNER = 1;
}

// ReplicaDescriptor describes a replica location by node ID
// (corresponds to a host:port via lookup on gossip network) and store
// ID (identifies the device).
// TODO(jeffreyxiao): All nullable fields in ReplicaDescriptor can be made
// non-nullable if #38302 is guaranteed to be on all nodes (I.E. 20.1).
message ReplicaDescriptor {
  option (gogoproto.goproto_stringer) = false;
  option (gogoproto.equal) = true;
  option (gogoproto.populate) = true;

  optional int32 node_id = 1 [(gogoproto.nullable) = false,
      (gogoproto.customname) = "NodeID", (gogoproto.casttype) = "NodeID"];
  optional int32 store_id = 2 [(gogoproto.nullable) = false,
      (gogoproto.customname) = "StoreID", (gogoproto.casttype) = "StoreID"];

  // replica_id uniquely identifies a replica instance. If a range is removed from
  // a store and then re-added to the same store, the new instance will have a
  // higher replica_id.
  optional int32 replica_id = 3 [(gogoproto.nullable) = false,
      (gogoproto.customname) = "ReplicaID", (gogoproto.casttype) = "ReplicaID"];

  // Type indicates which raft activities a replica participates in. A nil type
  // is equivalent to VOTER.
  optional ReplicaType type = 4;
}

// ReplicaIdent uniquely identifies a specific replica.
message ReplicaIdent {
  optional int64 range_id = 1 [(gogoproto.nullable) = false,
      (gogoproto.customname) = "RangeID", (gogoproto.casttype) = "RangeID"];
  optional ReplicaDescriptor replica = 2 [(gogoproto.nullable) = false];
}

// RangeDescriptor is the value stored in a range metadata key.
// A range is described using an inclusive start key, a non-inclusive end key,
// and a list of replicas where the range is stored.
//
// NOTE: Care must be taken when changing the encoding of this proto
// because it is used as part of conditional put operations.
// TODO(jeffreyxiao): All nullable fields in RangeDescriptor can be made
// non-nullable if #38302 is guaranteed to be on all nodes (I.E. 20.1).
message RangeDescriptor {
  option (gogoproto.goproto_stringer) = false;
  option (gogoproto.equal) = true;
  option (gogoproto.populate) = true;

  optional int64 range_id = 1 [(gogoproto.nullable) = false,
      (gogoproto.customname) = "RangeID", (gogoproto.casttype) = "RangeID"];
  // start_key is the first key which may be contained by this range.
  optional bytes start_key = 2 [(gogoproto.casttype) = "RKey"];
  // end_key marks the end of the range's possible keys.  EndKey itself is not
  // contained in this range - it will be contained in the immediately
  // subsequent range.
  optional bytes end_key = 3 [(gogoproto.casttype) = "RKey"];

  // InternalReplicas is the is the set of nodes/stores on which replicas of
  // this range are stored. DO NOT USE this field directly, use the `Replicas`
  // method instead. The ordering is arbitrary and subject to permutation.
  repeated ReplicaDescriptor internal_replicas = 4 [(gogoproto.nullable) = false];

  // next_replica_id is a counter used to generate replica IDs.
  optional int32 next_replica_id = 5 [(gogoproto.nullable) = false,
      (gogoproto.customname) = "NextReplicaID", (gogoproto.casttype) = "ReplicaID"];

  // generation is incremented on every split, merge, and every replica change,
  // i.e., whenever the span of the range or replica set changes. It is
  // initialized to zero when the range is first created. The generation
  // counter was first introduced to allow the range descriptor resulting from
  // a split and then merge to be distinguishable from the initial range
  // descriptor. This is important since changes to the range descriptors use
  // CPuts to ensure mutual exclusion.
  //
  // See #28071 for details on the above.
  //
  // Generations are also useful to make local replicaGC decisions when applying
  // a snapshot on keyspace that has overlapping replicas (but note that we do
  // not use this at the time of writing due to migration concerns; see below).
  //
  // We want to be able to compare the snapshot range's generation counter to
  // that of the overlapping replicas to draw a conclusion about whether the
  // snapshot can be applied (in which case the overlapping replicas need to be
  // safely removable). To that end, on a split, not only do we increment the
  // left hand side's generation, we also copy the resultant generation to the
  // newly created right hand side. On merges, we update the left hand side's
  // generation so that it exceeds by one the maximum of the left hand side and
  // the right hand side's generations from before the merge.
  //
  // If two replicas (perhaps one of them represented by a raft or preemptive
  // snapshot) as defined by their full range descriptor (including, notably,
  // the generation) overlap, then one of them has to be stale. This is because
  // the keyspace cleanly shards into non-overlapping ranges at all times (i.e.
  // for all consistent snapshots). Since meta ranges (or more generally, range
  // descriptors) are only ever updated transactionally, mutations to the meta
  // ranges can be serialized (i.e. put into some sequential ordering). We know
  // that the descriptors corresponding to both of our replicas can't be from
  // the same consistent snapshot of the meta ranges, so there is a version of
  // the meta ranges that includes only the first replica, and there is a
  // version that includes only the second replica. Without loss of generality,
  // assume that the first version is "older". This means that there is a finite
  // sequence of splits and merges that were applied to the consistent snapshot
  // corresponding to the first version which resulted in the second version of
  // the meta ranges.
  //
  // Each individual operation, thanks to the generational semantics above, has
  // the invariant that the resulting descriptors have a strictly larger
  // generation than any descriptors from the previous version that they cover.
  // For example, if a descriptor [a,c) at generation 5 is split into [a,b) and
  // [b,c), both of those latter range descriptors have generation 6. If [c,d)
  // is at generation 12 and [d, f) is at generation 17, then the resulting
  // merged range [c,f) will have generation 18.
  //
  // At the end of the day, for incoming snapshots, this means that we only have
  // to collect the overlapping replicas and their generations. Any replica with
  // a smaller generation is stale by the above argument and can be replicaGC'ed
  // right away. Any replica with a larger generation indicates that the snapshot
  // is stale and should be discarded. A replica with the same generation is
  // necessarily a replica of the range the snapshot is addressing (this is the
  // usual case, in which a snapshot "overlaps" precisely one replica, which is
  // the replica it's supposed to update, and no splits and merges have taken
  // place at all).
  //
  // For a third note, observe that the generational semantics above may
  // possibly allow range merges without colocation, at least in the sense that
  // the counter examples in #28071 are defused. This is because the
  // generational counter can answer the question whether the overlapping
  // replica is gc'able or not. If it is not gc'able, then by definition the
  // replica applying the merge is.
  optional int64 generation = 6 [(gogoproto.nullable) = false];
  // The presence of the sticky_bit indicates that the range should not be
  // automatically merged by the merge queue with the range to its left. It is
  // set during a split operation and unset during an unsplit operation. Note
  // that the unsplit operation is a different operation from the merge
  // operation. Unsplit only unsets sticky_bit. It is represented by a
  // timestamp that indicates when it expires. After the expiration time has
  // passed, the split is eligible for automatic merging. A nil sticky bit is
  // equivalent to hlc.Timestamp{}.
  //
  // The reason the sticky_bit exists is because when the merge queue is
  // enabled and a manual split happens, the split ranges would immediately be
  // merged by the merge queue. Previous, we threw an error when a user
  // attempted to execute ALTER TABLE/INDEX ... SPLIT AT ... when the merge
  // queue is enabled. With sticky_bit, users can manually split ranges without
  // diabling the merge queue.
  optional util.hlc.Timestamp sticky_bit = 7;

  reserved 8;
}

// Percentiles contains a handful of hard-coded percentiles meant to summarize
// a distribution.
message Percentiles {
  option (gogoproto.goproto_stringer) = false;

  optional double p10 = 1 [(gogoproto.nullable) = false];
  optional double p25 = 2 [(gogoproto.nullable) = false];
  optional double p50 = 3 [(gogoproto.nullable) = false];
  optional double p75 = 4 [(gogoproto.nullable) = false];
  optional double p90 = 5 [(gogoproto.nullable) = false];
  optional double pMax = 6 [(gogoproto.nullable) = false];
}

// StoreCapacity contains capacity information for a storage device.
message StoreCapacity {
  option (gogoproto.goproto_stringer) = false;

  // Total capacity of the disk used by the store, including space used by the
  // operating system and other applications.
  optional int64 capacity = 1 [(gogoproto.nullable) = false];
  // Available space remaining on the disk used by the store.
  optional int64 available = 2 [(gogoproto.nullable) = false];
  // Amount of disk space used by the data in the CockroachDB store. Note that
  // this is going to be less than (capacity - available), because those two
  // fields consider the entire disk and everything on it, while this only
  // tracks the store's disk usage.
  optional int64 used = 8 [(gogoproto.nullable) = false];
  // Amount of logical bytes stored in the store, ignoring RocksDB space
  // overhead. Useful for rebalancing so that moving a replica from one store
  // to another actually removes its bytes from the source store even though
  // RocksDB may not actually reclaim the physical disk space for a while.
  optional int64 logical_bytes = 9 [(gogoproto.nullable) = false];
  optional int32 range_count = 3 [(gogoproto.nullable) = false];
  optional int32 lease_count = 4 [(gogoproto.nullable) = false];
  // queries_per_second tracks the average number of queries processed per
  // second by replicas in the store. The stat is tracked over the time period
  // defined in storage/replica_stats.go, which as of July 2018 is 30 minutes.
  optional double queries_per_second = 10 [(gogoproto.nullable) = false];
  // writes_per_second tracks the average number of keys written per second
  // by ranges in the store. The stat is tracked over the time period defined
  // in storage/replica_stats.go, which as of July 2018 is 30 minutes.
  optional double writes_per_second = 5 [(gogoproto.nullable) = false];
  // bytes_per_replica and writes_per_replica contain percentiles for the
  // number of bytes and writes-per-second to each replica in the store.
  // This information can be used for rebalancing decisions.
  optional Percentiles bytes_per_replica = 6 [(gogoproto.nullable) = false];
  optional Percentiles writes_per_replica = 7 [(gogoproto.nullable) = false];
}

// NodeDescriptor holds details on node physical/network topology.
message NodeDescriptor {
  optional int32 node_id = 1 [(gogoproto.nullable) = false,
      (gogoproto.customname) = "NodeID", (gogoproto.casttype) = "NodeID"];
  optional util.UnresolvedAddr address = 2 [(gogoproto.nullable) = false];
  optional Attributes attrs = 3 [(gogoproto.nullable) = false];
  optional Locality locality = 4 [(gogoproto.nullable) = false];
  optional Version ServerVersion = 5 [(gogoproto.nullable) = false];
  optional string build_tag = 6 [(gogoproto.nullable) = false];
  optional int64 started_at = 7 [(gogoproto.nullable) = false];
  repeated LocalityAddress locality_address = 8 [(gogoproto.nullable) = false];
  optional string cluster_name = 9 [(gogoproto.nullable) = false];
  // The SQL address. If empty, indicates that the base address field
  // is also used to accept SQL connections.
  optional util.UnresolvedAddr sql_address = 10 [(gogoproto.nullable) = false, (gogoproto.customname) = "SQLAddress"];
}

// LocalityAddress holds the private address accessible only from other nodes
// in the corresponding locality.
message LocalityAddress {
  optional util.UnresolvedAddr address = 1 [(gogoproto.nullable) = false];
  optional Tier locality_tier = 2 [(gogoproto.nullable) = false];
}

// StoreDescriptor holds store information including store attributes, node
// descriptor and store capacity.
message StoreDescriptor {
  optional int32 store_id = 1 [(gogoproto.nullable) = false,
      (gogoproto.customname) = "StoreID", (gogoproto.casttype) = "StoreID"];
  optional Attributes attrs = 2 [(gogoproto.nullable) = false];
  optional NodeDescriptor node = 3 [(gogoproto.nullable) = false];
  optional StoreCapacity capacity = 4 [(gogoproto.nullable) = false];
}

// StoreDeadReplicas holds a storeID and a list of dead replicas on that store.
// Used to let the range lease holder know about corrupted or otherwise
// destroyed replicas that should be transferred to a different store.
message StoreDeadReplicas {
  optional int32 store_id = 1 [(gogoproto.nullable) = false,
      (gogoproto.customname) = "StoreID", (gogoproto.casttype) = "StoreID"];
  repeated ReplicaIdent replicas = 2 [(gogoproto.nullable) = false];
}

// Locality is an ordered set of key value Tiers that describe a node's
// location. The tier keys should be the same across all nodes.
message Locality {
  option (gogoproto.goproto_stringer) = false;

  repeated Tier tiers = 1 [(gogoproto.nullable) = false];
}

// Tier represents one level of the locality hierarchy.
message Tier {
  option (gogoproto.goproto_stringer) = false;

  // Key is the name of tier and should match all other nodes.
  optional string key = 1 [(gogoproto.nullable) = false];
  // Value is node specific value corresponding to the key.
  optional string value = 2 [(gogoproto.nullable) = false];
}

message Version {
  option (gogoproto.goproto_stringer) = false;

  // The names "major" and "minor" are reserved in C in
  // some platforms (e.g. FreeBSD).

  optional int32 major_val = 1 [(gogoproto.nullable) = false, (gogoproto.customname) = "Major"];
  optional int32 minor_val = 2 [(gogoproto.nullable) = false, (gogoproto.customname) = "Minor"];
  // Note that patch is a placeholder and will always be zero.
  optional int32 patch = 3 [(gogoproto.nullable) = false];
  // The unstable version is used to migrate during development.
  // Users of stable, public releases will only use binaries
  // with unstable set to 0.
  optional int32 unstable = 4 [(gogoproto.nullable) = false];
}