github.com/matrixorigin/matrixone@v1.2.0/proto/logservice.proto

/* 
 * Copyright 2021 Matrix Origin
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

syntax = "proto3";
package logservice;
option go_package = "github.com/matrixorigin/matrixone/pkg/pb/logservice";

import "github.com/gogo/protobuf/gogoproto/gogo.proto";
import "metadata.proto";

option (gogoproto.goproto_enum_prefix_all) = false;

enum UpdateType {
  LeaseHolderIDUpdate = 0;
  TruncateLSNUpdate   = 1;
  UserEntryUpdate     = 2;
  TSOUpdate           = 3;
}

enum NodeState {
  NormalState  = 0;
  TimeoutState = 1;
}

message CNStore {
  string          UUID               = 1;
  string          ServiceAddress     = 2;
  string          SQLAddress         = 3;
  string          LockServiceAddress = 4;
  reserved 5;
  metadata.CNRole Role               = 6;
  uint64          Tick               = 7;
  NodeState       State              = 8;
  map<string, metadata.LabelList> Labels = 9 [(gogoproto.nullable) = false];
  metadata.WorkState WorkState       = 10;
  string          QueryAddress       = 11;
  ConfigData ConfigData = 12;
  Resource Resource = 13 [(gogoproto.nullable) = false];
  int64 UpTime = 14;
}

message TNStore {
  string    UUID           = 1;
  string    ServiceAddress = 2;
  uint64    Tick           = 3;
  NodeState State          = 4;

  repeated TNShardInfo Shards = 5 [(gogoproto.nullable) = false];

  // Server address for logtail push model
  string LogtailServerAddress = 6;
  // LockServiceAddress lock service address for lock table allocator
  string LockServiceAddress = 7;
  reserved 8;
  ConfigData ConfigData = 9;
  // QueryAddress is the address of the queryservice on tn
  string QueryAddress = 10;
}

message LogStore {
  string    UUID           = 1;
  string    ServiceAddress = 2;
  uint64    Tick           = 3;
  NodeState State          = 4;

  repeated LogReplicaInfo Replicas = 5 [(gogoproto.nullable) = false];
  ConfigData ConfigData = 6;
}

// LogShardInfo contains information a log shard.
message LogShardInfo {
  // ShardID is the ID of a Log shard.
  uint64 ShardID               = 1;
  // Replicas is a map of ReplicaID to LogStore UUID, it describe the member
  // replicas of the shard at the given Epoch.
  map<uint64, string> Replicas = 2;
  // Epoch is the epoch value of the Shard, member replicas of the Shard can
  // change across epochs.
  uint64 Epoch                 = 3;
  // LeaderID is the ReplicaID of the leader replica at the given term. When
  // LeaderID is 0, it means there is no leader or the leader is unknown.
  uint64 LeaderID              = 4;
  // Term is the Raft term value.
  uint64 Term                  = 5;

  // TODO: per shard stats like CPU/memory/network usage can be added here
};

// LogReplicaInfo contains information of a log replica.
message LogReplicaInfo {
  LogShardInfo LogShardInfo = 1 [(gogoproto.nullable) = false, (gogoproto.embed) = true];
  // ReplicaID is the ID of a replica within the Log shard.
  uint64 ReplicaID = 2;
}

message Resource {
  uint64 CPUTotal = 1;
  double CPUAvailable = 2;
  uint64 MemTotal = 3;
  uint64 MemAvailable = 4;
}

// CNStoreHeartbeat is the periodic message sent tp the HAKeeper by CN stores.
message CNStoreHeartbeat {
  string          UUID               = 1;
  string          ServiceAddress     = 2;
  string          SQLAddress         = 3;
  string          LockServiceAddress = 4;
  reserved 5;
  metadata.CNRole Role               = 6;
  bool            TaskServiceCreated = 7;
  string          QueryAddress       = 8;
  string          InitWorkState      = 9;
  string          GossipAddress      = 10;
  bool            GossipJoined       = 11;
  ConfigData      ConfigData         = 12;
  Resource        Resource           = 13 [(gogoproto.nullable) = false];
}

// CNAllocateID is the periodic message sent tp the HAKeeper by CN stores.
message CNAllocateID {
  string key = 1;
  uint64 Batch = 2;
}

// LogStoreHeartbeat is the periodic message sent to the HAKeeper by Log Stores.
message LogStoreHeartbeat {
  // UUID is the uuid of the Log Store.
  string UUID = 1;

  string RaftAddress    = 2;
  string ServiceAddress = 3;
  string GossipAddress  = 4;

  // Replicas is a list of LogReplicaInfo instances collected on the specified
  // LogStore. Details in Shards are based on the local knowledge of each
  // replica running on the current LogStore, it may not be accurate or
  // update to date due to various reasons.
  repeated LogReplicaInfo Replicas = 5 [(gogoproto.nullable) = false];
  // TaskServiceCreated task service is created at the current log node
  bool            TaskServiceCreated    = 6;

  ConfigData ConfigData = 7;
};

// TNShardInfo contains information of a launched TN shard.
message TNShardInfo {
  // ShardID uniquely identifies a TN shard. Each TN shard manages a Primary
  // Key range or hashed Primary Key value range.
  uint64 ShardID                 = 1;
  // ReplicaID uniquely identifies a TN shard instance. After repairing a
  // failed TN shard, a new TN shard instance is created with a new ReplicaID
  // value.
  uint64 ReplicaID               = 2;

  // TODO: per shard stats like CPU/memory/network usage can be added here
};

// TNStoreHeartbeat is the periodic message sent to the HAKeeper by TN stores.
message TNStoreHeartbeat {
  // UUID is the uuid of the TN Store.
  string UUID                 = 1;
  string ServiceAddress       = 2;
  // Shards is a list of TNShardInfo instances collected on the specified
  // TN store.
  repeated TNShardInfo Shards = 3 [(gogoproto.nullable) = false];
  bool            TaskServiceCreated = 4;
  // Server address for logtail push model
  string LogtailServerAddress = 5;
  // LockServiceAddress lock service address for lock table allocator
  string LockServiceAddress = 6;
  reserved 7;
  ConfigData ConfigData = 8;
  // QueryAddress is the address of queryservice on tn
  string QueryAddress = 9;
};

message RSMState {
  uint64 Tso = 1;
  uint64 Index = 2;
  uint64 LeaseHolderID = 3;
  uint64 TruncatedLsn = 4;
  map<uint64, uint64> LeaseHistory = 5; // log lsn -> truncate lsn
}

enum MethodType {
  TSO_UPDATE     = 0;
  APPEND         = 1;
  READ           = 2;
  TRUNCATE       = 3;
  GET_TRUNCATE   = 4;
  CONNECT        = 5;
  CONNECT_RO     = 6;
  LOG_HEARTBEAT  = 7;
  CN_HEARTBEAT   = 8;
  TN_HEARTBEAT   = 9;
  CHECK_HAKEEPER = 10;
  GET_CLUSTER_DETAILS = 11;
  GET_SHARD_INFO = 12;
  CN_ALLOCATE_ID = 13;
  GET_CLUSTER_STATE = 14;
  UPDATE_CN_LABEL = 15;
  UPDATE_CN_WORK_STATE = 16;
  PATCH_CN_STORE = 17;
  DELETE_CN_STORE = 18;
  PROXY_HEARTBEAT = 19;
};

enum RecordType {
  UserRecord    = 0;
  Internal      = 1;
  LeaseUpdate   = 2;
  LeaseRejected = 3;
}

// LogRecord is what we store into the LogService.
message LogRecord {
  uint64 Lsn    = 1;
  RecordType Type = 2;
  bytes Data      = 3;
};

message LogRequest {
  string Name         = 1;
  uint64 ShardID      = 2;
  uint64 Lsn          = 3;
  uint64 MaxSize      = 4;
  uint64 TNShardID    = 5;
  uint64 TNID         = 6;
}

message TsoRequest {
  uint64 Count        = 1;
}

enum CNLabelOp {
  SetLabel = 0;
  DeleteLabel = 1;
}

message CNStoreLabel {
  // UUID is the uuid of the CN store.
  string UUID = 1;
  // Operation is the CN label operation.
  // TODO(volgariver6): Unused field.
  CNLabelOp Operation = 2;
  // Labels is the labels of the CN store.
  map<string, metadata.LabelList> Labels = 3 [(gogoproto.nullable) = false];
}

message CNWorkState {
  // UUID is the uuid of the CN store.
  string UUID     = 1;
  // State is the CN work state.
  metadata.WorkState State = 2;
}

message CNStateLabel {
  // UUID is the uuid of the CN store.
  string UUID     = 1;
  // State is the CN work state.
  metadata.WorkState State = 2;
  // Labels is the labels of the CN store.
  map<string, metadata.LabelList> Labels = 3 [(gogoproto.nullable) = false];
}

message Request {
  uint64 RequestID               = 1;
  MethodType Method              = 2;
  LogRequest LogRequest          = 3 [(gogoproto.nullable) = false];
  LogStoreHeartbeat LogHeartbeat = 4;
  CNStoreHeartbeat CNHeartbeat   = 5;
  TNStoreHeartbeat TNHeartbeat   = 6;
  TsoRequest TsoRequest          = 7;
  CNAllocateID CNAllocateID      = 8;
  CNStoreLabel CNStoreLabel      = 9;
  CNWorkState CNWorkState        = 10;
  CNStateLabel CNStateLabel      = 11;
  DeleteCNStore DeleteCNStore    = 12;
  ProxyHeartbeat ProxyHeartbeat  = 13;
};

message LogResponse {
  uint64 ShardID      = 1;
  uint64 Lsn          = 2;
  uint64 LastLsn      = 3;
}

message AllocateIDResponse {
  uint64 FirstID = 1;
}

message Response {
  uint64 RequestID               = 1;
  MethodType Method              = 2;
  uint32 ErrorCode               = 3;
  string ErrorMessage            = 4;
  bool IsHAKeeper                = 5;
  LogResponse LogResponse        = 6 [(gogoproto.nullable) = false];
  CommandBatch CommandBatch      = 7;
  TsoResponse TsoResponse        = 8;
  ClusterDetails ClusterDetails  = 9;
  ShardInfoQueryResult ShardInfo = 10;
  AllocateIDResponse AllocateID  = 11;
  CheckerState CheckerState      = 12;
};

message LogRecordResponse {
  repeated LogRecord Records = 1 [(gogoproto.nullable) = false];
};

message TsoResponse {
  uint64 Value = 1;
}

//
// HAKeeper related pb
//

enum HAKeeperUpdateType {
  TickUpdate              = 0;
  CNHeartbeatUpdate       = 1;
  TNHeartbeatUpdate       = 2;
  LogHeartbeatUpdate      = 3;
  GetIDUpdate             = 4;
  ScheduleCommandUpdate   = 5;
  SetStateUpdate          = 6;
  InitialClusterUpdate    = 7;
  SetTaskSchedulerStateUpdate  = 8;
  SetTaskTableUserUpdate       = 9;
  UpdateCNLabel = 10;
  UpdateCNWorkState = 11;
  PatchCNStore = 12;
  RemoveCNStore = 13;
  ProxyHeartbeatUpdate = 14;
}

// HAKeeperState state transition diagram
//               HAKeeperCreated
//                     │
//                     │ cluster init info set
//                     ▼
//           HAKeeperBootstrapping
//                     │
//                     │ init schedule commands received
//                     ▼
//     HAKeeperBootstrapCommandsReceived
//                     │
//                     │
//         ┌───────────┴───────────┐
//         │ time out              │ bootstrap successfully
//         ▼                       ▼
// HAKeeperBootstrapFailed     HAKeeperRunning
//
enum HAKeeperState {
  HAKeeperCreated                    = 0;
  HAKeeperBootstrapping              = 1;
  HAKeeperBootstrapCommandsReceived  = 2;
  HAKeeperBootstrapFailed            = 3;
  HAKeeperRunning                    = 4;
}

// TaskSchedulerState state transition diagram
//         TaskSchedulerCreated
//                  │
//                  │  setTaskTableUser()
//                  ▼
//         TaskSchedulerStarted
//                 │ ▲
//                 │ │
//                 ▼ │
//          TaskSchedulerStopped
enum TaskSchedulerState {
  TaskSchedulerCreated  = 0;
  TaskSchedulerRunning  = 1;
  TaskSchedulerStopped  = 2;
}

message TaskTableUser {
  // Username for task table user
  string Username = 1;
  // Password for task table user
  string Password = 2;
}

// Replica of the shard
message Replica {
  // UUID which store the Replica is located in
  string UUID = 1;

  uint64 ShardID   = 2;
  uint64 ReplicaID = 3;
  uint64 Epoch     = 4;

  // LogShardID only used for TNShard.
  uint64 LogShardID = 5;
}

// ConfigChangeType indicates config change command type.
enum ConfigChangeType {
  AddReplica    = 0;
  RemoveReplica = 1;
  StartReplica  = 2;
  StopReplica   = 3;
  KillZombie    = 4;
}

// ConfigChange is the detail of a config change.
message ConfigChange {
  Replica Replica             = 1 [(gogoproto.nullable) = false];
  ConfigChangeType ChangeType = 2;

  // only used for bootstrap
  map<uint64, string> InitialMembers = 3;
}

// ShutdownStore would stop store.
message ShutdownStore {
  string StoreID = 1;
}

// ServiceType specifies type of service
enum ServiceType {
  LogService = 0;
  TNService  = 1;
  CNService  = 2;
  ProxyService = 3;
}

// ScheduleCommand contains a shard schedule command.
message ScheduleCommand {
  // UUID which store the ScheduleCommand is sent to
  string            UUID            = 1;
  bool              Bootstrapping   = 2;
  ServiceType       ServiceType     = 3;
  ConfigChange      ConfigChange    = 4;
  ShutdownStore     ShutdownStore   = 5;
  CreateTaskService CreateTaskService = 6;
  DeleteCNStore     DeleteCNStore     = 7;
  JoinGossipCluster JoinGossipCluster = 8;
  DeleteProxyStore  DeleteProxyStore  = 9;
}

message JoinGossipCluster {
  repeated string Existing = 1;
}

// CreateTaskService start task service at current node
message CreateTaskService {
  // User used to connect to the task database.
  TaskTableUser User         = 1 [(gogoproto.nullable) = false];
  // TaskDatabase task framework database.
  string        TaskDatabase = 2;
}

// DeleteCNStore deletes CN store from CNState to save hakeeper's memory.
message DeleteCNStore {
  string StoreID = 1;
}

// DeleteProxyStore deletes Proxy store from ProxyState to save hakeeper's memory.
message DeleteProxyStore {
  string StoreID = 1;
}

message CommandBatch {
  uint64 Term                       = 1;
  repeated ScheduleCommand Commands = 2 [(gogoproto.nullable) = false];
}

// CNStoreInfo contains information on a CN store.
message CNStoreInfo {
  uint64          Tick               = 1;
  string          ServiceAddress     = 2;
  string          SQLAddress         = 3;
  string          LockServiceAddress = 4;
  reserved 5;
  metadata.CNRole Role               = 6;
  bool            TaskServiceCreated = 7;
  map<string, metadata.LabelList> Labels = 8 [(gogoproto.nullable) = false];
  metadata.WorkState WorkState       = 9;
  string          QueryAddress       = 10;
  string          GossipAddress      = 11;
  bool            GossipJoined       = 12;
  ConfigData      ConfigData         = 13;
  Resource        Resource           = 14 [(gogoproto.nullable) = false];
  int64           UpTime             = 15;
}

// CNState contains all CN details known to the HAKeeper.
message CNState {
  // Stores is keyed by CN store UUID. 
  map<string, CNStoreInfo> Stores = 1 [(gogoproto.nullable) = false];
}


// TNStoreInfo contains information on a TN store.
message TNStoreInfo {
  uint64 Tick                 = 1;
  string ServiceAddress       = 2;
  repeated TNShardInfo Shards = 3 [(gogoproto.nullable) = false];

  bool TaskServiceCreated = 4;

  // Server address for logtail push model
  string LogtailServerAddress = 5;
  // LockServiceAddress lock service address for lock table allocator
  string LockServiceAddress = 6;
  reserved 7;
  ConfigData ConfigData = 8;
  // QueryAddress is the address of queryservice on tn
  string          QueryAddress       = 9;
}

// TNState contains all TN details known to the HAKeeper.
message TNState {
  // Stores is keyed by TN store UUID.
  map<string, TNStoreInfo> Stores = 1 [(gogoproto.nullable) = false];
}

message ProxyStore {
  string UUID = 1;
  uint64 Tick = 2;
  string ListenAddress = 3;
  ConfigData ConfigData = 4;
}

message ProxyState {
  map<string, ProxyStore> Stores = 1 [(gogoproto.nullable) = false];
}

message ProxyHeartbeat {
  string UUID = 1;
  string ListenAddress = 2;
  ConfigData ConfigData = 3;
};

message ClusterDetails {
  repeated TNStore  TNStores    = 1 [(gogoproto.nullable) = false];
  repeated CNStore  CNStores    = 2 [(gogoproto.nullable) = false];
  repeated LogStore LogStores   = 3 [(gogoproto.nullable) = false];
  repeated ProxyStore ProxyStores = 4 [(gogoproto.nullable) = false];
  repeated DeletedStore DeletedStores = 5 [(gogoproto.nullable) = false];
}

// ClusterInfo provides a global view of all shards in the cluster. It
// describes the logical sharding of the system, rather than physical
// distribution of all replicas that belong to those shards.
message ClusterInfo {
  repeated metadata.TNShardRecord TNShards = 1 [(gogoproto.nullable) = false];
  repeated metadata.LogShardRecord LogShards = 2 [(gogoproto.nullable) = false];
}

message InitialClusterRequest {
  uint64 NumOfLogShards   = 1;
  uint64 NumOfTNShards    = 2;
  uint64 NumOfLogReplicas = 3;
  uint64 NextID           = 4;
  map<string, uint64> NextIDByKey = 5;
}

// LogStoreInfo contains information of all replicas found on a Log store.
message LogStoreInfo {
  uint64 Tick                                 = 1;
  string RaftAddress                          = 2;
  string ServiceAddress                       = 3;
  string GossipAddress                        = 4;
  repeated LogReplicaInfo Replicas = 5 [(gogoproto.nullable) = false];

  bool TaskServiceCreated = 6;
  ConfigData ConfigData = 7;
}

message LogState {
  // Shards is keyed by ShardID, it contains details aggregated from all Log
  // stores. Each pb.LogShardInfo here contains data aggregated from
  // different replicas and thus reflect a more accurate description on each
  // shard.
  map<uint64, LogShardInfo> Shards = 1 [(gogoproto.nullable) = false];
  // Stores is keyed by log store UUID, it contains details found on each
  // store. Each LogStoreInfo here reflects what was last reported by each Log
  // store.
  map<string, LogStoreInfo> Stores = 2 [(gogoproto.nullable) = false];
}

// CheckerState contains all HAKeeper state required for making schedule
// commands.
message CheckerState {
  uint64 Tick             = 1;
  ClusterInfo ClusterInfo = 2 [(gogoproto.nullable) = false];
  TNState TNState         = 3 [(gogoproto.nullable) = false];
  LogState LogState       = 4 [(gogoproto.nullable) = false];
  CNState CNState         = 5 [(gogoproto.nullable) = false];
  HAKeeperState State     = 6;
  TaskSchedulerState TaskSchedulerState = 7;
  TaskTableUser TaskTableUser           = 8 [(gogoproto.nullable) = false];
  uint64 NextId                         = 9;
  map<string, uint64> NextIDByKey       = 10;
  ProxyState ProxyState = 11 [(gogoproto.nullable) = false];
  Resource Resource = 12 [(gogoproto.nullable) = false];
}

message DeletedStore {
  string UUID = 1;
  // StoreType indicates the type of the store: CN, TN or proxy.
  string StoreType = 2;
  string Address = 3;
  int64 UpTime = 4;
  int64 DownTime = 5;
}

// HAKeeperRSMState contains state maintained by HAKeeper's RSM.
message HAKeeperRSMState {
  uint64 Index                               = 1;
  uint64 Tick                                = 2;
  // NextID is a shared, global ID.
  uint64 NextID                              = 3;
  // NextIDByKey is IDs isolated by keys.
  map<string, uint64> NextIDByKey            = 4;
  uint64 Term                                = 5;
  HAKeeperState State                        = 6;
  TaskSchedulerState TaskSchedulerState      = 7;
  map<string, CommandBatch> ScheduleCommands = 8 [(gogoproto.nullable) = false];
  map<string, uint64> LogShards = 9;
  CNState CNState               = 10  [(gogoproto.nullable) = false];
  TNState TNState               = 11 [(gogoproto.nullable) = false];
  LogState LogState             = 12 [(gogoproto.nullable) = false];
  ClusterInfo ClusterInfo       = 13 [(gogoproto.nullable) = false];
  TaskTableUser TaskTableUser   = 14 [(gogoproto.nullable) = false];
  ProxyState ProxyState         = 15 [(gogoproto.nullable) = false];
  repeated DeletedStore DeletedStores = 16 [(gogoproto.nullable) = false];
}

// ReplicaInfo contains details of a replica
message ReplicaInfo {
  string UUID           = 1;
  string ServiceAddress = 2;
}

// ShardInfoQueryResult contains the result of the shard info query. 
message ShardInfoQueryResult {
  uint64 ShardID                    = 1;
  map<uint64, ReplicaInfo> Replicas = 2 [(gogoproto.nullable) = false];
  uint64 Epoch                      = 3;
  uint64 LeaderID                   = 4;
  uint64 Term                       = 5;
}

// BackupData is the information that needs to backup, including NextID and
// NextIDByKey in HAKeeperRSMState.
message BackupData {
  // NextID is a shared, global ID.
  uint64 NextID                   = 1;
  // NextIDByKey is IDs isolated by keys.
  map<string, uint64> NextIDByKey = 2;
}

message ConfigItem{
  // Name is the name of config item
  string Name = 1;
  // CurrentValue is the current value of config item
  string CurrentValue = 2;
  // DefaultValue is the default value of config item
  string DefaultValue = 3;
  // Internal denotes the config item is internal or not
  string Internal = 4;
}

// ConfigData is the content of config on cn,tn,log and proxy
message ConfigData{
  // Content is the key value of the config
  map<string, ConfigItem> Content = 1;
}