github.com/matrixorigin/matrixone@v0.7.0/proto/logservice.proto

/* 
 * Copyright 2021 Matrix Origin
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

syntax = "proto3";
package logservice;
option go_package = "github.com/matrixorigin/matrixone/pkg/pb/logservice";

import "github.com/gogo/protobuf/gogoproto/gogo.proto";
import "metadata.proto";

option (gogoproto.goproto_enum_prefix_all) = false;

enum UpdateType {
  LeaseHolderIDUpdate = 0;
  TruncateLSNUpdate   = 1;
  UserEntryUpdate     = 2;
  TSOUpdate           = 3;
}

enum NodeState {
  NormalState  = 0;
  TimeoutState = 1;
}

message CNStore {
  string          UUID           = 1;
  string          ServiceAddress = 2;
  string          SQLAddress     = 3;
  metadata.CNRole Role           = 4;
  uint64          Tick           = 5;
  NodeState       State          = 6;
}

message DNStore {
  string    UUID           = 1;
  string    ServiceAddress = 2;
  uint64    Tick           = 3;
  NodeState State          = 4;

  repeated DNShardInfo Shards = 5 [(gogoproto.nullable) = false];

  // Server address for logtail push model
  string LogtailServerAddress = 6;
}

message LogStore {
  string    UUID           = 1;
  string    ServiceAddress = 2;
  uint64    Tick           = 3;
  NodeState State          = 4;

  repeated LogReplicaInfo Replicas = 5 [(gogoproto.nullable) = false];
}

// LogShardInfo contains information a log shard.
message LogShardInfo {
  // ShardID is the ID of a Log shard.
  uint64 ShardID               = 1;
  // Replicas is a map of ReplicaID to LogStore UUID, it describe the member
  // replicas of the shard at the given Epoch. 
  map<uint64, string> Replicas = 2;
  // Epoch is the epoch value of the Shard, member replicas of the Shard can
  // change across epochs. 
  uint64 Epoch                 = 3;
  // LeaderID is the ReplicaID of the leader replica at the given term. When
  // LeaderID is 0, it means there is no leader or the leader is unknown.
  uint64 LeaderID              = 4;
  // Term is the Raft term value. 
  uint64 Term                  = 5;

  // TODO: per shard stats like CPU/memory/network usage can be added here
};

// LogReplicaInfo contains information of a log replica.
message LogReplicaInfo {
  LogShardInfo LogShardInfo = 1 [(gogoproto.nullable) = false, (gogoproto.embed) = true];
  // ReplicaID is the ID of a replica within the Log shard.
  uint64 ReplicaID = 2;
}

// CNStoreHeartbeat is the periodic message sent tp the HAKeeper by CN stores.
message CNStoreHeartbeat {
  string          UUID           = 1;
  string          ServiceAddress = 2;
  string          SQLAddress     = 3;
  metadata.CNRole Role           = 4;
  bool            TaskServiceCreated    = 5;
}


// CNAllocateID is the periodic message sent tp the HAKeeper by CN stores.
message CNAllocateID {
  uint64 Batch = 1;
}

// LogStoreHeartbeat is the periodic message sent to the HAKeeper by Log Stores.
message LogStoreHeartbeat {
  // UUID is the uuid of the Log Store.
  string UUID = 1;

  string RaftAddress    = 2;
  string ServiceAddress = 3;
  string GossipAddress  = 4;

  // Replicas is a list of LogReplicaInfo instances collected on the specified
  // LogStore. Details in Shards are based on the local knowledge of each
  // replica running on the current LogStore, it may not be accurate or
  // update to date due to various reasons. 
  repeated LogReplicaInfo Replicas = 5 [(gogoproto.nullable) = false];
  // TaskServiceCreated task service is created at the current log node
  bool            TaskServiceCreated    = 6;
};

// DNShardInfo contains information of a launched DN shard. 
message DNShardInfo {
  // ShardID uniquely identifies a DN shard. Each DN shard manages a Primary
  // Key range or hashed Primary Key value range. 
  uint64 ShardID                 = 1;
  // ReplicaID uniquely identifies a DN shard instance. After repairing a
  // failed DN shard, a new DN shard instance is created with a new ReplicaID
  // value.
  uint64 ReplicaID               = 2;

  // TODO: per shard stats like CPU/memory/network usage can be added here
};

// DNStoreHeartbeat is the periodic message sent to the HAKeeper by DN stores.
message DNStoreHeartbeat {
  // UUID is the uuid of the DN Store.
  string UUID                 = 1; 
  string ServiceAddress       = 2;
  // Shards is a list of DNShardInfo instances collected on the specified
  // DN store. 
  repeated DNShardInfo Shards = 3 [(gogoproto.nullable) = false];
  bool            TaskServiceCreated = 4;

  // Server address for logtail push model
  string LogtailServerAddress = 5;
};

message RSMState {
  uint64 Tso = 1;
  uint64 Index = 2;
  uint64 LeaseHolderID = 3;
  uint64 TruncatedLsn = 4;
  map<uint64, uint64> LeaseHistory = 5; // log lsn -> truncate lsn
}

enum MethodType {
  TSO_UPDATE     = 0;
  APPEND         = 1;
  READ           = 2;
  TRUNCATE       = 3;
  GET_TRUNCATE   = 4;
  CONNECT        = 5;
  CONNECT_RO     = 6;
  LOG_HEARTBEAT  = 7;
  CN_HEARTBEAT   = 8;
  DN_HEARTBEAT   = 9;
  CHECK_HAKEEPER = 10;
  GET_CLUSTER_DETAILS = 11;
  GET_SHARD_INFO = 12;
  CN_ALLOCATE_ID = 13;
  GET_CLUSTER_STATE = 14;
};

enum RecordType {
  UserRecord    = 0;
  Internal      = 1;
  LeaseUpdate   = 2;
  LeaseRejected = 3;
}

// LogRecord is what we store into the LogService.
message LogRecord {
  uint64 Lsn    = 1;
  RecordType Type = 2;
  bytes Data      = 3;
};

message LogRequest {
  string Name         = 1;
  uint64 ShardID      = 2;
  uint64 Lsn          = 3;
  uint64 MaxSize      = 4;
  uint64 DNShardID    = 5;
  uint64 DNID         = 6;
}

message TsoRequest {
  uint64 Count        = 1;
}

message Request {
  uint64 RequestID               = 1;
  MethodType Method              = 2;
  LogRequest LogRequest          = 3 [(gogoproto.nullable) = false];
  LogStoreHeartbeat LogHeartbeat = 4;
  CNStoreHeartbeat CNHeartbeat   = 5;
  DNStoreHeartbeat DNHeartbeat   = 6;
  TsoRequest TsoRequest          = 7;
  CNAllocateID CNAllocateID      = 8;
};

message LogResponse {
  uint64 ShardID      = 1;
  uint64 Lsn          = 2;
  uint64 LastLsn      = 3;
}

message AllocateIDResponse {
  uint64 FirstID = 1;
}

message Response {
  uint64 RequestID               = 1;
  MethodType Method              = 2;
  uint32 ErrorCode               = 3;
  string ErrorMessage            = 4;
  bool IsHAKeeper                = 5;
  LogResponse LogResponse        = 6 [(gogoproto.nullable) = false];
  CommandBatch CommandBatch      = 7;
  TsoResponse TsoResponse        = 8;
  ClusterDetails ClusterDetails  = 9;
  ShardInfoQueryResult ShardInfo = 10;
  AllocateIDResponse AllocateID  = 11;
  CheckerState CheckerState      = 12;
};

message LogRecordResponse {
  repeated LogRecord Records = 1 [(gogoproto.nullable) = false];
};

message TsoResponse {
  uint64 Value = 1;
}

//
// HAKeeper related pb
//

enum HAKeeperUpdateType {
  TickUpdate              = 0;
  CNHeartbeatUpdate       = 1;
  DNHeartbeatUpdate       = 2;
  LogHeartbeatUpdate      = 3;
  GetIDUpdate             = 4;
  ScheduleCommandUpdate   = 5;
  SetStateUpdate          = 6;
  InitialClusterUpdate    = 7;
  SetTaskSchedulerStateUpdate  = 8;
  SetTaskTableUserUpdate       = 9;
}

// HAKeeperState state transition diagram
//               HAKeeperCreated
//                     │
//                     │ cluster init info set
//                     ▼
//           HAKeeperBootstrapping
//                     │
//                     │ init schedule commands received
//                     ▼
//     HAKeeperBootstrapCommandsReceived
//                     │
//                     │
//         ┌───────────┴───────────┐
//         │ time out              │ bootstrap successfully
//         ▼                       ▼
// HAKeeperBootstrapFailed     HAKeeperRunning
//
enum HAKeeperState {
  HAKeeperCreated                    = 0;
  HAKeeperBootstrapping              = 1;
  HAKeeperBootstrapCommandsReceived  = 2;
  HAKeeperBootstrapFailed            = 3;
  HAKeeperRunning                    = 4;
}

// TaskSchedulerState state transition diagram
//         TaskSchedulerCreated
//                  │
//                  │  setTaskTableUser()
//                  ▼
//         TaskSchedulerStarted
//                 │ ▲
//                 │ │
//                 ▼ │
//          TaskSchedulerStopped
enum TaskSchedulerState {
  TaskSchedulerCreated  = 0;
  TaskSchedulerRunning  = 1;
  TaskSchedulerStopped  = 2;
}

message TaskTableUser {
  // Username for task table user
  string Username = 1;
  // Password for task table user
  string Password = 2;
}

// Replica of the shard
message Replica {
  // UUID which store the Replica is located in
  string UUID = 1;

  uint64 ShardID   = 2;
  uint64 ReplicaID = 3;
  uint64 Epoch     = 4;

  // LogShardID only used for DNShard.
  uint64 LogShardID = 5;
}

// ConfigChangeType indicates config change command type.
enum ConfigChangeType {
  AddReplica    = 0;
  RemoveReplica = 1;
  StartReplica  = 2;
  StopReplica   = 3;
  KillZombie    = 4;
}

// ConfigChange is the detail of a config change.
message ConfigChange {
  Replica Replica             = 1 [(gogoproto.nullable) = false];
  ConfigChangeType ChangeType = 2;

  // only used for bootstrap
  map<uint64, string> InitialMembers = 3;
}

// ShutdownStore would stop store.
message ShutdownStore {
  string StoreID = 1;
}

// ServiceType specifies type of service
enum ServiceType {
  LogService = 0;
  DNService  = 1;
  CNService  = 2;
}

// ScheduleCommand contains a shard schedule command.
message ScheduleCommand {
  // UUID which store the ScheduleCommand is sent to
  string            UUID            = 1;
  bool              Bootstrapping   = 2;
  ServiceType       ServiceType     = 3;
  ConfigChange      ConfigChange    = 4;
  ShutdownStore     ShutdownStore   = 5;
  CreateTaskService CreateTaskService = 6;
  DeleteCNStore     DeleteCNStore     = 7;
}

// CreateTaskService start task service at current node
message CreateTaskService {
  // User used to connect to the task database.
  TaskTableUser User         = 1 [(gogoproto.nullable) = false];
  // TaskDatabase task framework database.
  string        TaskDatabase = 2;
}

// DeleteCNStore deletes CN store from CNState to save hakeeper's memory.
message DeleteCNStore {
  string StoreID = 1;
}

message CommandBatch {
  uint64 Term                       = 1;
  repeated ScheduleCommand Commands = 2 [(gogoproto.nullable) = false];
}

// CNStoreInfo contains information on a CN store.
message CNStoreInfo {
  uint64          Tick           = 1;
  string          ServiceAddress = 2;
  string          SQLAddress     = 3;
  metadata.CNRole Role           = 4;

  bool TaskServiceCreated = 5;
}

// CNState contains all CN details known to the HAKeeper.
message CNState {
  // Stores is keyed by CN store UUID. 
  map<string, CNStoreInfo> Stores = 1 [(gogoproto.nullable) = false];
}


// DNStoreInfo contains information on a DN store.
message DNStoreInfo {
  uint64 Tick                 = 1;
  string ServiceAddress       = 2;
  repeated DNShardInfo Shards = 3 [(gogoproto.nullable) = false];

  bool TaskServiceCreated = 4;

  // Server address for logtail push model
  string LogtailServerAddress = 5;
}

// DNState contains all DN details known to the HAKeeper.
message DNState {
  // Stores is keyed by DN store UUID.
  map<string, DNStoreInfo> Stores = 1 [(gogoproto.nullable) = false];
}

message ClusterDetails {
  repeated DNStore  DNStores    = 1 [(gogoproto.nullable) = false];
  repeated CNStore  CNStores    = 2 [(gogoproto.nullable) = false];
  repeated LogStore LogStores   = 3 [(gogoproto.nullable) = false];
}

// ClusterInfo provides a global view of all shards in the cluster. It
// describes the logical sharding of the system, rather than physical
// distribution of all replicas that belong to those shards.
message ClusterInfo {
  repeated metadata.DNShardRecord DNShards = 1 [(gogoproto.nullable) = false];
  repeated metadata.LogShardRecord LogShards = 2 [(gogoproto.nullable) = false];
}

message InitialClusterRequest {
  uint64 NumOfLogShards   = 1;
  uint64 NumOfDNShards    = 2;
  uint64 NumOfLogReplicas = 3;
}

// LogStoreInfo contains information of all replicas found on a Log store.
message LogStoreInfo {
  uint64 Tick                                 = 1;
  string RaftAddress                          = 2;
  string ServiceAddress                       = 3;
  string GossipAddress                        = 4;
  repeated LogReplicaInfo Replicas = 5 [(gogoproto.nullable) = false];

  bool TaskServiceCreated = 6;
}

message LogState {
  // Shards is keyed by ShardID, it contains details aggregated from all Log
  // stores. Each pb.LogShardInfo here contains data aggregated from
  // different replicas and thus reflect a more accurate description on each
  // shard.
  map<uint64, LogShardInfo> Shards = 1 [(gogoproto.nullable) = false];
  // Stores is keyed by log store UUID, it contains details found on each
  // store. Each LogStoreInfo here reflects what was last reported by each Log
  // store.
  map<string, LogStoreInfo> Stores = 2 [(gogoproto.nullable) = false];
}

// CheckerState contains all HAKeeper state required for making schedule
// commands.
message CheckerState {
  uint64 Tick             = 1;
  ClusterInfo ClusterInfo = 2 [(gogoproto.nullable) = false];
  DNState DNState         = 3 [(gogoproto.nullable) = false];
  LogState LogState       = 4 [(gogoproto.nullable) = false];
  CNState CNState         = 5 [(gogoproto.nullable) = false];
  HAKeeperState State     = 6;
  TaskSchedulerState TaskSchedulerState = 7;
  TaskTableUser TaskTableUser           = 8 [(gogoproto.nullable) = false];
}

// HAKeeperRSMState contains state maintained by HAKeeper's RSM.
message HAKeeperRSMState {
  uint64 Tick                                = 1;
  uint64 NextID                              = 2;
  uint64 Term                                = 3;
  HAKeeperState State                        = 4;
  TaskSchedulerState TaskSchedulerState      = 5;
  map<string, CommandBatch> ScheduleCommands = 6 [(gogoproto.nullable) = false];
  map<string, uint64> LogShards = 7;
  CNState CNState               = 8  [(gogoproto.nullable) = false];
  DNState DNState               = 9  [(gogoproto.nullable) = false];
  LogState LogState             = 10  [(gogoproto.nullable) = false];
  ClusterInfo ClusterInfo       = 11 [(gogoproto.nullable) = false];
  TaskTableUser TaskTableUser   = 12 [(gogoproto.nullable) = false];
}

// ReplicaInfo contains details of a replica
message ReplicaInfo {
  string UUID           = 1;
  string ServiceAddress = 2;
}

// ShardInfoQueryResult contains the result of the shard info query. 
message ShardInfoQueryResult {
  uint64 ShardID                    = 1;
  map<uint64, ReplicaInfo> Replicas = 2 [(gogoproto.nullable) = false];
  uint64 Epoch                      = 3;
  uint64 LeaderID                   = 4;
  uint64 Term                       = 5;
}