github.com/matrixorigin/matrixone@v1.2.0/proto/api.proto

/*
 * Copyright 2021 Matrix Origin
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

syntax = "proto3";
package api;

import "github.com/gogo/protobuf/gogoproto/gogo.proto";
import "timestamp.proto";
import "plan.proto";

option go_package            = "github.com/matrixorigin/matrixone/pkg/pb/api";
option (gogoproto.sizer_all) = false;
option (gogoproto.protosizer_all) = true;

enum OpCode {
    Nop          = 0;
    OpGetLogTail = 1000;
    OpPreCommit  = 1001;

    OpPing             = 2000;
    OpFlush            = 2001;
    OpCheckpoint       = 2003;
    OpInspect          = 2004;
    OpAddFaultPoint    = 2005;
    OpBackup           = 2006;
    OpTraceSpan        = 2007;
    OpStorageUsage     = 2008;
    OpGlobalCheckpoint = 2009;
    OpInterceptCommit  = 2010;
    OpCommitMerge      = 2011;
}

// TNPingRequest ping request
message TNPingRequest {
    option (gogoproto.sizer) = true;
    option (gogoproto.protosizer) = false;
    string parameter = 1;
}

// TNPingResponse ping response
message TNPingResponse {
    option (gogoproto.sizer) = true;
    option (gogoproto.protosizer) = false;
    string storeID        = 1;
    string serviceAddress = 2;
    uint64 shardID        = 3;
    uint64 replicaID      = 4;
    uint64 logShardID     = 5;
}

// TNStringResponse string type response
message TNStringResponse {
    option (gogoproto.sizer) = true;
    option (gogoproto.protosizer) = false;
    string returnStr = 1;
}

message Vector {
    bytes data     = 1;
    plan.Type type = 2;
    bool nullable  = 3;
    bytes nsp      = 4;
    bool is_const  = 5;
    uint32 len     = 6;
    bytes area     = 7;
};

message Batch {
    repeated string attrs = 1 [(gogoproto.nullable) = false];
    repeated Vector vecs  = 2 [(gogoproto.nullable) = false];
};

message TableID {
    uint64 db_id        = 1;
    uint64 tb_id        = 2;
    uint64 partition_id = 3;
}

// CN pull the log tail of table from TN.
message SyncLogTailReq {
    timestamp.Timestamp cn_have = 1;
    timestamp.Timestamp cn_want = 2;
    TableID table               = 3;
};

message SyncLogTailResp {
    // ckp_location is a checkpoint location list.
    // How to get checkpoint data on S3 by the ckp_location ,
    // pls ref to disttae/logtail.go/consumeLogTail function.
    string ckp_location = 1;
    // commands is log tail for a system or user table,
    // pls ref to tae/logtail/handle.go/HandleSyncLogTailReq function.
    repeated Entry commands = 2;
};

// How to parse and handle PrecommiWriteCmd , pls ref to
// tae/rpc/handle.go/HandlePreCommit function
message PrecommitWriteCmd {
    repeated Entry entry_list = 1;
};

// CN--->TN, DDL
// create database test:
//      Entry {
//          entry_type = Insert
//          table_id = 0
//          database_id = 0
//          table_name = mo_database
//          database_name = mo_catalog
//          file_name = ""
//          bat = "test, 0, ..."
//      }
// drop database test:
//      Entry {
//          entry_type = Delete
//          table_id = 0
//          database_id = 0
//          table_name = mo_database
//          database_name = mo_catalog
//          file_name = ""
//          bat= "test, 0, ..."
//      }

// create a user table:
//  insert entry into mo_tables and ...;
//      Entry {
//          entry_type = Insert
//          table_id = 1
//          database_id =  0
//          table_name =  mo_tables
//          database_name = mo_catalog
//          file_name = ""
//          bat.attrs = {"relid", "relname", "reldatabase", ...}
//      }

// drop table:
//  delete entry from mo_tables and ...;
//      Entry {
//          entry_type = delete
//          table_id = 1
//          database_id = 0
//          table_name =  mo_tables
//          database_name = mo_catalog
//          file_name = ""
//          bat.attrs = {"relid", "relname",...}
//      }

//  bulk load:
// bulk loads a block into S3.
//      Entry {
//          entry_type = Insert
//          table_id = tid
//          database_id = dbId
//          table_name =  xxx
//          database_name = xxx
//          // one S3 object file can hold many blocks.
//          file_name = "s3 file name"
//
//          //  bat holds meta locations for many blocks.
//          //  "meta_loc" : the location of block data which contains BF and ZM
//          index and be sorted by primary key. bat.attrs = {"meta_loc"}
//      }

// deletes/updates involves many rows need to write S3.
//      Entry {
//          entry_type = Delete
//          table_id = tid
//          database_id = dbId
//          table_name =  xxx
//          database_name = xxx
//
//          // one S3 object file can contains many blocks
//          file_name = "s3 file name"

//          // "delta_loc" : the location of deleted row-ids for a block
//          bat.attrs = {"delta_loc"}
//      }

//  DML:
// append a batch of data into table;
//            Entry {
//          entry_type = Insert
//          table_id =  tid
//          database_id = dbId
//          table_name =  xxx
//          database_name = xxx
//          file_name = ""
//          bat.attrs = {"primary column value",  "column1 value", ...}
//      }

//  delete batch of data from table;
//            Entry {
//          entry_type = Delete
//          table_id =  tid
//          database_id = dbId
//          table_name =  xxx
//          database_name = xxx
//          file_name = ""
//          bat.attrs = {"rowid"}
//      }

message Entry {
    enum EntryType {
        Insert = 0;
        Delete = 1;
        Update = 2;
        Alter = 3;
        SpecialDelete = 4;
    }
    EntryType entry_type = 1;
    uint64 table_id      = 2;
    uint64 database_id   = 3;
    string table_name    = 4;
    string database_name = 5;
    string file_name     = 6;
    Batch bat            = 7;
    // whether TN do the PK uniqueness check against txn's workspace or not.
    int32 pk_check_by_tn = 8;
};

// There are two kinds of checkpoint: delta checkpoint and base checkpoint,
// base checkpoint is a snapshot at a timestamp for catalog and block meta.
// delta checkpoint is a mini checkpoint represents changes during a period.

// CatalogCkp contains information about database and tables in the system,and
// MetadataCkp contains information about blocks.
message Checkpoint {
    // min_ts TN is the lower bounds of the checkpoint
    //  CN maybe don't care about it.
    timestamp.Timestamp min_ts = 1;
    // max_ts is the upper bounds of the checkpoint.
    //  CN maybe don't care about it.
    timestamp.Timestamp max_ts = 2;
    Batch bat                  = 3;
};
// catalog checkpoint:
// one Batch represents a table, such as : mo_databases, mo_tables,
// mo_columns,... etc. knowing more about system tables, pls ref to
// pkg/vm/engine/tae/catalog/model.go
message CatalogCkp {
    timestamp.Timestamp min_ts = 1;
    timestamp.Timestamp max_ts = 2;
    Batch bat                  = 3;
};

// metadata checkpoint:
//  Batch is a batch of block metadata for a table,
//  one row of Batch represents a block meta data.
//  TODO::
//  knowing more about block meta data , pls ref to ...
message MetadataCkp {
    timestamp.Timestamp min_ts = 1;
    timestamp.Timestamp max_ts = 2;
    // block meta data for a table;
    Batch bat = 3;
};

// For AlterTable

enum AlterKind {
    Invalid          = 0;
    AddColumn        = 1;
    DropColumn       = 2;
    RenameTable      = 3;
    UpdateComment    = 4;
    UpdateConstraint = 5;
    UpdatePolicy     = 6;
    AddPartition     = 7;
    RenameColumn     = 8;
}

message AlterTablePolicy {
    uint32 min_osize_quailifed = 1;
    uint32 max_obj_onerun = 2;
    uint32 max_osize_merged_obj = 3; 
    repeated MergeHint hints = 4;
    uint64 min_cn_merge_size = 5;
}

message AlterTableConstraint {
    bytes constraints = 1;
}

message AlterTableComment {
    string comment = 1;
}

message AlterTableRenameTable {
    string old_name = 1;
    string new_name = 2;
}

message AlterTableRenameCol {
    string old_name = 1;
    string new_name = 2;
    uint32 sequence_num = 3;  // used to double check
}

message AlterTableAddColumn {
    plan.ColDef column    = 1;
    int32 insert_position = 2;  // if the position can be found in the schema,
                                // the column will be added to the last
}

message AlterTableAddPartition {
    plan.PartitionByDef  partition_def  = 1;
}

message AlterTableDropColumn {
    uint32 logical_idx  = 1;
    uint32 sequence_num = 2;  // used to double check
}

message AlterTableReq {
    uint64 table_id = 1;
    uint64 db_id    = 2;
    AlterKind kind  = 3;

    oneof operation {
        AlterTableAddColumn add_column       = 4;
        AlterTableDropColumn drop_column     = 5;
        AlterTableRenameTable rename_table   = 6;
        AlterTableComment update_comment     = 7;
        AlterTableConstraint update_cstr     = 8;
        AlterTablePolicy update_policy       = 9;
        AlterTableAddPartition add_partition = 10;
        AlterTableRenameCol rename_col       = 11;
    }
}


enum MergeHint {
    Auto             = 0;
    NoDeletes        = 1;
    BigTable         = 2;
    ContinuousWrite  = 3;
}

message SchemaExtra {
    uint32 next_col_seqnum = 1;
    // sending mo_columns deletes according to this.
    repeated string dropped_attrs = 2;
    bool column_changed = 3; 
    // sending mo_tables deletes by this.
    string old_name = 4;

    uint32 min_osize_quailifed = 5;
    uint32 max_obj_onerun = 6;
    uint32 max_osize_merged_obj = 7;
    repeated MergeHint hints = 8;
    uint64 min_cn_merge_size = 9;
}

// Int64Map mainly used in unit test
message Int64Map {
    map<int64, int64> m = 1;
}

message TransDestPos {
    int32 objIdx = 1;
    int32 blkIdx = 2;
    int32 rowIdx = 3;
}

message BlkTransMap {
    map<int32, TransDestPos> m = 1[(gogoproto.nullable) = false];
}

message BlkTransferBooking {
    repeated BlkTransMap mappings = 1[(gogoproto.nullable) = false];
}


message MergeCommitEntry {
    uint64 db_id = 1;
    uint64 tbl_id = 2;
    string table_name = 3;
    timestamp.Timestamp start_ts = 4[(gogoproto.nullable) = false];
    repeated bytes merged_objs = 5;
    repeated bytes created_objs = 6;
    BlkTransferBooking booking = 7;
    bytes booking_loc = 8;
    string err = 9;
}

message MergeTaskEntry {
    uint64 db_id = 1;
    uint64 tbl_id = 2;
    string table_name = 3;
    string db_name = 4;
    // ObjectStats
    repeated bytes to_merge_objs = 5;
    uint64 estimated_mem_usage = 6;
    uint32 account_id = 7;
    uint32 user_id = 8;
    uint32 role_id = 9;
}