github.com/matrixorigin/matrixone@v0.7.0/proto/api.proto

/* 
 * Copyright 2021 Matrix Origin
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
syntax = "proto3";
package api;

import "github.com/gogo/protobuf/gogoproto/gogo.proto";
import "timestamp.proto";
import "plan.proto";


option go_package = "github.com/matrixorigin/matrixone/pkg/pb/api";
option (gogoproto.sizer_all) = false;
option (gogoproto.protosizer_all) = true;

enum OpCode {
    Nop = 0;
    OpGetLogTail = 1000;
    OpWrite = 1001;
    OpPreCommit = 1002;
}

message Vector {
    bytes data = 1;
    plan.Type  type = 2;
    bool  nullable = 3;
    bytes nsp = 4;
    bool is_const = 5;
    uint32 len = 6;
    bytes area = 7;
};

message Batch {
    repeated string attrs = 1;
    repeated Vector vecs = 2;
};

message TableID {
    uint64 db_id  = 1; 
    uint64 tb_id  = 2;
    uint64 partition_id = 3;
}

// CN pull the log tail of table from DN.
message SyncLogTailReq {
    timestamp.Timestamp cn_have = 1;
    timestamp.Timestamp cn_want = 2;
    TableID   table = 3;
};

message SyncLogTailResp {
  // ckp_location is a checkpoint location list.
  // How to get checkpoint data on S3 by the ckp_location ,
  // pls ref to disttae/logtail.go/consumeLogTail function.
  string     ckp_location = 1;
  // commands is log tail for a system or user table,
  // pls ref to tae/logtail/handle.go/HandleSyncLogTailReq function.
  repeated Entry commands = 2;
};

//How to parse and handle PrecommiWriteCmd , pls ref to tae/rpc/handle.go/HandlePreCommit function
message PrecommitWriteCmd {
    uint32 user_id = 1;
    uint32 role_id = 2;
    uint32 account_id = 3;
    repeated Entry entry_list = 4;
};

// CN--->DN, DDL
// create database test: 
//      Entry {
//          entry_type = Insert
//          table_id = 0
//          database_id = 0
//          table_name = mo_database
//          database_name = mo_catalog
//          file_name = "" 
//          bat = "test, 0, ..."
//      }
// drop database test:
//      Entry {
//          entry_type = Delete
//          table_id = 0
//          database_id = 0
//          table_name = mo_database
//          database_name = mo_catalog
//          file_name = "" 
//          bat= "test, 0, ..."
//      }

// create a user table: 
//  insert entry into mo_tables and ...;
//      Entry {
//          entry_type = Insert
//          table_id = 1
//          database_id =  0
//          table_name =  mo_tables
//          database_name = mo_catalog
//          file_name = "" 
//          bat.attrs = {"relid", "relname", "reldatabase", ...}
//      }

// drop table: 
//  delete entry from mo_tables and ...;
//      Entry {
//          entry_type = delete
//          table_id = 1
//          database_id = 0
//          table_name =  mo_tables
//          database_name = mo_catalog
//          file_name = "" 
//          bat.attrs = {"relid", "relname",...}
//      }

//  bulk load:
// bulk loads a block into S3.
//      Entry {
//          entry_type = Insert
//          table_id = tid
//          database_id = dbId
//          table_name =  xxx
//          database_name = xxx
//          // one S3 object file can hold many blocks.
//          file_name = "s3 file name"
//
//          //  bat holds meta locations for many blocks.
//          //  "meta_loc" : the location of block data which contains BF and ZM index and be sorted by primary key.
//          bat.attrs = {"meta_loc"}
//      }

// deletes/updates involves many rows need to write S3.
//      Entry {
//          entry_type = Delete
//          table_id = tid
//          database_id = dbId
//          table_name =  xxx
//          database_name = xxx
//
//          // one S3 object file can contains many blocks
//          file_name = "s3 file name"

//          // "delta_loc" : the location of deleted row-ids for a block
//          bat.attrs = {"delta_loc"}
//      }


//  DML:
// append a batch of data into table;
//            Entry {
    //          entry_type = Insert
    //          table_id =  tid
    //          database_id = dbId
    //          table_name =  xxx
    //          database_name = xxx
    //          file_name = "" 
    //          bat.attrs = {"primary column value",  "column1 value", ...}
    //      }


//  delete batch of data from table;
//            Entry {
    //          entry_type = Delete
    //          table_id =  tid
    //          database_id = dbId
    //          table_name =  xxx
    //          database_name = xxx
    //          file_name = ""
    //          bat.attrs = {"rowid"}
    //      }


message Entry {
    enum EntryType {
        Insert = 0;
        Delete = 1;
        Update = 2;
    }
    EntryType entry_type = 1;
    uint64    table_id = 2;
    uint64    database_id = 3;
    string    table_name = 4;
    string    database_name = 5;
    string    file_name = 6;
    Batch     bat = 8;
};

// There are two kinds of checkpoint: delta checkpoint and base checkpoint,
// base checkpoint is a snapshot at a timestamp for catalog and block meta.
// delta checkpoint is a mini checkpoint represents changes during a period.

// CatalogCkp contains information about database and tables in the system,and
// MetadataCkp contains information about blocks.
message Checkpoint {
    //min_ts DN is the lower bounds of the checkpoint
    // CN maybe don't care about it.
    timestamp.Timestamp min_ts = 1;
    //max_ts is the upper bounds of the checkpoint.
    // CN maybe don't care about it.
    timestamp.Timestamp max_ts = 2;
    Batch      bat   = 3;

};
// catalog checkpoint:
// one Batch represents a table, such as : mo_databases, mo_tables, mo_columns,... etc.
// knowing more about system tables, pls ref to pkg/vm/engine/tae/catalog/model.go
message CatalogCkp {
   timestamp.Timestamp min_ts = 1;
   timestamp.Timestamp max_ts = 2;
   Batch      bat   = 3;
};

//metadata checkpoint:
// Batch is a batch of block metadata for a table,  
// one row of Batch represents a block meta data.
// TODO::
// knowing more about block meta data , pls ref to ...
message MetadataCkp {
    timestamp.Timestamp min_ts = 1;
    timestamp.Timestamp max_ts = 2;
    //block meta data for a table;
    Batch      bat = 3;
};