github.com/matrixorigin/matrixone@v1.2.0/proto/pipeline.proto

/* 
 * Copyright 2021 Matrix Origin
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

syntax = "proto3";
package pipeline;

import "github.com/gogo/protobuf/gogoproto/gogo.proto";
option go_package = "github.com/matrixorigin/matrixone/pkg/pb/pipeline";
option (gogoproto.sizer_all) = false;
option (gogoproto.protosizer_all) = true;

import "plan.proto";
import "timestamp.proto";
import "lock.proto";

enum Method {
    UnknownMethod = 0;
    PipelineMessage = 1;
    BatchMessage = 2;
    PrepareDoneNotifyMessage = 3;// for dispatch
}

enum Status {
    UnknownStatus = 0;
    WaitingNext = 1;
    Last = 2;
    MessageEnd = 3;
}

message Message {
  Status  sid = 1;
  Method  cmd = 2;
  bytes   err = 3;
  bytes   data = 4;
  bytes   proc_info_data = 5;
  bytes   analyse = 6;
  uint64  id = 7;
  bytes   uuid = 8;
  uint64  batch_cnt = 9;
  uint32  checksum = 10;
  uint64  sequence = 11;
}

message Connector {
  int32 pipeline_id = 1;
  int32 connector_index = 2;
}

message Shuffle{
  int32 ShuffleColIdx=1;
  int32 ShuffleType = 2;
  int64 ShuffleColMin = 3;
  int64 ShuffleColMax = 4;
  int32 AliveRegCnt = 5;
  repeated uint64 shuffle_ranges_uint64 = 6;
  repeated int64 shuffle_ranges_int64=7;
}

message Dispatch {
  int32 func_id = 1;
  repeated Connector local_connector = 2;
  repeated WrapNode remote_connector = 3;
  repeated int32 shuffle_reg_idx_local=4;
  repeated int32 shuffle_reg_idx_remote=5;
  int32 shuffle_type=6;
  bool  is_sink = 7;
  bool  rec_sink = 8;
}

message Merge {
  bool sinkScan = 1;
}

message MultiArguemnt{
  bool Dist = 1;
  repeated plan.Expr GroupExpr = 2;
  repeated plan.Expr OrderByExpr = 3;
  string Separator = 4;
  int32 OrderId = 5;
}

message Aggregate {
  int64 op = 1;
  bool dist = 2;
  repeated plan.Expr expr = 3;
  bytes config =4;
}

message Group {
  bool need_eval = 1;
  uint64 ibucket = 2;
  uint64 nbucket = 3;
  repeated plan.Expr  exprs = 4;
  repeated plan.Type  types = 5 [(gogoproto.nullable) = false];
  repeated Aggregate  aggs = 6;
  repeated MultiArguemnt MultiAggs = 7;
  bool isShuffle = 8;
  uint64 preAllocSize = 9;
  bytes PartialResults = 11;
  repeated uint32 PartialResultTypes = 10;
}

message Insert{
  uint64 affected              = 1;
  bool ToWriteS3 = 2;
  bool add_affected_rows       = 3;
  plan.ObjectRef ref           = 4;
  repeated string attrs        = 5;
  // Align array index with the partition number
  repeated uint64 partition_table_ids = 6;
  repeated string partition_table_names = 7;
  int32 partition_idx = 8;
  bool is_end         = 9;
  plan.TableDef table_def = 10;
}

message Array{
  repeated int32 array = 1;
}

message Map{
  map<string, int32> mp = 1;
}

message Deletion{
  uint64 AffectedRows = 1;
  bool RemoteDelete = 2;
  uint32 IBucket = 3;
  uint32 NBucket = 4;
  // DeleteCtx

  int32 row_id_idx                      = 5;
  repeated uint64 partition_table_ids   = 6;
  repeated string partition_table_names = 7;
  int32 partition_index_in_batch        = 8;
  plan.ObjectRef ref                    = 9;
  bool add_affected_rows                = 10;
  map<string,int32> SegmentMap          = 11;
  bool can_truncate                     = 12;
  bool is_end                           = 13;
  int32 primary_key_idx                 = 14;
}

message PreInsert {
  string schema_name         = 1;
  plan.TableDef table_def    = 2;
  repeated int32 idx         = 3;
  repeated string attrs      = 4;
  bool has_auto_col          = 5;
  bool is_update             = 6;
  int64 estimated_row_count  = 7;
}

message LockTarget {
  uint64 table_id           	  = 1;
  int32 primary_col_idx_in_bat  = 2;
  plan.Type primary_col_typ     = 3 [(gogoproto.nullable) = false];
  int32 refresh_ts_idx_in_bat   = 4;
  int32 filter_col_idx_in_bat   = 5;
  bool lock_table               = 6;
  bool ChangeDef               = 7;
  lock.LockMode Mode         = 8;
}

message LockOp {
  repeated LockTarget targets = 1;
  bool                block   = 2;
}

message PreInsertUnique {
  plan.PreInsertUkCtx pre_insert_uk_ctx = 1;
}

message PreInsertSecondaryIndex{
  plan.PreInsertUkCtx pre_insert_sk_ctx = 1;
}

message OnDuplicateKey{
  repeated string attrs = 1;
  int32 insert_col_count = 2;
  repeated plan.Expr unique_col_check_expr = 3;
  repeated string unique_cols = 4;
  repeated int32 on_duplicate_idx		= 5;
  map<string, plan.Expr> on_duplicate_expr	= 6;
  bool is_ignore 						= 7;
}

message FuzzyFilter {
  float N = 1;
  string PkName = 2;
  plan.Type PkTyp = 3 [(gogoproto.nullable) = false];
}

message Join {
  repeated int32 rel_list = 1;
  repeated int32 col_list = 2;
  plan.Expr  expr = 3;
  repeated plan.Type  types = 4 [(gogoproto.nullable) = false];
  repeated plan.Expr left_cond = 5;
  repeated plan.Expr right_cond = 6;
  repeated plan.RuntimeFilterSpec runtime_filter_build_list = 7;
  bool hash_on_pk = 8;
  bool is_shuffle =9;
}

message AntiJoin{
  repeated int32 result = 1;
  plan.Expr  expr = 2;
  repeated plan.Type  types = 3 [(gogoproto.nullable) = false];
  repeated plan.Expr left_cond = 4;
  repeated plan.Expr right_cond = 5;
  bool hash_on_pk = 6;
  bool is_shuffle = 7;
}

message LeftJoin {
  repeated int32 rel_list = 1;
  repeated int32 col_list = 2;
  plan.Expr  expr = 3;
  repeated plan.Type  types = 4 [(gogoproto.nullable) = false];
  repeated plan.Expr left_cond = 5;
  repeated plan.Expr right_cond = 6;
  repeated plan.RuntimeFilterSpec runtime_filter_build_list = 7;
  bool hash_on_pk = 8;
  bool is_shuffle = 9;
}

message RightJoin {
  repeated int32 rel_list = 1;
  repeated int32 col_list = 2;
  plan.Expr  expr = 3;
  repeated plan.Type  left_types = 4 [(gogoproto.nullable) = false];
  repeated plan.Type  right_types = 5 [(gogoproto.nullable) = false];
  repeated plan.Expr left_cond = 6;
  repeated plan.Expr right_cond = 7;
  repeated plan.RuntimeFilterSpec runtime_filter_build_list = 8;
  bool hash_on_pk = 9;
  bool is_shuffle = 10;
}

message RightSemiJoin {
  repeated int32 result = 1;
  plan.Expr  expr = 2;
  repeated plan.Type right_types = 3 [(gogoproto.nullable) = false];
  repeated plan.Expr left_cond = 4;
  repeated plan.Expr right_cond = 5;
  repeated plan.RuntimeFilterSpec runtime_filter_build_list = 6;
  bool hash_on_pk = 7;
  bool is_shuffle = 8;
}

message RightAntiJoin {
  repeated int32 result = 1;
  plan.Expr  expr = 2;
  repeated plan.Type right_types = 3 [(gogoproto.nullable) = false];
  repeated plan.Expr left_cond = 4;
  repeated plan.Expr right_cond = 5;
  repeated plan.RuntimeFilterSpec runtime_filter_build_list = 6;
  bool hash_on_pk = 7;
  bool is_shuffle = 8;
}

message SemiJoin {
  repeated int32 result = 1;
  plan.Expr  expr = 2;
  repeated plan.Type  types = 3 [(gogoproto.nullable) = false];
  repeated plan.Expr left_cond = 4;
  repeated plan.Expr right_cond = 5;
  repeated plan.RuntimeFilterSpec runtime_filter_build_list = 6;
  bool hash_on_pk = 7;
  bool is_shuffle = 8;
}

message SingleJoin {
  repeated int32 rel_list = 1;
  repeated int32 col_list = 2;
  plan.Expr  expr = 3;
  repeated plan.Type  types = 4 [(gogoproto.nullable) = false];
  repeated plan.Expr left_cond = 5;
  repeated plan.Expr right_cond = 6;
  repeated plan.RuntimeFilterSpec runtime_filter_build_list = 7;
  bool hash_on_pk = 8;
  bool is_shuffle = 9;
}

message MarkJoin {
  repeated int32 result = 1;
  plan.Expr expr = 2;
  repeated plan.Type types = 3 [(gogoproto.nullable) = false];
  repeated plan.Expr left_cond = 4;
  repeated plan.Expr right_cond = 5;
  repeated plan.Expr on_list = 6;
  bool hash_on_pk = 7;
  bool is_shuffle = 8;
}

message Product {
  repeated int32 rel_list = 1;
  repeated int32 col_list = 2;
  repeated plan.Type  types = 3 [(gogoproto.nullable) = false];
  bool is_shuffle =4;
}

message IndexJoin {
  repeated int32 result = 1;
  repeated plan.Type  types = 2 [(gogoproto.nullable) = false];
  repeated plan.RuntimeFilterSpec runtime_filter_build_list = 3;
}

message TableFunction {
  repeated string attrs = 1;
  repeated plan.ColDef rets = 2;
  repeated plan.Expr args = 3;
  bytes params = 4;
  string name = 5;
}

message HashBuild {
  bool need_expr = 1;
  bool need_hash = 2;
  uint64 ibucket = 3;
  uint64 nbucket = 4;
  repeated plan.Type types = 5 [(gogoproto.nullable) = false];
  repeated plan.Expr conds = 6;
  bool hash_on_pk = 7;
  bool need_merged_batch = 8;
  bool need_allocate_sels = 9;
}

message ExternalName2ColIndex {
  string name = 1;
  int32 index = 2;
}

message file_offset {
  repeated int64 offset = 1;
}

message ExternalScan {
  repeated string attrs = 1;
  repeated int64 file_size = 2;
  repeated file_offset file_offset_total = 3;
  repeated plan.ColDef cols = 4;
  repeated ExternalName2ColIndex name2_col_index = 5;
  string create_sql = 6;
  repeated string file_list = 7;
  repeated plan.ColDef origin_cols = 8;
  plan.Expr    filter = 9;
}

message StreamScan {
  plan.TableDef tbl_def = 1;
  int64 offset = 2;
  int64 limit = 3;
}

message SampleFunc {
  enum SampleType {
    Rows = 0;
    Percent = 1;
    MergeRows = 2;
  }
  SampleType sample_type = 1;
  int32 sample_rows = 2;
  double sample_percent = 3;
  repeated plan.Expr sample_columns = 4;
}

message Instruction{
  // Op specified the operator code of an instruction.
  int32 op = 1;
  // Idx specified the anaylze information index.
  int32 idx = 2;
  AntiJoin  anti = 3;
  Connector   connect = 4;
  Dispatch  dispatch = 5;
  Group     agg = 6;
  LeftJoin  left_join = 8;
  SemiJoin  semi_join = 9;
  SingleJoin single_join = 10;
  MarkJoin    mark_join = 11;
  Join    join = 12;
  Product    product = 13;
  TableFunction table_function = 14;
  HashBuild hash_build = 15;
  ExternalScan external_scan = 16;
  Insert insert = 17;
  OnDuplicateKey on_duplicate_key = 18;
  PreInsert pre_insert               = 19;
  PreInsertUnique pre_insert_unique  = 20;
  repeated plan.OrderBySpec order_by = 21;
  repeated plan.Expr    project_list = 22;
  plan.Expr    filter = 23;
  uint64    limit = 24;
  uint64    offset = 25;
  // isFirst identifies whether it is the first instruction of analyzeInfo corresponding to idx
  bool isFirst = 26;
  // isLast identifies whether it is the last instruction of analyzeInfo corresponding to idx
  bool isLast = 27;
  
  RightJoin right_join = 28;
  RightSemiJoin right_semi_join = 29;
  RightAntiJoin right_anti_join = 30;

  Deletion delete = 31;
  LockOp lock_op = 32;
  Shuffle shuffle = 33;
  Merge merge = 34;
  StreamScan stream_scan = 35;
  PreInsertSecondaryIndex pre_insert_secondary_index  = 36;
  SampleFunc sample_func = 37;
  FuzzyFilter fuzzy_filter = 38;

  string cn_addr = 39;
  int32 operator_id = 40;
  int32 parallel_id = 41;
  int32 max_parallel = 42;
  IndexJoin index_join = 43;
}

message AnalysisList {
  repeated plan.AnalyzeInfo list = 1;
}

message Source {
  string schema_name = 1;
  string table_name = 2;
  repeated string col_list = 3;
  string    block = 4;
  uint64    pushdown_id = 5;
  string    pushdown_addr = 6;
  plan.Expr  expr = 7;
  plan.TableDef tableDef = 8;
  timestamp.Timestamp timestamp = 9;
  repeated plan.RuntimeFilterSpec runtime_filter_probe_list = 10;
}

message NodeInfo {
  int32 mcpu = 1;
  string id = 2;
  string addr = 3;
  string payload = 4;
  uint32 type = 5;
}

message ProcessLimitation {
  int64 size = 1;
  int64 batch_rows = 2;
  int64 batch_size = 3;
  int64 partition_rows = 4;
  int64 reader_size = 5;
}

message ProcessInfo {
  string id = 1;
  string sql = 2;
  ProcessLimitation lim = 3;
  int64 unix_time = 4;
  uint32 account_id = 5;
  string snapshot = 6;
  SessionInfo session_info = 7;
  repeated int32 analysis_node_list = 8;
}

message SessionInfo {
  string user = 1;
  string host = 2;
  string role = 3;
  uint64 connection_id = 4;
  string database = 5;
  string version = 6;
  bytes  time_zone = 7;
  string  account = 8;
  repeated string query_id = 9;
}

message Pipeline {
  enum PipelineType{
    Merge = 0;
    Normal = 1;
    Remote = 2;
    Parallel = 3;
  }

  PipelineType pipeline_type = 1;
  int32 pipeline_id = 2;
  plan.Plan qry = 3;
  Source data_source = 4;
  repeated Pipeline children = 5;
  repeated Instruction instruction_list = 6;

  bool is_end = 7;
  bool is_join = 8;
  bool is_load = 9;
  NodeInfo node = 10;
  int32 push_down_info = 11;
  int32 children_count = 12;

  repeated UuidToRegIdx uuids_to_reg_idx = 13;
  int32 build_idx = 14;
  int32 shuffle_cnt = 15;
}

message WrapNode {
  string node_addr = 1;
  bytes uuid = 2;
}

message UuidToRegIdx {
  int32 idx = 1;
  bytes uuid = 2;
  string from_addr = 3;
}