github.com/matrixorigin/matrixone@v0.7.0/proto/pipeline.proto

/* 
 * Copyright 2021 Matrix Origin
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

syntax = "proto3";
package pipeline;

import "github.com/gogo/protobuf/gogoproto/gogo.proto";
option go_package = "github.com/matrixorigin/matrixone/pkg/pb/pipeline";
option (gogoproto.sizer_all) = false;
option (gogoproto.protosizer_all) = true;

import "plan.proto";
import "timestamp.proto";

message Message {
  uint64  sid = 1;
  uint64  cmd = 2;
  bytes   err = 3;
  bytes   data = 4;
  bytes   proc_info_data = 5;
  bytes   analyse = 6;
  uint64  id = 7;
  bytes   uuid = 8;
  uint64  batch_cnt = 9;
  uint32  checksum = 10;
  uint64  sequence = 11;
}

message Connector {
  int32 pipeline_id = 1;
  int32 connector_index = 2;
}

message Dispatch {
  int32 func_id = 1;
  repeated Connector local_connector = 2;
  repeated WrapNode remote_connector = 3;
}

message Aggregate {
  int32 op = 1;
  bool dist = 2;
  plan.Expr expr = 3;
}

message Group {
  bool need_eval = 1;
  uint64 ibucket = 2;
  uint64 nbucket = 3;
  repeated plan.Expr  exprs = 4;
  repeated plan.Type  types = 5;
  repeated Aggregate  aggs = 6;
}

message Insert{
  uint64 affected   = 1;
  bool IsRemote     = 2;

  // InsertCtx
  repeated int32 idx              = 3;
  plan.ObjectRef ref              = 4;
  plan.TableDef table_def         = 5;
  repeated int32 idx_idx          = 6;
  map<string, int32> parent_idx   = 7;
  plan.ClusterTable ClusterTable  = 8;
}

message Join {
  uint64 ibucket = 1;
  uint64 nbucket = 2;
  repeated int32 rel_list = 3;
  repeated int32 col_list = 4;
  plan.Expr  expr = 5;
  repeated plan.Type  types = 6;
  repeated plan.Expr left_cond = 7;
  repeated plan.Expr right_cond = 8;
}

message AntiJoin{
  uint64 ibucket = 1;
  uint64 nbucket = 2;
  repeated int32 result = 3;
  plan.Expr  expr = 4;
  repeated plan.Type  types = 5;
  repeated plan.Expr left_cond = 6;
  repeated plan.Expr right_cond = 7;
}

message InnerJoin {
  uint64 ibucket = 1;
  uint64 nbucket = 2;
  repeated int32 rel_list = 3;
  repeated int32 col_list = 4;
  plan.Expr  expr = 5;
  repeated plan.Type  types = 6;
  repeated plan.Expr left_cond = 7;
  repeated plan.Expr right_cond = 8;
}

message LeftJoin {
  uint64 ibucket = 1;
  uint64 nbucket = 2;
  repeated int32 rel_list = 3;
  repeated int32 col_list = 4;
  plan.Expr  expr = 5;
  repeated plan.Type  types = 6;
  repeated plan.Expr left_cond = 7;
  repeated plan.Expr right_cond = 8;
}

message SemiJoin {
  uint64 ibucket = 1;
  uint64 nbucket = 2;
  repeated int32 result = 3;
  plan.Expr  expr = 4;
  repeated plan.Type  types = 5;
  repeated plan.Expr left_cond = 6;
  repeated plan.Expr right_cond = 7;
}

message SingleJoin {
  uint64 ibucket = 1;
  uint64 nbucket = 2;
  repeated int32 rel_list = 3;
  repeated int32 col_list = 4;
  plan.Expr  expr = 5;
  repeated plan.Type  types = 6;
  repeated plan.Expr left_cond = 7;
  repeated plan.Expr right_cond = 8;
}

message MarkJoin {
  uint64 ibucket = 1;
  uint64 nbucket = 2;
  repeated int32 result = 3;
  plan.Expr expr = 4;
  repeated plan.Type types = 5;
  repeated plan.Expr left_cond = 6;
  repeated plan.Expr right_cond = 7;
  repeated plan.Expr on_list = 8;
}

message Product {
  repeated int32 rel_list = 1;
  repeated int32 col_list = 2;
  repeated plan.Type  types = 3;
}

message TableFunction {
  repeated string attrs = 1;
  repeated plan.ColDef rets = 2;
  repeated plan.Expr args = 3;
  bytes params = 4;
  string name = 5;
}

message HashBuild {
  bool need_expr = 1;
  bool need_hash = 2;
  uint64 ibucket = 3;
  uint64 nbucket = 4;
  repeated plan.Type types = 5;
  repeated plan.Expr conds = 6;
}

message ExternalName2ColIndex {
  string name = 1;
  int32 index = 2;
}

message ExternalScan {
  repeated string attrs = 1;
  repeated plan.ColDef cols = 2;
  repeated ExternalName2ColIndex name2_col_index = 3;
  string create_sql = 4;
  repeated string file_list = 5;
  repeated plan.ColDef origin_cols = 6;
}

message Instruction{
  // Op specified the operator code of an instruction.
  int32 op = 1;
  // Idx specified the anaylze information index.
  int32 idx = 2;
  AntiJoin  anti = 3;
  Connector   connect = 4;
  Dispatch  dispatch = 5;
  Group     agg = 6;
  InnerJoin inner_join = 7;
  LeftJoin  left_join = 8;
  SemiJoin  semi_join = 9;
  SingleJoin single_join = 10;
  MarkJoin    mark_join = 11;
  Join    join = 12;
  Product    product = 13;
  TableFunction table_function = 14;
  HashBuild hash_build = 15;
  ExternalScan external_scan = 16;
  Insert insert = 17;
  repeated plan.OrderBySpec order_by = 18;
  repeated plan.Expr    project_list = 19;
  plan.Expr    filter = 20;
  uint64    limit = 21;
  uint64    offset = 22;
  // isFirst identifies whether it is the first instruction of analyzeInfo corresponding to idx
  bool isFirst = 23;
  // isLast identifies whether it is the last instruction of analyzeInfo corresponding to idx
  bool isLast = 24;
}

message AnalysisList {
  repeated plan.AnalyzeInfo list = 1;
}

message Source {
  string schema_name = 1;
  string table_name = 2;
  repeated string col_list = 3;
  string    block = 4;
  uint64    pushdown_id = 5;
  string    pushdown_addr = 6;
  plan.Expr  expr = 7;
  plan.TableDef tableDef = 8;
  timestamp.Timestamp timestamp = 9;
}

message NodeInfo {
  int32 mcpu = 1;
  string id = 2;
  string addr = 3;
  repeated string payload = 4;
}

message ProcessLimitation {
  int64 size = 1;
  int64 batch_rows = 2;
  int64 batch_size = 3;
  int64 partition_rows = 4;
  int64 reader_size = 5;
}

message ProcessInfo {
  string id = 1;
  ProcessLimitation lim = 2;
  int64 unix_time = 3;
  string snapshot = 4;
  SessionInfo session_info = 5;
  repeated int32 analysis_node_list = 6;
}

message SessionInfo {
  string user = 1;
  string host = 2;
  string role = 3;
  uint64 connection_id = 4;
  string database = 5;
  string version = 6;
  bytes  time_zone = 7;
  string  account = 8;
}

message Pipeline {
  enum PipelineType{
    Merge = 0;
    Normal = 1;
    Remote = 2;
    Parallel = 3;
  }

  PipelineType pipeline_type = 1;
  int32 pipeline_id = 2;
  plan.Plan qry = 3;
  Source data_source = 4;
  repeated Pipeline children = 5;
  repeated Instruction instruction_list = 6;

  bool is_end = 7;
  bool is_join = 8;
  NodeInfo node = 9;
  int32 push_down_info = 10;
  int32 children_count = 11;

  repeated UuidToRegIdx uuids_to_reg_idx = 12;
}

message WrapNode {
  string node_addr = 1;
  bytes uuid = 2;
}

message UuidToRegIdx {
  int32 idx = 1;
  bytes uuid = 2;
  string from_addr = 3;
}