github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/docs/data-flow.dot (about)

     1  // The data-flow diagram of TiCDC.
     2  //
     3  // Run `make data-flow-diagram` to generate data-flow.svg
     4  //
     5  // Metrics in TiCDC
     6  // TiKV -> KV client: ticdc_kvclient_pull_event_count | counter
     7  //     puller event chan size: ticdc_puller_event_chan_size | gauge
     8  // KV client -> puller memBuffer: ticdc_puller_kv_event_count | counter
     9  //     puller memBuffer size: ticdc_puller_mem_buffer_size | gauge
    10  // puller memBuffer -> puller outputChan: ticdc_puller_txn_collect_event_count | counter
    11  //     puller outputChan size: ticdc_puller_output_chan_size | gauge
    12  // puller outputCnan -> sorter unsorted
    13  //     sorter unsorted size: ticdc_puller_entry_sorter_unsorted_size | gauge
    14  //     sorter sort duration: ticdc_puller_entry_sorter_sort | duration
    15  //     sorter merge duration: ticdc_puller_entry_sorter_merge | duration
    16  // sorter unsorted -> sorter outputCh: missing
    17  //     sorter outputCh size: ticdc_puller_entry_sorter_output_chan_size | gauge
    18  // sorter outputCh -> Rectifier outputCh: missing
    19  // Rectifier outputCh -> processor outputCh: missing
    20  //                    `-> mounter rawRowChangedChs
    21  //                             mounter unmarshal duration: ticdc_mounter_unmarshal_and_mount | duration
    22  // processor outputCh -> sink.EmitRowChangedEvents:
    23  //     processor outputCh size: ticdc_processor_txn_output_chan_size | gague
    24  
    25  digraph Dataflow {
    26      subgraph cluster_legends {
    27          label = "Legend";
    28          fontsize = 10;
    29          color = lightgrey;
    30          style = filled;
    31  
    32          node [
    33              style = filled,
    34              color = white,
    35              shape = rect,
    36              fontsize = 10,
    37              width = 0.5,
    38              height = 0.5,
    39          ];
    40  
    41          edge [
    42              labelfontsize = 10,
    43          ]
    44  
    45          flow [
    46              label = "Go routine #1",
    47          ]
    48          flow_ [
    49              label = "Go routine #2",
    50          ]
    51          flow -> flow_ [
    52              headlabel = "Flow\ndirection",
    53              taillabel = "Channel\n/Buffer",
    54          ]
    55  
    56          wait [
    57              label = "Go routine #1",
    58          ]
    59          wait_ [
    60              label = "Go routine #2",
    61          ]
    62          wait -> wait_ [
    63              taillabel = "#1 wiats #2",
    64              style = "dashed",
    65          ]
    66      }
    67  
    68      subgraph cluster_tikv {
    69          label = "TiKV";
    70          color = lightgrey;
    71          node [
    72              style = filled,
    73              color = lightgrey,
    74              shape = rect,
    75          ];
    76  
    77          Raftstore -> CDC [label = "channel"]
    78      }
    79  
    80      subgraph cluster_ticdc {
    81          label = "TiCDC";
    82          color = lightgrey;
    83          node [
    84              style = filled,
    85              color = white,
    86              shape = rect,
    87          ];
    88  
    89          CDC -> kv_client [constraint = false, label = "gRPC"]
    90  
    91          subgraph cluster_kvclient {
    92              label = "KV client";
    93              style = filled;
    94  
    95              kv_client
    96          }
    97  
    98          kv_client -> puller_run_step1 [label = "eventCh", constraint = false]
    99  
   100          subgraph cluster_processor {
   101              label = "Processor";
   102              style = filled;
   103  
   104              processor_sorter_consume [label = "sorterConsume"]
   105              processor_puller_consume [label = "pullerConsume"]
   106  
   107          }
   108              processor_sorter_consume -> table_sink_run [label = "P.output"]
   109  
   110          subgraph cluster_puller {
   111              label = "Puller";
   112              style = filled;
   113  
   114              puller_run_step1 [label = "Run #1\n(Add to memory buffer)"]
   115              puller_run_step2 [label = "Run #2\n(Output to output channel)"]
   116  
   117              puller_run_step1 -> puller_run_step2 [label = "P.buffer"]
   118              puller_run_step2 -> processor_puller_consume [label = "P.Output"]
   119          }
   120  
   121          processor_puller_consume -> sorter_run [label = "E.AddEntry"]
   122  
   123          subgraph cluster_sorter {
   124              label = "Sorter";
   125              style = filled;
   126  
   127              subgraph cluster_entry_sorter {
   128                  label = "Entry sorter";
   129  
   130                  sorter_run [label = "Run"]
   131              }
   132  
   133          }
   134  
   135          sorter_run -> processor_sorter_consume [label = "S.Ouput"]
   136          processor_sorter_consume -> mounter_unmarshal [label = "unmarshal"]
   137  
   138          subgraph cluster_mounter {
   139              label = "Mounter";
   140              style = filled;
   141  
   142              mounter_unmarshal [label = "Unmarshal"]
   143          }
   144  
   145          subgraph cluster_sink {
   146              label = "Sink Manager";
   147              style = filled;
   148  
   149              subgraph table_sink {
   150                  label = "Table Sink (N)";
   151  
   152                  table_sink_run [label = "Table Sink (N)\nFlushEvents"]
   153              }
   154  
   155              subgraph buffer_sink {
   156                  label = "Buffer Sink (1)";
   157  
   158                  buffer_sink_run [label = "Buffer Sink (1)\nFlushEvents"]
   159              }
   160  
   161              subgraph backend_sink {
   162                  label = "Backend Sink (1)";
   163  
   164                  backend_sink_run [label = "Backend Sink (1)\nFlushEvents"]
   165              }
   166  
   167              table_sink_run -> buffer_sink_run [label = "S.EmitEvents"]
   168              buffer_sink_run -> backend_sink_run [label = "S.EmitEvents"]
   169          }
   170      }
   171  
   172      data_in [label = "Data in", shape = oval]
   173      data_in -> Raftstore
   174  
   175      data_out [label = "Data out", shape = oval]
   176      backend_sink_run -> data_out
   177  }