github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/docs/data-flow.dot (about)

     1  // The data-flow diagram of TiCDC.
     2  //
     3  // Run `make data-flow-diagram` to generate data-flow.svg
     4  //
     5  // Metrics in TiCDC
     6  // TiKV -> KV client: ticdc_kvclient_pull_event_count | counter
     7  //     puller event chan size: ticdc_puller_event_chan_size | gauge
     8  // KV client -> puller memBuffer: ticdc_puller_kv_event_count | counter
     9  //     puller memBuffer size: ticdc_puller_mem_buffer_size | gauge
    10  // puller memBuffer -> puller outputChan: ticdc_puller_txn_collect_event_count | counter
    11  //     puller outputChan size: ticdc_puller_output_chan_size | gauge
    12  // puller outputCnan -> sorter unsorted
    13  //     sorter unsorted size: ticdc_puller_entry_sorter_unsorted_size | gauge
    14  //     sorter sort duration: ticdc_puller_entry_sorter_sort | duration
    15  //     sorter merge duration: ticdc_puller_entry_sorter_merge | duration
    16  // sorter unsorted -> sorter outputCh: missing
    17  //     sorter outputCh size: ticdc_puller_entry_sorter_output_chan_size | gauge
    18  // sorter outputCh -> Rectifier outputCh: missing
    19  // Rectifier outputCh -> processor outputCh: missing
    20  //                    `-> mounter rawRowChangedChs
    21  //                             mounter unmarshal duration: ticdc_mounter_unmarshal_and_mount | duration
    22  //                             mounter rawRowCHangedChs size: ticdc_mounter_input_chan_size | gauge
    23  // processor outputCh -> sink.EmitRowChangedEvents:
    24  //     processor outputCh size: ticdc_processor_txn_output_chan_size | gague
    25  
    26  digraph Dataflow {
    27      subgraph cluster_legends {
    28          label = "Legend";
    29          fontsize = 10;
    30          color = lightgrey;
    31          style = filled;
    32  
    33          node [
    34              style = filled,
    35              color = white,
    36              shape = rect,
    37              fontsize = 10,
    38              width = 0.5,
    39              height = 0.5,
    40          ];
    41  
    42          edge [
    43              labelfontsize = 10,
    44          ]
    45  
    46          flow [
    47              label = "Go routine #1",
    48          ]
    49          flow_ [
    50              label = "Go routine #2",
    51          ]
    52          flow -> flow_ [
    53              headlabel = "Flow\ndirection",
    54              taillabel = "Channel\n/Buffer",
    55          ]
    56  
    57          wait [
    58              label = "Go routine #1",
    59          ]
    60          wait_ [
    61              label = "Go routine #2",
    62          ]
    63          wait -> wait_ [
    64              taillabel = "#1 wiats #2",
    65              style = "dashed",
    66          ]
    67      }
    68  
    69      subgraph cluster_tikv {
    70          label = "TiKV";
    71          color = lightgrey;
    72          node [
    73              style = filled,
    74              color = lightgrey,
    75              shape = rect,
    76          ];
    77  
    78          Raftstore -> CDC [label = "channel"]
    79      }
    80  
    81      subgraph cluster_ticdc {
    82          label = "TiCDC";
    83          color = lightgrey;
    84          node [
    85              style = filled,
    86              color = white,
    87              shape = rect,
    88          ];
    89  
    90          CDC -> kv_client [constraint = false, label = "gRPC"]
    91  
    92          subgraph cluster_kvclient {
    93              label = "KV client";
    94              style = filled;
    95  
    96              kv_client
    97          }
    98  
    99          kv_client -> puller_run_step1 [label = "eventCh", constraint = false]
   100  
   101          subgraph cluster_processor {
   102              label = "Processor";
   103              style = filled;
   104  
   105              processor_sorter_consume [label = "sorterConsume"]
   106              processor_puller_consume [label = "pullerConsume"]
   107  
   108          }
   109              processor_sorter_consume -> table_sink_run [label = "P.output"]
   110  
   111          subgraph cluster_puller {
   112              label = "Puller";
   113              style = filled;
   114  
   115              puller_run_step1 [label = "Run #1\n(Add to memory buffer)"]
   116              puller_run_step2 [label = "Run #2\n(Output to output channel)"]
   117  
   118              puller_run_step1 -> puller_run_step2 [label = "P.buffer"]
   119              puller_run_step2 -> processor_puller_consume [label = "P.Output"]
   120          }
   121  
   122          processor_puller_consume -> sorter_run [label = "E.AddEntry"]
   123  
   124          subgraph cluster_sorter {
   125              label = "Sorter";
   126              style = filled;
   127  
   128              subgraph cluster_entry_sorter {
   129                  label = "Entry sorter";
   130  
   131                  sorter_run [label = "Run"]
   132              }
   133  
   134          }
   135  
   136          sorter_run -> processor_sorter_consume [label = "S.Ouput"]
   137          processor_sorter_consume -> mounter_run [label = "M.Input"]
   138          mounter_run -> table_sink_run [
   139              label = "wait unmarshal",
   140              style = "dashed",
   141              dir = back,
   142          ]
   143  
   144          subgraph cluster_mounter {
   145              label = "Mounter";
   146              style = filled;
   147  
   148              mounter_run [label = "Run"]
   149          }
   150  
   151          subgraph cluster_sink {
   152              label = "Sink Manager";
   153              style = filled;
   154  
   155              subgraph table_sink {
   156                  label = "Table Sink (N)";
   157  
   158                  table_sink_run [label = "Table Sink (N)\nFlushEvents"]
   159              }
   160  
   161              subgraph buffer_sink {
   162                  label = "Buffer Sink (1)";
   163  
   164                  buffer_sink_run [label = "Buffer Sink (1)\nFlushEvents"]
   165              }
   166  
   167              subgraph backend_sink {
   168                  label = "Backend Sink (1)";
   169  
   170                  backend_sink_run [label = "Backend Sink (1)\nFlushEvents"]
   171              }
   172  
   173              table_sink_run -> buffer_sink_run [label = "S.EmitEvents"]
   174              buffer_sink_run -> backend_sink_run [label = "S.EmitEvents"]
   175          }
   176      }
   177  
   178      data_in [label = "Data in", shape = oval]
   179      data_in -> Raftstore
   180  
   181      data_out [label = "Data out", shape = oval]
   182      backend_sink_run -> data_out
   183  }