github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/docs/data-flow.svg (about) 1 <?xml version="1.0" encoding="UTF-8" standalone="no"?> 2 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" 3 "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> 4 <!-- Generated by graphviz version 2.47.0 (20210316.0004) 5 --> 6 <!-- Title: Dataflow Pages: 1 --> 7 <svg width="655pt" height="735pt" 8 viewBox="0.00 0.00 655.00 735.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"> 9 <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 731)"> 10 <title>Dataflow</title> 11 <polygon fill="white" stroke="transparent" points="-4,4 -4,-731 651,-731 651,4 -4,4"/> 12 <g id="clust1" class="cluster"> 13 <title>cluster_legends</title> 14 <polygon fill="lightgrey" stroke="lightgrey" points="8,-503 8,-683 190,-683 190,-503 8,-503"/> 15 <text text-anchor="middle" x="99" y="-671" font-family="Times,serif" font-size="10.00">Legend</text> 16 </g> 17 <g id="clust2" class="cluster"> 18 <title>cluster_tikv</title> 19 <polygon fill="none" stroke="lightgrey" points="557,-394 557,-578 639,-578 639,-394 557,-394"/> 20 <text text-anchor="middle" x="598" y="-562.8" font-family="Times,serif" font-size="14.00">TiKV</text> 21 </g> 22 <g id="clust3" class="cluster"> 23 <title>cluster_ticdc</title> 24 <polygon fill="none" stroke="lightgrey" points="198,-63 198,-719 549,-719 549,-63 198,-63"/> 25 <text text-anchor="middle" x="373.5" y="-703.8" font-family="Times,serif" font-size="14.00">TiCDC</text> 26 </g> 27 <g id="clust4" class="cluster"> 28 <title>cluster_kvclient</title> 29 <polygon fill="lightgrey" stroke="lightgrey" points="457,-612 457,-687 541,-687 541,-612 457,-612"/> 30 <text text-anchor="middle" x="499" y="-671.8" font-family="Times,serif" font-size="14.00">KV client</text> 31 </g> 32 <g id="clust5" class="cluster"> 33 <title>cluster_processor</title> 34 <polygon fill="lightgrey" stroke="lightgrey" points="267,-394 267,-469 501,-469 501,-394 267,-394"/> 35 <text text-anchor="middle" x="384" y="-453.8" font-family="Times,serif" font-size="14.00">Processor</text> 36 </g> 37 <g id="clust6" class="cluster"> 38 <title>cluster_puller</title> 39 <polygon fill="lightgrey" stroke="lightgrey" points="241,-502 241,-688 419,-688 419,-502 241,-502"/> 40 <text text-anchor="middle" x="330" y="-672.8" font-family="Times,serif" font-size="14.00">Puller</text> 41 </g> 42 <g id="clust7" class="cluster"> 43 <title>cluster_sorter</title> 44 <polygon fill="lightgrey" stroke="lightgrey" points="444,-247 444,-361 541,-361 541,-247 444,-247"/> 45 <text text-anchor="middle" x="492.5" y="-345.8" font-family="Times,serif" font-size="14.00">Sorter</text> 46 </g> 47 <g id="clust8" class="cluster"> 48 <title>cluster_entry_sorter</title> 49 <polygon fill="lightgrey" stroke="lightgrey" points="452,-255 452,-330 533,-330 533,-255 452,-255"/> 50 <text text-anchor="middle" x="492.5" y="-314.8" font-family="Times,serif" font-size="14.00">Entry sorter</text> 51 </g> 52 <g id="clust9" class="cluster"> 53 <title>cluster_mounter</title> 54 <polygon fill="lightgrey" stroke="lightgrey" points="342,-255 342,-330 436,-330 436,-255 342,-255"/> 55 <text text-anchor="middle" x="389" y="-314.8" font-family="Times,serif" font-size="14.00">Mounter</text> 56 </g> 57 <g id="clust10" class="cluster"> 58 <title>cluster_sink</title> 59 <polygon fill="lightgrey" stroke="lightgrey" points="206,-71 206,-331 334,-331 334,-71 206,-71"/> 60 <text text-anchor="middle" x="270" y="-315.8" font-family="Times,serif" font-size="14.00">Sink Manager</text> 61 </g> 62 <!-- flow --> 63 <g id="node1" class="node"> 64 <title>flow</title> 65 <polygon fill="white" stroke="white" points="182,-656 108,-656 108,-620 182,-620 182,-656"/> 66 <text text-anchor="middle" x="145" y="-635.5" font-family="Times,serif" font-size="10.00">Go routine #1</text> 67 </g> 68 <!-- flow_ --> 69 <g id="node2" class="node"> 70 <title>flow_</title> 71 <polygon fill="white" stroke="white" points="182,-547 108,-547 108,-511 182,-511 182,-547"/> 72 <text text-anchor="middle" x="145" y="-526.5" font-family="Times,serif" font-size="10.00">Go routine #2</text> 73 </g> 74 <!-- flow->flow_ --> 75 <g id="edge1" class="edge"> 76 <title>flow->flow_</title> 77 <path fill="none" stroke="black" d="M145,-619.81C145,-602.96 145,-577.02 145,-557.34"/> 78 <polygon fill="black" stroke="black" points="148.5,-557.15 145,-547.15 141.5,-557.15 148.5,-557.15"/> 79 <text text-anchor="middle" x="126.5" y="-561.15" font-family="Times,serif" font-size="10.00">Flow</text> 80 <text text-anchor="middle" x="126.5" y="-550.15" font-family="Times,serif" font-size="10.00">direction</text> 81 <text text-anchor="middle" x="127.5" y="-611.81" font-family="Times,serif" font-size="10.00">Channel</text> 82 <text text-anchor="middle" x="127.5" y="-600.81" font-family="Times,serif" font-size="10.00">/Buffer</text> 83 </g> 84 <!-- wait --> 85 <g id="node3" class="node"> 86 <title>wait</title> 87 <polygon fill="white" stroke="white" points="90,-656 16,-656 16,-620 90,-620 90,-656"/> 88 <text text-anchor="middle" x="53" y="-635.5" font-family="Times,serif" font-size="10.00">Go routine #1</text> 89 </g> 90 <!-- wait_ --> 91 <g id="node4" class="node"> 92 <title>wait_</title> 93 <polygon fill="white" stroke="white" points="90,-547 16,-547 16,-511 90,-511 90,-547"/> 94 <text text-anchor="middle" x="53" y="-526.5" font-family="Times,serif" font-size="10.00">Go routine #2</text> 95 </g> 96 <!-- wait->wait_ --> 97 <g id="edge2" class="edge"> 98 <title>wait->wait_</title> 99 <path fill="none" stroke="black" stroke-dasharray="5,2" d="M53,-619.81C53,-602.96 53,-577.02 53,-557.34"/> 100 <polygon fill="black" stroke="black" points="56.5,-557.15 53,-547.15 49.5,-557.15 56.5,-557.15"/> 101 <text text-anchor="middle" x="29" y="-611.81" font-family="Times,serif" font-size="10.00">#1 wiats #2</text> 102 </g> 103 <!-- Raftstore --> 104 <g id="node5" class="node"> 105 <title>Raftstore</title> 106 <polygon fill="lightgrey" stroke="lightgrey" points="631,-547 565,-547 565,-511 631,-511 631,-547"/> 107 <text text-anchor="middle" x="598" y="-525.3" font-family="Times,serif" font-size="14.00">Raftstore</text> 108 </g> 109 <!-- CDC --> 110 <g id="node6" class="node"> 111 <title>CDC</title> 112 <polygon fill="lightgrey" stroke="lightgrey" points="619,-438 565,-438 565,-402 619,-402 619,-438"/> 113 <text text-anchor="middle" x="592" y="-416.3" font-family="Times,serif" font-size="14.00">CDC</text> 114 </g> 115 <!-- Raftstore->CDC --> 116 <g id="edge3" class="edge"> 117 <title>Raftstore->CDC</title> 118 <path fill="none" stroke="black" d="M597.04,-510.81C596.09,-493.96 594.64,-468.02 593.53,-448.34"/> 119 <polygon fill="black" stroke="black" points="597.02,-447.94 592.96,-438.15 590.03,-448.33 597.02,-447.94"/> 120 <text text-anchor="middle" x="616.5" y="-480.8" font-family="Times,serif" font-size="14.00">channel</text> 121 </g> 122 <!-- kv_client --> 123 <g id="node7" class="node"> 124 <title>kv_client</title> 125 <polygon fill="white" stroke="white" points="532.5,-656 465.5,-656 465.5,-620 532.5,-620 532.5,-656"/> 126 <text text-anchor="middle" x="499" y="-634.3" font-family="Times,serif" font-size="14.00">kv_client</text> 127 </g> 128 <!-- CDC->kv_client --> 129 <g id="edge4" class="edge"> 130 <title>CDC->kv_client</title> 131 <path fill="none" stroke="black" d="M572.39,-438.27C556.48,-453.44 534.82,-476.94 523,-502 506.59,-536.78 501.34,-581.43 499.69,-609.64"/> 132 <polygon fill="black" stroke="black" points="496.18,-609.79 499.21,-619.94 503.17,-610.11 496.18,-609.79"/> 133 <text text-anchor="middle" x="539.5" y="-525.3" font-family="Times,serif" font-size="14.00">gRPC</text> 134 </g> 135 <!-- puller_run_step1 --> 136 <g id="node8" class="node"> 137 <title>puller_run_step1</title> 138 <polygon fill="white" stroke="white" points="402,-657 252,-657 252,-619 402,-619 402,-657"/> 139 <text text-anchor="middle" x="327" y="-641.8" font-family="Times,serif" font-size="14.00">Run #1</text> 140 <text text-anchor="middle" x="327" y="-626.8" font-family="Times,serif" font-size="14.00">(Add to memory buffer)</text> 141 </g> 142 <!-- kv_client->puller_run_step1 --> 143 <g id="edge5" class="edge"> 144 <title>kv_client->puller_run_step1</title> 145 <path fill="none" stroke="black" d="M465.32,-638C450.02,-638 431.15,-638 412.32,-638"/> 146 <polygon fill="black" stroke="black" points="412.19,-634.5 402.19,-638 412.19,-641.5 412.19,-634.5"/> 147 <text text-anchor="middle" x="433.75" y="-644.8" font-family="Times,serif" font-size="14.00">eventCh</text> 148 </g> 149 <!-- puller_run_step2 --> 150 <g id="node12" class="node"> 151 <title>puller_run_step2</title> 152 <polygon fill="white" stroke="white" points="410.5,-548 249.5,-548 249.5,-510 410.5,-510 410.5,-548"/> 153 <text text-anchor="middle" x="330" y="-532.8" font-family="Times,serif" font-size="14.00">Run #2</text> 154 <text text-anchor="middle" x="330" y="-517.8" font-family="Times,serif" font-size="14.00">(Output to output channel)</text> 155 </g> 156 <!-- puller_run_step1->puller_run_step2 --> 157 <g id="edge7" class="edge"> 158 <title>puller_run_step1->puller_run_step2</title> 159 <path fill="none" stroke="black" d="M327.51,-618.85C327.97,-602.32 328.66,-577.66 329.2,-558.5"/> 160 <polygon fill="black" stroke="black" points="332.71,-558.33 329.49,-548.24 325.71,-558.14 332.71,-558.33"/> 161 <text text-anchor="middle" x="350.5" y="-589.8" font-family="Times,serif" font-size="14.00">P.buffer</text> 162 </g> 163 <!-- processor_sorter_consume --> 164 <g id="node9" class="node"> 165 <title>processor_sorter_consume</title> 166 <polygon fill="white" stroke="white" points="374.5,-438 275.5,-438 275.5,-402 374.5,-402 374.5,-438"/> 167 <text text-anchor="middle" x="325" y="-416.3" font-family="Times,serif" font-size="14.00">sorterConsume</text> 168 </g> 169 <!-- table_sink_run --> 170 <g id="node11" class="node"> 171 <title>table_sink_run</title> 172 <polygon fill="white" stroke="white" points="312,-300 214,-300 214,-262 312,-262 312,-300"/> 173 <text text-anchor="middle" x="263" y="-284.8" font-family="Times,serif" font-size="14.00">Table Sink (N)</text> 174 <text text-anchor="middle" x="263" y="-269.8" font-family="Times,serif" font-size="14.00">FlushEvents</text> 175 </g> 176 <!-- processor_sorter_consume->table_sink_run --> 177 <g id="edge6" class="edge"> 178 <title>processor_sorter_consume->table_sink_run</title> 179 <path fill="none" stroke="black" d="M276.15,-401.81C268.91,-397.14 262.42,-391.3 258,-384 244.52,-361.75 248.14,-331.49 253.68,-309.71"/> 180 <polygon fill="black" stroke="black" points="257.05,-310.65 256.4,-300.07 250.32,-308.75 257.05,-310.65"/> 181 <text text-anchor="middle" x="280.5" y="-372.8" font-family="Times,serif" font-size="14.00">P.output</text> 182 </g> 183 <!-- mounter_unmarshal --> 184 <g id="node14" class="node"> 185 <title>mounter_unmarshal</title> 186 <polygon fill="white" stroke="white" points="427.5,-299 350.5,-299 350.5,-263 427.5,-263 427.5,-299"/> 187 <text text-anchor="middle" x="389" y="-277.3" font-family="Times,serif" font-size="14.00">Unmarshal</text> 188 </g> 189 <!-- processor_sorter_consume->mounter_unmarshal --> 190 <g id="edge11" class="edge"> 191 <title>processor_sorter_consume->mounter_unmarshal</title> 192 <path fill="none" stroke="black" d="M321.55,-401.78C320.4,-391.64 320.57,-378.9 326,-369 329.08,-363.38 333.5,-365.57 338,-361 353.33,-345.45 366.56,-324.58 375.7,-308.2"/> 193 <polygon fill="black" stroke="black" points="378.92,-309.61 380.6,-299.15 372.76,-306.28 378.92,-309.61"/> 194 <text text-anchor="middle" x="355" y="-372.8" font-family="Times,serif" font-size="14.00">unmarshal</text> 195 </g> 196 <!-- processor_puller_consume --> 197 <g id="node10" class="node"> 198 <title>processor_puller_consume</title> 199 <polygon fill="white" stroke="white" points="493,-438 393,-438 393,-402 493,-402 493,-438"/> 200 <text text-anchor="middle" x="443" y="-416.3" font-family="Times,serif" font-size="14.00">pullerConsume</text> 201 </g> 202 <!-- sorter_run --> 203 <g id="node13" class="node"> 204 <title>sorter_run</title> 205 <polygon fill="white" stroke="white" points="514,-299 460,-299 460,-263 514,-263 514,-299"/> 206 <text text-anchor="middle" x="487" y="-277.3" font-family="Times,serif" font-size="14.00">Run</text> 207 </g> 208 <!-- processor_puller_consume->sorter_run --> 209 <g id="edge9" class="edge"> 210 <title>processor_puller_consume->sorter_run</title> 211 <path fill="none" stroke="black" d="M448.48,-401.93C455.96,-378.65 469.43,-336.7 478.3,-309.1"/> 212 <polygon fill="black" stroke="black" points="481.72,-309.88 481.45,-299.29 475.06,-307.74 481.72,-309.88"/> 213 <text text-anchor="middle" x="492" y="-372.8" font-family="Times,serif" font-size="14.00">E.AddEntry</text> 214 </g> 215 <!-- buffer_sink_run --> 216 <g id="node15" class="node"> 217 <title>buffer_sink_run</title> 218 <polygon fill="white" stroke="white" points="314,-206 214,-206 214,-168 314,-168 314,-206"/> 219 <text text-anchor="middle" x="264" y="-190.8" font-family="Times,serif" font-size="14.00">Buffer Sink (1)</text> 220 <text text-anchor="middle" x="264" y="-175.8" font-family="Times,serif" font-size="14.00">FlushEvents</text> 221 </g> 222 <!-- table_sink_run->buffer_sink_run --> 223 <g id="edge12" class="edge"> 224 <title>table_sink_run->buffer_sink_run</title> 225 <path fill="none" stroke="black" d="M263.2,-261.82C263.34,-248.85 263.53,-231.13 263.69,-216.31"/> 226 <polygon fill="black" stroke="black" points="267.19,-216.18 263.8,-206.14 260.19,-216.1 267.19,-216.18"/> 227 <text text-anchor="middle" x="302" y="-227.8" font-family="Times,serif" font-size="14.00">S.EmitEvents</text> 228 </g> 229 <!-- puller_run_step2->processor_puller_consume --> 230 <g id="edge8" class="edge"> 231 <title>puller_run_step2->processor_puller_consume</title> 232 <path fill="none" stroke="black" d="M349.17,-509.85C367.93,-492.08 396.63,-464.91 417.35,-445.29"/> 233 <polygon fill="black" stroke="black" points="420.01,-447.59 424.86,-438.18 415.19,-442.51 420.01,-447.59"/> 234 <text text-anchor="middle" x="408" y="-480.8" font-family="Times,serif" font-size="14.00">P.Output</text> 235 </g> 236 <!-- sorter_run->processor_sorter_consume --> 237 <g id="edge10" class="edge"> 238 <title>sorter_run->processor_sorter_consume</title> 239 <path fill="none" stroke="black" d="M481.35,-299.31C474.5,-317.5 461.16,-345.59 440,-361 427.1,-370.39 419.55,-362.46 405,-369 394.54,-373.7 393.61,-377.74 384,-384 377.62,-388.16 370.75,-392.42 364.03,-396.48"/> 240 <polygon fill="black" stroke="black" points="361.81,-393.73 355.01,-401.86 365.4,-399.74 361.81,-393.73"/> 241 <text text-anchor="middle" x="428" y="-372.8" font-family="Times,serif" font-size="14.00">S.Ouput</text> 242 </g> 243 <!-- backend_sink_run --> 244 <g id="node16" class="node"> 245 <title>backend_sink_run</title> 246 <polygon fill="white" stroke="white" points="326,-117 214,-117 214,-79 326,-79 326,-117"/> 247 <text text-anchor="middle" x="270" y="-101.8" font-family="Times,serif" font-size="14.00">Backend Sink (1)</text> 248 <text text-anchor="middle" x="270" y="-86.8" font-family="Times,serif" font-size="14.00">FlushEvents</text> 249 </g> 250 <!-- buffer_sink_run->backend_sink_run --> 251 <g id="edge13" class="edge"> 252 <title>buffer_sink_run->backend_sink_run</title> 253 <path fill="none" stroke="black" d="M265.24,-167.97C266.06,-156.19 267.13,-140.56 268.06,-127.16"/> 254 <polygon fill="black" stroke="black" points="271.56,-127.22 268.76,-117 264.58,-126.74 271.56,-127.22"/> 255 <text text-anchor="middle" x="306" y="-138.8" font-family="Times,serif" font-size="14.00">S.EmitEvents</text> 256 </g> 257 <!-- data_out --> 258 <g id="node18" class="node"> 259 <title>data_out</title> 260 <ellipse fill="none" stroke="black" cx="270" cy="-18" rx="40.89" ry="18"/> 261 <text text-anchor="middle" x="270" y="-14.3" font-family="Times,serif" font-size="14.00">Data out</text> 262 </g> 263 <!-- backend_sink_run->data_out --> 264 <g id="edge15" class="edge"> 265 <title>backend_sink_run->data_out</title> 266 <path fill="none" stroke="black" d="M270,-78.91C270,-69.26 270,-57.18 270,-46.4"/> 267 <polygon fill="black" stroke="black" points="273.5,-46.19 270,-36.19 266.5,-46.19 273.5,-46.19"/> 268 </g> 269 <!-- data_in --> 270 <g id="node17" class="node"> 271 <title>data_in</title> 272 <ellipse fill="none" stroke="black" cx="598" cy="-638" rx="36.29" ry="18"/> 273 <text text-anchor="middle" x="598" y="-634.3" font-family="Times,serif" font-size="14.00">Data in</text> 274 </g> 275 <!-- data_in->Raftstore --> 276 <g id="edge14" class="edge"> 277 <title>data_in->Raftstore</title> 278 <path fill="none" stroke="black" d="M598,-619.81C598,-602.96 598,-577.02 598,-557.34"/> 279 <polygon fill="black" stroke="black" points="601.5,-557.15 598,-547.15 594.5,-557.15 601.5,-557.15"/> 280 </g> 281 </g> 282 </svg>