github.com/pingcap/ticdc@v0.0.0-20220526033649-485a10ef2652/docs/data-flow.svg (about) 1 <?xml version="1.0" encoding="UTF-8" standalone="no"?> 2 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" 3 "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> 4 <!-- Generated by graphviz version 2.40.1 (20161225.0304) 5 --> 6 <!-- Title: Dataflow Pages: 1 --> 7 <svg width="625pt" height="1448pt" 8 viewBox="0.00 0.00 625.00 1448.40" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"> 9 <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 1444.4)"> 10 <title>Dataflow</title> 11 <polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-1444.4 621,-1444.4 621,4 -4,4"/> 12 <g id="clust1" class="cluster"> 13 <title>cluster_legends</title> 14 <polygon fill="#d3d3d3" stroke="#d3d3d3" points="8,-1132.6 8,-1392 186,-1392 186,-1132.6 8,-1132.6"/> 15 <text text-anchor="middle" x="97" y="-1379" font-family="Times,serif" font-size="10.00" fill="#000000">Legend</text> 16 </g> 17 <g id="clust2" class="cluster"> 18 <title>cluster_tikv</title> 19 <polygon fill="none" stroke="#d3d3d3" points="525,-1059.6 525,-1298.2 609,-1298.2 609,-1059.6 525,-1059.6"/> 20 <text text-anchor="middle" x="567" y="-1281.6" font-family="Times,serif" font-size="14.00" fill="#000000">TiKV</text> 21 </g> 22 <g id="clust3" class="cluster"> 23 <title>cluster_ticdc</title> 24 <polygon fill="none" stroke="#d3d3d3" points="194,-63 194,-1432.4 517,-1432.4 517,-63 194,-63"/> 25 <text text-anchor="middle" x="355.5" y="-1415.8" font-family="Times,serif" font-size="14.00" fill="#000000">TiCDC</text> 26 </g> 27 <g id="clust4" class="cluster"> 28 <title>cluster_kvclient</title> 29 <polygon fill="#d3d3d3" stroke="#d3d3d3" points="425,-1320 425,-1396.8 509,-1396.8 509,-1320 425,-1320"/> 30 <text text-anchor="middle" x="467" y="-1380.2" font-family="Times,serif" font-size="14.00" fill="#000000">KV client</text> 31 </g> 32 <g id="clust5" class="cluster"> 33 <title>cluster_processor</title> 34 <polygon fill="#d3d3d3" stroke="#d3d3d3" points="202,-831.4 202,-908.2 438,-908.2 438,-831.4 202,-831.4"/> 35 <text text-anchor="middle" x="320" y="-891.6" font-family="Times,serif" font-size="14.00" fill="#000000">Processor</text> 36 </g> 37 <g id="clust6" class="cluster"> 38 <title>cluster_puller</title> 39 <polygon fill="#d3d3d3" stroke="#d3d3d3" points="202,-981 202,-1399.6 382,-1399.6 382,-981 202,-981"/> 40 <text text-anchor="middle" x="292" y="-1383" font-family="Times,serif" font-size="14.00" fill="#000000">Puller</text> 41 </g> 42 <g id="clust7" class="cluster"> 43 <title>cluster_sorter</title> 44 <polygon fill="#d3d3d3" stroke="#d3d3d3" points="365,-641 365,-758.6 463,-758.6 463,-641 365,-641"/> 45 <text text-anchor="middle" x="414" y="-742" font-family="Times,serif" font-size="14.00" fill="#000000">Sorter</text> 46 </g> 47 <g id="clust8" class="cluster"> 48 <title>cluster_entry_sorter</title> 49 <polygon fill="#d3d3d3" stroke="#d3d3d3" points="373,-649 373,-725.8 455,-725.8 455,-649 373,-649"/> 50 <text text-anchor="middle" x="414" y="-709.2" font-family="Times,serif" font-size="14.00" fill="#000000">Entry sorter</text> 51 </g> 52 <g id="clust9" class="cluster"> 53 <title>cluster_mounter</title> 54 <polygon fill="#d3d3d3" stroke="#d3d3d3" points="278,-649 278,-725.8 348,-725.8 348,-649 278,-649"/> 55 <text text-anchor="middle" x="313" y="-709.2" font-family="Times,serif" font-size="14.00" fill="#000000">Mounter</text> 56 </g> 57 <g id="clust10" class="cluster"> 58 <title>cluster_sink</title> 59 <polygon fill="#d3d3d3" stroke="#d3d3d3" points="228,-71 228,-570.2 358,-570.2 358,-71 228,-71"/> 60 <text text-anchor="middle" x="293" y="-553.6" font-family="Times,serif" font-size="14.00" fill="#000000">Sink Manager</text> 61 </g> 62 <!-- flow --> 63 <g id="node1" class="node"> 64 <title>flow</title> 65 <polygon fill="#ffffff" stroke="#ffffff" points="177.5464,-1364 106.4536,-1364 106.4536,-1328 177.5464,-1328 177.5464,-1364"/> 66 <text text-anchor="middle" x="142" y="-1343" font-family="Times,serif" font-size="10.00" fill="#000000">Go routine #1</text> 67 </g> 68 <!-- flow_ --> 69 <g id="node2" class="node"> 70 <title>flow_</title> 71 <polygon fill="#ffffff" stroke="#ffffff" points="177.5464,-1176.6 106.4536,-1176.6 106.4536,-1140.6 177.5464,-1140.6 177.5464,-1176.6"/> 72 <text text-anchor="middle" x="142" y="-1155.6" font-family="Times,serif" font-size="10.00" fill="#000000">Go routine #2</text> 73 </g> 74 <!-- flow->flow_ --> 75 <g id="edge1" class="edge"> 76 <title>flow->flow_</title> 77 <path fill="none" stroke="#000000" d="M142,-1327.6901C142,-1294.8582 142,-1225.7745 142,-1186.8363"/> 78 <polygon fill="#000000" stroke="#000000" points="145.5001,-1186.7083 142,-1176.7083 138.5001,-1186.7084 145.5001,-1186.7083"/> 79 <text text-anchor="middle" x="124.228" y="-1191.7083" font-family="Times,serif" font-size="10.00" fill="#000000">Flow</text> 80 <text text-anchor="middle" x="124.228" y="-1179.7083" font-family="Times,serif" font-size="10.00" fill="#000000">direction</text> 81 <text text-anchor="middle" x="125.337" y="-1318.6901" font-family="Times,serif" font-size="10.00" fill="#000000">Channel</text> 82 <text text-anchor="middle" x="125.337" y="-1306.6901" font-family="Times,serif" font-size="10.00" fill="#000000">/Buffer</text> 83 </g> 84 <!-- wait --> 85 <g id="node3" class="node"> 86 <title>wait</title> 87 <polygon fill="#ffffff" stroke="#ffffff" points="87.5464,-1364 16.4536,-1364 16.4536,-1328 87.5464,-1328 87.5464,-1364"/> 88 <text text-anchor="middle" x="52" y="-1343" font-family="Times,serif" font-size="10.00" fill="#000000">Go routine #1</text> 89 </g> 90 <!-- wait_ --> 91 <g id="node4" class="node"> 92 <title>wait_</title> 93 <polygon fill="#ffffff" stroke="#ffffff" points="87.5464,-1176.6 16.4536,-1176.6 16.4536,-1140.6 87.5464,-1140.6 87.5464,-1176.6"/> 94 <text text-anchor="middle" x="52" y="-1155.6" font-family="Times,serif" font-size="10.00" fill="#000000">Go routine #2</text> 95 </g> 96 <!-- wait->wait_ --> 97 <g id="edge2" class="edge"> 98 <title>wait->wait_</title> 99 <path fill="none" stroke="#000000" stroke-dasharray="5,2" d="M52,-1327.6901C52,-1294.8582 52,-1225.7745 52,-1186.8363"/> 100 <polygon fill="#000000" stroke="#000000" points="55.5001,-1186.7083 52,-1176.7083 48.5001,-1186.7084 55.5001,-1186.7083"/> 101 <text text-anchor="middle" x="28.9475" y="-1318.6901" font-family="Times,serif" font-size="10.00" fill="#000000">#1 wiats #2</text> 102 </g> 103 <!-- Raftstore --> 104 <g id="node5" class="node"> 105 <title>Raftstore</title> 106 <polygon fill="#d3d3d3" stroke="#d3d3d3" points="600.8135,-1265.4 533.1865,-1265.4 533.1865,-1229.4 600.8135,-1229.4 600.8135,-1265.4"/> 107 <text text-anchor="middle" x="567" y="-1243.2" font-family="Times,serif" font-size="14.00" fill="#000000">Raftstore</text> 108 </g> 109 <!-- CDC --> 110 <g id="node6" class="node"> 111 <title>CDC</title> 112 <polygon fill="#d3d3d3" stroke="#d3d3d3" points="587,-1103.6 533,-1103.6 533,-1067.6 587,-1067.6 587,-1103.6"/> 113 <text text-anchor="middle" x="560" y="-1081.4" font-family="Times,serif" font-size="14.00" fill="#000000">CDC</text> 114 </g> 115 <!-- Raftstore->CDC --> 116 <g id="edge3" class="edge"> 117 <title>Raftstore->CDC</title> 118 <path fill="none" stroke="#000000" d="M566.2106,-1229.154C565.0028,-1201.2351 562.6851,-1147.6651 561.248,-1114.4464"/> 119 <polygon fill="#000000" stroke="#000000" points="564.7263,-1113.8669 560.7972,-1104.0276 557.7328,-1114.1695 564.7263,-1113.8669"/> 120 <text text-anchor="middle" x="584.7672" y="-1154.4" font-family="Times,serif" font-size="14.00" fill="#000000">channel</text> 121 </g> 122 <!-- kv_client --> 123 <g id="node7" class="node"> 124 <title>kv_client</title> 125 <polygon fill="#ffffff" stroke="#ffffff" points="501.1011,-1364 432.8989,-1364 432.8989,-1328 501.1011,-1328 501.1011,-1364"/> 126 <text text-anchor="middle" x="467" y="-1341.8" font-family="Times,serif" font-size="14.00" fill="#000000">kv_client</text> 127 </g> 128 <!-- CDC->kv_client --> 129 <g id="edge4" class="edge"> 130 <title>CDC->kv_client</title> 131 <path fill="none" stroke="#000000" d="M553.4676,-1103.8906C537.4335,-1148.7861 496.0542,-1264.6483 476.9442,-1318.1563"/> 132 <polygon fill="#000000" stroke="#000000" points="473.6069,-1317.0946 473.5396,-1327.6892 480.1991,-1319.449 473.6069,-1317.0946"/> 133 <text text-anchor="middle" x="537.7279" y="-1198.8" font-family="Times,serif" font-size="14.00" fill="#000000">gRPC</text> 134 </g> 135 <!-- puller_run_step1 --> 136 <g id="node8" class="node"> 137 <title>puller_run_step1</title> 138 <polygon fill="#ffffff" stroke="#ffffff" points="361.6716,-1366.6019 210.3284,-1366.6019 210.3284,-1325.3981 361.6716,-1325.3981 361.6716,-1366.6019"/> 139 <text text-anchor="middle" x="286" y="-1350.2" font-family="Times,serif" font-size="14.00" fill="#000000">Run #1</text> 140 <text text-anchor="middle" x="286" y="-1333.4" font-family="Times,serif" font-size="14.00" fill="#000000">(Add to memory buffer)</text> 141 </g> 142 <!-- kv_client->puller_run_step1 --> 143 <g id="edge5" class="edge"> 144 <title>kv_client->puller_run_step1</title> 145 <path fill="none" stroke="#000000" d="M432.4825,-1346C415.1645,-1346 393.3422,-1346 371.8685,-1346"/> 146 <polygon fill="#000000" stroke="#000000" points="371.7116,-1342.5001 361.7116,-1346 371.7115,-1349.5001 371.7116,-1342.5001"/> 147 <text text-anchor="middle" x="397.3926" y="-1353.2" font-family="Times,serif" font-size="14.00" fill="#000000">eventCh</text> 148 </g> 149 <!-- puller_run_step2 --> 150 <g id="node12" class="node"> 151 <title>puller_run_step2</title> 152 <polygon fill="#ffffff" stroke="#ffffff" points="374.4152,-1030.4019 209.5848,-1030.4019 209.5848,-989.1981 374.4152,-989.1981 374.4152,-1030.4019"/> 153 <text text-anchor="middle" x="292" y="-1014" font-family="Times,serif" font-size="14.00" fill="#000000">Run #2</text> 154 <text text-anchor="middle" x="292" y="-997.2" font-family="Times,serif" font-size="14.00" fill="#000000">(Output to output channel)</text> 155 </g> 156 <!-- puller_run_step1->puller_run_step2 --> 157 <g id="edge7" class="edge"> 158 <title>puller_run_step1->puller_run_step2</title> 159 <path fill="none" stroke="#000000" d="M286.999,-1325.2309C287.8717,-1305.2809 289,-1274.2878 289,-1247.4 289,-1247.4 289,-1247.4 289,-1085.6 289,-1070.9473 289.5666,-1054.7138 290.2074,-1041.0411"/> 160 <polygon fill="#000000" stroke="#000000" points="293.7205,-1040.8674 290.7277,-1030.704 286.7293,-1040.5154 293.7205,-1040.8674"/> 161 <text text-anchor="middle" x="311.7395" y="-1154.4" font-family="Times,serif" font-size="14.00" fill="#000000">P.buffer</text> 162 </g> 163 <!-- processor_sorter_consume --> 164 <g id="node9" class="node"> 165 <title>processor_sorter_consume</title> 166 <polygon fill="#ffffff" stroke="#ffffff" points="310.2577,-875.4 209.7423,-875.4 209.7423,-839.4 310.2577,-839.4 310.2577,-875.4"/> 167 <text text-anchor="middle" x="260" y="-853.2" font-family="Times,serif" font-size="14.00" fill="#000000">sorterConsume</text> 168 </g> 169 <!-- table_sink_run --> 170 <g id="node11" class="node"> 171 <title>table_sink_run</title> 172 <polygon fill="#ffffff" stroke="#ffffff" points="335.9734,-537.2019 236.0266,-537.2019 236.0266,-495.9981 335.9734,-495.9981 335.9734,-537.2019"/> 173 <text text-anchor="middle" x="286" y="-520.8" font-family="Times,serif" font-size="14.00" fill="#000000">Table Sink (N)</text> 174 <text text-anchor="middle" x="286" y="-504" font-family="Times,serif" font-size="14.00" fill="#000000">FlushEvents</text> 175 </g> 176 <!-- processor_sorter_consume->table_sink_run --> 177 <g id="edge6" class="edge"> 178 <title>processor_sorter_consume->table_sink_run</title> 179 <path fill="none" stroke="#000000" d="M241.4138,-839.1919C231.3689,-827.381 221,-811.058 221,-794 221,-794 221,-794 221,-605.6 221,-581.873 236.2522,-560.2492 251.98,-544.2586"/> 180 <polygon fill="#000000" stroke="#000000" points="254.41,-546.7776 259.1867,-537.3206 249.5552,-541.7347 254.41,-546.7776"/> 181 <text text-anchor="middle" x="244.5319" y="-670.8" font-family="Times,serif" font-size="14.00" fill="#000000">P.output</text> 182 </g> 183 <!-- mounter_run --> 184 <g id="node14" class="node"> 185 <title>mounter_run</title> 186 <polygon fill="#ffffff" stroke="#ffffff" points="340,-693 286,-693 286,-657 340,-657 340,-693"/> 187 <text text-anchor="middle" x="313" y="-670.8" font-family="Times,serif" font-size="14.00" fill="#000000">Run</text> 188 </g> 189 <!-- processor_sorter_consume->mounter_run --> 190 <g id="edge11" class="edge"> 191 <title>processor_sorter_consume->mounter_run</title> 192 <path fill="none" stroke="#000000" d="M262.253,-839.206C264.2833,-824.574 267.6919,-803.5664 272.5042,-785.6 280.2576,-756.6529 292.4055,-724.6018 301.3753,-702.4876"/> 193 <polygon fill="#000000" stroke="#000000" points="304.6804,-703.6528 305.2457,-693.0731 298.2062,-700.9911 304.6804,-703.6528"/> 194 <text text-anchor="middle" x="295.7479" y="-789.8" font-family="Times,serif" font-size="14.00" fill="#000000">M.Input</text> 195 </g> 196 <!-- processor_puller_consume --> 197 <g id="node10" class="node"> 198 <title>processor_puller_consume</title> 199 <polygon fill="#ffffff" stroke="#ffffff" points="429.5438,-875.4 328.4562,-875.4 328.4562,-839.4 429.5438,-839.4 429.5438,-875.4"/> 200 <text text-anchor="middle" x="379" y="-853.2" font-family="Times,serif" font-size="14.00" fill="#000000">pullerConsume</text> 201 </g> 202 <!-- sorter_run --> 203 <g id="node13" class="node"> 204 <title>sorter_run</title> 205 <polygon fill="#ffffff" stroke="#ffffff" points="441,-693 387,-693 387,-657 441,-657 441,-693"/> 206 <text text-anchor="middle" x="414" y="-670.8" font-family="Times,serif" font-size="14.00" fill="#000000">Run</text> 207 </g> 208 <!-- processor_puller_consume->sorter_run --> 209 <g id="edge9" class="edge"> 210 <title>processor_puller_consume->sorter_run</title> 211 <path fill="none" stroke="#000000" d="M382.718,-839.0768C384.874,-828.3857 387.624,-814.6357 390,-802.4 396.6367,-768.2227 404.0158,-728.8328 408.8169,-703.0084"/> 212 <polygon fill="#000000" stroke="#000000" points="412.2618,-703.6268 410.6458,-693.1559 405.3794,-702.3491 412.2618,-703.6268"/> 213 <text text-anchor="middle" x="426.6315" y="-789.8" font-family="Times,serif" font-size="14.00" fill="#000000">E.AddEntry</text> 214 </g> 215 <!-- buffer_sink_run --> 216 <g id="node15" class="node"> 217 <title>buffer_sink_run</title> 218 <polygon fill="#ffffff" stroke="#ffffff" points="337.5285,-328.8019 236.4715,-328.8019 236.4715,-287.5981 337.5285,-287.5981 337.5285,-328.8019"/> 219 <text text-anchor="middle" x="287" y="-312.4" font-family="Times,serif" font-size="14.00" fill="#000000">Buffer Sink (1)</text> 220 <text text-anchor="middle" x="287" y="-295.6" font-family="Times,serif" font-size="14.00" fill="#000000">FlushEvents</text> 221 </g> 222 <!-- table_sink_run->buffer_sink_run --> 223 <g id="edge13" class="edge"> 224 <title>table_sink_run->buffer_sink_run</title> 225 <path fill="none" stroke="#000000" d="M279.0433,-495.8009C275.9525,-484.7852 273,-470.9515 273,-458.3 273,-458.3 273,-458.3 273,-366.5 273,-357.3724 274.6426,-347.6576 276.8024,-338.8288"/> 226 <polygon fill="#000000" stroke="#000000" points="280.2395,-339.5293 279.5082,-328.9598 273.4886,-337.6784 280.2395,-339.5293"/> 227 <text text-anchor="middle" x="311.3047" y="-408.2" font-family="Times,serif" font-size="14.00" fill="#000000">S.EmitEvents</text> 228 </g> 229 <!-- puller_run_step2->processor_puller_consume --> 230 <g id="edge8" class="edge"> 231 <title>puller_run_step2->processor_puller_consume</title> 232 <path fill="none" stroke="#000000" d="M303.9146,-988.929C319.3679,-961.8591 346.3595,-914.5771 363.4228,-884.6869"/> 233 <polygon fill="#000000" stroke="#000000" points="366.6686,-886.0608 368.5868,-875.6411 360.5894,-882.5904 366.6686,-886.0608"/> 234 <text text-anchor="middle" x="360.0852" y="-939.4" font-family="Times,serif" font-size="14.00" fill="#000000">P.Output</text> 235 </g> 236 <!-- sorter_run->processor_sorter_consume --> 237 <g id="edge10" class="edge"> 238 <title>sorter_run->processor_sorter_consume</title> 239 <path fill="none" stroke="#000000" d="M401.9115,-693.0988C384.5791,-718.5433 351.0946,-765.8642 318,-802.4 308.7377,-812.6254 297.9217,-823.204 288.1558,-832.3191"/> 240 <polygon fill="#000000" stroke="#000000" points="285.5821,-829.9312 280.5986,-839.2808 290.3249,-835.0797 285.5821,-829.9312"/> 241 <text text-anchor="middle" x="356.1399" y="-789.8" font-family="Times,serif" font-size="14.00" fill="#000000">S.Ouput</text> 242 </g> 243 <!-- mounter_run->table_sink_run --> 244 <g id="edge12" class="edge"> 245 <title>mounter_run->table_sink_run</title> 246 <path fill="none" stroke="#000000" stroke-dasharray="5,2" d="M308.2195,-646.9546C302.8649,-615.5406 294.3113,-565.3598 289.5293,-537.3055"/> 247 <polygon fill="#000000" stroke="#000000" points="304.7721,-647.5598 309.9028,-656.8295 311.6726,-646.3835 304.7721,-647.5598"/> 248 <text text-anchor="middle" x="345.959" y="-601.4" font-family="Times,serif" font-size="14.00" fill="#000000">wait unmarshal</text> 249 </g> 250 <!-- backend_sink_run --> 251 <g id="node16" class="node"> 252 <title>backend_sink_run</title> 253 <polygon fill="#ffffff" stroke="#ffffff" points="349.9748,-120.4019 236.0252,-120.4019 236.0252,-79.1981 349.9748,-79.1981 349.9748,-120.4019"/> 254 <text text-anchor="middle" x="293" y="-104" font-family="Times,serif" font-size="14.00" fill="#000000">Backend Sink (1)</text> 255 <text text-anchor="middle" x="293" y="-87.2" font-family="Times,serif" font-size="14.00" fill="#000000">FlushEvents</text> 256 </g> 257 <!-- buffer_sink_run->backend_sink_run --> 258 <g id="edge14" class="edge"> 259 <title>buffer_sink_run->backend_sink_run</title> 260 <path fill="none" stroke="#000000" d="M279.5082,-287.4402C276.1796,-276.4352 273,-262.5993 273,-249.9 273,-249.9 273,-249.9 273,-158.1 273,-148.6927 275.3587,-138.8711 278.456,-130.0292"/> 261 <polygon fill="#000000" stroke="#000000" points="281.786,-131.1244 282.1805,-120.5369 275.2697,-128.5675 281.786,-131.1244"/> 262 <text text-anchor="middle" x="311.3047" y="-199.8" font-family="Times,serif" font-size="14.00" fill="#000000">S.EmitEvents</text> 263 </g> 264 <!-- data_out --> 265 <g id="node18" class="node"> 266 <title>data_out</title> 267 <ellipse fill="none" stroke="#000000" cx="293" cy="-18" rx="44.0396" ry="18"/> 268 <text text-anchor="middle" x="293" y="-13.8" font-family="Times,serif" font-size="14.00" fill="#000000">Data out</text> 269 </g> 270 <!-- backend_sink_run->data_out --> 271 <g id="edge16" class="edge"> 272 <title>backend_sink_run->data_out</title> 273 <path fill="none" stroke="#000000" d="M293,-79.1584C293,-69.2193 293,-57.1448 293,-46.3586"/> 274 <polygon fill="#000000" stroke="#000000" points="296.5001,-46.1287 293,-36.1287 289.5001,-46.1287 296.5001,-46.1287"/> 275 </g> 276 <!-- data_in --> 277 <g id="node17" class="node"> 278 <title>data_in</title> 279 <ellipse fill="none" stroke="#000000" cx="567" cy="-1346" rx="39.3836" ry="18"/> 280 <text text-anchor="middle" x="567" y="-1341.8" font-family="Times,serif" font-size="14.00" fill="#000000">Data in</text> 281 </g> 282 <!-- data_in->Raftstore --> 283 <g id="edge15" class="edge"> 284 <title>data_in->Raftstore</title> 285 <path fill="none" stroke="#000000" d="M567,-1327.9152C567,-1313.4126 567,-1292.7876 567,-1276.0435"/> 286 <polygon fill="#000000" stroke="#000000" points="570.5001,-1275.704 567,-1265.704 563.5001,-1275.7041 570.5001,-1275.704"/> 287 </g> 288 </g> 289 </svg>