github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/dm/master/task_advanced.yaml (about)

     1  ---
     2  name: test # global unique
     3  task-mode: all  # full/incremental/all
     4  is-sharding: true  # whether multi dm-worker do one sharding job
     5  meta-schema: "dm_meta"  # meta schema in downstreaming database to store meta informaton of dm
     6  enable-heartbeat: false  # whether to enable heartbeat for calculating lag between master and syncer
     7  # heartbeat-update-interval: 1  # interval to do heartbeat and save timestamp, default 1s
     8  # heartbeat-report-interval: 10 # interval to report time lap to prometheus, default 10s
     9  
    10  target-database:
    11    host: "192.168.0.1"
    12    port: 4000
    13    user: "root"
    14    password: ""
    15  
    16  mysql-instances:             # one or more source database, config more source database for sharding merge
    17    -
    18      source-id: "instance118-4306" # unique in all instances, used as id when save checkpoints, configs, etc.
    19  
    20      # binlog pos used to as start pos for syncer, for different task-mode, this maybe used or not
    21      # `full` / `all`:
    22      #    never be used
    23      # `incremental`:
    24      #    if checkpoints already exists in `meta-schema`, this will not be used
    25      #    otherwise, this will be used
    26      meta:
    27        binlog-name: mysql-bin.000001
    28        binlog-pos: 4
    29      route-rules: ["user-route-rules-schema", "user-route-rules"]
    30      filter-rules: ["user-filter-1", "user-filter-2"]
    31      block-allow-list:  "instance"
    32  
    33      # `mydumper-config-name` and `mydumper` should only set one
    34      mydumper-config-name: "global"   # ref `mydumpers` config
    35      # `mydumper-thread` is alias for `threads` in `mydumper` config, and its priority is higher than `threads`
    36      #mydumper-thread: 16
    37      #mydumper:
    38      #  threads: 16
    39  
    40      # `loader-config-name` and `loader` should only set one
    41      loader-config-name: "global"    # ref `loaders` config
    42      # `loader-thread` is alias for `pool-size` in `loader` config, and its priority is higher than `pool-size`
    43      #loader-thread: 32
    44      #loader:                  # local loader rule
    45      #  pool-size: 32
    46      # `syncer-config-name` and `syncer` should only set one
    47  
    48      syncer-config-name: "global"    # ref `syncers` config
    49      # `syncer-thread` is alias for `worker-count` in `syncer` config, and its priority is higher than `worker-count`
    50      #syncer-thread: 32
    51      #syncer:
    52      #  worker-count: 32
    53  
    54    -
    55      source-id: "instance118-5306"
    56      meta:
    57        binlog-name: mysql-bin.000001
    58        binlog-pos: 320
    59      route-rules: ["user-route-rules-schema", "user-route-rules"]
    60      filter-rules: ["user-filter-2"]
    61      block-allow-list: "instance"
    62  
    63      mydumper:
    64        threads: 4
    65        chunk-filesize: 64
    66        skip-tz-utc: true
    67  #      extra-args: "-B test -T t1,t2"
    68  
    69      loader:                  # local loader rule
    70        pool-size: 16
    71        dir: "./dumped_data"   # if not ends with `.{name}` (`{name}` is the task name), then `.{name}` will be appended
    72  
    73      syncer:
    74        worker-count: 16
    75        batch: 100
    76  
    77  # other common configs shared by all instances
    78  
    79  routes:                      # schema/table route mapping
    80    user-route-rules-schema:
    81      schema-pattern: "test_*"    # pattern of the upstream schema name, wildcard characters (*?) are supported
    82      target-schema: "test"       # downstream schema name
    83    user-route-rules:
    84      schema-pattern: "test_*"
    85      table-pattern: "t_*"        # pattern of the upstream table name, wildcard characters (*?) are supported
    86      target-schema: "test"
    87      target-table: "t"           # downstream table name
    88  
    89  filters:                     # filter rules, mysql instance can ref rules in it
    90    user-filter-1:
    91      schema-pattern: "test_*"    # pattern of the upstream schema name, wildcard characters (*?) are supported
    92      table-pattern: "t_*"        # pattern of the upstream table name, wildcard characters (*?) are supported
    93      events: ["truncate table", "drop table"]  # ignore truncate/drop table ddl
    94      # pattern of the SQL statements, regular expression is supported
    95      # when using global filtering rule (`schema-pattern: "*"`), it will try to match the original statement executed in the upstream
    96      # when not using global filtering rule, it will try to match the statement parsed/restored by DM without route-rules applied (can find it in the log)
    97      sql-pattern: ["^DROP\\s+PROCEDURE", "^CREATE\\s+PROCEDURE"]
    98      action: Ignore
    99    user-filter-2:
   100      schema-pattern: "test_*"
   101      events: ["all dml"]             # only do all DML events
   102      action: Do
   103  
   104  block-allow-list:
   105    instance:
   106      do-dbs: ["~^test.*", "do"]        # allow list of upstream schemas needs to be replicated, regular expression (starts with ~) is supported
   107      ignore-dbs: ["mysql", "ignored"]  # block list of upstream schemas needs to be replicated, regular expression (starts with ~) is supported
   108      do-tables:                        # allow list of upstream tables needs to be replicated, (db-name, tbl-name) regular expression (starts with ~) is supported
   109      - db-name: "~^test.*"
   110        tbl-name: "~^t.*"
   111      - db-name: "do"
   112        tbl-name: "do"
   113      ignore-tables:                    # block list of upstream tables needs to be replicated, (db-name, tbl-name) regular expression (starts with ~) is supported
   114      - db-name: "test"
   115        tbl-name: "log"
   116  
   117  mydumpers:                   # mydumper process unit specific configs, mysql instance can ref one config in it
   118    global:
   119      threads: 4
   120      chunk-filesize: 64
   121      skip-tz-utc: true
   122      extra-args: "-B test -T t1,t2 --consistency none"
   123  
   124  loaders:                     # loader process unit specific configs, mysql instance can ref one config in it
   125    global:
   126      pool-size: 16
   127      dir: "./dumped_data"
   128  
   129  syncers:                     # syncer process unit specific configs, mysql instance can ref one config in it
   130    global:
   131      worker-count: 16
   132      batch: 100