github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/deployments/ticdc/docker-compose/docker-compose-avro.yml (about)

     1  ---
     2  version: '3.5'
     3  
     4  services:
     5    controller:
     6      image: ticdc:latest
     7      container_name: ticdc_controller
     8      build:
     9        context: ../../../
    10        dockerfile: ./deployments/ticdc/docker/dev.Dockerfile
    11      volumes:
    12        - /data
    13        - ./logs:/logs
    14        - ./configs:/configs
    15      command:
    16        - /usr/bin/socat
    17        - -v
    18        - tcp-l:1234,fork
    19        - exec:'/bin/cat'
    20      ports:
    21        - "1234:1234"
    22      depends_on:
    23        - "upstream-pd"
    24        - "schema-registry"
    25        - "kafka-connect-01"
    26        - "kafka-connect-healthcheck"
    27        - "kafka"
    28        - "capturer0"
    29        - "capturer1"
    30        - "capturer2"
    31      restart: on-failure
    32  
    33    capturer0:
    34      image: ticdc:latest
    35      build:
    36        context: ../../../
    37        dockerfile: ./deployments/ticdc/docker/dev.Dockerfile
    38      volumes:
    39        - /data
    40        - ./logs:/logs
    41      ports:
    42        - "8300:8300"
    43      entrypoint: "/cdc server"
    44      command:
    45        - --addr=0.0.0.0:8300
    46        - --pd=http://upstream-pd:2379
    47        - --log-file=/logs/capturer0.log
    48        - --log-level=debug
    49        - --advertise-addr=capturer0:8300
    50        - --tz=${CDC_TIME_ZONE:-SYSTEM}
    51        - --data-dir=/data
    52      depends_on:
    53        - "upstream-tidb"
    54        - "downstream-tidb"
    55        - "kafka"
    56      restart: on-failure
    57  
    58    capturer1:
    59      image: ticdc:latest
    60      build:
    61        context: ../../../
    62        dockerfile: ./deployments/ticdc/docker/dev.Dockerfile
    63      volumes:
    64        - /data
    65        - ./logs:/logs
    66      ports:
    67        - "8301:8300"
    68      entrypoint: "/cdc server"
    69      command:
    70        - --addr=0.0.0.0:8300
    71        - --pd=http://upstream-pd:2379
    72        - --log-file=/logs/capturer1.log
    73        - --log-level=debug
    74        - --advertise-addr=capturer1:8300
    75        - --tz=${CDC_TIME_ZONE:-SYSTEM}
    76        - --data-dir=/data
    77      depends_on:
    78        - "upstream-tidb"
    79        - "downstream-tidb"
    80        - "kafka"
    81      restart: on-failure
    82  
    83    capturer2:
    84      image: ticdc:latest
    85      build:
    86        context: ../../../
    87        dockerfile: ./deployments/ticdc/docker/dev.Dockerfile
    88      volumes:
    89        - /data
    90        - ./logs:/logs
    91      ports:
    92        - "8302:8300"
    93      entrypoint: "/cdc server"
    94      command:
    95        - --addr=0.0.0.0:8300
    96        - --pd=http://upstream-pd:2379
    97        - --log-file=/logs/capturer2.log
    98        - --log-level=debug
    99        - --advertise-addr=capturer2:8300
   100        - --tz=${CDC_TIME_ZONE:-SYSTEM}
   101        - --data-dir=/data
   102      depends_on:
   103        - "upstream-tidb"
   104        - "downstream-tidb"
   105        - "kafka"
   106      restart: on-failure
   107  
   108    upstream-pd:
   109      image: pingcap/pd:nightly
   110      ports:
   111        - "2379:2379"
   112      volumes:
   113        - ./configs/pd.toml:/pd.toml:ro
   114        - /data
   115        - ./logs:/logs
   116      command:
   117        - --name=upstream-pd
   118        - --client-urls=http://0.0.0.0:2379
   119        - --peer-urls=http://0.0.0.0:2380
   120        - --advertise-client-urls=http://upstream-pd:2379
   121        - --advertise-peer-urls=http://upstream-pd:2380
   122        - --initial-cluster=upstream-pd=http://upstream-pd:2380
   123        - --data-dir=/data/upstream-pd
   124        - --config=/pd.toml
   125        - --log-file=/logs/upstream-pd.log
   126        - -L=info
   127      restart: on-failure
   128  
   129    upstream-tikv0:
   130      image: pingcap/tikv:nightly
   131      volumes:
   132        - ./configs/tikv.toml:/tikv.toml:ro
   133        - /data
   134        - ./logs:/logs
   135      command:
   136        - --addr=0.0.0.0:20160
   137        - --advertise-addr=upstream-tikv0:20160
   138        - --data-dir=/data/upstream-tikv0
   139        - --pd=upstream-pd:2379
   140        - --config=/tikv.toml
   141        - --log-file=/logs/upstream-tikv0.log
   142        - --log-level=debug
   143      depends_on:
   144        - "upstream-pd"
   145      restart: on-failure
   146  
   147    upstream-tikv1:
   148      image: pingcap/tikv:nightly
   149      volumes:
   150        - ./configs/tikv.toml:/tikv.toml:ro
   151        - /data
   152        - ./logs:/logs
   153      command:
   154        - --addr=0.0.0.0:20160
   155        - --advertise-addr=upstream-tikv1:20160
   156        - --data-dir=/data/upstream-tikv1
   157        - --pd=upstream-pd:2379
   158        - --config=/tikv.toml
   159        - --log-file=/logs/upstream-tikv1.log
   160        - --log-level=debug
   161      depends_on:
   162        - "upstream-pd"
   163      restart: on-failure
   164  
   165    upstream-tikv2:
   166      image: pingcap/tikv:nightly
   167      volumes:
   168        - ./configs/tikv.toml:/tikv.toml:ro
   169        - /data
   170        - ./logs:/logs
   171      command:
   172        - --addr=0.0.0.0:20160
   173        - --advertise-addr=upstream-tikv2:20160
   174        - --data-dir=/data/upstream-tikv2
   175        - --pd=upstream-pd:2379
   176        - --config=/tikv.toml
   177        - --log-file=/logs/upstream-tikv2.log
   178        - --log-level=debug
   179      depends_on:
   180        - "upstream-pd"
   181      restart: on-failure
   182  
   183    upstream-tidb:
   184      image: pingcap/tidb:nightly
   185      ports:
   186        - "4000:4000"
   187        - "10080:10080"
   188      volumes:
   189        - ./configs/tidb.toml:/tidb.toml:ro
   190        - ./logs:/logs
   191      command:
   192        - --store=tikv
   193        - --path=upstream-pd:2379
   194        - --config=/tidb.toml
   195        - --log-file=/logs/upstream-tidb.log
   196        - --advertise-address=upstream-tidb
   197        - -L=info
   198      depends_on:
   199        - "upstream-tikv0"
   200        - "upstream-tikv1"
   201        - "upstream-tikv2"
   202      restart: on-failure
   203  
   204    downstream-pd:
   205      image: pingcap/pd:nightly
   206      ports:
   207        - "3379:2379"
   208      volumes:
   209        - ./configs/pd.toml:/pd.toml:ro
   210        - /data
   211        - ./logs:/logs
   212      command:
   213        - --name=downstream-pd
   214        - --client-urls=http://0.0.0.0:2379
   215        - --peer-urls=http://0.0.0.0:2380
   216        - --advertise-client-urls=http://downstream-pd:2379
   217        - --advertise-peer-urls=http://downstream-pd:2380
   218        - --initial-cluster=downstream-pd=http://downstream-pd:2380
   219        - --data-dir=/data/downstream-pd
   220        - --config=/pd.toml
   221        - --log-file=/logs/downstream-pd.log
   222        - -L=info
   223      restart: on-failure
   224  
   225    downstream-tikv0:
   226      image: pingcap/tikv:nightly
   227      volumes:
   228        - ./configs/tikv.toml:/tikv.toml:ro
   229        - /data
   230        - ./logs:/logs
   231      command:
   232        - --addr=0.0.0.0:20160
   233        - --advertise-addr=downstream-tikv0:20160
   234        - --data-dir=/data/downstream-tikv0
   235        - --pd=downstream-pd:2379
   236        - --config=/tikv.toml
   237        - --log-file=/logs/downstream-tikv0.log
   238        - --log-level=info
   239      depends_on:
   240        - "downstream-pd"
   241      restart: on-failure
   242  
   243    downstream-tikv1:
   244      image: pingcap/tikv:nightly
   245      volumes:
   246        - ./configs/tikv.toml:/tikv.toml:ro
   247        - /data
   248        - ./logs:/logs
   249      command:
   250        - --addr=0.0.0.0:20160
   251        - --advertise-addr=downstream-tikv1:20160
   252        - --data-dir=/data/downstream-tikv1
   253        - --pd=downstream-pd:2379
   254        - --config=/tikv.toml
   255        - --log-file=/logs/downstream-tikv1.log
   256        - --log-level=info
   257      depends_on:
   258        - "downstream-pd"
   259      restart: on-failure
   260  
   261    downstream-tikv2:
   262      image: pingcap/tikv:nightly
   263      volumes:
   264        - ./configs/tikv.toml:/tikv.toml:ro
   265        - /data
   266        - ./logs:/logs
   267      command:
   268        - --addr=0.0.0.0:20160
   269        - --advertise-addr=downstream-tikv2:20160
   270        - --data-dir=/data/downstream-tikv2
   271        - --pd=downstream-pd:2379
   272        - --config=/tikv.toml
   273        - --log-file=/logs/downstream-tikv2.log
   274        - --log-level=info
   275      depends_on:
   276        - "downstream-pd"
   277      restart: on-failure
   278  
   279    downstream-tidb:
   280      image: pingcap/tidb:nightly
   281      ports:
   282        - "5000:4000"
   283        - "20080:10080"
   284      volumes:
   285        - ./configs/tidb.toml:/tidb.toml:ro
   286        - ./logs:/logs
   287      command:
   288        - --store=tikv
   289        - --path=downstream-pd:2379
   290        - --config=/tidb.toml
   291        - --log-file=/logs/downstream-tidb.log
   292        - --advertise-address=downstream-tidb
   293        - -L=info
   294      depends_on:
   295        - "downstream-tikv0"
   296        - "downstream-tikv1"
   297        - "downstream-tikv2"
   298      restart: on-failure
   299  
   300  # The rest of the file is adapted from https://github.com/confluentinc/demo-scene/blob/master/connect-jdbc/docker-compose.yml
   301  
   302    zookeeper:
   303      image: confluentinc/cp-zookeeper:5.5.1
   304      container_name: zookeeper
   305      environment:
   306        ZOOKEEPER_CLIENT_PORT: 2181
   307        ZOOKEEPER_TICK_TIME: 2000
   308  
   309    kafka:
   310      image: confluentinc/cp-enterprise-kafka:5.5.1
   311      container_name: kafka
   312      depends_on:
   313        - zookeeper
   314      ports:
   315        # Exposes 9092 for external connections to the broker
   316        # Use kafka:29092 for connections internal on the docker network
   317        # See https://rmoff.net/2018/08/02/kafka-listeners-explained/ for details
   318        - 9092:9092
   319      environment:
   320        KAFKA_BROKER_ID: 1
   321        KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
   322        KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
   323        KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
   324        KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://kafka:9092
   325        KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
   326        KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
   327        KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
   328        KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 100
   329        CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092
   330        CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
   331        CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
   332        CONFLUENT_METRICS_ENABLE: 'true'
   333        CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
   334  
   335    schema-registry:
   336      image: confluentinc/cp-schema-registry:5.5.1
   337      container_name: schema-registry
   338      ports:
   339        - 8081:8081
   340      depends_on:
   341        - zookeeper
   342        - kafka
   343      environment:
   344        SCHEMA_REGISTRY_HOST_NAME: schema-registry
   345        SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
   346  
   347    kafka-connect-01:
   348      image: confluentinc/cp-kafka-connect:5.5.1
   349      container_name: kafka-connect-01
   350      depends_on:
   351        - zookeeper
   352        - kafka
   353        - schema-registry
   354        - downstream-tidb
   355      ports:
   356        - 8083:8083
   357      environment:
   358        CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n"
   359        CONNECT_BOOTSTRAP_SERVERS: "kafka:29092"
   360        CONNECT_REST_PORT: 8083
   361        CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect-01"
   362        CONNECT_GROUP_ID: compose-connect-group
   363        CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
   364        CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
   365        CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
   366        CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
   367        CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
   368        CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
   369        CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
   370        CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
   371        CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
   372        CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
   373        CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
   374        CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
   375        CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
   376        CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
   377        CONNECT_PLUGIN_PATH: '/usr/share/java'
   378      command:
   379        - /bin/bash
   380        - -c
   381        - |
   382          # JDBC Drivers
   383          # ------------
   384          # MySQL
   385          cd /usr/share/java/kafka-connect-jdbc/
   386          wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-8.0.21.tar.gz
   387          tar -xf mysql-connector-java-8.0.21.tar.gz
   388          mv mysql-connector-java-8.0.21/mysql-connector-java-8.0.21.jar ./
   389          # Now launch Kafka Connect
   390          sleep infinity &
   391          /etc/confluent/docker/run
   392  
   393    kafka-connect-healthcheck:
   394      image: devshawn/kafka-connect-healthcheck:0.1.0
   395      container_name: kafka-connect-healthcheck
   396      depends_on:
   397        - kafka-connect-01
   398      ports:
   399        - 18083:18083
   400      environment:
   401        HEALTHCHECK_CONNECT_URL: 'http://kafka-connect-01:8083'