update

Former-commit-id: 6a6c706b3beee3af4dfbc52150b6d2fa1201dae9
Former-commit-id: c379c6e3979e30077979ec6745f8263a15bd3ee1

labs/doris/conf/canal/canal.properties

@@ -1,23 +0,0 @@
-# ...
-# 可选项: tcp(默认), kafka, RocketMQ
-canal.serverMode = kafka
-# ...
-# kafka/rocketmq 集群配置: 192.168.1.117:9092,192.168.1.118:9092,192.168.1.119:9092 
-canal.mq.servers = kafka:9092
-canal.mq.retries = 0
-# flagMessage模式下可以调大该值, 但不要超过MQ消息体大小上限
-canal.mq.batchSize = 16384
-canal.mq.maxRequestSize = 1048576
-# flatMessage模式下请将该值改大, 建议50-200
-canal.mq.lingerMs = 1
-canal.mq.bufferMemory = 33554432
-# Canal的batch size, 默认50K, 由于kafka最大消息体限制请勿超过1M(900K以下)
-canal.mq.canalBatchSize = 50
-# Canal get数据的超时时间, 单位: 毫秒, 空为不限超时
-canal.mq.canalGetTimeout = 100
-# 是否为flat json格式对象
-canal.mq.flatMessage = false
-canal.mq.compressionType = none
-canal.mq.acks = all
-# kafka消息投递是否使用事务
-canal.mq.transaction = false

labs/doris/docker-compose.yml

@@ -1,6 +1,6 @@
 version: "3.8"
 services:
-    mysql: 
+    mysql:
         image: mysql:8.0.23
         restart: always
         environment:
@@ -17,6 +17,8 @@ services:
         restart: always
         ports:
             - 2181:2181
+        volumes:
+            - ./data/zookeeper:/data
     #/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic mysql-server.example.User
     kafka:
         image: wurstmeister/kafka:2.13-2.7.0
@@ -30,8 +32,12 @@ services:
             - ./data/kafka:/kafka
         ports:
             - 9092:9092
-        depends_on: 
+        depends_on:
             - zookeeper
+    kafkacat:
+        image: edenhill/kafkacat:1.6.0
+        restart: always
+        command: ["kafkacat", "-C", "-b", "kafka:9092", "-t", "mysql.example.User"]
     kafka-manager:
         image: kafkamanager/kafka-manager:3.0.0.4
         restart: always
@@ -40,7 +46,7 @@ services:
             KAFKA_BROKERS: kafka:9092
         ports:
             - 9090:9000
-        depends_on: 
+        depends_on:
             - zookeeper
             - kafka
     # cockroachdb to kafka :https://www.cockroachlabs.com/docs/v21.1/stream-data-out-of-cockroachdb-using-changefeeds.html#create-a-core-changefeed
@@ -48,7 +54,7 @@ services:
     # https://github.com/debezium/docker-images/blob/master/examples/mysql/1.5/inventory.sql
     # https://github.com/debezium/debezium-examples/blob/master/tutorial/register-mysql.json
     # https://debezium.io/documentation/reference/1.5/connectors/mysql.html
-    debezium: 
+    debezium:
         image: debezium/connect:1.5
         environment:
             - BOOTSTRAP_SERVERS=$IP:9092
@@ -56,44 +62,26 @@ services:
             - CONFIG_STORAGE_TOPIC=my_connect_configs
             - OFFSET_STORAGE_TOPIC=my_connect_offsets
             - STATUS_STORAGE_TOPIC=my_connect_statuses
-        ports: 
+        ports:
             - 8083:8083
-    debezium-ui: 
+    debezium-ui:
         image: debezium/debezium-ui:1.5
         restart: always
         environment:
             - KAFKA_CONNECT_URI=http://debezium:8083
             - ui.base.uri=http://localhost:8080/api
-        ports: 
+        ports:
             - 8080:8080
-    canal: 
-        image: canal/canal-server:v1.1.5
-        privileged: true
-        restart: always
-        ports: 
-            - 11111:11111
-        environment:
-            - canal.auto.scan=false
-            - canal.instance.master.address=$IP:3306
-            - canal.instance.dbUsername=slave
-            - canal.instance.dbPassword=aA123456!
-            - canal.instance.filter.regex=.*\\..*  
-            - canal.instance.filter.black.regex=mysql\\.slave_.*
-            - canal.instance.connectionCharset=UTF-8
-            - canal.instance.tsdb.enable=true
-        depends_on: 
-            - mysql
-        command: bash -c "run.sh"
     doris-be:
         image: primetoninc/jdk:1.8
-        ports: 
+        ports:
             - 8040:8040
             - 8060:8060
             - 9050:9050
             - 9060:9060
-        environment: 
+        environment:
             - priority_networks=$IP/28
-        volumes: 
+        volumes:
             - ./apps/doris:/doris
             - ./conf/doris/be.conf:/doris/be/conf/be.conf
             - ./data/doris/be/storage:/doris/be/storage
@@ -103,14 +91,14 @@ services:
     #启动耗时长
     doris-fe:
         image: primetoninc/jdk:1.8
-        ports: 
+        ports:
             - 8030:8030
             - 9010:9010
             - 9020:9020
             - 9030:9030
-        environment: 
+        environment:
             - priority_networks=$CIDR
-        volumes: 
+        volumes:
             - ./apps/doris:/doris
             - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf
             - ./log/doris/fe:/doris/fe/log
@@ -123,15 +111,15 @@ services:
     #superset db upgrade
     #superset load_examples
     #superset init
-    #superset: 
+    #superset:
     #    image: apache/superset:4d2c9324be3979ecbc3d97d0d029b19e91c67c87-dev
-    #    ports: 
+    #    ports:
     #       - 8088:8088
-    #metabase: 
+    #metabase:
     #    image: metabase/metabase:v0.39.0.1
-    #    ports: 
+    #    ports:
     #        - 3000:3000
-    #    volumes: 
+    #    volumes:
     #        - ./data/metabase:/metabase-data
     #    environment:
     #        - MB_DB_FILE=/metabase-data/metabase.db