diff --git a/labs/doris/conf/canal/canal.properties b/labs/doris/conf/canal/canal.properties deleted file mode 100644 index 33dc60e3..00000000 --- a/labs/doris/conf/canal/canal.properties +++ /dev/null @@ -1,23 +0,0 @@ -# ... -# 可选项: tcp(默认), kafka, RocketMQ -canal.serverMode = kafka -# ... -# kafka/rocketmq 集群配置: 192.168.1.117:9092,192.168.1.118:9092,192.168.1.119:9092 -canal.mq.servers = kafka:9092 -canal.mq.retries = 0 -# flagMessage模式下可以调大该值, 但不要超过MQ消息体大小上限 -canal.mq.batchSize = 16384 -canal.mq.maxRequestSize = 1048576 -# flatMessage模式下请将该值改大, 建议50-200 -canal.mq.lingerMs = 1 -canal.mq.bufferMemory = 33554432 -# Canal的batch size, 默认50K, 由于kafka最大消息体限制请勿超过1M(900K以下) -canal.mq.canalBatchSize = 50 -# Canal get数据的超时时间, 单位: 毫秒, 空为不限超时 -canal.mq.canalGetTimeout = 100 -# 是否为flat json格式对象 -canal.mq.flatMessage = false -canal.mq.compressionType = none -canal.mq.acks = all -# kafka消息投递是否使用事务 -canal.mq.transaction = false \ No newline at end of file diff --git a/labs/doris/docker-compose.yml b/labs/doris/docker-compose.yml index d6417071..936b20bd 100644 --- a/labs/doris/docker-compose.yml +++ b/labs/doris/docker-compose.yml @@ -1,6 +1,6 @@ version: "3.8" services: - mysql: + mysql: image: mysql:8.0.23 restart: always environment: @@ -17,6 +17,8 @@ services: restart: always ports: - 2181:2181 + volumes: + - ./data/zookeeper:/data #/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic mysql-server.example.User kafka: image: wurstmeister/kafka:2.13-2.7.0 @@ -30,8 +32,12 @@ services: - ./data/kafka:/kafka ports: - 9092:9092 - depends_on: + depends_on: - zookeeper + kafkacat: + image: edenhill/kafkacat:1.6.0 + restart: always + command: ["kafkacat", "-C", "-b", "kafka:9092", "-t", "mysql.example.User"] kafka-manager: image: kafkamanager/kafka-manager:3.0.0.4 restart: always @@ -40,7 +46,7 @@ services: KAFKA_BROKERS: kafka:9092 ports: - 9090:9000 - depends_on: + depends_on: - zookeeper - kafka # cockroachdb to kafka :https://www.cockroachlabs.com/docs/v21.1/stream-data-out-of-cockroachdb-using-changefeeds.html#create-a-core-changefeed @@ -48,7 +54,7 @@ services: # https://github.com/debezium/docker-images/blob/master/examples/mysql/1.5/inventory.sql # https://github.com/debezium/debezium-examples/blob/master/tutorial/register-mysql.json # https://debezium.io/documentation/reference/1.5/connectors/mysql.html - debezium: + debezium: image: debezium/connect:1.5 environment: - BOOTSTRAP_SERVERS=$IP:9092 @@ -56,44 +62,26 @@ services: - CONFIG_STORAGE_TOPIC=my_connect_configs - OFFSET_STORAGE_TOPIC=my_connect_offsets - STATUS_STORAGE_TOPIC=my_connect_statuses - ports: + ports: - 8083:8083 - debezium-ui: + debezium-ui: image: debezium/debezium-ui:1.5 restart: always environment: - KAFKA_CONNECT_URI=http://debezium:8083 - ui.base.uri=http://localhost:8080/api - ports: + ports: - 8080:8080 - canal: - image: canal/canal-server:v1.1.5 - privileged: true - restart: always - ports: - - 11111:11111 - environment: - - canal.auto.scan=false - - canal.instance.master.address=$IP:3306 - - canal.instance.dbUsername=slave - - canal.instance.dbPassword=aA123456! - - canal.instance.filter.regex=.*\\..* - - canal.instance.filter.black.regex=mysql\\.slave_.* - - canal.instance.connectionCharset=UTF-8 - - canal.instance.tsdb.enable=true - depends_on: - - mysql - command: bash -c "run.sh" doris-be: image: primetoninc/jdk:1.8 - ports: + ports: - 8040:8040 - 8060:8060 - 9050:9050 - 9060:9060 - environment: + environment: - priority_networks=$IP/28 - volumes: + volumes: - ./apps/doris:/doris - ./conf/doris/be.conf:/doris/be/conf/be.conf - ./data/doris/be/storage:/doris/be/storage @@ -103,14 +91,14 @@ services: #启动耗时长 doris-fe: image: primetoninc/jdk:1.8 - ports: + ports: - 8030:8030 - 9010:9010 - 9020:9020 - 9030:9030 - environment: + environment: - priority_networks=$CIDR - volumes: + volumes: - ./apps/doris:/doris - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf - ./log/doris/fe:/doris/fe/log @@ -123,15 +111,15 @@ services: #superset db upgrade #superset load_examples #superset init - #superset: + #superset: # image: apache/superset:4d2c9324be3979ecbc3d97d0d029b19e91c67c87-dev - # ports: + # ports: # - 8088:8088 - #metabase: + #metabase: # image: metabase/metabase:v0.39.0.1 - # ports: + # ports: # - 3000:3000 - # volumes: + # volumes: # - ./data/metabase:/metabase-data # environment: # - MB_DB_FILE=/metabase-data/metabase.db