diff --git a/labs/doris/conf/debezium/start.sh b/labs/doris/conf/debezium/start.sh index 0849fddd..dbf10e8c 100644 --- a/labs/doris/conf/debezium/start.sh +++ b/labs/doris/conf/debezium/start.sh @@ -1 +1 @@ -curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @config.json \ No newline at end of file +curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://debezium:8083/connectors/ -d @config.json \ No newline at end of file diff --git a/labs/doris/conf/doris/example.sql b/labs/doris/conf/doris/example.sql index d43024ce..e69de29b 100644 --- a/labs/doris/conf/doris/example.sql +++ b/labs/doris/conf/doris/example.sql @@ -1,19 +0,0 @@ ---创建数据库 - - - -CREATE ROUTINE LOAD db1.job1 on tbl1 -PROPERTIES -( - "desired_concurrent_number"="1" -) -FROM KAFKA -( - "kafka_broker_list"= "broker1:9091,broker2:9091", - "kafka_topic" = "my_topic", - "property.security.protocol" = "ssl", - "property.ssl.ca.location" = "FILE:ca.pem", - "property.ssl.certificate.location" = "FILE:client.pem", - "property.ssl.key.location" = "FILE:client.key", - "property.ssl.key.password" = "abcdefg" -); \ No newline at end of file diff --git a/labs/doris/docker-compose.yml b/labs/doris/docker-compose.yml index 2a375b61..d7340d35 100644 --- a/labs/doris/docker-compose.yml +++ b/labs/doris/docker-compose.yml @@ -19,7 +19,6 @@ services: - 2181:2181 volumes: - ./data/zookeeper:/data - #/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic mysql-server.example.User kafka: image: wurstmeister/kafka:2.13-2.7.0 restart: always @@ -65,6 +64,8 @@ services: - STATUS_STORAGE_TOPIC=my_connect_statuses ports: - 8083:8083 + volumes: + - ./conf/debezium:/start debezium-ui: image: debezium/debezium-ui:1.5 restart: always @@ -91,7 +92,7 @@ services: CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1" CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1" CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1" - volumes: + volumes: - ./data/kafka-connect/:/data - ./conf/kafka/connectors/confluentinc-kafka-connect-elasticsearch:/usr/share/java/confluentinc-kafka-connect-elasticsearch #command: bash -c "tail -f /dev/null" @@ -109,29 +110,13 @@ services: volumes: #chmod 777 elasticsearch - ./data/elasticsearch:/usr/share/elasticsearch/data - kibana: + kibana: image: kibana:7.12.0 restart: always - ports: + ports: - 9601:5601 depends_on: - elasticsearch - doris-be: - image: primetoninc/jdk:1.8 - ports: - - 8040:8040 - - 8060:8060 - - 9050:9050 - - 9060:9060 - environment: - - priority_networks=$IP/28 - volumes: - - ./apps/doris:/doris - - ./conf/doris/be.conf:/doris/be/conf/be.conf - - ./data/doris/be/storage:/doris/be/storage - command: bash -c "/doris/be/bin/start_be.sh" - #SET PASSWORD FOR 'root' = PASSWORD('aA123456!'); - #启动耗时长 doris-fe: image: primetoninc/jdk:1.8 ports: @@ -140,6 +125,7 @@ services: - 9020:9020 - 9030:9030 environment: + - TZ="Asia/Shanghai" - priority_networks=$CIDR volumes: - ./apps/doris:/doris @@ -147,6 +133,22 @@ services: - ./log/doris/fe:/doris/fe/log - ./data/doris/fe/doris-meta:/doris/fe/doris-meta command: bash -c "/doris/fe/bin/start_fe.sh" + doris-be: + image: primetoninc/jdk:1.8 + ports: + - 8040:8040 + - 8060:8060 + - 9050:9050 + - 9060:9060 + environment: + - TZ="Asia/Shanghai" + - priority_networks=$CIDR + volumes: + - ./apps/doris:/doris + - ./conf/doris/be.conf:/doris/be/conf/be.conf + - ./data/doris/be/storage:/doris/be/storage + - ./log/doris/be/:/doris/be/log + command: bash -c "/doris/be/bin/start_be.sh" #doris kafka Routine Load https://cloud.baidu.com/doc/PALO/s/ykizgdgv8 #doris 不支持 superset 和 metabase，以后通过二次开发解决 #https://hub.docker.com/r/apache/superset diff --git a/labs/doris/readme.md b/labs/doris/readme.md index 1ac141df..3275c450 100644 --- a/labs/doris/readme.md +++ b/labs/doris/readme.md @@ -4,20 +4,98 @@ mysql配置了主从，创建了数据库example，新建了User表 +## debezium + +进入容器执行 /start 目录，执行 ./start.sh + +## debezium-ui + +查看debezium状态: http://localhost:8080 + +## kafka + +通过 kafkacat 查看实时信息 + +## kafaka-connect + +进入容器执行 /usr/share/java/confluentinc-kafka-connect-elasticsearch 目录，执行 ./start.sh + +## kibana + +查看 elasticsearch 状态: http://localhost:9601 + ## doris -进入 mysql 容器内，连接 doris fe： +### 进入 mysql 容器内，连接 doris fe： mysql -h doris-fe -P 9030 -u root -设置root密码： +### 设置root密码： SET PASSWORD FOR 'root' = PASSWORD('aA123456!'); -添加 doris be： +### 添加 doris be： + +查看 be 的 log 可以查看到当前的ip，测试时使用该ip替换hostname或固定容器ip设置.env +ALTER SYSTEM ADD BACKEND "hostname:9050"; + +### 移除 doris be： + +ALTER SYSTEM DROPP BACKEND "hostname:9050"; + +### 查看be状态： + +SHOW PROC '/backends'; + +### 创建数据库： + +CREATE DATABASE example; + +### 切换数据库： + +USE example; + +### 创建表： -ALTER SYSTEM ADD BACKEND "192.168.100.144:9050"; +CREATE TABLE `User` ( + `Id` char(36) NOT NULL COMMENT 'Id', + `UserName` varchar(255) NOT NULL COMMENT 'UserName', + `SecurityStamp` varchar(255) REPLACE NULL COMMENT 'SecurityStamp', + `PasswordHash` varchar(255) REPLACE NULL COMMENT 'PasswordHash', + `PasswordConfirmed` tinyint(1) REPLACE NOT NULL COMMENT 'PasswordConfirmed', + `Email` varchar(255) REPLACE NULL DEFAULT NULL COMMENT 'Email', + `EmailConfirmed` tinyint(1) REPLACE NOT NULL COMMENT 'EmailConfirmed', + `PhoneNumber` varchar(255) REPLACE NULL DEFAULT NULL COMMENT 'PhoneNumber', + `PhoneNumberConfirmed` tinyint(1) REPLACE NOT NULL COMMENT 'PhoneNumberConfirmed', + `RealName` varchar(255) REPLACE NULL COMMENT 'RealName', + `IdentityNumber` varchar(255) REPLACE NULL COMMENT 'IdentityNumber', + `IdentityConfirmed` tinyint(1) REPLACE NOT NULL COMMENT 'IdentityConfirmed', + `NickName` varchar(255) REPLACE NULL COMMENT 'NickName', + `Avatar` varchar(255) REPLACE NULL COMMENT 'Avatar', + `Sex` int(0) REPLACE NULL DEFAULT NULL COMMENT 'Sex', + `Birthday` datetime REPLACE NULL DEFAULT NULL COMMENT 'Birthday', + `LockoutEnabled` tinyint(1) REPLACE NOT NULL COMMENT 'LockoutEnabled', + `AccessFailedCount` int(0) REPLACE NOT NULL COMMENT 'AccessFailedCount', + `LockoutEnd` datetime REPLACE NULL DEFAULT NULL COMMENT 'LockoutEnd', + `RowVersion` varchar(255) REPLACE NULL COMMENT 'RowVersion', + `Created` datetime REPLACE NOT NULL COMMENT 'Created', + `Modified` datetime REPLACE NULL DEFAULT NULL COMMENT 'Modified', + `Deleted` datetime REPLACE NULL DEFAULT NULL COMMENT 'Deleted' +) +AGGREGATE KEY(Id,UserName) +DISTRIBUTED BY HASH(Id) BUCKETS 10 +PROPERTIES("replication_num" = "1"); -查看be状态： +======================================================================= +从kafka导入： -SHOW PROC '/backends'; \ No newline at end of file +CREATE ROUTINE LOAD example.job1 on User +PROPERTIES +( + "desired_concurrent_number"="1" +) +FROM KAFKA +( + "kafka_broker_list"= "kafka:9092", + "kafka_topic" = "mysql.example.User" +); \ No newline at end of file