diff --git a/labs/doris/Kafka2Doris/appsettings.json b/labs/doris/Kafka2Doris/appsettings.json index 4302547b..2cdd854f 100644 --- a/labs/doris/Kafka2Doris/appsettings.json +++ b/labs/doris/Kafka2Doris/appsettings.json @@ -6,7 +6,7 @@ } }, "kafka": { - "host": "192.168.100.144:9092", + "host": "localhost:9992", "topics": "mysql.example.User" }, "doris": { diff --git a/labs/doris/conf/kafka-connect/kafka2elasticsearch.json b/labs/doris/conf/kafka-connect/kafka2elasticsearch.json index e1e46633..0cc62cb8 100644 --- a/labs/doris/conf/kafka-connect/kafka2elasticsearch.json +++ b/labs/doris/conf/kafka-connect/kafka2elasticsearch.json @@ -6,7 +6,9 @@ "topics.regex": "mysql.example.*", "key.ignore": false, "schema.ignore": "false", - "key.converter.schemas.enable": false, + "schemas.enable": "false", + "key.converter.schemas.enable": "false", + "value.converter.schemas.enable": "false", "key.converter": "org.apache.kafka.connect.storage.StringConverter", "connection.url": "http://elasticsearch:9200", "type.name": "kafka-connect", diff --git a/labs/doris/conf/kafka-connect/mysql2kafka.json b/labs/doris/conf/kafka-connect/mysql2kafka.json index 1ab861ae..e9ab020c 100644 --- a/labs/doris/conf/kafka-connect/mysql2kafka.json +++ b/labs/doris/conf/kafka-connect/mysql2kafka.json @@ -11,7 +11,8 @@ "database.server.name": "mysql", "database.include": "example", "database.history.kafka.bootstrap.servers": "kafka:9092", - "database.history.kafka.topic": "mysql.schema", - "include.schema.changes": "true" + "database.history.kafka.topic": "mysql.schema.example", + "include.schema.changes": "true", + "time.precision.mode":"connect" } } \ No newline at end of file diff --git a/labs/doris/conf/kafka-connect/start.sh b/labs/doris/conf/kafka-connect/start.sh index aecaf74b..35cb2384 100644 --- a/labs/doris/conf/kafka-connect/start.sh +++ b/labs/doris/conf/kafka-connect/start.sh @@ -1,7 +1,8 @@ #!/bin/bash -echo 'init connectors start:' -echo 'mysql2kafka' +echo 'init connectors start:\n' +echo 'mysql2kafka\n' curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @mysql2kafka.json -echo 'kafka2elasticsearch' +echo 'kafka2elasticsearch\n' curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @kafka2elasticsearch.json -echo 'init connectors end!' \ No newline at end of file +echo 'init connectors end!\n +# curl -X DELETE http://localhost:8083/connectors/elasticsearch-sink' \ No newline at end of file diff --git a/labs/doris/conf/mysql/initdb.d/init.sql b/labs/doris/conf/mysql/initdb.d/init.sql index 6ccec089..e7ceb974 100644 --- a/labs/doris/conf/mysql/initdb.d/init.sql +++ b/labs/doris/conf/mysql/initdb.d/init.sql @@ -48,4 +48,27 @@ INSERT INTO `User` VALUES ('49fa11e7-4404-4fe5-9873-e89a01d8e529', 'super', '123 INSERT INTO `User` VALUES ('74fab867-a65a-4cda-8b2f-63e12d003b79', 'user', '123456', '71dd07494c5ee54992a27746d547e25dee01bd97', 1, 'user@test.com', 1, NULL, 0, NULL, NULL, 0, '普通用户', NULL, NULL, NULL, 0, 0, NULL, '2b691577-e96c-40fe-897e-3adc9a4488bf', '2021-04-14 05:10:36.611679', NULL, NULL); INSERT INTO `User` VALUES ('e6a2e59c-a4e1-4ced-856c-036b6e4786d1', 'admin', '123456', '579f889441b4a55d667233941d72a83ed644f7e5', 1, 'admin@test.com', 1, NULL, 0, NULL, NULL, 0, '管理员', NULL, NULL, NULL, 0, 0, NULL, '7d53094e-569d-4689-a6d1-d2c1f99da85b', '2021-04-14 05:10:36.611663', NULL, NULL); +-- ---------------------------- +-- Table structure for Role +-- ---------------------------- +DROP TABLE IF EXISTS `Role`; +CREATE TABLE `Role` ( + `Id` char(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'Id', + `Name` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT 'Name', + `Number` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT 'Number', + `IsReadOnly` tinyint(1) NOT NULL COMMENT 'IsReadOnly', + `Created` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT 'Created', + `Modified` datetime(6) NULL DEFAULT NULL COMMENT 'Modified', + `Deleted` datetime(6) NULL DEFAULT NULL COMMENT 'Deleted', + PRIMARY KEY (`Id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = '角色' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Records of Role +-- ---------------------------- +INSERT INTO `Role` VALUES ('74ce76b4-c239-4ba7-938c-27a069a9e8f5', '机构管理员', 'organ', 0, '2021-04-14 05:10:25.509670', NULL, NULL); +INSERT INTO `Role` VALUES ('a7964dbe-5dbf-41c9-9906-3266f47f5746', '管理员', 'admin', 0, '2021-04-14 05:10:25.509716', NULL, NULL); +INSERT INTO `Role` VALUES ('a9784fe1-b830-46f2-b1aa-f7620a92ed4a', '超级管理员', 'super', 1, '2021-04-14 05:10:25.509719', NULL, NULL); + + SET FOREIGN_KEY_CHECKS = 1; \ No newline at end of file diff --git a/labs/doris/conf/mysql/my.cnf b/labs/doris/conf/mysql/my.cnf index 61dfb90b..6d7fdb06 100644 --- a/labs/doris/conf/mysql/my.cnf +++ b/labs/doris/conf/mysql/my.cnf @@ -8,7 +8,8 @@ default-character-set=utf8mb4 character-set-server=utf8mb4 collation-server=utf8mb4_general_ci server_id=1 -log_bin = mysql-bin -binlog_format = row -binlog_row_image = full -expire_logs_days = 30 +log_bin=mysql-bin +binlog_format=row +binlog_row_image=full +expire_logs_days=30 +binlog_expire_logs_seconds=2592000 diff --git a/labs/doris/conf/nginx/nginx.conf b/labs/doris/conf/nginx/nginx.conf new file mode 100644 index 00000000..0d928997 --- /dev/null +++ b/labs/doris/conf/nginx/nginx.conf @@ -0,0 +1,38 @@ +user root; +worker_processes 4; + +events { + worker_connections 1024; +} + +stream { + server { + listen 9092; + proxy_pass 172.172.0.21:9092; + } +} + + +http { + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 80; + server_name default; + absolute_redirect off; + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root html; + } + } + include conf.d/*.conf; +} \ No newline at end of file diff --git a/labs/doris/docker-compose.yml b/labs/doris/docker-compose.yml index 194549cd..6b58a630 100644 --- a/labs/doris/docker-compose.yml +++ b/labs/doris/docker-compose.yml @@ -41,7 +41,7 @@ services: environment: KAFKA_HEAP_OPTS: "-Xmx512m -Xms512m" KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_HOST_NAME: ${ip} + KAFKA_ADVERTISED_HOST_NAME: 172.172.0.21 num.partitions: 1 volumes: - ./data/kafka:/kafka @@ -56,7 +56,14 @@ services: image: edenhill/kafkacat:1.6.0 restart: always command: - ["kafkacat", "-C", "-b", "kafka:9092", "-t", "mysql.example.User"] + [ + "kafkacat", + "-C", + "-b", + "172.172.0.21:9092", + "-t", + "mysql.example.User", + ] networks: default: ipv4_address: 172.172.0.22 @@ -67,7 +74,7 @@ services: - 8083:8083 environment: KAFKA_HEAP_OPTS: -Xms512m -Xmx512m - CONNECT_BOOTSTRAP_SERVERS: "kafka:9092" + CONNECT_BOOTSTRAP_SERVERS: "172.172.0.21:9092" CONNECT_GROUP_ID: kafka-connect CONNECT_CONFIG_STORAGE_TOPIC: _connect-configs CONNECT_OFFSET_STORAGE_TOPIC: _connect-offsets @@ -89,6 +96,19 @@ services: networks: default: ipv4_address: 172.172.0.23 + nginx: + image: nginx:1.20.0 + restart: always + environment: + TZ: "Asia/Shanghai" + ports: + - 9992:9092 + volumes: + - ./conf/nginx/nginx.conf:/etc/nginx/nginx.conf + - ./log/nginx:/var/log/nginx + networks: + default: + ipv4_address: 172.172.0.24 elasticsearch: image: elasticsearch:7.12.0 healthcheck: diff --git a/labs/doris/readme.md b/labs/doris/readme.md index 55bd4777..256713c6 100644 --- a/labs/doris/readme.md +++ b/labs/doris/readme.md @@ -21,6 +21,7 @@ CPU需要支持AVX2指令集，cat /proc/cpuinfo |grep avx2 确认有输出即 1. https://www.confluent.io/hub/debezium/debezium-connector-mysql 1. https://github.com/zendesk/maxwell + ## cockroachdb cdc 1. cockroachdb to kafka :https://www.cockroachlabs.com/docs/v21.1/stream-data-out-of-cockroachdb-using-changefeeds.html#create-a-core-changefeed