update

Former-commit-id: ccded7078a9db535bb9ec9012651604a121cce03
Former-commit-id: 9df3d2a76ee4deefd3db78fab7ee34ab60eaf535

labs/doris/Kafka2Doris/appsettings.json

@@ -6,7 +6,7 @@
     }
   },
   "kafka": {
-    "host": "192.168.100.144:9092",
+    "host": "localhost:9992",
     "topics": "mysql.example.User"
   },
   "doris": {

labs/doris/conf/kafka-connect/kafka2elasticsearch.json

@@ -6,7 +6,9 @@
         "topics.regex": "mysql.example.*",
         "key.ignore": false,
         "schema.ignore": "false",
-        "key.converter.schemas.enable": false,
+        "schemas.enable": "false",
+        "key.converter.schemas.enable": "false",
+        "value.converter.schemas.enable": "false",
         "key.converter": "org.apache.kafka.connect.storage.StringConverter",
         "connection.url": "http://elasticsearch:9200",
         "type.name": "kafka-connect",

labs/doris/conf/kafka-connect/mysql2kafka.json

@@ -11,7 +11,8 @@
         "database.server.name": "mysql",
         "database.include": "example",
         "database.history.kafka.bootstrap.servers": "kafka:9092",
-        "database.history.kafka.topic": "mysql.schema",
+        "database.history.kafka.topic": "mysql.schema.example",
-        "include.schema.changes": "true"
+        "include.schema.changes": "true",
+        "time.precision.mode":"connect"
     }
 }

labs/doris/conf/kafka-connect/start.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
-echo 'init connectors start:'
+echo 'init connectors start:\n'
-echo 'mysql2kafka'
+echo 'mysql2kafka\n'
 curl -i -X POST -H "Accept:application/json" -H  "Content-Type:application/json" http://localhost:8083/connectors/ -d @mysql2kafka.json
-echo 'kafka2elasticsearch'
+echo 'kafka2elasticsearch\n'
 curl -i -X POST -H "Accept:application/json" -H  "Content-Type:application/json" http://localhost:8083/connectors/ -d @kafka2elasticsearch.json
-echo 'init connectors end!'
+echo 'init connectors end!\n
+# curl -X DELETE  http://localhost:8083/connectors/elasticsearch-sink'

labs/doris/conf/mysql/initdb.d/init.sql

@@ -48,4 +48,27 @@ INSERT INTO `User` VALUES ('49fa11e7-4404-4fe5-9873-e89a01d8e529', 'super', '123
 INSERT INTO `User` VALUES ('74fab867-a65a-4cda-8b2f-63e12d003b79', 'user', '123456', '71dd07494c5ee54992a27746d547e25dee01bd97', 1, 'user@test.com', 1, NULL, 0, NULL, NULL, 0, '普通用户', NULL, NULL, NULL, 0, 0, NULL, '2b691577-e96c-40fe-897e-3adc9a4488bf', '2021-04-14 05:10:36.611679', NULL, NULL);
 INSERT INTO `User` VALUES ('e6a2e59c-a4e1-4ced-856c-036b6e4786d1', 'admin', '123456', '579f889441b4a55d667233941d72a83ed644f7e5', 1, 'admin@test.com', 1, NULL, 0, NULL, NULL, 0, '管理员', NULL, NULL, NULL, 0, 0, NULL, '7d53094e-569d-4689-a6d1-d2c1f99da85b', '2021-04-14 05:10:36.611663', NULL, NULL);
 
+-- ----------------------------
+-- Table structure for Role
+-- ----------------------------
+DROP TABLE IF EXISTS `Role`;
+CREATE TABLE `Role`  (
+  `Id` char(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT 'Id',
+  `Name` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT 'Name',
+  `Number` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT 'Number',
+  `IsReadOnly` tinyint(1) NOT NULL COMMENT 'IsReadOnly',
+  `Created` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) COMMENT 'Created',
+  `Modified` datetime(6) NULL DEFAULT NULL COMMENT 'Modified',
+  `Deleted` datetime(6) NULL DEFAULT NULL COMMENT 'Deleted',
+  PRIMARY KEY (`Id`) USING BTREE
+) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = '角色' ROW_FORMAT = Dynamic;
+
+-- ----------------------------
+-- Records of Role
+-- ----------------------------
+INSERT INTO `Role` VALUES ('74ce76b4-c239-4ba7-938c-27a069a9e8f5', '机构管理员', 'organ', 0, '2021-04-14 05:10:25.509670', NULL, NULL);
+INSERT INTO `Role` VALUES ('a7964dbe-5dbf-41c9-9906-3266f47f5746', '管理员', 'admin', 0, '2021-04-14 05:10:25.509716', NULL, NULL);
+INSERT INTO `Role` VALUES ('a9784fe1-b830-46f2-b1aa-f7620a92ed4a', '超级管理员', 'super', 1, '2021-04-14 05:10:25.509719', NULL, NULL);
+
+
 SET FOREIGN_KEY_CHECKS = 1;

labs/doris/conf/mysql/my.cnf

@@ -8,7 +8,8 @@ default-character-set=utf8mb4
 character-set-server=utf8mb4
 collation-server=utf8mb4_general_ci
 server_id=1
-log_bin = mysql-bin
+log_bin=mysql-bin
-binlog_format = row
+binlog_format=row
-binlog_row_image = full
+binlog_row_image=full
-expire_logs_days = 30
+expire_logs_days=30
+binlog_expire_logs_seconds=2592000

labs/doris/conf/nginx/nginx.conf

@@ -0,0 +1,38 @@
+user root;
+worker_processes 4;
+
+events {
+    worker_connections 1024;
+}
+
+stream {
+    server {
+        listen 9092;
+        proxy_pass 172.172.0.21:9092;
+    }
+}
+
+
+http {
+    include mime.types;
+    default_type application/octet-stream;
+    sendfile on;
+    keepalive_timeout 65;
+
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        '' close;
+    }
+
+    server {
+        listen 80;
+        server_name default;
+        absolute_redirect off;
+
+        error_page 500 502 503 504 /50x.html;
+        location = /50x.html {
+            root html;
+        }
+    }
+    include conf.d/*.conf;
+}

labs/doris/docker-compose.yml

@@ -41,7 +41,7 @@ services:
         environment:
             KAFKA_HEAP_OPTS: "-Xmx512m -Xms512m"
             KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-            KAFKA_ADVERTISED_HOST_NAME: ${ip}
+            KAFKA_ADVERTISED_HOST_NAME: 172.172.0.21
             num.partitions: 1
         volumes:
             - ./data/kafka:/kafka
@@ -56,7 +56,14 @@ services:
         image: edenhill/kafkacat:1.6.0
         restart: always
         command:
-            ["kafkacat", "-C", "-b", "kafka:9092", "-t", "mysql.example.User"]
+            [
+                "kafkacat",
+                "-C",
+                "-b",
+                "172.172.0.21:9092",
+                "-t",
+                "mysql.example.User",
+            ]
         networks:
             default:
                 ipv4_address: 172.172.0.22
@@ -67,7 +74,7 @@ services:
             - 8083:8083
         environment:
             KAFKA_HEAP_OPTS: -Xms512m -Xmx512m
-            CONNECT_BOOTSTRAP_SERVERS: "kafka:9092"
+            CONNECT_BOOTSTRAP_SERVERS: "172.172.0.21:9092"
             CONNECT_GROUP_ID: kafka-connect
             CONNECT_CONFIG_STORAGE_TOPIC: _connect-configs
             CONNECT_OFFSET_STORAGE_TOPIC: _connect-offsets
@@ -89,6 +96,19 @@ services:
         networks:
             default:
                 ipv4_address: 172.172.0.23
+    nginx:
+        image: nginx:1.20.0
+        restart: always
+        environment:
+            TZ: "Asia/Shanghai"
+        ports:
+            - 9992:9092
+        volumes:
+            - ./conf/nginx/nginx.conf:/etc/nginx/nginx.conf
+            - ./log/nginx:/var/log/nginx
+        networks:
+            default:
+                ipv4_address: 172.172.0.24
     elasticsearch:
         image: elasticsearch:7.12.0
         healthcheck:

labs/doris/readme.md

@@ -21,6 +21,7 @@ CPU需要支持AVX2指令集，cat /proc/cpuinfo |grep avx2 确认有输出即
 1. https://www.confluent.io/hub/debezium/debezium-connector-mysql
 1. https://github.com/zendesk/maxwell
 
+
 ## cockroachdb cdc
 
 1. cockroachdb to kafka :https://www.cockroachlabs.com/docs/v21.1/stream-data-out-of-cockroachdb-using-changefeeds.html#create-a-core-changefeed