update

Former-commit-id: 67f3142f26bbe0c1779ac9bbc3d1392c60a0729c
Former-commit-id: 47df6461b214ca2e8edd9660195a6f00f0a6cc6a

labs/doris/conf/kafka-connect/boot.sh

@@ -13,4 +13,4 @@ while : ; do
 done
 cd /usr/share/confluent-hub-components/
 ./start.sh
-sleep infinity
+sleep infinity

labs/doris/conf/kafka-connect/kafka2elasticsearch.json

@@ -4,15 +4,17 @@
         "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector",
         "tasks.max": 1,
         "topics": "mysql.example.User",
-        "key.ignore": true,
+        "key.ignore": false,
         "key.converter.schemas.enable": false,
         "key.converter": "org.apache.kafka.connect.storage.StringConverter",
         "connection.url": "http://elasticsearch:9200",
         "type.name": "kafka-connect",
         "write.method": "upsert",
         "behavior.on.null.values": "delete",
-        "transforms": "ExtractFieldObject",
+        "transforms": "ValueToKey,ExtractFieldObject",
         "transforms.ExtractFieldObject.type": "org.apache.kafka.connect.transforms.ExtractField$Value",
-        "transforms.ExtractFieldObject.field": "after"
+        "transforms.ExtractFieldObject.field": "after",
+        "transforms.ValueToKey.type":"org.apache.kafka.connect.transforms.ValueToKey",
+        "transforms.ValueToKey.fields":"Id"
     }
 }

labs/doris/docker-compose.yml

@@ -29,8 +29,8 @@ services:
         image: wurstmeister/kafka:2.13-2.7.0
         restart: always
         environment:
-            KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
             KAFKA_HEAP_OPTS: "-Xmx512m -Xms512m"
+            KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
             KAFKA_ADVERTISED_HOST_NAME: 172.172.0.21
             num.partitions: 1
         volumes:
@@ -56,6 +56,7 @@ services:
         ports:
             - 8083:8083
         environment:
+            KAFKA_HEAP_OPTS: -Xms512m -Xmx512m
             CONNECT_BOOTSTRAP_SERVERS: "kafka:9092"
             CONNECT_GROUP_ID: kafka-connect
             CONNECT_CONFIG_STORAGE_TOPIC: _connect-configs
@@ -82,8 +83,8 @@ services:
         image: elasticsearch:7.12.0
         restart: always
         environment:
-            - discovery.type=single-node
             - ES_JAVA_OPTS=-Xms512m -Xmx512m
+            - discovery.type=single-node
             - http.cors.enabled=true
             - http.cors.allow-origin=*
         ports:
@@ -105,42 +106,42 @@ services:
         networks:
             default:
                 ipv4_address: 172.172.0.31
-    doris-fe:
+    # doris-fe:
-        image: primetoninc/jdk:1.8
+    #     image: primetoninc/jdk:1.8
-        ports:
+    #     ports:
-            - 8030:8030
+    #         - 8030:8030
-            - 9010:9010
+    #         - 9010:9010
-            - 9020:9020
+    #         - 9020:9020
-            - 9030:9030
+    #         - 9030:9030
-        environment:
+    #     environment:
-            - priority_networks=172.172.0.0/24
+    #         - priority_networks=172.172.0.0/24
-        volumes:
+    #     volumes:
-            - ./apps/doris:/doris
+    #         - ./apps/doris:/doris
-            - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf
+    #         - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf
-            - ./log/doris/fe:/doris/fe/log
+    #         - ./log/doris/fe:/doris/fe/log
-            - ./data/doris/fe/doris-meta:/doris/fe/doris-meta
+    #         - ./data/doris/fe/doris-meta:/doris/fe/doris-meta
-        command: bash -c "/doris/fe/bin/start_fe.sh"
+    #     command: bash -c "/doris/fe/bin/start_fe.sh"
-        networks:
+    #     networks:
-            default:
+    #         default:
-                ipv4_address: 172.172.0.40
+    #             ipv4_address: 172.172.0.40
-    doris-be:
+    # doris-be:
-        image: primetoninc/jdk:1.8
+    #     image: primetoninc/jdk:1.8
-        ports:
+    #     ports:
-            - 8040:8040
+    #         - 8040:8040
-            - 8060:8060
+    #         - 8060:8060
-            - 9050:9050
+    #         - 9050:9050
-            - 9060:9060
+    #         - 9060:9060
-        environment:
+    #     environment:
-            - priority_networks=172.172.0.0/24
+    #         - priority_networks=172.172.0.0/24
-        volumes:
+    #     volumes:
-            - ./apps/doris:/doris
+    #         - ./apps/doris:/doris
-            - ./conf/doris/be.conf:/doris/be/conf/be.conf
+    #         - ./conf/doris/be.conf:/doris/be/conf/be.conf
-            - ./data/doris/be/storage:/doris/be/storage
+    #         - ./data/doris/be/storage:/doris/be/storage
-            - ./log/doris/be/:/doris/be/log
+    #         - ./log/doris/be/:/doris/be/log
-        command: bash -c "/doris/be/bin/start_be.sh"
+    #     command: bash -c "/doris/be/bin/start_be.sh"
-        networks:
+    #     networks:
-            default:
+    #         default:
-                ipv4_address: 172.172.0.41
+    #             ipv4_address: 172.172.0.41
 networks:
     default:
         driver: bridge