diff --git a/labs/doris/conf/kafka-connect/boot.sh b/labs/doris/conf/kafka-connect/boot.sh index 119860ed..a5005955 100644 --- a/labs/doris/conf/kafka-connect/boot.sh +++ b/labs/doris/conf/kafka-connect/boot.sh @@ -13,4 +13,4 @@ while : ; do done cd /usr/share/confluent-hub-components/ ./start.sh -sleep infinity \ No newline at end of file +sleep infinity diff --git a/labs/doris/conf/kafka-connect/kafka2elasticsearch.json b/labs/doris/conf/kafka-connect/kafka2elasticsearch.json index 744757d6..3ccbe784 100644 --- a/labs/doris/conf/kafka-connect/kafka2elasticsearch.json +++ b/labs/doris/conf/kafka-connect/kafka2elasticsearch.json @@ -4,15 +4,17 @@ "connector.class": "io.confluent.connect.elasticsearch.ElasticsearchSinkConnector", "tasks.max": 1, "topics": "mysql.example.User", - "key.ignore": true, + "key.ignore": false, "key.converter.schemas.enable": false, "key.converter": "org.apache.kafka.connect.storage.StringConverter", "connection.url": "http://elasticsearch:9200", "type.name": "kafka-connect", "write.method": "upsert", "behavior.on.null.values": "delete", - "transforms": "ExtractFieldObject", + "transforms": "ValueToKey,ExtractFieldObject", "transforms.ExtractFieldObject.type": "org.apache.kafka.connect.transforms.ExtractField$Value", - "transforms.ExtractFieldObject.field": "after" + "transforms.ExtractFieldObject.field": "after", + "transforms.ValueToKey.type":"org.apache.kafka.connect.transforms.ValueToKey", + "transforms.ValueToKey.fields":"Id" } } \ No newline at end of file diff --git a/labs/doris/docker-compose.yml b/labs/doris/docker-compose.yml index eaef2bc1..967f0839 100644 --- a/labs/doris/docker-compose.yml +++ b/labs/doris/docker-compose.yml @@ -29,8 +29,8 @@ services: image: wurstmeister/kafka:2.13-2.7.0 restart: always environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_HEAP_OPTS: "-Xmx512m -Xms512m" + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_ADVERTISED_HOST_NAME: 172.172.0.21 num.partitions: 1 volumes: @@ -56,6 +56,7 @@ services: ports: - 8083:8083 environment: + KAFKA_HEAP_OPTS: -Xms512m -Xmx512m CONNECT_BOOTSTRAP_SERVERS: "kafka:9092" CONNECT_GROUP_ID: kafka-connect CONNECT_CONFIG_STORAGE_TOPIC: _connect-configs @@ -82,8 +83,8 @@ services: image: elasticsearch:7.12.0 restart: always environment: - - discovery.type=single-node - ES_JAVA_OPTS=-Xms512m -Xmx512m + - discovery.type=single-node - http.cors.enabled=true - http.cors.allow-origin=* ports: @@ -105,42 +106,42 @@ services: networks: default: ipv4_address: 172.172.0.31 - doris-fe: - image: primetoninc/jdk:1.8 - ports: - - 8030:8030 - - 9010:9010 - - 9020:9020 - - 9030:9030 - environment: - - priority_networks=172.172.0.0/24 - volumes: - - ./apps/doris:/doris - - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf - - ./log/doris/fe:/doris/fe/log - - ./data/doris/fe/doris-meta:/doris/fe/doris-meta - command: bash -c "/doris/fe/bin/start_fe.sh" - networks: - default: - ipv4_address: 172.172.0.40 - doris-be: - image: primetoninc/jdk:1.8 - ports: - - 8040:8040 - - 8060:8060 - - 9050:9050 - - 9060:9060 - environment: - - priority_networks=172.172.0.0/24 - volumes: - - ./apps/doris:/doris - - ./conf/doris/be.conf:/doris/be/conf/be.conf - - ./data/doris/be/storage:/doris/be/storage - - ./log/doris/be/:/doris/be/log - command: bash -c "/doris/be/bin/start_be.sh" - networks: - default: - ipv4_address: 172.172.0.41 + # doris-fe: + # image: primetoninc/jdk:1.8 + # ports: + # - 8030:8030 + # - 9010:9010 + # - 9020:9020 + # - 9030:9030 + # environment: + # - priority_networks=172.172.0.0/24 + # volumes: + # - ./apps/doris:/doris + # - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf + # - ./log/doris/fe:/doris/fe/log + # - ./data/doris/fe/doris-meta:/doris/fe/doris-meta + # command: bash -c "/doris/fe/bin/start_fe.sh" + # networks: + # default: + # ipv4_address: 172.172.0.40 + # doris-be: + # image: primetoninc/jdk:1.8 + # ports: + # - 8040:8040 + # - 8060:8060 + # - 9050:9050 + # - 9060:9060 + # environment: + # - priority_networks=172.172.0.0/24 + # volumes: + # - ./apps/doris:/doris + # - ./conf/doris/be.conf:/doris/be/conf/be.conf + # - ./data/doris/be/storage:/doris/be/storage + # - ./log/doris/be/:/doris/be/log + # command: bash -c "/doris/be/bin/start_be.sh" + # networks: + # default: + # ipv4_address: 172.172.0.41 networks: default: driver: bridge