From 46e481ba2a8bf12223ce7c0aef2dc8df829cfef8 Mon Sep 17 00:00:00 2001 From: wanggang <76527413@qq.com> Date: Wed, 12 May 2021 11:34:31 +0800 Subject: [PATCH] =?UTF-8?q?kafka=E5=86=85=E5=A4=96=E7=BD=91=E8=AE=BF?= =?UTF-8?q?=E9=97=AE=E5=92=8C=E4=BB=A3=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Former-commit-id: 4f4cb1da5bfebb5dd5deca89b33b62f0450b6874 Former-commit-id: d1be473d269d331a9e0c0b8bbebf8e36034e6fb3 --- labs/doris/Kafka2Doris/Program.cs | 14 +--- .../Properties/launchSettings.json | 5 +- .../Kafka2Doris/appsettings.Development.json | 2 +- labs/doris/Kafka2Doris/appsettings.json | 2 +- labs/doris/conf/kafka-connect/boot.sh | 10 ++- labs/doris/conf/nginx/nginx.conf | 16 +++- labs/doris/docker-compose.yml | 73 ++++++++++--------- 7 files changed, 70 insertions(+), 52 deletions(-) diff --git a/labs/doris/Kafka2Doris/Program.cs b/labs/doris/Kafka2Doris/Program.cs index e84e4eb5..61495041 100644 --- a/labs/doris/Kafka2Doris/Program.cs +++ b/labs/doris/Kafka2Doris/Program.cs @@ -1,10 +1,8 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using System; using System.IO; -using System.Reflection; using System.Runtime.InteropServices; namespace Kafka2Doris @@ -16,15 +14,9 @@ namespace Kafka2Doris var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); var rid = isWindows ? "win-x64" : "linux-x64"; var file = isWindows ? "librdkafka.dll" : "librdkafka.so"; - var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "runtimes",rid, "native", file); + var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "runtimes", rid, "native", file); Confluent.Kafka.Library.Load(path); - var config = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile("appsettings.json") - .AddEnvironmentVariables() - .AddCommandLine(args) - .Build(); Host.CreateDefaultBuilder(args) .ConfigureLogging(o => { @@ -33,7 +25,7 @@ namespace Kafka2Doris .ConfigureServices((hostingContext, services) => { services.AddLogging(); - services.AddSingleton(config); + //services.AddSingleton(config); services.AddHttpClient(); services.AddHostedService(); }) diff --git a/labs/doris/Kafka2Doris/Properties/launchSettings.json b/labs/doris/Kafka2Doris/Properties/launchSettings.json index f32249bd..846f5d5e 100644 --- a/labs/doris/Kafka2Doris/Properties/launchSettings.json +++ b/labs/doris/Kafka2Doris/Properties/launchSettings.json @@ -1,7 +1,10 @@ { "profiles": { "Kafka2Doris": { - "commandName": "Project" + "commandName": "Project", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } } } } \ No newline at end of file diff --git a/labs/doris/Kafka2Doris/appsettings.Development.json b/labs/doris/Kafka2Doris/appsettings.Development.json index fe897830..0d0ea22d 100644 --- a/labs/doris/Kafka2Doris/appsettings.Development.json +++ b/labs/doris/Kafka2Doris/appsettings.Development.json @@ -6,7 +6,7 @@ } }, "kafka": { - "host": "kafka:9092", + "host": "localhost:9092", "topics": "mysql.example.User" }, "doris": { diff --git a/labs/doris/Kafka2Doris/appsettings.json b/labs/doris/Kafka2Doris/appsettings.json index 2cdd854f..b3278d08 100644 --- a/labs/doris/Kafka2Doris/appsettings.json +++ b/labs/doris/Kafka2Doris/appsettings.json @@ -6,7 +6,7 @@ } }, "kafka": { - "host": "localhost:9992", + "host": "kafka:9092", "topics": "mysql.example.User" }, "doris": { diff --git a/labs/doris/conf/kafka-connect/boot.sh b/labs/doris/conf/kafka-connect/boot.sh index e15c7532..4edf5ca0 100644 --- a/labs/doris/conf/kafka-connect/boot.sh +++ b/labs/doris/conf/kafka-connect/boot.sh @@ -11,6 +11,14 @@ while :; do fi sleep 5 done + cd /usr/share/confluent-hub-components/ -./start.sh +#1:mysql-source +if [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors/mysql-source) -eq 404 ]; then + curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @mysql2kafka.json +fi +#2:elasticsearch-sink +if [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors/elasticsearch-sink) -eq 404 ]; then + curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @kafka2elasticsearch.json +fi sleep infinity diff --git a/labs/doris/conf/nginx/nginx.conf b/labs/doris/conf/nginx/nginx.conf index 0d928997..68bf231b 100644 --- a/labs/doris/conf/nginx/nginx.conf +++ b/labs/doris/conf/nginx/nginx.conf @@ -6,9 +6,21 @@ events { } stream { + log_format proxy '$remote_addr [$time_local] ' + '$protocol $status $bytes_sent $bytes_received ' + '$session_time "$upstream_addr" ' + '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"'; + + access_log /var/log/nginx/tcp-access.log proxy; + open_log_file_cache off; + + upstream kafka_brokers { + server 172.172.0.21:9092; + } + server { listen 9092; - proxy_pass 172.172.0.21:9092; + proxy_pass kafka_brokers; } } @@ -34,5 +46,5 @@ http { root html; } } - include conf.d/*.conf; + #include conf.d/*.conf; } \ No newline at end of file diff --git a/labs/doris/docker-compose.yml b/labs/doris/docker-compose.yml index 6b58a630..f58d9909 100644 --- a/labs/doris/docker-compose.yml +++ b/labs/doris/docker-compose.yml @@ -41,7 +41,8 @@ services: environment: KAFKA_HEAP_OPTS: "-Xmx512m -Xms512m" KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_HOST_NAME: 172.172.0.21 + KAFKA_LISTENERS: PLAINTEXT://172.172.0.21:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${ip}:9092 num.partitions: 1 volumes: - ./data/kafka:/kafka @@ -67,6 +68,8 @@ services: networks: default: ipv4_address: 172.172.0.22 + depends_on: + - kafka kafka-connect: image: confluentinc/cp-kafka-connect:6.1.1 restart: always @@ -141,40 +144,40 @@ services: networks: default: ipv4_address: 172.172.0.31 - doris-fe: - image: 76527413/doris:0.14.7 - ports: - - 8030:8030 - - 9010:9010 - - 9020:9020 - - 9030:9030 - environment: - - priority_networks=172.172.0.0/24 - volumes: - - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf - - ./log/doris/fe:/doris/fe/log - - ./data/doris/fe/doris-meta:/doris/fe/doris-meta - command: bash -c "/doris/fe/bin/start_fe.sh" - networks: - default: - ipv4_address: 172.172.0.40 - doris-be: - image: 76527413/doris:0.14.7 - ports: - - 8040:8040 - - 8060:8060 - - 9050:9050 - - 9060:9060 - environment: - - priority_networks=172.172.0.0/24 - volumes: - - ./conf/doris/be.conf:/doris/be/conf/be.conf - - ./data/doris/be/storage:/doris/be/storage - - ./log/doris/be/:/doris/be/log - command: bash -c "/doris/be/bin/start_be.sh" - networks: - default: - ipv4_address: 172.172.0.41 + # doris-fe: + # image: 76527413/doris:0.14.7 + # ports: + # - 8030:8030 + # - 9010:9010 + # - 9020:9020 + # - 9030:9030 + # environment: + # - priority_networks=172.172.0.0/24 + # volumes: + # - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf + # - ./log/doris/fe:/doris/fe/log + # - ./data/doris/fe/doris-meta:/doris/fe/doris-meta + # command: bash -c "/doris/fe/bin/start_fe.sh" + # networks: + # default: + # ipv4_address: 172.172.0.40 + # doris-be: + # image: 76527413/doris:0.14.7 + # ports: + # - 8040:8040 + # - 8060:8060 + # - 9050:9050 + # - 9060:9060 + # environment: + # - priority_networks=172.172.0.0/24 + # volumes: + # - ./conf/doris/be.conf:/doris/be/conf/be.conf + # - ./data/doris/be/storage:/doris/be/storage + # - ./log/doris/be/:/doris/be/log + # command: bash -c "/doris/be/bin/start_be.sh" + # networks: + # default: + # ipv4_address: 172.172.0.41 networks: default: driver: bridge