kafka内外网访问和代理

Former-commit-id: 4f4cb1da5bfebb5dd5deca89b33b62f0450b6874
Former-commit-id: d1be473d269d331a9e0c0b8bbebf8e36034e6fb3

labs/doris/Kafka2Doris/Program.cs

@@ -1,10 +1,8 @@
-using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using System;
 using System.IO;
-using System.Reflection;
 using System.Runtime.InteropServices;
 
 namespace Kafka2Doris
@@ -16,15 +14,9 @@ namespace Kafka2Doris
             var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
             var rid = isWindows ? "win-x64" : "linux-x64";
             var file = isWindows ? "librdkafka.dll" : "librdkafka.so";
-            var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "runtimes",rid, "native", file);
+            var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "runtimes", rid, "native", file);
             Confluent.Kafka.Library.Load(path);
 
-            var config = new ConfigurationBuilder()
-                .SetBasePath(Directory.GetCurrentDirectory())
-                .AddJsonFile("appsettings.json")
-                .AddEnvironmentVariables()
-                .AddCommandLine(args)
-                .Build();
             Host.CreateDefaultBuilder(args)
                 .ConfigureLogging(o =>
                 {
@@ -33,7 +25,7 @@ namespace Kafka2Doris
                 .ConfigureServices((hostingContext, services) =>
                 {
                     services.AddLogging();
-                    services.AddSingleton(config);
+                    //services.AddSingleton(config);
                     services.AddHttpClient();
                     services.AddHostedService<Worker>();
                 })

labs/doris/Kafka2Doris/Properties/launchSettings.json

@@ -1,7 +1,10 @@
 {
   "profiles": {
     "Kafka2Doris": {
-      "commandName": "Project"
+      "commandName": "Project",
+      "environmentVariables": {
+        "DOTNET_ENVIRONMENT": "Development"
+      }
     }
   }
 }

labs/doris/Kafka2Doris/appsettings.Development.json

@@ -6,7 +6,7 @@
     }
   },
   "kafka": {
-    "host": "kafka:9092",
+    "host": "localhost:9092",
     "topics": "mysql.example.User"
   },
   "doris": {

labs/doris/Kafka2Doris/appsettings.json

@@ -6,7 +6,7 @@
     }
   },
   "kafka": {
-    "host": "localhost:9992",
+    "host": "kafka:9092",
     "topics": "mysql.example.User"
   },
   "doris": {

labs/doris/conf/kafka-connect/boot.sh

@@ -11,6 +11,14 @@ while :; do
     fi
     sleep 5
 done
+
 cd /usr/share/confluent-hub-components/
-./start.sh
+#1:mysql-source
+if [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors/mysql-source) -eq 404 ]; then
+    curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @mysql2kafka.json
+fi
+#2:elasticsearch-sink
+if [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors/elasticsearch-sink) -eq 404 ]; then
+    curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @kafka2elasticsearch.json
+fi
 sleep infinity

labs/doris/conf/nginx/nginx.conf

@@ -6,9 +6,21 @@ events {
 }
 
 stream {
+    log_format proxy '$remote_addr [$time_local] '
+    '$protocol $status $bytes_sent $bytes_received '
+    '$session_time "$upstream_addr" '
+    '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
+
+    access_log /var/log/nginx/tcp-access.log proxy;
+    open_log_file_cache off;
+
+    upstream kafka_brokers {
+        server 172.172.0.21:9092;
+    }
+
     server {
         listen 9092;
-        proxy_pass 172.172.0.21:9092;
+        proxy_pass kafka_brokers;
     }
 }
 
@@ -34,5 +46,5 @@ http {
             root html;
         }
     }
-    include conf.d/*.conf;
+    #include conf.d/*.conf;
 }

labs/doris/docker-compose.yml

@@ -41,7 +41,8 @@ services:
         environment:
             KAFKA_HEAP_OPTS: "-Xmx512m -Xms512m"
             KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-            KAFKA_ADVERTISED_HOST_NAME: 172.172.0.21
+            KAFKA_LISTENERS: PLAINTEXT://172.172.0.21:9092
+            KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${ip}:9092
             num.partitions: 1
         volumes:
             - ./data/kafka:/kafka
@@ -67,6 +68,8 @@ services:
         networks:
             default:
                 ipv4_address: 172.172.0.22
+        depends_on: 
+            - kafka
     kafka-connect:
         image: confluentinc/cp-kafka-connect:6.1.1
         restart: always
@@ -141,40 +144,40 @@ services:
         networks:
             default:
                 ipv4_address: 172.172.0.31
-    doris-fe:
+    # doris-fe:
-        image: 76527413/doris:0.14.7
+    #     image: 76527413/doris:0.14.7
-        ports:
+    #     ports:
-            - 8030:8030
+    #         - 8030:8030
-            - 9010:9010
+    #         - 9010:9010
-            - 9020:9020
+    #         - 9020:9020
-            - 9030:9030
+    #         - 9030:9030
-        environment:
+    #     environment:
-            - priority_networks=172.172.0.0/24
+    #         - priority_networks=172.172.0.0/24
-        volumes:
+    #     volumes:
-            - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf
+    #         - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf
-            - ./log/doris/fe:/doris/fe/log
+    #         - ./log/doris/fe:/doris/fe/log
-            - ./data/doris/fe/doris-meta:/doris/fe/doris-meta
+    #         - ./data/doris/fe/doris-meta:/doris/fe/doris-meta
-        command: bash -c "/doris/fe/bin/start_fe.sh"
+    #     command: bash -c "/doris/fe/bin/start_fe.sh"
-        networks:
+    #     networks:
-            default:
+    #         default:
-                ipv4_address: 172.172.0.40
+    #             ipv4_address: 172.172.0.40
-    doris-be:
+    # doris-be:
-        image: 76527413/doris:0.14.7
+    #     image: 76527413/doris:0.14.7
-        ports:
+    #     ports:
-            - 8040:8040
+    #         - 8040:8040
-            - 8060:8060
+    #         - 8060:8060
-            - 9050:9050
+    #         - 9050:9050
-            - 9060:9060
+    #         - 9060:9060
-        environment:
+    #     environment:
-            - priority_networks=172.172.0.0/24
+    #         - priority_networks=172.172.0.0/24
-        volumes:
+    #     volumes:
-            - ./conf/doris/be.conf:/doris/be/conf/be.conf
+    #         - ./conf/doris/be.conf:/doris/be/conf/be.conf
-            - ./data/doris/be/storage:/doris/be/storage
+    #         - ./data/doris/be/storage:/doris/be/storage
-            - ./log/doris/be/:/doris/be/log
+    #         - ./log/doris/be/:/doris/be/log
-        command: bash -c "/doris/be/bin/start_be.sh"
+    #     command: bash -c "/doris/be/bin/start_be.sh"
-        networks:
+    #     networks:
-            default:
+    #         default:
-                ipv4_address: 172.172.0.41
+    #             ipv4_address: 172.172.0.41
 networks:
     default:
         driver: bridge