kafka内外网访问和代理

Former-commit-id: 4f4cb1da5bfebb5dd5deca89b33b62f0450b6874
Former-commit-id: d1be473d269d331a9e0c0b8bbebf8e36034e6fb3
1.0
wanggang 4 years ago
parent 2c27eca18a
commit 46e481ba2a

@ -1,10 +1,8 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.IO;
using System.Reflection;
using System.Runtime.InteropServices;
namespace Kafka2Doris
@ -16,15 +14,9 @@ namespace Kafka2Doris
var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
var rid = isWindows ? "win-x64" : "linux-x64";
var file = isWindows ? "librdkafka.dll" : "librdkafka.so";
var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "runtimes",rid, "native", file);
var path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "runtimes", rid, "native", file);
Confluent.Kafka.Library.Load(path);
var config = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json")
.AddEnvironmentVariables()
.AddCommandLine(args)
.Build();
Host.CreateDefaultBuilder(args)
.ConfigureLogging(o =>
{
@ -33,7 +25,7 @@ namespace Kafka2Doris
.ConfigureServices((hostingContext, services) =>
{
services.AddLogging();
services.AddSingleton(config);
//services.AddSingleton(config);
services.AddHttpClient();
services.AddHostedService<Worker>();
})

@ -1,7 +1,10 @@
{
"profiles": {
"Kafka2Doris": {
"commandName": "Project"
"commandName": "Project",
"environmentVariables": {
"DOTNET_ENVIRONMENT": "Development"
}
}
}
}

@ -6,7 +6,7 @@
}
},
"kafka": {
"host": "kafka:9092",
"host": "localhost:9092",
"topics": "mysql.example.User"
},
"doris": {

@ -6,7 +6,7 @@
}
},
"kafka": {
"host": "localhost:9992",
"host": "kafka:9092",
"topics": "mysql.example.User"
},
"doris": {

@ -11,6 +11,14 @@ while :; do
fi
sleep 5
done
cd /usr/share/confluent-hub-components/
./start.sh
#1:mysql-source
if [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors/mysql-source) -eq 404 ]; then
curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @mysql2kafka.json
fi
#2:elasticsearch-sink
if [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors/elasticsearch-sink) -eq 404 ]; then
curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @kafka2elasticsearch.json
fi
sleep infinity

@ -6,9 +6,21 @@ events {
}
stream {
log_format proxy '$remote_addr [$time_local] '
'$protocol $status $bytes_sent $bytes_received '
'$session_time "$upstream_addr" '
'"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
access_log /var/log/nginx/tcp-access.log proxy;
open_log_file_cache off;
upstream kafka_brokers {
server 172.172.0.21:9092;
}
server {
listen 9092;
proxy_pass 172.172.0.21:9092;
proxy_pass kafka_brokers;
}
}
@ -34,5 +46,5 @@ http {
root html;
}
}
include conf.d/*.conf;
#include conf.d/*.conf;
}

@ -41,7 +41,8 @@ services:
environment:
KAFKA_HEAP_OPTS: "-Xmx512m -Xms512m"
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_HOST_NAME: 172.172.0.21
KAFKA_LISTENERS: PLAINTEXT://172.172.0.21:9092
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${ip}:9092
num.partitions: 1
volumes:
- ./data/kafka:/kafka
@ -67,6 +68,8 @@ services:
networks:
default:
ipv4_address: 172.172.0.22
depends_on:
- kafka
kafka-connect:
image: confluentinc/cp-kafka-connect:6.1.1
restart: always
@ -141,40 +144,40 @@ services:
networks:
default:
ipv4_address: 172.172.0.31
doris-fe:
image: 76527413/doris:0.14.7
ports:
- 8030:8030
- 9010:9010
- 9020:9020
- 9030:9030
environment:
- priority_networks=172.172.0.0/24
volumes:
- ./conf/doris/fe.conf:/doris/fe/conf/fe.conf
- ./log/doris/fe:/doris/fe/log
- ./data/doris/fe/doris-meta:/doris/fe/doris-meta
command: bash -c "/doris/fe/bin/start_fe.sh"
networks:
default:
ipv4_address: 172.172.0.40
doris-be:
image: 76527413/doris:0.14.7
ports:
- 8040:8040
- 8060:8060
- 9050:9050
- 9060:9060
environment:
- priority_networks=172.172.0.0/24
volumes:
- ./conf/doris/be.conf:/doris/be/conf/be.conf
- ./data/doris/be/storage:/doris/be/storage
- ./log/doris/be/:/doris/be/log
command: bash -c "/doris/be/bin/start_be.sh"
networks:
default:
ipv4_address: 172.172.0.41
# doris-fe:
# image: 76527413/doris:0.14.7
# ports:
# - 8030:8030
# - 9010:9010
# - 9020:9020
# - 9030:9030
# environment:
# - priority_networks=172.172.0.0/24
# volumes:
# - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf
# - ./log/doris/fe:/doris/fe/log
# - ./data/doris/fe/doris-meta:/doris/fe/doris-meta
# command: bash -c "/doris/fe/bin/start_fe.sh"
# networks:
# default:
# ipv4_address: 172.172.0.40
# doris-be:
# image: 76527413/doris:0.14.7
# ports:
# - 8040:8040
# - 8060:8060
# - 9050:9050
# - 9060:9060
# environment:
# - priority_networks=172.172.0.0/24
# volumes:
# - ./conf/doris/be.conf:/doris/be/conf/be.conf
# - ./data/doris/be/storage:/doris/be/storage
# - ./log/doris/be/:/doris/be/log
# command: bash -c "/doris/be/bin/start_be.sh"
# networks:
# default:
# ipv4_address: 172.172.0.41
networks:
default:
driver: bridge

Loading…
Cancel
Save