wanggang
/
iot
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

debezium kafka doris :

Browse Source 
1.timestamp => datetime ?
2.string +> varchar(?)


Former-commit-id: 2752c0dfa5fed8333b29940e3a366264f13e9e9e
Former-commit-id: 189494ebe91dd16d2540a0e34bded71c42255d8c
1.0
wanggang 4 years ago
parent 46e481ba2a
commit 2b92089c9f
8 changed files with 125 additions and 64 deletions
Show Stats Download Patch File Download Diff File

94
labs/doris/Kafka2Doris/Worker.cs
Unescape View File

@ -3,6 +3,7 @@ using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using System; using System;
using System.Linq;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.IO; using System.IO;
@ -10,6 +11,7 @@ using System.Net.Http;
using System.Net.Http.Headers; using System.Net.Http.Headers;
using System.Text; using System.Text;
using System.Text.Json; using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -33,18 +35,26 @@ namespace Kafka2Doris
while (!stoppingToken.IsCancellationRequested) while (!stoppingToken.IsCancellationRequested)
{ {
_logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now); _logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now);
//
var topics = _config.GetSection("kafka").GetValue("topics", "").Split(','); var conf = new ConsumerConfig
{
BootstrapServers = _config.GetSection("kafka").GetValue("host", "localhost:9092"),
GroupId = $"doris-v1",
AutoOffsetReset = AutoOffsetReset.Earliest,
EnableAutoCommit = false
};
var topicsRegex = _config.GetSection("kafka").GetValue("topics.regex", "mysql.example.*");
var timeout = TimeSpan.FromSeconds(_config.GetSection("kafka").GetValue("timeout", 20));
var topics = new List<string>();
using (var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = conf.BootstrapServers }).Build())
{
var meta = adminClient.GetMetadata(timeout);
topics = meta.Topics.Where(o => Regex.IsMatch(o.Topic, topicsRegex)).Select(o => o.Topic).ToList();
}
foreach (var topic in topics) foreach (var topic in topics)
{ {
var conf = new ConsumerConfig
{
BootstrapServers = _config.GetSection("kafka").GetValue("host", "localhost:9092"),
GroupId = $"doris-v1",
AutoOffsetReset = AutoOffsetReset.Earliest,
EnableAutoCommit = false
};
var timeout = TimeSpan.FromSeconds(_config.GetValue("timeout", 5));
try try
{ {
using (var consumer = new ConsumerBuilder<Ignore, string>(conf).Build()) using (var consumer = new ConsumerBuilder<Ignore, string>(conf).Build())
@ -62,6 +72,7 @@ namespace Kafka2Doris
{ {
var max = _config.GetValue("max", 1000); var max = _config.GetValue("max", 1000);
var list = new List<string>(max); var list = new List<string>(max);
JsonElement schema = default;
while (max > 0) while (max > 0)
{ {
try try
@ -71,7 +82,9 @@ namespace Kafka2Doris
{ {
break; break;
} }
Debug.WriteLine(consumeResult.Message.Value);
var json = JsonDocument.Parse(consumeResult.Message.Value); var json = JsonDocument.Parse(consumeResult.Message.Value);
schema = json.RootElement.GetProperty("schema");
var after = json.RootElement.GetProperty("payload").GetProperty("after"); var after = json.RootElement.GetProperty("payload").GetProperty("after");
if (after.ValueKind == JsonValueKind.Object) if (after.ValueKind == JsonValueKind.Object)
{ {
@ -87,6 +100,7 @@ namespace Kafka2Doris
} }
if (list.Count > 0) if (list.Count > 0)
{ {
CreateIfNotExists(topic, schema,_config);
var httpClient = this._httpClientFactory.CreateClient(); var httpClient = this._httpClientFactory.CreateClient();
var username = _config.GetSection("doris").GetValue("username", "root"); var username = _config.GetSection("doris").GetValue("username", "root");
var password = _config.GetSection("doris").GetValue("password", "aA123456!"); var password = _config.GetSection("doris").GetValue("password", "aA123456!");
@ -133,10 +147,10 @@ namespace Kafka2Doris
var responseText = result.Content.ReadAsStringAsync().Result; var responseText = result.Content.ReadAsStringAsync().Result;
Debug.WriteLine(responseText); Debug.WriteLine(responseText);
var response = JsonDocument.Parse(responseText); var response = JsonDocument.Parse(responseText);
var label = response.RootElement.GetProperty("Label").GetRawText(); var label = response.RootElement.GetProperty("Label").GetString();
var message = response.RootElement.GetProperty("Message").GetRawText(); var message = response.RootElement.GetProperty("Message").GetString();
this._logger.LogInformation($"{label}:{message}"); this._logger.LogInformation($"{label}:{message}");
var status = response.RootElement.GetProperty("Status").GetRawText(); var status = response.RootElement.GetProperty("Status").GetString();
if (status == "Success" || status == "Publish Timeout") if (status == "Success" || status == "Publish Timeout")
{ {
consumer.Commit(); consumer.Commit();
@ -186,5 +200,59 @@ namespace Kafka2Doris
await Task.Delay(this._config.GetValue("delay", 1000 * 60), stoppingToken); await Task.Delay(this._config.GetValue("delay", 1000 * 60), stoppingToken);
} }
} }
private void CreateIfNotExists(string topic, JsonElement schema,IConfiguration config)
{
var tableName = topic.Replace(".", "_");
var sql = $"CREATE TABLE IF NOT EXISTS `{tableName}` (";
sql += "\n";
var fields = schema.GetProperty("fields")[0].GetProperty("fields");
var length = fields.GetArrayLength();
var index = 0;
var buckets = _config.GetSection("doris").GetValue("buckets", 1);
var replication_num = _config.GetSection("doris").GetValue("replication_num", 1);
foreach (var item in fields.EnumerateArray())
{
var type = item.GetProperty("type").GetString();
var optional = item.GetProperty("optional").GetBoolean();
var field = item.GetProperty("field").GetString();
var name = item.TryGetProperty("name", out var nameElement)?nameElement.GetString():null;
var notNull = !optional;
if (!string.IsNullOrEmpty(name))
{
if(name== "org.apache.kafka.connect.data.Timestamp")
{
type = "DATETIME";
}
Debug.WriteLine($"{type}:{name}");
}
if(type=="string")
{
type = "VARCHAR(255)";
}
else if(type=="int16")
{
if(notNull)
{
notNull = false;
}
}
if(type.StartsWith("int"))
{
if(int.Parse(type.Substring(3)) <= 32)
{
type = "INT";
}
else
{
type = "BIGINT";
}
}
index += 1;
sql += $"`{field}` {type.ToUpper()}{(notNull ? " NOT NULL" : "")} {(index<length?",":"")}\n";
}
sql += $")\nUNIQUE KEY(Id)\nDISTRIBUTED BY HASH(Id) BUCKETS {buckets}\nPROPERTIES(\"replication_num\" = \"{replication_num}\");";
Debug.WriteLine(sql);
}
} }
} }

2
labs/doris/Kafka2Doris/appsettings.json
Unescape View File

@ -7,7 +7,7 @@
}, },
"kafka": { "kafka": {
"host": "kafka:9092", "host": "kafka:9092",
"topics": "mysql.example.User" "topics.regex": "mysql.example.*"
}, },
"doris": { "doris": {
"server": "http://doris-fe:8030", "server": "http://doris-fe:8030",

2
labs/doris/conf/kafka-connect/boot.sh
Unescape View File

@ -17,8 +17,10 @@ cd /usr/share/confluent-hub-components/
if [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors/mysql-source) -eq 404 ]; then if [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors/mysql-source) -eq 404 ]; then
curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @mysql2kafka.json curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @mysql2kafka.json
fi fi
# curl -X DELETE http://localhost:8083/connectors/mysql-source
#2:elasticsearch-sink #2:elasticsearch-sink
if [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors/elasticsearch-sink) -eq 404 ]; then if [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors/elasticsearch-sink) -eq 404 ]; then
curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @kafka2elasticsearch.json curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @kafka2elasticsearch.json
fi fi
# curl -X DELETE http://localhost:8083/connectors/elasticsearch-sink
sleep infinity sleep infinity

9
labs/doris/conf/kafka-connect/example.json
Unescape View File

@ -1,6 +1,5 @@
[ [
{"Id":"49fa11e7-4404-4fe5-9873-e89a01d8e529","UserName":"super","SecurityStamp":"123456","PasswordHash":"579f889441b4a55d667233941d72a83ed644f7e5","PasswordConfirmed":1,"Email":"super@test.com","EmailConfirmed":1,"PhoneNumber":null,"PhoneNumberConfirmed":0,"RealName":null,"IdentityNumber":null,"IdentityConfirmed":0,"NickName":"超级管理员","Avatar":null,"Sex":null,"Birthday":null,"LockoutEnabled":0,"AccessFailedCount":0,"LockoutEnd":null,"RowVersion":"1e59e461-af12-446f-b876-eac8f58d3c79","Created":1618377036611,"Modified":null,"Deleted":null}, {"Id":"49fa11e7-4404-4fe5-9873-e89a01d8e529","UserName":"super1","SecurityStamp":"123456","PasswordHash":"579f889441b4a55d667233941d72a83ed644f7e5","PasswordConfirmed":1,"Email":"super@test.com","EmailConfirmed":1,"PhoneNumber":null,"PhoneNumberConfirmed":0,"RealName":null,"IdentityNumber":null,"IdentityConfirmed":0,"NickName":"超级管理员","Avatar":null,"Sex":null,"Birthday":null,"LockoutEnabled":0,"AccessFailedCount":0,"LockoutEnd":null,"RowVersion":"1e59e461-af12-446f-b876-eac8f58d3c79","Created":"2007-11-30 10:30-19","Modified":null,"Deleted":null},
{"Id":"74fab867-a65a-4cda-8b2f-63e12d003b79","UserName":"user","SecurityStamp":"123456","PasswordHash":"71dd07494c5ee54992a27746d547e25dee01bd97","PasswordConfirmed":1,"Email":"user@test.com","EmailConfirmed":1,"PhoneNumber":null,"PhoneNumberConfirmed":0,"RealName":null,"IdentityNumber":null,"IdentityConfirmed":0,"NickName":"普通用户","Avatar":null,"Sex":null,"Birthday":null,"LockoutEnabled":0,"AccessFailedCount":0,"LockoutEnd":null,"RowVersion":"2b691577-e96c-40fe-897e-3adc9a4488bf","Created":1618377036611,"Modified":null,"Deleted":null}, {"Id":"74fab867-a65a-4cda-8b2f-63e12d003b79","UserName":"user","SecurityStamp":"123456","PasswordHash":"71dd07494c5ee54992a27746d547e25dee01bd97","PasswordConfirmed":1,"Email":"user@test.com","EmailConfirmed":1,"PhoneNumber":null,"PhoneNumberConfirmed":0,"RealName":null,"IdentityNumber":null,"IdentityConfirmed":0,"NickName":"普通用户","Avatar":null,"Sex":null,"Birthday":null,"LockoutEnabled":0,"AccessFailedCount":0,"LockoutEnd":null,"RowVersion":"2b691577-e96c-40fe-897e-3adc9a4488bf","Created":"2007-11-30 10:30-19","Modified":null,"Deleted":null},
{"Id":"e6a2e59c-a4e1-4ced-856c-036b6e4786d1","UserName":"admin","SecurityStamp":"123456","PasswordHash":"579f889441b4a55d667233941d72a83ed644f7e5","PasswordConfirmed":1,"Email":"admin@test.com","EmailConfirmed":1,"PhoneNumber":null,"PhoneNumberConfirmed":0,"RealName":null,"IdentityNumber":null,"IdentityConfirmed":0,"NickName":"管理员","Avatar":null,"Sex":null,"Birthday":null,"LockoutEnabled":0,"AccessFailedCount":0,"LockoutEnd":null,"RowVersion":"7d53094e-569d-4689-a6d1-d2c1f99da85b","Created":1618377036611,"Modified":null,"Deleted":null} {"Id":"e6a2e59c-a4e1-4ced-856c-036b6e4786d1","UserName":"admin","SecurityStamp":"123456","PasswordHash":"579f889441b4a55d667233941d72a83ed644f7e5","PasswordConfirmed":1,"Email":"admin@test.com","EmailConfirmed":1,"PhoneNumber":null,"PhoneNumberConfirmed":0,"RealName":null,"IdentityNumber":null,"IdentityConfirmed":0,"NickName":"管理员","Avatar":null,"Sex":null,"Birthday":null,"LockoutEnabled":0,"AccessFailedCount":0,"LockoutEnd":null,"RowVersion":"7d53094e-569d-4689-a6d1-d2c1f99da85b","Created":"2007-11-30 10:30-19","Modified":null,"Deleted":null}
] ]

2
labs/doris/conf/kafka-connect/mysql2kafka.json
Unescape View File

@ -13,6 +13,6 @@
"database.history.kafka.bootstrap.servers": "kafka:9092", "database.history.kafka.bootstrap.servers": "kafka:9092",
"database.history.kafka.topic": "mysql.schema.example", "database.history.kafka.topic": "mysql.schema.example",
"include.schema.changes": "true", "include.schema.changes": "true",
"time.precision.mode":"connect" "time.precision.mode": "connect"
} }
} }

8
labs/doris/conf/kafka-connect/start.sh
Unescape View File

@ -1,8 +0,0 @@
#!/bin/bash
echo 'init connectors start:\n'
echo 'mysql2kafka\n'
curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @mysql2kafka.json
echo 'kafka2elasticsearch\n'
curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @kafka2elasticsearch.json
echo 'init connectors end!\n
# curl -X DELETE http://localhost:8083/connectors/elasticsearch-sink'

70
labs/doris/docker-compose.yml
Unescape View File

@ -68,7 +68,7 @@ services:
networks: networks:
default: default:
ipv4_address: 172.172.0.22 ipv4_address: 172.172.0.22
depends_on: depends_on:
- kafka - kafka
kafka-connect: kafka-connect:
image: confluentinc/cp-kafka-connect:6.1.1 image: confluentinc/cp-kafka-connect:6.1.1
@ -144,40 +144,40 @@ services:
networks: networks:
default: default:
ipv4_address: 172.172.0.31 ipv4_address: 172.172.0.31
# doris-fe: doris-fe:
# image: 76527413/doris:0.14.7 image: 76527413/doris:0.14.7
# ports: ports:
# - 8030:8030 - 8030:8030
# - 9010:9010 - 9010:9010
# - 9020:9020 - 9020:9020
# - 9030:9030 - 9030:9030
# environment: environment:
# - priority_networks=172.172.0.0/24 - priority_networks=172.172.0.0/24
# volumes: volumes:
# - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf
# - ./log/doris/fe:/doris/fe/log - ./log/doris/fe:/doris/fe/log
# - ./data/doris/fe/doris-meta:/doris/fe/doris-meta - ./data/doris/fe/doris-meta:/doris/fe/doris-meta
# command: bash -c "/doris/fe/bin/start_fe.sh" command: bash -c "/doris/fe/bin/start_fe.sh"
# networks: networks:
# default: default:
# ipv4_address: 172.172.0.40 ipv4_address: 172.172.0.40
# doris-be: doris-be:
# image: 76527413/doris:0.14.7 image: 76527413/doris:0.14.7
# ports: ports:
# - 8040:8040 - 8040:8040
# - 8060:8060 - 8060:8060
# - 9050:9050 - 9050:9050
# - 9060:9060 - 9060:9060
# environment: environment:
# - priority_networks=172.172.0.0/24 - priority_networks=172.172.0.0/24
# volumes: volumes:
# - ./conf/doris/be.conf:/doris/be/conf/be.conf - ./conf/doris/be.conf:/doris/be/conf/be.conf
# - ./data/doris/be/storage:/doris/be/storage - ./data/doris/be/storage:/doris/be/storage
# - ./log/doris/be/:/doris/be/log - ./log/doris/be/:/doris/be/log
# command: bash -c "/doris/be/bin/start_be.sh" command: bash -c "/doris/be/bin/start_be.sh"
# networks: networks:
# default: default:
# ipv4_address: 172.172.0.41 ipv4_address: 172.172.0.41
networks: networks:
default: default:
driver: bridge driver: bridge

2
labs/doris/readme.md
Unescape View File

@ -91,7 +91,7 @@ USE example;
### 创建表： ### 创建表：
//curl -i -v --location-trusted -u root:aA123456! -H "format: json" -H "strip_outer_array: true" -T example.json http://doris-fe:8030/api/example/User/_stream_load //curl -i -v --location-trusted -u root:aA123456! -H "format: json" -H "strip_outer_array: true" -T example.json http://doris-fe:8030/api/example/mysql_example.User/_stream_load
CREATE TABLE `User` ( CREATE TABLE `User` (
`Id` char(36) NOT NULL COMMENT 'Id', `Id` char(36) NOT NULL COMMENT 'Id',
`UserName` varchar(255) NOT NULL COMMENT 'UserName', `UserName` varchar(255) NOT NULL COMMENT 'UserName',

Write Preview
Loading…
Cancel
Save