wanggang
/
iot
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

debezium kafka doris :

Browse Source 
1.timestamp => datetime ?
2.string +> varchar(?)


Former-commit-id: 2752c0dfa5fed8333b29940e3a366264f13e9e9e
Former-commit-id: 189494ebe91dd16d2540a0e34bded71c42255d8c
1.0
wanggang 4 years ago
parent 46e481ba2a
commit 2b92089c9f
8 changed files with 125 additions and 64 deletions
Show Stats Download Patch File Download Diff File

94
labs/doris/Kafka2Doris/Worker.cs
Unescape View File

@ -3,6 +3,7 @@ using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.Linq;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
@ -10,6 +11,7 @@ using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
@ -33,18 +35,26 @@ namespace Kafka2Doris
while (!stoppingToken.IsCancellationRequested)
{
_logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now);
//
var topics = _config.GetSection("kafka").GetValue("topics", "").Split(',');
var conf = new ConsumerConfig
{
BootstrapServers = _config.GetSection("kafka").GetValue("host", "localhost:9092"),
GroupId = $"doris-v1",
AutoOffsetReset = AutoOffsetReset.Earliest,
EnableAutoCommit = false
};
var topicsRegex = _config.GetSection("kafka").GetValue("topics.regex", "mysql.example.*");
var timeout = TimeSpan.FromSeconds(_config.GetSection("kafka").GetValue("timeout", 20));
var topics = new List<string>();
using (var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = conf.BootstrapServers }).Build())
{
var meta = adminClient.GetMetadata(timeout);
topics = meta.Topics.Where(o => Regex.IsMatch(o.Topic, topicsRegex)).Select(o => o.Topic).ToList();
}
foreach (var topic in topics)
{
var conf = new ConsumerConfig
{
BootstrapServers = _config.GetSection("kafka").GetValue("host", "localhost:9092"),
GroupId = $"doris-v1",
AutoOffsetReset = AutoOffsetReset.Earliest,
EnableAutoCommit = false
};
var timeout = TimeSpan.FromSeconds(_config.GetValue("timeout", 5));
try
{
using (var consumer = new ConsumerBuilder<Ignore, string>(conf).Build())
@ -62,6 +72,7 @@ namespace Kafka2Doris
{
var max = _config.GetValue("max", 1000);
var list = new List<string>(max);
JsonElement schema = default;
while (max > 0)
{
try
@ -71,7 +82,9 @@ namespace Kafka2Doris
{
break;
}
Debug.WriteLine(consumeResult.Message.Value);
var json = JsonDocument.Parse(consumeResult.Message.Value);
schema = json.RootElement.GetProperty("schema");
var after = json.RootElement.GetProperty("payload").GetProperty("after");
if (after.ValueKind == JsonValueKind.Object)
{
@ -87,6 +100,7 @@ namespace Kafka2Doris
}
if (list.Count > 0)
{
CreateIfNotExists(topic, schema,_config);
var httpClient = this._httpClientFactory.CreateClient();
var username = _config.GetSection("doris").GetValue("username", "root");
var password = _config.GetSection("doris").GetValue("password", "aA123456!");
@ -133,10 +147,10 @@ namespace Kafka2Doris
var responseText = result.Content.ReadAsStringAsync().Result;
Debug.WriteLine(responseText);
var response = JsonDocument.Parse(responseText);
var label = response.RootElement.GetProperty("Label").GetRawText();
var message = response.RootElement.GetProperty("Message").GetRawText();
var label = response.RootElement.GetProperty("Label").GetString();
var message = response.RootElement.GetProperty("Message").GetString();
this._logger.LogInformation($"{label}:{message}");
var status = response.RootElement.GetProperty("Status").GetRawText();
var status = response.RootElement.GetProperty("Status").GetString();
if (status == "Success" || status == "Publish Timeout")
{
consumer.Commit();
@ -186,5 +200,59 @@ namespace Kafka2Doris
await Task.Delay(this._config.GetValue("delay", 1000 * 60), stoppingToken);
}
}
private void CreateIfNotExists(string topic, JsonElement schema,IConfiguration config)
{
var tableName = topic.Replace(".", "_");
var sql = $"CREATE TABLE IF NOT EXISTS `{tableName}` (";
sql += "\n";
var fields = schema.GetProperty("fields")[0].GetProperty("fields");
var length = fields.GetArrayLength();
var index = 0;
var buckets = _config.GetSection("doris").GetValue("buckets", 1);
var replication_num = _config.GetSection("doris").GetValue("replication_num", 1);
foreach (var item in fields.EnumerateArray())
{
var type = item.GetProperty("type").GetString();
var optional = item.GetProperty("optional").GetBoolean();
var field = item.GetProperty("field").GetString();
var name = item.TryGetProperty("name", out var nameElement)?nameElement.GetString():null;
var notNull = !optional;
if (!string.IsNullOrEmpty(name))
{
if(name== "org.apache.kafka.connect.data.Timestamp")
{
type = "DATETIME";
}
Debug.WriteLine($"{type}:{name}");
}
if(type=="string")
{
type = "VARCHAR(255)";
}
else if(type=="int16")
{
if(notNull)
{
notNull = false;
}
}
if(type.StartsWith("int"))
{
if(int.Parse(type.Substring(3)) <= 32)
{
type = "INT";
}
else
{
type = "BIGINT";
}
}
index += 1;
sql += $"`{field}` {type.ToUpper()}{(notNull ? " NOT NULL" : "")} {(index<length?",":"")}\n";
}
sql += $")\nUNIQUE KEY(Id)\nDISTRIBUTED BY HASH(Id) BUCKETS {buckets}\nPROPERTIES(\"replication_num\" = \"{replication_num}\");";
Debug.WriteLine(sql);
}
}
}

2
labs/doris/Kafka2Doris/appsettings.json
Unescape View File

@ -7,7 +7,7 @@
},
"kafka": {
"host": "kafka:9092",
"topics": "mysql.example.User"
"topics.regex": "mysql.example.*"
},
"doris": {
"server": "http://doris-fe:8030",

2
labs/doris/conf/kafka-connect/boot.sh
Unescape View File

@ -17,8 +17,10 @@ cd /usr/share/confluent-hub-components/
if [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors/mysql-source) -eq 404 ]; then
curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @mysql2kafka.json
fi
# curl -X DELETE http://localhost:8083/connectors/mysql-source
#2:elasticsearch-sink
if [ $(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors/elasticsearch-sink) -eq 404 ]; then
curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @kafka2elasticsearch.json
fi
# curl -X DELETE http://localhost:8083/connectors/elasticsearch-sink
sleep infinity

9
labs/doris/conf/kafka-connect/example.json
Unescape View File

@ -1,6 +1,5 @@
[
{"Id":"49fa11e7-4404-4fe5-9873-e89a01d8e529","UserName":"super","SecurityStamp":"123456","PasswordHash":"579f889441b4a55d667233941d72a83ed644f7e5","PasswordConfirmed":1,"Email":"super@test.com","EmailConfirmed":1,"PhoneNumber":null,"PhoneNumberConfirmed":0,"RealName":null,"IdentityNumber":null,"IdentityConfirmed":0,"NickName":"超级管理员","Avatar":null,"Sex":null,"Birthday":null,"LockoutEnabled":0,"AccessFailedCount":0,"LockoutEnd":null,"RowVersion":"1e59e461-af12-446f-b876-eac8f58d3c79","Created":1618377036611,"Modified":null,"Deleted":null},
{"Id":"74fab867-a65a-4cda-8b2f-63e12d003b79","UserName":"user","SecurityStamp":"123456","PasswordHash":"71dd07494c5ee54992a27746d547e25dee01bd97","PasswordConfirmed":1,"Email":"user@test.com","EmailConfirmed":1,"PhoneNumber":null,"PhoneNumberConfirmed":0,"RealName":null,"IdentityNumber":null,"IdentityConfirmed":0,"NickName":"普通用户","Avatar":null,"Sex":null,"Birthday":null,"LockoutEnabled":0,"AccessFailedCount":0,"LockoutEnd":null,"RowVersion":"2b691577-e96c-40fe-897e-3adc9a4488bf","Created":1618377036611,"Modified":null,"Deleted":null},
{"Id":"e6a2e59c-a4e1-4ced-856c-036b6e4786d1","UserName":"admin","SecurityStamp":"123456","PasswordHash":"579f889441b4a55d667233941d72a83ed644f7e5","PasswordConfirmed":1,"Email":"admin@test.com","EmailConfirmed":1,"PhoneNumber":null,"PhoneNumberConfirmed":0,"RealName":null,"IdentityNumber":null,"IdentityConfirmed":0,"NickName":"管理员","Avatar":null,"Sex":null,"Birthday":null,"LockoutEnabled":0,"AccessFailedCount":0,"LockoutEnd":null,"RowVersion":"7d53094e-569d-4689-a6d1-d2c1f99da85b","Created":1618377036611,"Modified":null,"Deleted":null}
]
{"Id":"49fa11e7-4404-4fe5-9873-e89a01d8e529","UserName":"super1","SecurityStamp":"123456","PasswordHash":"579f889441b4a55d667233941d72a83ed644f7e5","PasswordConfirmed":1,"Email":"super@test.com","EmailConfirmed":1,"PhoneNumber":null,"PhoneNumberConfirmed":0,"RealName":null,"IdentityNumber":null,"IdentityConfirmed":0,"NickName":"超级管理员","Avatar":null,"Sex":null,"Birthday":null,"LockoutEnabled":0,"AccessFailedCount":0,"LockoutEnd":null,"RowVersion":"1e59e461-af12-446f-b876-eac8f58d3c79","Created":"2007-11-30 10:30-19","Modified":null,"Deleted":null},
{"Id":"74fab867-a65a-4cda-8b2f-63e12d003b79","UserName":"user","SecurityStamp":"123456","PasswordHash":"71dd07494c5ee54992a27746d547e25dee01bd97","PasswordConfirmed":1,"Email":"user@test.com","EmailConfirmed":1,"PhoneNumber":null,"PhoneNumberConfirmed":0,"RealName":null,"IdentityNumber":null,"IdentityConfirmed":0,"NickName":"普通用户","Avatar":null,"Sex":null,"Birthday":null,"LockoutEnabled":0,"AccessFailedCount":0,"LockoutEnd":null,"RowVersion":"2b691577-e96c-40fe-897e-3adc9a4488bf","Created":"2007-11-30 10:30-19","Modified":null,"Deleted":null},
{"Id":"e6a2e59c-a4e1-4ced-856c-036b6e4786d1","UserName":"admin","SecurityStamp":"123456","PasswordHash":"579f889441b4a55d667233941d72a83ed644f7e5","PasswordConfirmed":1,"Email":"admin@test.com","EmailConfirmed":1,"PhoneNumber":null,"PhoneNumberConfirmed":0,"RealName":null,"IdentityNumber":null,"IdentityConfirmed":0,"NickName":"管理员","Avatar":null,"Sex":null,"Birthday":null,"LockoutEnabled":0,"AccessFailedCount":0,"LockoutEnd":null,"RowVersion":"7d53094e-569d-4689-a6d1-d2c1f99da85b","Created":"2007-11-30 10:30-19","Modified":null,"Deleted":null}
]

2
labs/doris/conf/kafka-connect/mysql2kafka.json
Unescape View File

@ -13,6 +13,6 @@
"database.history.kafka.bootstrap.servers": "kafka:9092",
"database.history.kafka.topic": "mysql.schema.example",
"include.schema.changes": "true",
"time.precision.mode":"connect"
"time.precision.mode": "connect"
}
}

8
labs/doris/conf/kafka-connect/start.sh
Unescape View File

@ -1,8 +0,0 @@
#!/bin/bash
echo 'init connectors start:\n'
echo 'mysql2kafka\n'
curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @mysql2kafka.json
echo 'kafka2elasticsearch\n'
curl -i -X POST -H "Accept:application/json" -H "Content-Type:application/json" http://localhost:8083/connectors/ -d @kafka2elasticsearch.json
echo 'init connectors end!\n
# curl -X DELETE http://localhost:8083/connectors/elasticsearch-sink'

70
labs/doris/docker-compose.yml
Unescape View File

@ -68,7 +68,7 @@ services:
networks:
default:
ipv4_address: 172.172.0.22
depends_on:
depends_on:
- kafka
kafka-connect:
image: confluentinc/cp-kafka-connect:6.1.1
@ -144,40 +144,40 @@ services:
networks:
default:
ipv4_address: 172.172.0.31
# doris-fe:
# image: 76527413/doris:0.14.7
# ports:
# - 8030:8030
# - 9010:9010
# - 9020:9020
# - 9030:9030
# environment:
# - priority_networks=172.172.0.0/24
# volumes:
# - ./conf/doris/fe.conf:/doris/fe/conf/fe.conf
# - ./log/doris/fe:/doris/fe/log
# - ./data/doris/fe/doris-meta:/doris/fe/doris-meta
# command: bash -c "/doris/fe/bin/start_fe.sh"
# networks:
# default:
# ipv4_address: 172.172.0.40
# doris-be:
# image: 76527413/doris:0.14.7
# ports:
# - 8040:8040
# - 8060:8060
# - 9050:9050
# - 9060:9060
# environment:
# - priority_networks=172.172.0.0/24
# volumes:
# - ./conf/doris/be.conf:/doris/be/conf/be.conf
# - ./data/doris/be/storage:/doris/be/storage
# - ./log/doris/be/:/doris/be/log
# command: bash -c "/doris/be/bin/start_be.sh"
# networks:
# default:
# ipv4_address: 172.172.0.41
doris-fe:
image: 76527413/doris:0.14.7
ports:
- 8030:8030
- 9010:9010
- 9020:9020
- 9030:9030
environment:
- priority_networks=172.172.0.0/24
volumes:
- ./conf/doris/fe.conf:/doris/fe/conf/fe.conf
- ./log/doris/fe:/doris/fe/log
- ./data/doris/fe/doris-meta:/doris/fe/doris-meta
command: bash -c "/doris/fe/bin/start_fe.sh"
networks:
default:
ipv4_address: 172.172.0.40
doris-be:
image: 76527413/doris:0.14.7
ports:
- 8040:8040
- 8060:8060
- 9050:9050
- 9060:9060
environment:
- priority_networks=172.172.0.0/24
volumes:
- ./conf/doris/be.conf:/doris/be/conf/be.conf
- ./data/doris/be/storage:/doris/be/storage
- ./log/doris/be/:/doris/be/log
command: bash -c "/doris/be/bin/start_be.sh"
networks:
default:
ipv4_address: 172.172.0.41
networks:
default:
driver: bridge

2
labs/doris/readme.md
Unescape View File

@ -91,7 +91,7 @@ USE example;
### 创建表：
//curl -i -v --location-trusted -u root:aA123456! -H "format: json" -H "strip_outer_array: true" -T example.json http://doris-fe:8030/api/example/User/_stream_load
//curl -i -v --location-trusted -u root:aA123456! -H "format: json" -H "strip_outer_array: true" -T example.json http://doris-fe:8030/api/example/mysql_example.User/_stream_load
CREATE TABLE `User` (
`Id` char(36) NOT NULL COMMENT 'Id',
`UserName` varchar(255) NOT NULL COMMENT 'UserName',

Write Preview
Loading…
Cancel
Save