You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
|
|
|
|
|
input {
|
|
|
|
|
|
kafka {
|
|
|
|
|
|
#Kafka<6B><61>ַ
|
|
|
|
|
|
bootstrap_servers => "127.0.0.1:9092"
|
|
|
|
|
|
#<23><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
|
|
|
|
|
|
topics => ["tableData"]
|
|
|
|
|
|
#<23>ӵ<EFBFBD>һ<EFBFBD><D2BB><EFBFBD><EFBFBD>ʼ<EFBFBD><CABC>ȡ
|
|
|
|
|
|
auto_offset_reset => "earliest"
|
|
|
|
|
|
#<23><><EFBFBD><EFBFBD><EFBFBD>̣߳<DFB3>һ<EFBFBD><D2BB><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>е<EFBFBD>partitions<6E><73>
|
|
|
|
|
|
consumer_threads => 1
|
|
|
|
|
|
decorate_events => true
|
|
|
|
|
|
#<23><><EFBFBD><EFBFBD>id<69><64>kafak<61>ᱣ<EFBFBD><E1B1A3><EFBFBD><EFBFBD>Ϣ<EFBFBD><CFA2><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFA3AC><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ŷ<EFBFBD>ȡ
|
|
|
|
|
|
group_id => "ds-1"
|
|
|
|
|
|
codec => "json"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
filter {
|
|
|
|
|
|
mutate {
|
|
|
|
|
|
#logstash<73><68><EFBFBD>Զ<EFBFBD><D4B6><EFBFBD><EFBFBD><EFBFBD>һЩ<D2BB>ֶΣ<D6B6>ȥ<EFBFBD><C8A5><EFBFBD><EFBFBD>Щ<EFBFBD><D0A9><EFBFBD>õ<EFBFBD><C3B5>ֶ<EFBFBD>
|
|
|
|
|
|
remove_field => ["event","@version","@timestamp"]
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
output {
|
|
|
|
|
|
elasticsearch {
|
|
|
|
|
|
hosts => "127.0.0.1:9200"
|
|
|
|
|
|
index => "%{table_name}"
|
|
|
|
|
|
document_id => "%{id}"
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
##<23><><EFBFBD><EFBFBD>
|
|
|
|
|
|
/usr/local/logstash/bin/logstash -f /usr/local/logstash/config/kafka2es.conf -w 1 -b 5000
|
|
|
|
|
|
|
|
|
|
|
|
##<23><>̨<EFBFBD><CCA8><EFBFBD><EFBFBD>
|
|
|
|
|
|
nohup /usr/local/logstash/bin/logstash -f /usr/local/logstash/config/kafka2es.conf -w 1 -b 5000 > /dev/null 2>&1 &
|
