bigdata
/
dsMin
6
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

'commit'

Browse Source
master
wangshuai 5 years ago
parent ef6a5f03dc
commit e76c90a24b
9 changed files with 380 additions and 41 deletions
Show Stats Download Patch File Download Diff File

1
dsDataexV1.1/Config/Config.ini
Unescape View File

@ -33,6 +33,7 @@ KafkaAccessLogTopic = log_dataex
partition = 20 partition = 20
replication = 2 replication = 2
process_no = 1 process_no = 1
enable = 1
[elasticsearch] [elasticsearch]

2
dsDataexV1.1/Html/jquery-3.5.1.min.js vendored
View File

File diff suppressed because one or more lines are too long

138
dsDataexV1.1/Html/jquery.timers-1.2.js
Unescape View File

@ -0,0 +1,138 @@
/**
* jQuery.timers - Timer abstractions for jQuery
* Written by Blair Mitchelmore (blair DOT mitchelmore AT gmail DOT com)
* Licensed under the WTFPL (http://sam.zoy.org/wtfpl/).
* Date: 2009/10/16
*
* @author Blair Mitchelmore
* @version 1.2
*
**/
jQuery.fn.extend({
everyTime: function(interval, label, fn, times) {
return this.each(function() {
jQuery.timer.add(this, interval, label, fn, times);
});
},
oneTime: function(interval, label, fn) {
return this.each(function() {
jQuery.timer.add(this, interval, label, fn, 1);
});
},
stopTime: function(label, fn) {
return this.each(function() {
jQuery.timer.remove(this, label, fn);
});
}
});
jQuery.extend({
timer: {
global: [],
guid: 1,
dataKey: "jQuery.timer",
regex: /^([0-9]+(?:\.[0-9]*)?)\s*(.*s)?$/,
powers: {
// Yeah this is major overkill...
'ms': 1,
'cs': 10,
'ds': 100,
's': 1000,
'das': 10000,
'hs': 100000,
'ks': 1000000
},
timeParse: function(value) {
if (value == undefined || value == null)
return null;
var result = this.regex.exec(jQuery.trim(value.toString()));
if (result[2]) {
var num = parseFloat(result[1]);
var mult = this.powers[result[2]] || 1;
return num * mult;
} else {
return value;
}
},
add: function(element, interval, label, fn, times) {
var counter = 0;
if (jQuery.isFunction(label)) {
if (!times)
times = fn;
fn = label;
label = interval;
}
interval = jQuery.timer.timeParse(interval);
if (typeof interval != 'number' || isNaN(interval) || interval < 0)
return;
if (typeof times != 'number' || isNaN(times) || times < 0)
times = 0;
times = times || 0;
var timers = jQuery.data(element, this.dataKey) || jQuery.data(element, this.dataKey, {});
if (!timers[label])
timers[label] = {};
fn.timerID = fn.timerID || this.guid++;
var handler = function() {
if ((++counter > times && times !== 0) || fn.call(element, counter) === false)
jQuery.timer.remove(element, label, fn);
};
handler.timerID = fn.timerID;
if (!timers[label][fn.timerID])
timers[label][fn.timerID] = window.setInterval(handler,interval);
this.global.push( element );
},
remove: function(element, label, fn) {
var timers = jQuery.data(element, this.dataKey), ret;
if ( timers ) {
if (!label) {
for ( label in timers )
this.remove(element, label, fn);
} else if ( timers[label] ) {
if ( fn ) {
if ( fn.timerID ) {
window.clearInterval(timers[label][fn.timerID]);
delete timers[label][fn.timerID];
}
} else {
for ( var fn in timers[label] ) {
window.clearInterval(timers[label][fn]);
delete timers[label][fn];
}
}
for ( ret in timers[label] ) break;
if ( !ret ) {
ret = null;
delete timers[label];
}
}
for ( ret in timers ) break;
if ( !ret )
jQuery.removeData(element, this.dataKey);
}
}
}
});
jQuery(window).bind("unload", function() {
jQuery.each(jQuery.timer.global, function(index, item) {
jQuery.timer.remove(item);
});
});

187
dsDataexV1.1/Html/mock.html
Unescape View File

@ -0,0 +1,187 @@
<!DOCTYPE html>
<html>
<head>
<title>My Page</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="http://code.jquery.com/mobile/1.1.0/jquery.mobile-1.1.0.min.css" />
<script src="http://code.jquery.com/jquery-1.7.1.min.js" type="text/javascript"></script>
<script src="http://code.jquery.com/mobile/1.1.0/jquery.mobile-1.1.0.min.js" type="text/javascript"></script>
<script src="jquery.timers-1.2.js" type="text/javascript"></script>
<script type="text/javascript">
$(document).ready(function(){
//请求参数
var list = {
"access_token": "TEST_009##202011300904##4bac61eee0661173d2435a58827cf683",
"data_source": "log_login",
"event_datas": [
{
"event_name": "在线提交作业",
"event_seqno": "事件顺序号：1/2/3",
"event_time": "2020/08/08 18:19:20",
"event_type": "1001",
"event_uri": "https://edusoa.com/cloud/study/do_homework",
"event_userid": "用户ID",
"properties": "{res_id:12300,res_name:初一语文一单元作业,teacher_id:123001,time_spend:100s}"
}
],
"user_datas": [
{
"access_id": "访客ID（用户未登录）",
"access_ip": "10.10.8.88",
"access_way": "app/web/weixin/dingding",
"identity": "用户身份（1：教师，2：学生，3：家长，4：管理员，5：访客）",
"properties": "{group_id:12300,group_name:课外学习一组,user_level:2,login_time:2020/08/08 12:13:14}",
"user_id": "用户ID"
}
]
}
$(".button").toggle(function(){
var page = 1
var perPage = $('.per_page').val()
var num
var event_datas = []
var user_datas = []
var i, j
var d = new Date();
for (var i = 0; i < perPage; i++) {
var time = new Date().Format("yyyy/MM/dd hh:mm:ss");
event_datas[i] = {}
event_datas[i].event_name = "在线提交作业" + i
event_datas[i].event_seqno = "事件顺序号：" + i
event_datas[i].event_time = time
event_datas[i].event_type = "1001" + i
event_datas[i].event_uri = "https://edusoa.com/cloud/study/do_homework/" + i
event_datas[i].event_userid = "用户ID" + i
event_datas[i].properties = "{res_id:12300,res_name:初一语文一单元作业,teacher_id:123001,time_spend:100s}"
}
for (var j = 0; j < perPage; j++) {
var time = new Date().Format("yyyy/MM/dd hh:mm:ss");
user_datas[j] = {}
user_datas[j].access_id = "访客ID（用户未登录）" + j
user_datas[j].access_ip = "10.10.8.88"
user_datas[j].access_way = "app/web/weixin/dingding/" + j
user_datas[j].identity = randomNum(1, 5).toString()
user_datas[j].properties = "{group_id:12300,group_name:课外学习一组,user_level:2,login_time:" + time + "}"
user_datas[j].user_id = "用户ID" + j
}
list.event_datas = event_datas
list.user_datas = user_datas
var str
var cycle = 0
str = JSON.stringify(list)
$('body').everyTime('1s','A',function(){
$.ajax({
type : "post",
url: "http://127.0.0.1:9009/dataex/dataex/DataexCollect",
contentType:"application/json",
dataType: "json",
data : str,
beforeSend : function(result) {
num = perPage * page
$(".print_box").html("正在执行" + page + "页," + num + "条数据")
//$(".button").text("SENDING...");
//$(".button").prop("value", "SENDING...");
},
//请求成功
success : function(result) {
console.log(result);
},
//请求失败，包含具体的错误信息
error : function(e){
console.log(e.status);
console.log(e.responseText);
}
}).done(function() {
num = perPage * page
$(".print_box").html("已执行" + page + "页," + num + "条数据")
page++
//$(".button").text("DONE");
//$(".button").prop("value", "DONE");
});
}, cycle);
}, function(){
$("body").stopTime ("A");
//$(".mock").text("MOCK");
//$(".mock").prop("value", "MOCK");
});
});
Date.prototype.Format = function (fmt) { //author: meizz
var o = {
"M+": this.getMonth() + 1, //月份
"d+": this.getDate(), //日
"h+": this.getHours(), //小时
"m+": this.getMinutes(), //分
"s+": this.getSeconds(), //秒
"q+": Math.floor((this.getMonth() + 3) / 3), //季度
"S": this.getMilliseconds() //毫秒
};
if (/(y+)/.test(fmt)) fmt = fmt.replace(RegExp.$1, (this.getFullYear() + "").substr(4 - RegExp.$1.length));
for (var k in o)
if (new RegExp("(" + k + ")").test(fmt)) fmt = fmt.replace(RegExp.$1, (RegExp.$1.length == 1) ? (o[k]) : (("00" + o[k]).substr(("" + o[k]).length)));
return fmt;
}
// 生成从minNum到maxNum的随机数
function randomNum(minNum,maxNum){
switch(arguments.length){
case 1:
return parseInt(Math.random()*minNum+1,10);
break;
case 2:
return parseInt(Math.random()*(maxNum-minNum+1)+minNum,10);
break;
default:
return 0;
break;
}
}
// 生成len长度随机字符串
function randomString(len) {
len = len || 32;
var $chars = 'ABCDEFGHJKMNPQRSTWXYZabcdefhijkmnprstwxyz2345678'; /****默认去掉了容易混淆的字符oOLl,9gq,Vv,Uu,I1****/
var maxPos = $chars.length;
var pwd = '';
for (i = 0; i < len; i++) {
pwd += $chars.charAt(Math.floor(Math.random() * maxPos));
}
return pwd;
}
</script>
</head>
<body>
<div data-role="page">
<div data-role="header">
<h1>My Title</h1>
</div><!-- /header -->
<div data-role="content">
<p class="print_box">Hello world</p>
<select name="per_page" class="per_page">
<option value="100">100/Cycle</option>
<option value="200">200/Cycle</option>
<option value="300">300/Cycle</option>
<option value="500">500/Cycle</option>
<option value="1000">1000/Cycle</option>
</select>
<button class="button" type='button'>MOCK</button>
</div><!-- /content -->
</div><!-- /page -->
</body>
</html>

53
dsDataexV1.1/MiddleWare/Middleware.go
Unescape View File

@ -36,34 +36,39 @@ func stringEndWith(input string) bool {
} }
func init() { func init() {
var err error // add by wangshuai 2020-12-15
var hook *kafkalogrus.KafkaLogrusHook // 增加判断是否启用Kafka
if ConfigUtil.KafkaEnable == 1 {
var err error
var hook *kafkalogrus.KafkaLogrusHook
//add by zhangjun 2020-08-03 //add by zhangjun 2020-08-03
//初始化 topic，多个 partition，提高数据批处理性能。 //初始化 topic，多个 partition，提高数据批处理性能。
KafkaUtil.InitTopic(ConfigUtil.KafkaAccessLogTopic)
KafkaUtil.InitTopic(ConfigUtil.KafkaAccessLogTopic)
if hook, err = kafkalogrus.NewKafkaLogrusHook(
"kh", if hook, err = kafkalogrus.NewKafkaLogrusHook(
logrus.AllLevels, "kh",
&logrus.JSONFormatter{}, logrus.AllLevels,
//change by zhangjun 2020-08-03 &logrus.JSONFormatter{},
//[]string{ConfigUtil.KafkaAddress}, //change by zhangjun 2020-08-03
ConfigUtil.KafkaBrokers, //[]string{ConfigUtil.KafkaAddress},
ConfigUtil.KafkaAccessLogTopic, ConfigUtil.KafkaBrokers,
true, ConfigUtil.KafkaAccessLogTopic,
nil); err != nil { true,
panic(err) nil); err != nil {
} panic(err)
}
//设置日志格式 //设置日志格式
logrus.SetFormatter(CSTFormatter{&logrus.JSONFormatter{}}) logrus.SetFormatter(CSTFormatter{&logrus.JSONFormatter{}})
//add by zhangjun 2020-08-03 //add by zhangjun 2020-08-03
src, _:= os.OpenFile("./gin_server.log", os.O_APPEND|os.O_WRONLY, os.ModeAppend) src, _:= os.OpenFile("./gin_server.log", os.O_APPEND|os.O_WRONLY, os.ModeAppend)
logrus.SetOutput(src) logrus.SetOutput(src)
logrus.AddHook(hook) logrus.AddHook(hook)
}
} }
// 日志记录到 Kafka // 日志记录到 Kafka

5
dsDataexV1.1/MyTask/Kafka2ES/Kafka2ESService/Kafka2ESService.go
Unescape View File

@ -53,13 +53,10 @@ func ServiceStart() {
KafkaUtil.CountTopicProc = make(map[string]int) KafkaUtil.CountTopicProc = make(map[string]int)
ChanTopic = make(chan []string, 100) ChanTopic = make(chan []string, 100)
for topics := range ChanTopic { for topics := range ChanTopic {
for no := 0; no < len(topics); no++ { for no := 0; no < len(topics); no++ {
topic := topics[no] topic := topics[no]
_, f := KafkaUtil.CountTopicProc[topic] _, f := KafkaUtil.CountTopicProc[topic]
if f == false { if f == false {
//change by zhangjun 2020-08-02 //change by zhangjun 2020-08-02
@ -171,7 +168,6 @@ func KafkaProcess(topic string) {
* @return * @return
**/ **/
func LogProcess() { func LogProcess() {
for k, v := range KafkaUtil.CountTopicProc { for k, v := range KafkaUtil.CountTopicProc {
fmt.Println("[Kafka] ["+k+"] "+time.Now().Format("2006/01/02 15:04:05")+" Process message total:", v) fmt.Println("[Kafka] ["+k+"] "+time.Now().Format("2006/01/02 15:04:05")+" Process message total:", v)
} }
@ -185,7 +181,6 @@ func LogProcess() {
* @return * @return
**/ **/
func ESRefreshProcess() { func ESRefreshProcess() {
for k,_:=range KafkaUtil.CountTopicProc { for k,_:=range KafkaUtil.CountTopicProc {
ES7Util.IndexRefresh( k ) ES7Util.IndexRefresh( k )

3
dsDataexV1.1/Utils/ConfigUtil/ConfigUtil.go
Unescape View File

@ -56,6 +56,8 @@ var (
KafkaProcNo int64 KafkaProcNo int64
KafkaEnable int64
//ES Nodes 地址 //ES Nodes 地址
ESNodes string ESNodes string
@ -131,6 +133,7 @@ func init() {
KafkaReply =iniParser.GetInt64("kafka", "replication") KafkaReply =iniParser.GetInt64("kafka", "replication")
KafkaProcNo=iniParser.GetInt64("kafka", "process_no") KafkaProcNo=iniParser.GetInt64("kafka", "process_no")
KafkaParts=iniParser.GetInt64("kafka", "partition") KafkaParts=iniParser.GetInt64("kafka", "partition")
KafkaEnable=iniParser.GetInt64("kafka", "enable")
//add by zhangjun 2020-08-03 //add by zhangjun 2020-08-03
KafkaAccessLogTopic= iniParser.GetString("kafka", "KafkaAccessLogTopic") KafkaAccessLogTopic= iniParser.GetString("kafka", "KafkaAccessLogTopic")

25
dsDataexV1.1/Utils/KafkaUtil/KafkaUtil.go
Unescape View File

@ -32,20 +32,23 @@ var loc sync.Mutex
var CountTopicProc map[string]int var CountTopicProc map[string]int
func init() { func init() {
// add by wangshuai 2020-12-15
// 增加判断是否启用Kafka
if ConfigUtil.KafkaEnable == 1 {
KafkaClient, _ := kafka.DialLeader(context.Background(), "tcp", ConfigUtil.KafkaBrokers[0], "__consumer_offsets", 0)
brokers, _ := KafkaClient.Brokers()
//change by zhangjun 2020-08-14
for no:=0;no< len(brokers);no++{
KafkaBroker += brokers[no].Host + ":" + strconv.Itoa(brokers[no].Port) + "【" + strconv.Itoa(brokers[no].ID) + "】,"
}
//KafkaBroker = brokers[0].Host + ":" + strconv.Itoa(brokers[0].Port) + "【" + strconv.Itoa(brokers[0].ID) + "】"
KafkaClient, _ := kafka.DialLeader(context.Background(), "tcp", ConfigUtil.KafkaBrokers[0], "__consumer_offsets", 0) writerPool = make(map[string]*kafka.Writer)
brokers, _ := KafkaClient.Brokers() kafkaPool = make(map[string]map[int]*kafka.Conn)
//change by zhangjun 2020-08-14 kafkaParts = make(map[string]int)
for no:=0;no< len(brokers);no++{
KafkaBroker += brokers[no].Host + ":" + strconv.Itoa(brokers[no].Port) + "【" + strconv.Itoa(brokers[no].ID) + "】,"
} }
//KafkaBroker = brokers[0].Host + ":" + strconv.Itoa(brokers[0].Port) + "【" + strconv.Itoa(brokers[0].ID) + "】"
writerPool = make(map[string]*kafka.Writer)
kafkaPool = make(map[string]map[int]*kafka.Conn)
kafkaParts = make(map[string]int)
} }
/** /**

7
dsDataexV1.1/main.go
Unescape View File

@ -88,7 +88,12 @@ func main() {
fmt.Println("ES Server :" + ES7Util.ServerVersion) fmt.Println("ES Server :" + ES7Util.ServerVersion)
//fmt.Println("ES Server :" + ES7SqlUtil.ServerVersion) //fmt.Println("ES Server :" + ES7SqlUtil.ServerVersion)
fmt.Println("Kafka Server :" + KafkaUtil.KafkaBroker )
// add by wangshuai 2020-12-15
// 增加判断是否启用Kafka
if ConfigUtil.KafkaEnable == 1 {
fmt.Println("Kafka Server :" + KafkaUtil.KafkaBroker)
}
//KafkaUtil.CreateTopic2("log_test") //KafkaUtil.CreateTopic2("log_test")
//var token=MD5Util.MD5V1("TEST_007" + "20200707" + "DSDataex_Token_7ee1f0f76243449f8d75f40fdcc2b93d") //var token=MD5Util.MD5V1("TEST_007" + "20200707" + "DSDataex_Token_7ee1f0f76243449f8d75f40fdcc2b93d")

Write Preview
Loading…
Cancel
Save