|
|
|
|
|
import json
|
|
|
|
|
|
import os
|
|
|
|
|
|
from urllib.parse import urlencode
|
|
|
|
|
|
from kafka import KafkaProducer
|
|
|
|
|
|
import requests
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# 网站获取数据Api
|
|
|
|
|
|
from Util import CommonUtil
|
|
|
|
|
|
|
|
|
|
|
|
url = "http://10.10.14.199/dsideal_yy/admin/new_base/personInfo_GetPersonInfoList"
|
|
|
|
|
|
|
|
|
|
|
|
# POST构建请求
|
|
|
|
|
|
headers = {'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'} # 一种请求头，需要携带
|
|
|
|
|
|
|
|
|
|
|
|
# Kafka
|
|
|
|
|
|
bootstrap_servers = '10.10.14.237:9092'
|
|
|
|
|
|
|
|
|
|
|
|
# Topic
|
|
|
|
|
|
Topic = "SyncData"
|
|
|
|
|
|
|
|
|
|
|
|
# 记录的最后上报时间(文件名称不能重复噢~~)
|
|
|
|
|
|
logFile = "./Log/PJ.log"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# 获取指定页码的数据
|
|
|
|
|
|
def getData(pageNumber):
|
|
|
|
|
|
# 最后一次更新时间
|
|
|
|
|
|
update_ts = CommonUtil.read_txt(logFile)
|
|
|
|
|
|
|
|
|
|
|
|
# 构造数据
|
|
|
|
|
|
data = {
|
|
|
|
|
|
'bureau_id': 400195,
|
|
|
|
|
|
'pageNumber': pageNumber,
|
|
|
|
|
|
'pageSize': 20,
|
|
|
|
|
|
'org_id': 400195,
|
|
|
|
|
|
'person_name': '',
|
|
|
|
|
|
'query_child': True,
|
|
|
|
|
|
'query_bureau_child': False,
|
|
|
|
|
|
'b_use': 1,
|
|
|
|
|
|
'update_ts': update_ts
|
|
|
|
|
|
}
|
|
|
|
|
|
# 构建POST请求
|
|
|
|
|
|
response = requests.request("POST", url, headers=headers, data=urlencode(data))
|
|
|
|
|
|
decodejson = json.loads(response.text)
|
|
|
|
|
|
|
|
|
|
|
|
# 总页数
|
|
|
|
|
|
totalPage = decodejson['totalPage']
|
|
|
|
|
|
# 遍历每一条，进行打入Kafka
|
|
|
|
|
|
for x in decodejson['list']:
|
|
|
|
|
|
# 向Kafka发送的内容
|
|
|
|
|
|
parmas_message = json.dumps(x, ensure_ascii=False)
|
|
|
|
|
|
producer.send(Topic, parmas_message.encode('utf-8'))
|
|
|
|
|
|
|
|
|
|
|
|
# 完成一个列表的数据上报
|
|
|
|
|
|
print("%s 成功获取%d条" % (CommonUtil.getCurrentTime(), len(decodejson['list'])))
|
|
|
|
|
|
return totalPage
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
|
# 判断上报日志文件是否存在，不存在则创建
|
|
|
|
|
|
if not os.path.exists(logFile):
|
|
|
|
|
|
CommonUtil.write_txt(logFile, "2000-01-01 00:00:00")
|
|
|
|
|
|
|
|
|
|
|
|
# KafkaClient生产者
|
|
|
|
|
|
global producer
|
|
|
|
|
|
producer = KafkaProducer(bootstrap_servers=bootstrap_servers) # 连接kafka
|
|
|
|
|
|
# 调用一次
|
|
|
|
|
|
totalPage = getData(1)
|
|
|
|
|
|
for i in range(2, totalPage + 1):
|
|
|
|
|
|
getData(i)
|
|
|
|
|
|
# 关闭KafkaClient
|
|
|
|
|
|
producer.close()
|
|
|
|
|
|
|
|
|
|
|
|
# 写入成功上报的时间戳
|
|
|
|
|
|
CommonUtil.write_txt(logFile, CommonUtil.getCurrentTime())
|
