From cf8752e4445fb5648b578270b4214557b69a1b95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=84=E6=B5=B7?= <10402852@qq.com> Date: Thu, 16 Mar 2023 17:19:00 +0800 Subject: [PATCH] 'commit' --- ETL/.idea/.gitignore | 8 -- ETL/.idea/dataSources.xml | 12 --- ETL/.idea/encodings.xml | 8 -- .../inspectionProfiles/profiles_settings.xml | 6 -- ETL/.idea/misc.xml | 4 - ETL/.idea/modules.xml | 8 -- ETL/.idea/pythonProject.iml | 8 -- ETL/.idea/sqldialects.xml | 6 -- ETL/.idea/vcs.xml | 6 -- ETL/Mars/application.properties | 5 + ETL/Mars/pack.py | 102 ++++++++++++++++++ 11 files changed, 107 insertions(+), 66 deletions(-) delete mode 100644 ETL/.idea/.gitignore delete mode 100644 ETL/.idea/dataSources.xml delete mode 100644 ETL/.idea/encodings.xml delete mode 100644 ETL/.idea/inspectionProfiles/profiles_settings.xml delete mode 100644 ETL/.idea/misc.xml delete mode 100644 ETL/.idea/modules.xml delete mode 100644 ETL/.idea/pythonProject.iml delete mode 100644 ETL/.idea/sqldialects.xml delete mode 100644 ETL/.idea/vcs.xml create mode 100644 ETL/Mars/application.properties create mode 100644 ETL/Mars/pack.py diff --git a/ETL/.idea/.gitignore b/ETL/.idea/.gitignore deleted file mode 100644 index 4aa91ea..0000000 --- a/ETL/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/ETL/.idea/dataSources.xml b/ETL/.idea/dataSources.xml deleted file mode 100644 index 148de7d..0000000 --- a/ETL/.idea/dataSources.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - sqlite.xerial - true - org.sqlite.JDBC - jdbc:sqlite:D:\python\ETL\db\progress.db - $ProjectFileDir$ - - - \ No newline at end of file diff --git a/ETL/.idea/encodings.xml b/ETL/.idea/encodings.xml deleted file mode 100644 index c787d59..0000000 --- a/ETL/.idea/encodings.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/ETL/.idea/inspectionProfiles/profiles_settings.xml b/ETL/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 20fc29e..0000000 --- a/ETL/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/ETL/.idea/misc.xml b/ETL/.idea/misc.xml deleted file mode 100644 index 76db80a..0000000 --- a/ETL/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/ETL/.idea/modules.xml b/ETL/.idea/modules.xml deleted file mode 100644 index e1ceb69..0000000 --- a/ETL/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/ETL/.idea/pythonProject.iml b/ETL/.idea/pythonProject.iml deleted file mode 100644 index d9e6024..0000000 --- a/ETL/.idea/pythonProject.iml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/ETL/.idea/sqldialects.xml b/ETL/.idea/sqldialects.xml deleted file mode 100644 index b47a90f..0000000 --- a/ETL/.idea/sqldialects.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/ETL/.idea/vcs.xml b/ETL/.idea/vcs.xml deleted file mode 100644 index 2e3f692..0000000 --- a/ETL/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/ETL/Mars/application.properties b/ETL/Mars/application.properties new file mode 100644 index 0000000..1945b6d --- /dev/null +++ b/ETL/Mars/application.properties @@ -0,0 +1,5 @@ +user=root +password=DsideaL147258369 +host=10.10.14.230 +port=22066 +database=test \ No newline at end of file diff --git a/ETL/Mars/pack.py b/ETL/Mars/pack.py new file mode 100644 index 0000000..28d33f2 --- /dev/null +++ b/ETL/Mars/pack.py @@ -0,0 +1,102 @@ +# python -m pip install --upgrade pip -i http://pypi.douban.com/simple/ +# pip install pymysql -i http://pypi.douban.com/simple/ +import datetime +import sys + +import pymysql.cursors + + +class MySQLHelper: + myVersion = 0.1 + + def __init__(self, host, port, db, user, password, charset="utf8"): + self.host = host + self.user = user + self.port = port + self.password = password + self.charset = charset + self.db = db + + try: + self.conn = pymysql.connect(host=self.host, port=self.port, user=self.user, passwd=self.password, + db=self.db, charset=self.charset, cursorclass=pymysql.cursors.DictCursor) + self.cursor = self.conn.cursor() + + # 设置执行时间为8小时 + sql = "set session wait_timeout=288000" + self.cursor.execute(sql) + sql = "set session interactive_timeout=288000" + self.cursor.execute(sql) + + except Exception as e: + print('MySql Error : %d %s' % (e.args[0], e.args[1])) + + def query(self, sql: object) -> object: + try: + self.cursor.execute(sql) + result = self.cursor.fetchall() + return result + except Exception as e: + print('MySql Error: %s SQL: %s' % (e, sql)) + + def execute(self, sql): + try: + self.cursor.execute(sql) + self.conn.commit() + # 获取更新了多少条数据 + return self.cursor.rowcount + except Exception as e: + print('MySql Error: %s SQL: %s' % (e, sql)) + + def executeWithPara(self, sql, params): + try: + self.cursor.execute(sql, params) + self.conn.commit() + except Exception as e: + print('MySql Error: %s SQL: %s' % (e, sql)) + + def executemany(self, sql, data): + try: + self.cursor.executemany(sql, data) + self.conn.commit() + except Exception as e: + print('MySql Error: %s SQL: %s' % (e, sql)) + + def close(self): + self.cursor.close() + self.conn.close() + + +# 黄海定义的输出信息的办法,带当前时间 +def logInfo(msg): + i = datetime.datetime.now() + print(" %s %s" % (i, msg)) + + +# 准备扩展 +_dict = { + "t_resource_base": "update t_resource_base t1,t_dm_stage t2 set t1.CHECK_MESSAGE=t2.stage_name where t1.stage_id=t2.stage_id and t1.CHECK_MESSAGE is null limit 100"} + +if __name__ == '__main__': + if len(sys.argv) == 1: + print("没有正确输入参数,请检查!输入样例: python pack.py t_resource_base") + sys.exit() + tableName = sys.argv[1] + + # 配置 + user = 'root' + password = 'DsideaL147258369' + host = '10.10.14.230' + port = 22066 + database = 'test' + + db = MySQLHelper(user=user, host=host, port=port, password=password, db=database) + + while True: + sql = _dict[tableName] + cnt = db.execute(sql) + logInfo("成功更新%s条" % cnt) + if cnt == 0: + break + + db.close()