From 92601f455b45cdc725524988696ddd21e2c549f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=BB=84=E6=B5=B7?= <10402852@qq.com>
Date: Sun, 23 Feb 2025 09:39:06 +0800
Subject: [PATCH] 'commit'

---
 .../__pycache__/sync_service.cpython-310.pyc  | Bin 8297 -> 8332 bytes
 ClickHouse/services/sync_service.py           |  14 ++++++++------
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/ClickHouse/services/__pycache__/sync_service.cpython-310.pyc b/ClickHouse/services/__pycache__/sync_service.cpython-310.pyc
index e61ec9c893da627ca66bbdbfcf9a2f17a892adee..9f8c46a7ef86a94aec6c90dafa8bb667be124578 100644
GIT binary patch
delta 1940
zcmZvdO>7%Q6o6-Tz5ch8IzP@&qogTGV`xd*poX6&q@`}cuc}lK8X&8UXK^=KubtVo
z(u8#*hy({(fx5?D94bHvApt&-3kQ%ufP~=2o<Jzj+_?Y};=SGcTJhoY%$xUS-n@D5
z?bi6s^T}c&5hd_@>$Q!%(f?WUD}in~KMSwNnnB205UtKoPjiinmndy<z6kWw&f2em
zw`tm2Jcb&*fQ-mHaoksXBe+2?99-lQEV+^F^_*I=w4%X}<6KfXFLGBjEuN9Kt#OMJ
zh?BJ&;ooTR6pq^*H98*L-u~n0tk8w_{62)|Cwk!qb7gK8Iq!A)6Qi*qTs)09;~0tb
zpl^-yM-XV{#&gB8VadLoD_KQL(fQ-JJna0~n67=F_?LE{gZWClXL>R_mF>}3&sYzq
zAbO6EqT-md*)X?yh&%p17#Ti}IFGo1cmi<|vCl`Cx<_Pl*~rOU&8grOQ!FJD*%ub{
zdnTkSrX_34zxO5FWrB18eYbI*+(iS9Mx5Upb?48v#Of6IZ4ryw0;3&-32Ooq*My2>
z2_OkDi3M+om0+G!LM&7XTVhpQCzVKqtX%QOcBGB4$U=ZWXw&(?yu`vRdW%xBqiNG>
z5G<m0)I2*)j9Ht^hpJ&<#Ez~DmDQLXsz&YDI-0<EjoQ+J#C4lg<95gv*9A)KFpG_l
zO4tro6TS{=qf5Jo;D*Uo!a@szXWIalpaTc?1&GyX9eGt)zD-FbPRUBzN>!U!9M^3+
zQ|EItOZbLY2}@SuH{y%!{{IqPB8we%h~GWn+5wlq!IpJG!SV{pG&t2%I>VbV+$n%|
zPb8!qhtcM{H~;wL{d-^D+1&WR72#z+68VJlQ)++?I{%~|9c^{Rf^r?889V^`ZotwC
zsvFG}4MWXgskniXX5{@GYDJAuK`F_+xCB*^ZyK2G02$9B1OyyR{OSrVDck^4S2A@~
zExA%oFPds5;b)YWE)%HShwySM#g+}FpylK_#mZfiO>ISWerXy`-VYjX5RNUWj1S-L
zPA67CVom~@#U!nrYaXM|))l;$+Fn`sH3+(yz`kTY2?9Qhs7qJe^+V^ImUS9-wAQaz
zwd$Lcconx5M)g5JMsiyqY_evUD!1f0y_j3@+;vmiBBru(N!N0UrMfXxwXShhVW!(#
zQn;e)sxC7P`i?eNwp6yWk*Vqs^qEu<np6R9FBc8xhxDA{-D3t%pF+HZIPAWASKV%B
zv#5yyh8I=~hu-%^SpBs2PFs@zDRH->pN=};b#%wZ(DEt7)6QQVLwg+tx<p>J<ifJK
zsLNgu7{{3L%ZPTwZcFr=LS5(aS`y~kUQY4naoUD>0Wpn0$M{vm%Y-zLoy76e#reE5
zGvb}Imq@Rmu6JWh7%n5~9`{L*eOpT%eTH7Fb8@#)?#~i8fOX*CqHd=G?DTi-y{;4V
za@|YdW4RJj=gN7%!FmsO2Bf>6C>S657Fb4vI_=%r(W9s{fbhOR58>!B#971$!fWAP
z=)F+k-8j0%v(BCFhaWf<j)dF7r{f{s2to*-_YQkS>OFy%Pnz>zcP~Be9P63<A5T}V
Ar~m)}

delta 1933
zcmZ`)OKclu5Z-@ZemiNL#KCT4C!}etR3#yVMkPrLO-eu%Py&Pol-0(&2~O-?=l5Dk
ztaVM{g49+*+FpQ*eJCp8At8$cIluv^I06!vJpoesIB)}TqGDz@4<|CV_S@Ni9y|a1
z-^^Zr`oa0I9t!yx_^p?=SL!Fe41dRxjP(n9H4uSC+K$jsU*r}DX}7-S`$??+8-I;N
z9YqqgJpd_=#!&97zawswXAUc=04m!zy_r_?Mn>0YFFp$@r8>2_f<aTlt~ZA0M;NSM
z@%~Q4Qz%C*)qh^x4gaZsk?lr*`Y3{vV-h4=Oc!ZEr?kiF4^0FfL*)>{u%(5f7hG?A
z9z#GkJDApsnjyJ<wqWRnoTZbfejFjZ4=pH-)_)8=B#Ea$E(MeGGt;xv$&8YmOi}_%
zFVb-|ny_|SmJ&_UbK_#A=s8Pn?W!~+2#fYHsj#J`C90-XNJ)_mIpu4p2`0VZ$;t&o
z$|&xpr!m$=g#S(XNzg9BRd@ox&vud^#}Yr`Ntjsww$4_9xD3|?w3&o$7>vm(L}3-~
z23HnVn6jwwWsgCsq{fuJKBhG1%BHuAQN%T#_L*drUF8%(@!TMU*|RiBl?M->=_{O?
zXS79=Srw`xpvUyp*s|Xgt6tMzsiBL{6yGH#zb4Qnlc|Er<Z3KoOpoF{!<0RyPz|^?
zXpJfDzr+IpgPn}Rud$AA2t2_CUU-6MjIhzN%x=9$m~wzHrKr(bJ*N0jzn^ktFXuMJ
z?>b&)lt4LfC9vM&-Z#hwv)*co^n*jG9ZCrjjMNwco^wnpSZsq@K6v}XdpB?Iynp@f
z=eO>Db@kruYeg)zyKw*8k9M{{wmGT_=}$O%(fX@xfSk6vBWFk2DGz2gZ)9?+?N95P
zrlzr9Y(AgSR_HlU(hjun<m9|W^$lo^n~{NzM^G_^z#?!6z1Gi>cr560$YF@Xl|IxV
zo2*#-$WRIdTZCYFRcT6kWJU?D0y>A;dZm4mTsqR+4wD_qyZr{1DZYWbgRsyv__Lt6
zW%)a6#Akih`F&*}u%rUP)#YWK8ctWy(_oUh4;xAeb|-X&Dq(G;jFvAN5}4&=V<9eQ
zv-*alZst`AN)5VLk~KNIRZ^*aywOzBidoPVBt>18i&?`?<m62$FH<7}WsYbC2|_^d
zf=Uf(DXXW~tUsc9#o60D?&2kc*Abf4;9&c^<LnKz#ERQoPR%u~!K<MC0<OXUfS2_{
zd-@6M+*tyDmL>I{y8b4>kUOz{GH(4DO9UpsmOh8@ycLZPiSE3yF2$oUtSP)r-lScq
z-ye^35~faM${9dNvI8>FS$vEl%ptsjfN9a!5H16xLJh|`X8jsZopAym&}soqo$Vr}
z=$i;f=I0Eo-Kmcre}POjWQB(AR0OG!Ha3s^cW2v?NN<7ZKlLBFPm-6990L}@78G@<
zxI!ITI;S>_q4&N|AU8b-Y@dgWHJ+HBh@;I(1n2WJg3?)pQ3M>S6#If=t=&rD-YBL;
sr>t)hV}pZUpSR099Q4puScLMOM;(jcG^cX{4w)3IC)rDSt*PY9f1%}`q5uE@

diff --git a/ClickHouse/services/sync_service.py b/ClickHouse/services/sync_service.py
index 4a219392..cdc19186 100644
--- a/ClickHouse/services/sync_service.py
+++ b/ClickHouse/services/sync_service.py
@@ -121,6 +121,8 @@ class SyncService:
 
     def _sync_with_cursor(self, cursor, total, table, batch_size):
         """通用同步流程"""
+        # 根据数据量动态调整批次大小
+        dynamic_batch_size = max(min(total // 100, 5000), 1000)
         progress = tqdm(total=total, desc="同步进度", unit="rec")
         batch = []
         insert_count = 0
@@ -139,7 +141,7 @@ class SyncService:
                 mapped = self.mapper.map_row(self.columns, row)
                 batch.append(mapped)
 
-                if len(batch) >= batch_size:
+                if len(batch) >= dynamic_batch_size:
                     self._insert_batch(batch, table)
                     insert_count += 1
                     progress.update(len(batch))
@@ -184,11 +186,11 @@ class SyncService:
                 batch,
                 types_check=True,
                 settings={
-                    'date_time_input_format': 'best_effort',
-                    'allow_experimental_analyzer': 0,
-                    'input_format_null_as_default': 1,
-                    'max_partitions_per_insert_block': 1000,
-                    'optimize_on_insert': 1
+                    'max_insert_block_size': 100000,  # 增大插入块大小
+                    'insert_deduplicate': 0,  # 关闭插入时去重
+                    'max_threads': 8,  # 增加处理线程
+                    'parallel_distributed_insert_select': 1,
+                    'optimize_on_insert': 0  # 关闭插入时优化
                 }
             )
         except Exception as e: