'commit'

ClickHouse/config/db_config.py

@@ -4,7 +4,7 @@ MYSQL_CONFIG = {
     'port': 3306,
     'user': 'ylt',
     'password': 'Ycharge666',
-    'db': 'yltcharge',
+    'database': 'yltcharge',
     'charset': 'utf8mb4'
 }
 

ClickHouse/get_mysql_table_structure.py

@@ -0,0 +1,149 @@
+import pymysql
+import re
+from config.db_config import *
+from config.db_config import MYSQL_CONFIG
+
+def get_mysql_table_structure(config, table):
+    """获取MySQL表结构信息"""
+    conn = pymysql.connect(host=config["host"],
+                           port=config["port"],
+                           user=config["user"],
+                           password=config["password"],
+                           database=config["database"])
+
+    # 获取字段信息
+    with conn.cursor() as cursor:
+        cursor.execute(f"""
+            SELECT column_name, data_type, column_type, is_nullable, column_default
+            FROM information_schema.columns
+            WHERE table_schema = '{config["database"]}' AND table_name = '{table}'
+        """)
+        columns = cursor.fetchall()
+
+    # 获取索引信息
+    with conn.cursor() as cursor:
+        cursor.execute(f"""
+            SELECT index_name, column_name, non_unique
+            FROM information_schema.statistics
+            WHERE table_schema = '{config["database"]}' AND table_name = '{table}'
+        """)
+        indexes = cursor.fetchall()
+
+    # 获取主键信息
+    with conn.cursor() as cursor:
+        cursor.execute(f"""
+            SELECT column_name
+            FROM information_schema.key_column_usage
+            WHERE table_schema = '{config["database"]}' 
+            AND table_name = '{table}'
+            AND constraint_name = 'PRIMARY'
+        """)
+        primary_keys = [row[0] for row in cursor.fetchall()]
+
+    conn.close()
+    return columns, indexes, primary_keys
+
+
+def convert_to_clickhouse_type(mysql_type):
+    """将MySQL类型转换为ClickHouse类型，并处理无符号类型"""
+    # 去除unsigned标记并转换为小写
+    type_str = re.sub(r'\sunsigned', '', mysql_type, flags=re.IGNORECASE).lower()
+
+    # 类型映射
+    type_map = {
+        'tinyint': 'Int8',
+        'smallint': 'Int16',
+        'mediumint': 'Int32',
+        'int': 'Int32',
+        'bigint': 'Int64',
+        'float': 'Float32',
+        'double': 'Float64',
+        'decimal': 'Decimal',
+        'char': 'String',
+        'varchar': 'String',
+        'text': 'String',
+        'longtext': 'String',
+        'date': 'Date',
+        'datetime': 'DateTime',
+        'timestamp': 'DateTime',
+        'enum': 'String'
+    }
+
+    # 处理特殊类型
+    if 'tinyint(1)' in type_str:
+        return 'Int8'  # 处理MySQL的布尔类型
+
+    # 匹配数字精度
+    match = re.match(r'(\w+)(\(\d+,\d+\))?', type_str)
+    if match:
+        base_type = match.group(1)
+        precision = match.group(2) or ''
+        return type_map.get(base_type, 'String') + precision
+
+    return type_map.get(type_str, 'String')
+
+def generate_clickhouse_ddl(table_name, columns, indexes, primary_keys):
+    """生成ClickHouse建表语句"""
+    # 处理字段定义
+    column_defs = []
+    for col in columns:
+        name, data_type, column_type, is_nullable, default = col
+        ch_type = convert_to_clickhouse_type(column_type)
+        # 修正浮点型定义
+        if 'Float64' in ch_type and '(' in ch_type:
+            ch_type = 'Float64'
+        nullable = 'Nullable(' + ch_type + ')' if is_nullable == 'YES' else ch_type
+        column_defs.append(f'    {name} {nullable}')
+
+    # 处理主键
+    order_by = 'ORDER BY (' + ', '.join(primary_keys) + ')' if primary_keys else ''
+
+    # 处理索引（新增分组逻辑）
+    index_groups = {}
+    for idx in indexes:
+        index_name, column_name, non_unique = idx
+        if index_name == 'PRIMARY':
+            continue
+        if index_name not in index_groups:
+            index_groups[index_name] = {
+                'non_unique': non_unique,
+                'columns': []
+            }
+        index_groups[index_name]['columns'].append(column_name)
+
+    index_defs = []
+    for index_name, info in index_groups.items():
+        columns = info['columns']
+        # 修正索引语法（ClickHouse不支持UNIQUE KEY）
+        index_type = 'minmax'  # 默认索引类型
+        # 统一使用INDEX语法，ClickHouse 24.8+ 的MergeTree引擎不支持UNIQUE约束
+        index_defs.append(
+            f'    INDEX {index_name} ({", ".join(columns)}) TYPE {index_type} GRANULARITY 3'
+        )
+
+    # 组合DDL语句
+    ddl = f'CREATE TABLE {table_name} (\n'
+    ddl += ',\n'.join(column_defs)
+    if index_defs:
+        ddl += ',\n' + ',\n'.join(index_defs)
+    ddl += '\n) ENGINE = MergeTree()\n'
+    ddl += order_by
+    ddl += '\nSETTINGS index_granularity = 8192;'
+
+    return ddl
+
+
+# 使用示例
+if __name__ == '__main__':
+    table = 't_station'
+
+    # 获取表结构
+    columns, indexes, primary_keys = get_mysql_table_structure(MYSQL_CONFIG, table)
+
+    # 生成ClickHouse DDL
+    ddl = generate_clickhouse_ddl(table, columns, indexes, primary_keys)
+    print(ddl)
+    # 保存到文件中 schemas/t_station.sql
+    with open(f'schemas/{table}.sql', 'w',encoding='utf-8') as f:
+        f.write(ddl)
+    print(f'✅ DDL语句已保存至：schemas/{table}.sql')

ClickHouse/schemas/t_station.sql

@@ -0,0 +1,88 @@
+CREATE TABLE t_station (
+    id Int64,
+    station_sn String,
+    operator_id Int64,
+    equioment_owner_id String,
+    station_name String,
+    country_code String,
+    region_id Nullable(Int64),
+    region_code String,
+    regional_name Nullable(String),
+    regional_code Nullable(String),
+    address String,
+    station_tel Nullable(String),
+    service_tel String,
+    station_type Int16,
+    station_type_msg Nullable(String),
+    station_business_type Nullable(Int8),
+    station_business_type_msg Nullable(String),
+    station_status Int16,
+    station_status_msg Nullable(String),
+    park_nums Int16,
+    station_lng Float64,
+    station_lat Float64,
+    site_guide Nullable(String),
+    construction Int16,
+    construction_msg Nullable(String),
+    pictures Nullable(String),
+    match_cars Nullable(String),
+    park_info Nullable(String),
+    business_start_time Nullable(Int16),
+    business_end_time Nullable(Int16),
+    electricity_fee Nullable(String),
+    service_fee Nullable(String),
+    is_park_fee Nullable(Int8),
+    park_fee Nullable(String),
+    payment Nullable(Int8),
+    payment_msg Nullable(String),
+    support_order Nullable(Int8),
+    support_order_msg Nullable(String),
+    service_item Nullable(String),
+    service_item_image_url Nullable(String),
+    service_item_msg Nullable(String),
+    station_location_type Nullable(Int8),
+    station_location_type_msg Nullable(String),
+    station_charge_type Nullable(Int8),
+    station_charge_type_msg Nullable(String),
+    station_power Nullable(String),
+    station_voltage Nullable(String),
+    station_current Nullable(String),
+    is_open Nullable(Int8),
+    is_open_msg Nullable(String),
+    is_display Nullable(Int8),
+    is_display_msg Nullable(String),
+    sharing_code Nullable(String),
+    sharing_applet_picture Nullable(String),
+    theme_picture Nullable(String),
+    is_barrier_flag Nullable(Int8),
+    is_barrier_flag_msg Nullable(String),
+    charge_settle_type Nullable(Int8),
+    charge_settle_type_msg Nullable(String),
+    fast_connector_nums Nullable(Int32),
+    slow_connector_nums Nullable(Int32),
+    transformer_capacity Nullable(Float64),
+    installed_capacity Nullable(Float64),
+    remark Nullable(String),
+    record_unique_no String,
+    equipment_owner_name String,
+    supply_type Int8,
+    resident_no String,
+    watt_hour_meter_no String,
+    forward_power String,
+    operation_date DateTime,
+    charge_replace_type Int8,
+    is_hlht Int8,
+    is_subsidy Int8,
+    create_time Nullable(DateTime),
+    update_time Nullable(DateTime),
+    area_code_countryside Nullable(String),
+    business_expand_type Nullable(Int8),
+    capacity Float64,
+    video_monitor Nullable(Int8),
+    electricity_type Nullable(Int8),
+    INDEX station_sn (station_sn) TYPE minmax GRANULARITY 3,
+    INDEX operator_id (operator_id) TYPE minmax GRANULARITY 3,
+    INDEX idx_operator_id (operator_id) TYPE minmax GRANULARITY 3
+) ENGINE = MergeTree()
+ORDER BY (id)
+SETTINGS index_granularity = 8192;