临时保存一下

sync_tables_to_new_database.py

@@ -0,0 +1,458 @@
+"""
+从现有数据库同步三个表的数据到新数据库
+同步的表：f_polic_field, f_polic_file_config, f_polic_file_field
+同步前会先备份新数据库
+"""
+import os
+import sys
+import subprocess
+import pymysql
+from datetime import datetime
+from pathlib import Path
+from typing import List, Dict, Any
+from dotenv import load_dotenv
+
+# 设置输出编码为UTF-8（Windows控制台兼容）
+if sys.platform == 'win32':
+    import io
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
+    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace')
+
+# 加载环境变量
+load_dotenv()
+
+# 现有数据库配置（源数据库）
+SOURCE_DB_CONFIG = {
+    'host': os.getenv('DB_HOST', '152.136.177.240'),
+    'port': int(os.getenv('DB_PORT', 5012)),
+    'user': os.getenv('DB_USER', 'finyx'),
+    'password': os.getenv('DB_PASSWORD', '6QsGK6MpePZDE57Z'),
+    'database': os.getenv('DB_NAME', 'finyx'),
+    'charset': 'utf8mb4'
+}
+
+# 新数据库配置（目标数据库）
+TARGET_DB_CONFIG = {
+    'host': '10.100.31.21',
+    'port': 3306,
+    'user': 'finyx',
+    'password': 'FknJYz3FA5WDYtsd',
+    'database': 'finyx',
+    'charset': 'utf8mb4'
+}
+
+# 需要同步的表
+TABLES_TO_SYNC = ['f_polic_field', 'f_polic_file_config', 'f_polic_file_field']
+
+# 备份文件存储目录
+BACKUP_DIR = Path('backups')
+BACKUP_DIR.mkdir(exist_ok=True)
+
+
+def backup_target_database() -> str:
+    """
+    备份目标数据库
+    
+    Returns:
+        备份文件路径
+    """
+    print("=" * 60)
+    print("步骤 1: 备份新数据库")
+    print("=" * 60)
+    
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    backup_file = BACKUP_DIR / f"backup_target_db_{timestamp}.sql"
+    
+    # 构建mysqldump命令
+    cmd = [
+        'mysqldump',
+        f"--host={TARGET_DB_CONFIG['host']}",
+        f"--port={TARGET_DB_CONFIG['port']}",
+        f"--user={TARGET_DB_CONFIG['user']}",
+        f"--password={TARGET_DB_CONFIG['password']}",
+        '--single-transaction',
+        '--routines',
+        '--triggers',
+        '--events',
+        '--add-drop-table',
+        '--default-character-set=utf8mb4',
+        TARGET_DB_CONFIG['database']
+    ]
+    
+    try:
+        print(f"开始备份数据库 {TARGET_DB_CONFIG['database']}...")
+        print(f"备份文件: {backup_file}")
+        
+        # 执行备份命令
+        with open(backup_file, 'w', encoding='utf-8') as f:
+            result = subprocess.run(
+                cmd,
+                stdout=f,
+                stderr=subprocess.PIPE,
+                text=True
+            )
+        
+        if result.returncode != 0:
+            error_msg = result.stderr if result.stderr else '未知错误'
+            raise Exception(f"mysqldump执行失败: {error_msg}")
+        
+        # 检查文件大小
+        file_size = backup_file.stat().st_size
+        print(f"备份完成！文件大小: {file_size / 1024 / 1024:.2f} MB")
+        print(f"备份文件路径: {backup_file}")
+        
+        return str(backup_file)
+        
+    except FileNotFoundError:
+        print("警告: 未找到mysqldump命令，尝试使用Python方式备份...")
+        return backup_target_database_with_python(backup_file)
+    except Exception as e:
+        print(f"备份失败: {str(e)}")
+        raise
+
+
+def backup_target_database_with_python(backup_file: Path) -> str:
+    """
+    使用Python方式备份目标数据库（备用方式）
+    
+    Args:
+        backup_file: 备份文件路径
+        
+    Returns:
+        备份文件路径
+    """
+    try:
+        print(f"开始使用Python方式备份数据库 {TARGET_DB_CONFIG['database']}...")
+        
+        # 连接数据库
+        connection = pymysql.connect(**TARGET_DB_CONFIG)
+        cursor = connection.cursor()
+        
+        with open(backup_file, 'w', encoding='utf-8') as f:
+            # 写入文件头
+            f.write(f"-- MySQL数据库备份\n")
+            f.write(f"-- 数据库: {TARGET_DB_CONFIG['database']}\n")
+            f.write(f"-- 备份时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
+            f.write(f"-- 主机: {TARGET_DB_CONFIG['host']}:{TARGET_DB_CONFIG['port']}\n")
+            f.write("--\n\n")
+            f.write(f"SET NAMES utf8mb4;\n")
+            f.write(f"SET FOREIGN_KEY_CHECKS=0;\n\n")
+            
+            # 获取所有表
+            cursor.execute("SHOW TABLES")
+            tables = [table[0] for table in cursor.fetchall()]
+            
+            print(f"找到 {len(tables)} 个表")
+            
+            # 备份每个表
+            for table in tables:
+                print(f"备份表: {table}")
+                
+                # 获取表结构
+                cursor.execute(f"SHOW CREATE TABLE `{table}`")
+                create_table_sql = cursor.fetchone()[1]
+                
+                f.write(f"-- ----------------------------\n")
+                f.write(f"-- 表结构: {table}\n")
+                f.write(f"-- ----------------------------\n")
+                f.write(f"DROP TABLE IF EXISTS `{table}`;\n")
+                f.write(f"{create_table_sql};\n\n")
+                
+                # 获取表数据
+                cursor.execute(f"SELECT * FROM `{table}`")
+                rows = cursor.fetchall()
+                
+                if rows:
+                    # 获取列名
+                    cursor.execute(f"DESCRIBE `{table}`")
+                    columns = [col[0] for col in cursor.fetchall()]
+                    
+                    f.write(f"-- ----------------------------\n")
+                    f.write(f"-- 表数据: {table}\n")
+                    f.write(f"-- ----------------------------\n")
+                    
+                    # 分批写入数据
+                    batch_size = 1000
+                    for i in range(0, len(rows), batch_size):
+                        batch = rows[i:i+batch_size]
+                        values_list = []
+                        
+                        for row in batch:
+                            values = []
+                            for value in row:
+                                if value is None:
+                                    values.append('NULL')
+                                elif isinstance(value, (int, float)):
+                                    values.append(str(value))
+                                else:
+                                    # 转义特殊字符
+                                    escaped_value = str(value).replace('\\', '\\\\').replace("'", "\\'")
+                                    values.append(f"'{escaped_value}'")
+                            
+                            values_list.append(f"({', '.join(values)})")
+                        
+                        columns_str = ', '.join([f"`{col}`" for col in columns])
+                        values_str = ',\n'.join(values_list)
+                        
+                        f.write(f"INSERT INTO `{table}` ({columns_str}) VALUES\n")
+                        f.write(f"{values_str};\n\n")
+                
+                print(f"  完成: {len(rows)} 条记录")
+            
+            f.write("SET FOREIGN_KEY_CHECKS=1;\n")
+        
+        cursor.close()
+        connection.close()
+        
+        # 检查文件大小
+        file_size = backup_file.stat().st_size
+        print(f"备份完成！文件大小: {file_size / 1024 / 1024:.2f} MB")
+        
+        return str(backup_file)
+        
+    except Exception as e:
+        print(f"备份失败: {str(e)}")
+        raise
+
+
+def get_table_data(conn, table_name: str) -> List[Dict[str, Any]]:
+    """
+    从源数据库获取表的所有数据
+    
+    Args:
+        conn: 数据库连接
+        table_name: 表名
+        
+    Returns:
+        数据列表
+    """
+    cursor = conn.cursor(pymysql.cursors.DictCursor)
+    try:
+        cursor.execute(f"SELECT * FROM `{table_name}`")
+        return cursor.fetchall()
+    finally:
+        cursor.close()
+
+
+def get_table_columns(conn, table_name: str) -> List[str]:
+    """
+    获取表的列名
+    
+    Args:
+        conn: 数据库连接
+        table_name: 表名
+        
+    Returns:
+        列名列表
+    """
+    cursor = conn.cursor()
+    try:
+        cursor.execute(f"DESCRIBE `{table_name}`")
+        return [col[0] for col in cursor.fetchall()]
+    finally:
+        cursor.close()
+
+
+def clear_table(conn, table_name: str):
+    """
+    清空目标数据库中的表
+    
+    Args:
+        conn: 数据库连接
+        table_name: 表名
+    """
+    cursor = conn.cursor()
+    try:
+        # 禁用外键检查
+        cursor.execute("SET FOREIGN_KEY_CHECKS=0")
+        
+        # 清空表
+        cursor.execute(f"TRUNCATE TABLE `{table_name}`")
+        
+        # 恢复外键检查
+        cursor.execute("SET FOREIGN_KEY_CHECKS=1")
+        
+        conn.commit()
+        print(f"  已清空表: {table_name}")
+    except Exception as e:
+        conn.rollback()
+        raise Exception(f"清空表 {table_name} 失败: {str(e)}")
+    finally:
+        cursor.close()
+
+
+def insert_table_data(conn, table_name: str, columns: List[str], data: List[Dict[str, Any]]):
+    """
+    将数据插入到目标数据库
+    
+    Args:
+        conn: 数据库连接
+        table_name: 表名
+        columns: 列名列表
+        data: 数据列表
+    """
+    if not data:
+        print(f"  表 {table_name} 没有数据需要插入")
+        return
+    
+    cursor = conn.cursor()
+    try:
+        # 禁用外键检查
+        cursor.execute("SET FOREIGN_KEY_CHECKS=0")
+        
+        # 构建INSERT语句
+        columns_str = ', '.join([f"`{col}`" for col in columns])
+        placeholders = ', '.join(['%s'] * len(columns))
+        insert_sql = f"INSERT INTO `{table_name}` ({columns_str}) VALUES ({placeholders})"
+        
+        # 批量插入数据
+        batch_size = 1000
+        total_inserted = 0
+        
+        for i in range(0, len(data), batch_size):
+            batch = data[i:i+batch_size]
+            values_list = []
+            
+            for row in batch:
+                values = [row.get(col) for col in columns]
+                values_list.append(values)
+            
+            cursor.executemany(insert_sql, values_list)
+            total_inserted += len(batch)
+        
+        # 恢复外键检查
+        cursor.execute("SET FOREIGN_KEY_CHECKS=1")
+        
+        conn.commit()
+        print(f"  已插入 {total_inserted} 条记录到表: {table_name}")
+        
+    except Exception as e:
+        conn.rollback()
+        raise Exception(f"插入数据到表 {table_name} 失败: {str(e)}")
+    finally:
+        cursor.close()
+
+
+def sync_table(source_conn, target_conn, table_name: str):
+    """
+    同步单个表的数据
+    
+    Args:
+        source_conn: 源数据库连接
+        target_conn: 目标数据库连接
+        table_name: 表名
+    """
+    print(f"\n同步表: {table_name}")
+    print("-" * 60)
+    
+    try:
+        # 获取表的列名
+        columns = get_table_columns(source_conn, table_name)
+        print(f"  表列: {', '.join(columns)}")
+        
+        # 从源数据库获取数据
+        print(f"  从源数据库读取数据...")
+        source_data = get_table_data(source_conn, table_name)
+        print(f"  读取到 {len(source_data)} 条记录")
+        
+        # 清空目标表
+        print(f"  清空目标表...")
+        clear_table(target_conn, table_name)
+        
+        # 插入数据到目标表
+        if source_data:
+            print(f"  插入数据到目标表...")
+            insert_table_data(target_conn, table_name, columns, source_data)
+        else:
+            print(f"  表 {table_name} 没有数据需要同步")
+        
+        print(f"[OK] 表 {table_name} 同步完成")
+        
+    except Exception as e:
+        print(f"[ERROR] 表 {table_name} 同步失败: {str(e)}")
+        raise
+
+
+def main():
+    """主函数"""
+    print("=" * 60)
+    print("数据库表同步工具")
+    print("=" * 60)
+    print(f"源数据库: {SOURCE_DB_CONFIG['host']}:{SOURCE_DB_CONFIG['port']}/{SOURCE_DB_CONFIG['database']}")
+    print(f"目标数据库: {TARGET_DB_CONFIG['host']}:{TARGET_DB_CONFIG['port']}/{TARGET_DB_CONFIG['database']}")
+    print(f"同步表: {', '.join(TABLES_TO_SYNC)}")
+    print("=" * 60)
+    
+    # 步骤1: 备份目标数据库
+    try:
+        backup_file = backup_target_database()
+        print(f"\n[OK] 备份完成: {backup_file}\n")
+    except Exception as e:
+        print(f"\n[ERROR] 备份失败: {str(e)}")
+        response = input("是否继续同步？(y/n): ")
+        if response.lower() != 'y':
+            print("已取消同步")
+            sys.exit(1)
+    
+    # 步骤2: 连接数据库
+    print("=" * 60)
+    print("步骤 2: 连接数据库")
+    print("=" * 60)
+    
+    source_conn = None
+    target_conn = None
+    
+    try:
+        print("连接源数据库...")
+        source_conn = pymysql.connect(**SOURCE_DB_CONFIG)
+        print("[OK] 源数据库连接成功")
+        
+        print("连接目标数据库...")
+        try:
+            target_conn = pymysql.connect(**TARGET_DB_CONFIG, connect_timeout=10)
+            print("[OK] 目标数据库连接成功\n")
+        except pymysql.err.OperationalError as e:
+            if "timed out" in str(e) or "2003" in str(e):
+                print(f"[ERROR] 无法连接到目标数据库 {TARGET_DB_CONFIG['host']}:{TARGET_DB_CONFIG['port']}")
+                print("请检查：")
+                print("  1. 网络连接是否正常")
+                print("  2. 是否需要VPN连接")
+                print("  3. 数据库服务器是否可访问")
+                print("  4. 防火墙设置是否正确")
+            raise
+        
+        # 步骤3: 同步表数据
+        print("=" * 60)
+        print("步骤 3: 同步表数据")
+        print("=" * 60)
+        
+        for table_name in TABLES_TO_SYNC:
+            try:
+                sync_table(source_conn, target_conn, table_name)
+            except Exception as e:
+                print(f"\n[ERROR] 同步表 {table_name} 时发生错误: {str(e)}")
+                print("已停止同步")
+                sys.exit(1)
+        
+        print("\n" + "=" * 60)
+        print("[OK] 所有表同步完成！")
+        print("=" * 60)
+        
+    except pymysql.Error as e:
+        print(f"\n[ERROR] 数据库连接失败: {str(e)}")
+        if "timed out" in str(e) or "2003" in str(e):
+            print("\n提示：如果无法连接到目标数据库，请检查网络连接和VPN设置")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\n[ERROR] 发生错误: {str(e)}")
+        sys.exit(1)
+    finally:
+        if source_conn:
+            source_conn.close()
+        if target_conn:
+            target_conn.close()
+
+
+if __name__ == '__main__':
+    main()
+