# +====================================================================== # | DjangoAdmin 企业数字化开发平台 [ 赋能开发者,助力企业发展 ] # +---------------------------------------------------------------------- # | Copyright (c) 2021-2026 小蚂蚁云团队 版权所有 # +---------------------------------------------------------------------- # | 开源协议: Apache-2.0 # | 重要声明: 未经书面许可,不得移除或修改本框架相关版权标识。 # +---------------------------------------------------------------------- # | 官方网站: https://www.djangoadmin.cn # +---------------------------------------------------------------------- # | 研发团队: 小蚂蚁云团队 荣誉出品 # +====================================================================== # | 版权和免责声明: # | 一、知识产权声明 # | 本团队对该软件框架产品(以下简称“本产品”)拥有完整知识产权,包括但不限于商标权、专利权、 # | 著作权、商业秘密等。前述权利均受中华人民共和国及相关国际法律法规保护。未经本团队书面授权, # | 任何个人或单位不得基于本产品本身申请任何形式的知识产权。 # | 二、使用限制与禁止条款 # | 您理解并同意,禁止将本产品用于以下任何行为: # | 1. 任何违反中华人民共和国法律法规的用途; # | 2. 任何侵害他人合法权益(包括但不限于人身权、财产权、知识产权等)的行为; # | 3. 任何危害国家安全、公共利益及社会道德风尚的行为。 # | 三、责任豁免 # | 因使用本产品而产生的任何损失或法律纠纷(包括但不限于直接损失、间接损失、附带损失、衍生损失 # | 以及因违约、侵权等引发的责任),本团队均不承担任何法律责任。若因违规使用本产品导致任何第三 # | 方提起索赔或行政机关进行调查,您应独立承担全部责任。 # | 四、违约与法律责任 # | 如发现任何违反本声明的行为,本团队有权: # | 1. 无条件配合公安机关及其他有权机关的调查取证工作; # | 2. 依法追究相关方的民事责任,并要求赔偿全部损失; # | 3. 保留进一步采取刑事控告等法律措施的权利。 # | 五、许可范围说明 # | 1、被授权方应视本产品源代码为核心机密信息,负有最高标准的保密与保管义务。禁止以任何形式(包 # | 括但不限于直接或间接的分享、复制、传播、分发、转售、出租、开源、反向工程、反编译、反汇编或通 # | 过网络服务提供给任何第三方)泄露、披露或处置源代码的全部或部分内容。任何未经授权的复制、使用、 # | 传播行为均构成对本团队知识产权的严重侵权,本团队有权依法追究其全部法律责任,并要求赔偿因此造 # | 成的一切经济损失及商誉损害。 # | 2、本产品仅限于公司或个人在法律允许范围内,用于内部且合规的软件产品研发。具体权利义务以《软件 # | 许可协议》附件为准。 # +====================================================================== # +====================================================================== # | 模块: 跨库数据迁移脚本 # | 说明: 把源库的数据整体搬到目标库(MySQL → PostgreSQL / SQL Server / SQLite 等) # +====================================================================== """ 跨库数据迁移脚本(MySQL → PostgreSQL / SQL Server / SQLite 等) ============================================================= 把「源库」的数据整体搬到「目标库」: - 目标库结构用模型 metadata 生成(Base.metadata.create_all,方言可移植, 与 init_db.py / alembic 0001_baseline 同源,模型即 head,含 0002 的 city.parent_code nullable); - 数据按拓扑序逐表 SELECT ... ORDER BY id,显式带 id 批量插入,保住 无外键裸列引用关系(user.dept_id、role_menu.role_id/menu_id 等); - 最后按方言重置自增序列(PG setval / MSSQL IDENTITY_INSERT + DBCC CHECKIDENT / Oracle 为每表重建 sequence + BEFORE INSERT 触发器; MySQL / SQLite 显式插 id 后自动跟随,无需处理)。 与 init_db.py 的分工: - init_db.py 单库初始化(建库 + 导 MySQL dump + 补表 + alembic stamp) - migrate_db.py 双库迁移(源库 → 目标库,CLI 显式参数,不依赖 .env 连接串) 用法: python scripts/migrate_db.py \ --src-driver=mysql --src-host=127.0.0.1 --src-port=3306 \ --src-db=djangoadmin.fastapi.elevue --src-user=root --src-pass=root \ --dst-driver=postgresql --dst-host=127.0.0.1 --dst-port=5432 \ --dst-db=djangoadmin.fastapi.elevue --dst-user=root --dst-pass=root \ --drop-target-first 先干跑核对计划:加 --dry-run(只连源、逐表数行数、打印计划,不碰目标库)。 """ import argparse import os import sys import time from pathlib import Path from urllib.parse import quote_plus sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "src")) # Windows 控制台默认 GBK(cp936):✓ 等字符无法编码会崩溃、中文按终端编码显示成乱码。 # 统一把 stdout/stderr 切到 UTF-8,Git Bash / 重定向均正常;无 reconfigure 的环境跳过。 if sys.stdout and hasattr(sys.stdout, 'reconfigure'): sys.stdout.reconfigure(encoding='utf-8') sys.stderr.reconfigure(encoding='utf-8') from sqlalchemy import create_engine, inspect, select, text from sqlalchemy.pool import NullPool from core.config import DB_PREFIX from core.config.database import _normalize_driver, _sqlite_url from core.database import Base # 各驱动默认端口(sqlite 无端口概念) _DEFAULT_PORTS = {"mysql": 3306, "postgresql": 5432, "mssql": 1433, "oracle": 1521, "sqlite": None} # 迁移顺序:基础/被引用表在前,依赖表在后。项目无外键,表间关联是裸 Integer 列 # 由应用层维护,显式带 id 插入即保住引用关系;保序只是锦上添花(父先子后)。 _BASE_TABLES = [ "dept", "menu", "role", "level", "position", "city", "category", "dict", "config", "file_template", "param", "job", "link", ] _DEP_TABLES = [ "role_menu", "dict_item", "config_item", "job_log", "user", "user_role", "article", "notice", "operation_log", "login_log", ] # ============================================================ # CLI 参数解析 # ============================================================ def build_parser() -> argparse.ArgumentParser: """CLI 参数:源/目标两端连接显式给出,不依赖 .env""" p = argparse.ArgumentParser(description="跨库数据迁移:源库 → 目标库") def add_side(side: str, default_user: str): p.add_argument(f"--{side}-driver", choices=("mysql", "postgresql", "mssql", "oracle", "sqlite"), default="mysql", help="驱动(oracle 时 --{side}-db 即 service_name;sqlite 时即数据库文件路径)") p.add_argument(f"--{side}-host", default="127.0.0.1") p.add_argument(f"--{side}-port", type=int, default=None, help="端口,缺省按驱动默认(mysql 3306 / pg 5432 / mssql 1433 / oracle 1521)") p.add_argument(f"--{side}-db", required=True, help="数据库名(sqlite 时为文件路径)") p.add_argument(f"--{side}-user", default=default_user) p.add_argument(f"--{side}-pass", default="", help=f"口令,留空回落环境变量 {side.upper()}_DB_PASS(避免进 shell 历史)") add_side("src", "root") add_side("dst", "postgres") p.add_argument("--drop-target-first", action="store_true", help="先删目标库再重建(默认关)") p.add_argument("--dry-run", action="store_true", help="只连源、逐表数行数、打印计划,不碰目标库") p.add_argument("--chunk-size", type=int, default=1000, help="每批插入行数(默认 1000)") p.add_argument("--only-tables", default="", help="逗号分隔的表名子集(全名或去掉前缀的短名),续传用") p.add_argument("--skip-existing", action="store_true", help="目标已等于源行数的表跳过") p.add_argument("--skip-verify", action="store_true", help="跳过迁移后的逐表行数校验") p.add_argument("--single-transaction", action="store_true", help="全部表包进一个大事务(默认每表一个事务)") return p # ============================================================ # URL / 引擎 # ============================================================ def make_url(driver: str, host: str, port: int, database: str, username: str, password: str) -> str: """带显式参数的连接串构建器,URL 形态与 core.config.database.build_database_url 逐字等价""" drv = _normalize_driver(driver) if drv == 'sqlite': return _sqlite_url(database) base = quote_plus(username) + ':' + quote_plus(password) + '@' + host + ':' + str(port) if drv == 'oracle': return 'oracle+oracledb://' + base + '/?service_name=' + quote_plus(database) base = base + '/' + database if drv == 'mysql': return 'mysql+pymysql://' + base + '?charset=utf8mb4' if drv == 'postgresql': return 'postgresql+psycopg://' + base return 'mssql+pymssql://' + base + '?charset=utf8' def make_engine(driver: str, host: str, port: int, database: str, username: str, password: str): """迁移专用引擎:NullPool,避免连接池把目标/源的会话状态串起来""" return create_engine(make_url(driver, host, port, database, username, password), poolclass=NullPool) def _resolve_port(driver: str, port: int | None) -> int | None: return port if port is not None else _DEFAULT_PORTS[driver] def _fmt_duration(sec: float) -> str: """耗时格式化:<1s 显示毫秒,<60s 显示秒,否则分+秒""" if sec < 1: return f"{sec * 1000:.0f}ms" if sec < 60: return f"{sec:.1f}s" return f"{int(sec // 60)}m{int(sec % 60)}s" _BUSY = "[·]" _DONE = "[✓]" def _phase_start(n: int, total: int, desc: str, hint: bool = True): """阶段开始横幅""" tail = "(可能耗时较长,请勿中断)" if hint else "" print(f"{_BUSY} [{n}/{total}] {desc}...{tail}") def _phase_done(n: int, total: int, desc: str, t0: float, extra: str = ""): """阶段完成横幅""" print(f"{_DONE} [{n}/{total}] {desc} 完成(耗时 {_fmt_duration(time.perf_counter() - t0)}){extra}") # ============================================================ # 建目标库 # ============================================================ def create_database_if_missing(driver: str, host: str, port: int, database: str, username: str, password: str, drop_first: bool = False, tables: tuple = ()): """按方言连维护库建目标库(不存在才建)。sqlite 无建库概念,只确保父目录并可选删旧文件""" if driver == 'sqlite': file = Path(database).expanduser() print(f" · 检查/准备 sqlite 文件 {file}...") if file.parent.as_posix() != '.': file.parent.mkdir(parents=True, exist_ok=True) if drop_first and file.exists(): print(f" · 删除旧 sqlite 文件 {file}...") file.unlink() return if driver == 'mysql': import pymysql print(" · 正在连接 MySQL(无库连接)...") conn = pymysql.connect(host=host, port=int(port), user=username, password=password, autocommit=True) cur = conn.cursor() if drop_first: print(f" · 正在删除旧目标库 `{database}`...") cur.execute(f"DROP DATABASE IF EXISTS `{database}`") print(f" · 正在创建目标库 `{database}`(utf8mb4)...") cur.execute( f"CREATE DATABASE IF NOT EXISTS `{database}` " f"DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci" ) cur.close() conn.close() return if driver == 'postgresql': import psycopg print(" · 正在连接 postgres 维护库...") conn = psycopg.connect( host=host, port=int(port), user=username, password=password, dbname="postgres", autocommit=True, ) with conn.cursor() as cur: cur.execute("SELECT 1 FROM pg_database WHERE datname = %s", (database,)) exists = cur.fetchone() is not None if drop_first and exists: print(f" · 正在删除旧目标库 \"{database}\"(WITH FORCE)...") cur.execute(f'DROP DATABASE "{database}" WITH (FORCE)') exists = False if not exists: print(f" · 正在创建目标库 \"{database}\"(UTF8)...") cur.execute(f'CREATE DATABASE "{database}" ENCODING \'UTF8\' TEMPLATE template0') conn.close() return if driver == 'oracle': import oracledb print(f" · 正在连接 Oracle 服务 {database}(schema={username})...") conn = oracledb.connect(user=username, password=password, dsn=f"{host}:{port}/{database}") cur = conn.cursor() cur.execute("SELECT 1 FROM dual") cur.fetchone() if drop_first: print(f" · drop-first:清理 {len(tables)} 张旧项目表(DROP TABLE ... PURGE)...") for t in tables: try: cur.execute(f"DROP TABLE {t} PURGE") except Exception: pass conn.close() return # mssql import pymssql print(" · 正在连接 SQL Server(master 维护库)...") conn = pymssql.connect( server=host, user=username, password=password, port=int(port), database="master", autocommit=True, ) cur = conn.cursor() if drop_first: print(f" · 正在删除旧目标库 [{database}](SINGLE_USER + DROP)...") cur.execute( f"IF DB_ID(N'{database}') IS NOT NULL " f"ALTER DATABASE [{database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE" ) cur.execute(f"IF DB_ID(N'{database}') IS NOT NULL DROP DATABASE [{database}]") print(f" · 正在创建目标库 [{database}]...") cur.execute(f"IF DB_ID(N'{database}') IS NULL CREATE DATABASE [{database}]") cur.close() conn.close() # ============================================================ # 迁移顺序 # ============================================================ def ordered_tables(): """按拓扑序返回要迁移的 Table 对象(base 先、dep 后;metadata 中未列出的追加末尾)""" out, seen = [], set() for name in _BASE_TABLES + _DEP_TABLES: tbl = Base.metadata.tables.get(DB_PREFIX + name) if tbl is not None: out.append(tbl) seen.add(tbl.name) for tbl in Base.metadata.tables.values(): if tbl.name not in seen: out.append(tbl) return out # ============================================================ # 泵数据 # ============================================================ def sanitize(payload: list[dict], table) -> list[dict]: """把驱动读回的零日期/非法日期字符串统一置 None(防御性兜底)""" for row in payload: for col in table.columns: v = row.get(col.name) if isinstance(v, str) and v.startswith('0000-00-00'): row[col.name] = None return payload def pump_table(src, dst, table, chunk_size: int, driver: str) -> int: """逐 chunk 批量插入一张表,显式带 id 以保住裸列引用关系""" pk = next(iter(table.primary_key.columns)) sel = select(*table.columns).order_by(pk) use_identity = (driver == 'mssql') total = 0 t0 = time.perf_counter() if use_identity: dst.execute(text(f"SET IDENTITY_INSERT [{table.name}] ON")) try: result = src.execute(sel) while True: rows = result.fetchmany(chunk_size) if not rows: break payload = sanitize([dict(r._mapping) for r in rows], table) dst.execute(table.insert(), payload) total += len(payload) print(f" [{table.name}] +{len(payload)}(累计 {total})") finally: if use_identity: dst.execute(text(f"SET IDENTITY_INSERT [{table.name}] OFF")) print(f" [✓ {table.name}] 共 {total} 行(耗时 {_fmt_duration(time.perf_counter() - t0)})") return total # ============================================================ # 自增序列重置 # ============================================================ def pg_reset_sql(table_name: str) -> str: """PostgreSQL:把 SERIAL/IDENTITY 序列推进到 MAX(id)""" return ( f"SELECT setval(pg_get_serial_sequence('{table_name}', 'id'), " f"COALESCE((SELECT MAX(id) FROM \"{table_name}\"), 1), " f"(SELECT MAX(id) FROM \"{table_name}\") IS NOT NULL)" ) def mssql_reset_sql(table_name: str, max_id) -> str: """SQL Server:RESEED 必须是字面量,先查 MAX(id) 再拼""" return f"DBCC CHECKIDENT ('{table_name}', RESEED, {max_id or 0})" def oracle_sequence_sql(table_name: str, start_with: int) -> str: """Oracle:为 INTEGER 主键建 sequence""" return f"CREATE SEQUENCE {table_name}_id_seq START WITH {start_with} INCREMENT BY 1 NOCACHE" def oracle_trigger_sql(table_name: str) -> str: """Oracle:BEFORE INSERT 触发器,id 为 NULL 时从 sequence 取 NEXTVAL""" seq = f"{table_name}_id_seq" trg = f"{table_name}_id_trg" return ( f"CREATE OR REPLACE TRIGGER {trg}\n" f"BEFORE INSERT ON {table_name} FOR EACH ROW\n" f"WHEN (NEW.id IS NULL)\n" f"BEGIN SELECT {seq}.NEXTVAL INTO :NEW.id FROM dual; END;" ) def ensure_oracle_autoincrement(conn, table_name: str): """Oracle 迁移后补自增机制""" max_id = conn.execute(text(f"SELECT COALESCE(MAX(id), 0) FROM {table_name}")).scalar() seq = f"{table_name}_id_seq" exists = conn.execute(text( "SELECT COUNT(*) FROM user_sequences WHERE sequence_name = UPPER(:n)" ), {"n": seq}).scalar() if exists: conn.execute(text(f"DROP SEQUENCE {seq}")) conn.execute(text(oracle_sequence_sql(table_name, max_id + 1))) conn.execute(text(oracle_trigger_sql(table_name))) def reset_sequence(driver: str, conn, table): """迁移后重置自增序列""" if driver == 'postgresql': conn.execute(text(pg_reset_sql(table.name))) elif driver == 'mssql': max_id = conn.execute(text(f"SELECT MAX(id) FROM [{table.name}]")).scalar() conn.execute(text(mssql_reset_sql(table.name, max_id))) elif driver == 'oracle': ensure_oracle_autoincrement(conn, table.name) # ============================================================ # 校验 # ============================================================ def verify_counts(src, dst, tables) -> list[tuple[str, int, int]]: """逐表 count 对比,返回不一致的表列表""" failed = [] for t in tables: s = src.execute(text(f"SELECT COUNT(*) FROM {t.name}")).scalar() d = dst.execute(text(f"SELECT COUNT(*) FROM {t.name}")).scalar() mark = "PASS" if s == d else "FAIL" print(f" [{t.name}] source={s} target={d} {mark}") if s != d: failed.append((t.name, s, d)) return failed def _existing_equal(src, dst, table) -> bool: s = src.execute(text(f"SELECT COUNT(*) FROM {table.name}")).scalar() d = dst.execute(text(f"SELECT COUNT(*) FROM {table.name}")).scalar() return s == d # ============================================================ # 主流程 # ============================================================ def _dry_run(src_engine, tables, dst_driver: str, dst_db: str, t_start: float) -> int: print("=" * 62) print(f"[DRY-RUN 开始] 计划迁移 {len(tables)} 张表 → {dst_driver}:{dst_db}") print("=" * 62) with src_engine.connect() as src: for t in tables: n = src.execute(text(f"SELECT COUNT(*) FROM {t.name}")).scalar() print(f" - {t.name}: {n} 行") print(f"[DRY-RUN 完成] {len(tables)} 张表核对完毕(耗时 {_fmt_duration(time.perf_counter() - t_start)}),未触碰目标库") return 0 def run(args) -> int: """执行一次迁移。返回进程退出码(0 成功 / 1 校验失败)""" src_driver = _normalize_driver(args.src_driver) dst_driver = _normalize_driver(args.dst_driver) src_engine = make_engine( src_driver, args.src_host, _resolve_port(src_driver, args.src_port), args.src_db, args.src_user, args.src_pass or os.environ.get("SRC_DB_PASS", ""), ) dst_engine = make_engine( dst_driver, args.dst_host, _resolve_port(dst_driver, args.dst_port), args.dst_db, args.dst_user, args.dst_pass or os.environ.get("DST_DB_PASS", ""), ) # 加载全部模型填充 Base.metadata from core.app import create_app create_app() tables = ordered_tables() if args.only_tables: wanted = {x.strip() for x in args.only_tables.split(",") if x.strip()} tables = [t for t in tables if t.name in wanted or t.name[len(DB_PREFIX):] in wanted] try: src_insp = inspect(src_engine) present, missing = [], [] for t in tables: if src_insp.has_table(t.name): present.append(t) else: missing.append(t) for t in missing: print(f"[SKIP] 源库不存在表 {t.name},跳过") t_start = time.perf_counter() if args.dry_run: return _dry_run(src_engine, present, dst_driver, args.dst_db, t_start) print("=" * 62) print(f"[迁移开始] 源: {src_driver}:{args.src_db} → 目标: {dst_driver}:{args.dst_db}") note = "" if not missing else f",源端缺失 {len(missing)} 张已跳过" print(f"[迁移开始] 待迁移 {len(present)} 张表{note}") print("=" * 62) # 1) 建目标库 t0 = time.perf_counter() _phase_start(1, 5, "建目标库") create_database_if_missing( dst_driver, args.dst_host, _resolve_port(dst_driver, args.dst_port), args.dst_db, args.dst_user, args.dst_pass or os.environ.get("DST_DB_PASS", ""), drop_first=args.drop_target_first, tables=tuple(t.name for t in present), ) _phase_done(1, 5, "建目标库", t0) # 2) 建结构 t0 = time.perf_counter() _phase_start(2, 5, f"建表结构({len(present)} 张表 create_all)") Base.metadata.create_all(dst_engine) _phase_done(2, 5, f"建表结构({len(present)} 张表 create_all)", t0, extra=f",{len(present)} 张表") # 3) 逐表迁移数据 t0 = time.perf_counter() _phase_start(3, 5, "逐表迁移数据") total_rows = 0 n_present = len(present) if args.single_transaction: with dst_engine.begin() as dst, src_engine.connect() as src: for idx, t in enumerate(present, 1): if args.skip_existing and _existing_equal(src, dst, t): print(f" · [{idx}/{n_present}] 跳过 {t.name}(源=目标)") continue print(f" · [{idx}/{n_present}] 正在迁移 {t.name}...") total_rows += pump_table(src, dst, t, args.chunk_size, dst_driver) else: with src_engine.connect() as src: for idx, t in enumerate(present, 1): with dst_engine.begin() as dst: if args.skip_existing and _existing_equal(src, dst, t): print(f" · [{idx}/{n_present}] 跳过 {t.name}(源=目标)") continue print(f" · [{idx}/{n_present}] 正在迁移 {t.name}...") total_rows += pump_table(src, dst, t, args.chunk_size, dst_driver) _phase_done(3, 5, "逐表迁移数据", t0, extra=f",共 {total_rows} 行") # 4) 重置自增序列 reset_errors = [] t0 = time.perf_counter() _phase_start(4, 5, "重置自增序列") if dst_driver in ("postgresql", "mssql", "oracle"): with dst_engine.begin() as dst: for t in present: try: reset_sequence(dst_driver, dst, t) except Exception as e: reset_errors.append((t.name, str(e))) print(f" · [WARN] 序列重置失败 {t.name}: {e}") tail = f",{len(reset_errors)} 张失败" if reset_errors else "" _phase_done(4, 5, "重置自增序列", t0, extra=tail) else: print(" · MySQL/SQLite 自动跟随,无需处理") _phase_done(4, 5, "重置自增序列", t0, extra="(无需处理)") # 5) 校验 t0 = time.perf_counter() _phase_start(5, 5, "逐表行数校验") if not args.skip_verify: with src_engine.connect() as src, dst_engine.connect() as dst: failed = verify_counts(src, dst, present) if failed: _phase_done(5, 5, "逐表行数校验", t0, extra=f",FAIL {len(failed)} 张") for name, s, d in failed: print(f" - {name}: source={s} target={d}") return 1 _phase_done(5, 5, "逐表行数校验", t0, extra=f",{len(present)} 张表全部 PASS") else: print(" · 已跳过(--skip-verify)") _phase_done(5, 5, "逐表行数校验", t0, extra="(已跳过)") print("=" * 62) print(f"[迁移完成] 成功:{total_rows} 行 / {len(present)} 张表 / 总耗时 {_fmt_duration(time.perf_counter() - t_start)}") print("=" * 62) return 0 finally: src_engine.dispose() dst_engine.dispose() def main(): args = build_parser().parse_args() try: sys.exit(run(args)) except KeyboardInterrupt: sys.exit(130) if __name__ == "__main__": main()