403 lines
16 KiB
Python
403 lines
16 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
扫描 MySQL 数据库，分析表结构并生成配置
|
||
自动识别包含内容列的表（include 模式）：
|
||
- 只考虑表中包含 text、longtext、mediumtext、tinytext 类型的列
|
||
- 或者表中包含 blob、longblob、mediumblob、tinyblob 类型的列
|
||
支持通过 scan_exclude_config.json 配置文件额外排除不需要的数据库和表
|
||
"""
|
||
import pymysql
|
||
import json
|
||
import os
|
||
import re
|
||
from typing import List, Dict, Any, Set
|
||
from config import settings
|
||
|
||
def load_exclude_config(config_path: str = 'scan_exclude_config.json') -> Dict[str, Any]:
|
||
"""
|
||
加载排除配置文件
|
||
|
||
Args:
|
||
config_path: 配置文件路径
|
||
|
||
Returns:
|
||
排除配置字典，包含 exclude_databases, exclude_databases_patterns,
|
||
和 exclude_tables_patterns（仅支持正则表达式模式）
|
||
"""
|
||
default_config = {
|
||
"exclude_databases": ["information_schema", "performance_schema", "mysql", "sys"],
|
||
"exclude_databases_patterns": [],
|
||
"exclude_tables_patterns": {}
|
||
}
|
||
|
||
if not os.path.exists(config_path):
|
||
print(f"排除配置文件不存在: {config_path}，使用默认配置")
|
||
return default_config
|
||
|
||
try:
|
||
with open(config_path, 'r', encoding='utf-8') as f:
|
||
config = json.load(f)
|
||
# 确保配置格式正确
|
||
if 'exclude_databases' not in config:
|
||
config['exclude_databases'] = default_config['exclude_databases']
|
||
if 'exclude_databases_patterns' not in config:
|
||
config['exclude_databases_patterns'] = []
|
||
if 'exclude_tables_patterns' not in config:
|
||
config['exclude_tables_patterns'] = {}
|
||
return config
|
||
except Exception as e:
|
||
print(f"加载排除配置文件失败: {e}，使用默认配置")
|
||
return default_config
|
||
|
||
def matches_pattern(name: str, patterns: List[str]) -> bool:
|
||
"""
|
||
检查名称是否匹配任何正则表达式模式
|
||
|
||
Args:
|
||
name: 要检查的名称
|
||
patterns: 正则表达式模式列表
|
||
|
||
Returns:
|
||
如果匹配任何模式返回 True，否则返回 False
|
||
"""
|
||
for pattern in patterns:
|
||
try:
|
||
if re.search(pattern, name):
|
||
return True
|
||
except re.error as e:
|
||
print(f"警告: 正则表达式模式 '{pattern}' 无效: {e}")
|
||
continue
|
||
return False
|
||
|
||
def get_mysql_connection():
|
||
"""获取 MySQL 连接"""
|
||
return pymysql.connect(
|
||
host=settings.MYSQL_HOST,
|
||
port=settings.MYSQL_PORT,
|
||
user=settings.MYSQL_USER,
|
||
password=settings.MYSQL_PASSWORD,
|
||
charset=settings.MYSQL_CHARSET,
|
||
cursorclass=pymysql.cursors.DictCursor
|
||
)
|
||
|
||
def get_databases(conn, exclude_databases: Set[str] = None, exclude_patterns: List[str] = None) -> List[str]:
|
||
"""
|
||
获取所有数据库列表
|
||
|
||
Args:
|
||
conn: MySQL 连接
|
||
exclude_databases: 要排除的数据库集合（精确匹配）
|
||
exclude_patterns: 要排除的数据库正则表达式模式列表
|
||
|
||
Returns:
|
||
数据库列表
|
||
"""
|
||
if exclude_databases is None:
|
||
exclude_databases = set()
|
||
if exclude_patterns is None:
|
||
exclude_patterns = []
|
||
|
||
with conn.cursor() as cursor:
|
||
cursor.execute("SHOW DATABASES")
|
||
databases = [row['Database'] for row in cursor.fetchall()]
|
||
|
||
# 过滤掉排除的数据库
|
||
filtered_databases = []
|
||
for db in databases:
|
||
# 检查精确匹配
|
||
if db in exclude_databases:
|
||
continue
|
||
# 检查正则表达式匹配
|
||
if matches_pattern(db, exclude_patterns):
|
||
continue
|
||
filtered_databases.append(db)
|
||
|
||
return filtered_databases
|
||
|
||
def get_tables(conn, database: str) -> List[str]:
|
||
"""获取指定数据库的所有表"""
|
||
with conn.cursor() as cursor:
|
||
cursor.execute(f"USE `{database}`")
|
||
cursor.execute("SHOW TABLES")
|
||
# 表名在结果中的键名是 'Tables_in_{database}'
|
||
key = f"Tables_in_{database}"
|
||
return [row[key] for row in cursor.fetchall()]
|
||
|
||
def get_table_columns(conn, database: str, table: str) -> List[Dict[str, Any]]:
|
||
"""获取表的列信息"""
|
||
with conn.cursor() as cursor:
|
||
cursor.execute(f"USE `{database}`")
|
||
cursor.execute(f"DESCRIBE `{table}`")
|
||
return cursor.fetchall()
|
||
|
||
def should_include_table(table_name: str, columns: List[Dict[str, Any]]) -> bool:
|
||
"""
|
||
判断表是否应该被包含（仅根据列类型判断，并排除敏感信息表）
|
||
只考虑 text、longtext、mediumtext、tinytext 以及 blob 等类型的字段
|
||
排除包含敏感信息的表（authentication、private、key、secret、password 等）
|
||
|
||
Args:
|
||
table_name: 表名
|
||
columns: 表的列信息列表
|
||
|
||
Returns:
|
||
如果表应该被包含返回 True，否则返回 False
|
||
"""
|
||
# 排除敏感信息表（检查表名）
|
||
table_lower = table_name.lower()
|
||
sensitive_keywords = ['authentication', 'auth', 'private', 'key', 'secret', 'password',
|
||
'pwd', 'token', 'credential', 'credential', 'session', 'login',
|
||
'user_password', 'user_secret', 'api_key', 'access_key', 'secret_key']
|
||
|
||
# 检查表名是否包含敏感关键词
|
||
if any(keyword in table_lower for keyword in sensitive_keywords):
|
||
return False
|
||
|
||
# 检查列名是否包含敏感关键词
|
||
column_names = [col['Field'].lower() for col in columns]
|
||
if any(keyword in col_name for col_name in column_names for keyword in sensitive_keywords):
|
||
return False
|
||
|
||
column_types = [col['Type'].lower() for col in columns]
|
||
|
||
# 只检查是否有文本类型列（text、longtext、mediumtext、tinytext、blob、longblob、mediumblob、tinyblob）
|
||
text_types = ['text', 'longtext', 'mediumtext', 'tinytext', 'blob', 'longblob', 'mediumblob', 'tinyblob']
|
||
has_text_type = any(any(text_type in col_type for text_type in text_types) for col_type in column_types)
|
||
|
||
return has_text_type
|
||
|
||
def analyze_table_structure(columns: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||
"""分析表结构，推断配置项"""
|
||
column_names = [col['Field'] for col in columns]
|
||
column_types = {col['Field']: col['Type'].lower() for col in columns}
|
||
|
||
# 查找 ID 列（通常是 id, *_id）
|
||
id_column = None
|
||
for col in column_names:
|
||
if col.lower() == 'id':
|
||
id_column = col
|
||
break
|
||
elif col.lower().endswith('_id'):
|
||
id_column = col
|
||
break
|
||
|
||
# 查找内容列（只考虑 text、longtext、mediumtext、tinytext 以及 blob 等类型）
|
||
text_types = ['text', 'longtext', 'mediumtext', 'tinytext', 'blob', 'longblob', 'mediumblob', 'tinyblob']
|
||
content_columns = []
|
||
for col in column_names:
|
||
col_type = column_types.get(col, '').lower()
|
||
# 只包含 text 和 blob 类型的列
|
||
if any(text_type in col_type for text_type in text_types):
|
||
if col != id_column and not col.lower().endswith('_id'):
|
||
content_columns.append(col)
|
||
|
||
# 查找标题列（可能是 title, subject, name）
|
||
title_column = None
|
||
title_keywords = ['title', 'subject', 'name']
|
||
for col in column_names:
|
||
if col.lower() in title_keywords:
|
||
title_column = col
|
||
break
|
||
|
||
# 查找更新时间列（可能是 updated_at, updated_on, modified_at, update_time）
|
||
updated_at_column = None
|
||
update_keywords = ['updated_at', 'updated_on', 'modified_at', 'update_time', 'updated_time']
|
||
for col in column_names:
|
||
if col.lower() in update_keywords:
|
||
updated_at_column = col
|
||
break
|
||
|
||
# 查找元数据列（author, category 等，不包括时间相关列）
|
||
metadata_columns = []
|
||
# 时间相关的关键词，这些不应该出现在 metadata_columns 中
|
||
time_keywords = ['created_at', 'created_on', 'created_time', 'create_time', 'create_by',
|
||
'updated_at', 'updated_on', 'updated_time', 'update_time', 'modified_at',
|
||
'created_unix', 'updated_unix']
|
||
# 元数据关键词（不包括时间相关）
|
||
metadata_keywords = ['author', 'category', 'tag', 'tags', 'status', 'type', 'priority', 'source']
|
||
|
||
for col in column_names:
|
||
col_lower = col.lower()
|
||
# 排除时间相关的列
|
||
if any(time_keyword in col_lower for time_keyword in time_keywords):
|
||
continue
|
||
# 排除以 created_ 或 create_ 开头的列（时间相关）
|
||
if col_lower.startswith('created_') or col_lower.startswith('create_'):
|
||
continue
|
||
# 排除以 updated_ 或 update_ 开头的列（时间相关）
|
||
if col_lower.startswith('updated_') or col_lower.startswith('update_'):
|
||
continue
|
||
# 排除以 modified_ 开头的列（时间相关）
|
||
if col_lower.startswith('modified_'):
|
||
continue
|
||
# 添加符合条件的元数据列
|
||
if col_lower in metadata_keywords:
|
||
metadata_columns.append(col)
|
||
|
||
return {
|
||
'id_column': id_column or 'id',
|
||
'content_columns': content_columns[:3] if content_columns else ['content'], # 最多取3个
|
||
'title_column': title_column,
|
||
'updated_at_column': updated_at_column,
|
||
'metadata_columns': metadata_columns[:5] if metadata_columns else None # 最多取5个
|
||
}
|
||
|
||
def generate_config(conn, exclude_config: Dict[str, Any] = None) -> List[Dict[str, Any]]:
|
||
"""
|
||
生成配置列表
|
||
|
||
Args:
|
||
conn: MySQL 连接
|
||
exclude_config: 排除配置字典
|
||
|
||
Returns:
|
||
配置列表
|
||
"""
|
||
if exclude_config is None:
|
||
exclude_config = {
|
||
"exclude_databases": [],
|
||
"exclude_databases_patterns": [],
|
||
"exclude_tables_patterns": {}
|
||
}
|
||
|
||
configs = []
|
||
exclude_databases = set(exclude_config.get('exclude_databases', []))
|
||
exclude_db_patterns = exclude_config.get('exclude_databases_patterns', [])
|
||
exclude_table_patterns = exclude_config.get('exclude_tables_patterns', {})
|
||
|
||
databases = get_databases(conn, exclude_databases, exclude_db_patterns)
|
||
|
||
print(f"找到 {len(databases)} 个数据库: {', '.join(databases)}")
|
||
if exclude_databases:
|
||
print(f"排除的数据库（精确匹配）: {', '.join(sorted(exclude_databases))}")
|
||
if exclude_db_patterns:
|
||
print(f"排除的数据库（正则表达式）: {', '.join(exclude_db_patterns)}")
|
||
|
||
for database in databases:
|
||
print(f"\n扫描数据库: {database}")
|
||
try:
|
||
tables = get_tables(conn, database)
|
||
# 获取该数据库要排除的表（正则表达式模式）
|
||
db_exclude_patterns = exclude_table_patterns.get(database, [])
|
||
|
||
if db_exclude_patterns:
|
||
print(f" 排除的表（正则表达式）: {', '.join(db_exclude_patterns)}")
|
||
|
||
# 过滤掉排除的表（仅使用正则表达式模式）
|
||
filtered_tables = []
|
||
for table in tables:
|
||
# 检查正则表达式匹配
|
||
if matches_pattern(table, db_exclude_patterns):
|
||
continue
|
||
filtered_tables.append(table)
|
||
|
||
print(f" 找到 {len(tables)} 个表，排除后 {len(filtered_tables)} 个表")
|
||
|
||
# 进一步过滤：只包含有内容列的表
|
||
included_tables = []
|
||
excluded_by_content = []
|
||
|
||
for table in filtered_tables:
|
||
try:
|
||
columns = get_table_columns(conn, database, table)
|
||
if should_include_table(table, columns):
|
||
included_tables.append(table)
|
||
else:
|
||
excluded_by_content.append(table)
|
||
except Exception as e:
|
||
print(f" 检查表 {table} 时出错: {e}")
|
||
continue
|
||
|
||
print(f" 自动识别包含内容列的表: {len(included_tables)} 个")
|
||
if excluded_by_content:
|
||
print(f" 排除无内容列的表: {len(excluded_by_content)} 个")
|
||
if len(excluded_by_content) <= 10:
|
||
print(f" {', '.join(excluded_by_content)}")
|
||
else:
|
||
print(f" {', '.join(excluded_by_content[:10])} ... (共 {len(excluded_by_content)} 个)")
|
||
|
||
for table in included_tables:
|
||
print(f" 分析表: {table}")
|
||
try:
|
||
columns = get_table_columns(conn, database, table)
|
||
analysis = analyze_table_structure(columns)
|
||
|
||
# 生成配置项
|
||
config = {
|
||
'name': f"{database}_{table}",
|
||
'database': database,
|
||
'table_name': table,
|
||
'id_column': analysis['id_column'],
|
||
'content_column': ','.join(analysis['content_columns']),
|
||
'title_column': analysis['title_column'],
|
||
'metadata_columns': ','.join(analysis['metadata_columns']) if analysis['metadata_columns'] else None,
|
||
'content_separator': '\n',
|
||
'updated_at_column': analysis['updated_at_column']
|
||
}
|
||
|
||
# 清理 None 值
|
||
config = {k: v for k, v in config.items() if v is not None}
|
||
configs.append(config)
|
||
|
||
print(f" ID列: {analysis['id_column']}")
|
||
print(f" 内容列: {', '.join(analysis['content_columns'])}")
|
||
if analysis['title_column']:
|
||
print(f" 标题列: {analysis['title_column']}")
|
||
if analysis['updated_at_column']:
|
||
print(f" 更新时间列: {analysis['updated_at_column']}")
|
||
if analysis['metadata_columns']:
|
||
print(f" 元数据列: {', '.join(analysis['metadata_columns'])}")
|
||
|
||
except Exception as e:
|
||
print(f" 错误: {e}")
|
||
continue
|
||
|
||
except Exception as e:
|
||
print(f" 错误: {e}")
|
||
continue
|
||
|
||
return configs
|
||
|
||
def main():
|
||
"""主函数"""
|
||
try:
|
||
# 加载排除配置
|
||
exclude_config = load_exclude_config()
|
||
|
||
conn = get_mysql_connection()
|
||
print("成功连接到 MySQL 服务器")
|
||
print("=" * 60)
|
||
print("扫描模式: 自动识别包含内容列的表（include 模式）")
|
||
print("识别条件:")
|
||
print(" - 表中包含 text、longtext、mediumtext、tinytext 类型的列")
|
||
print(" - 或者表中包含 blob、longblob、mediumblob、tinyblob 类型的列")
|
||
print(" - 自动排除包含敏感信息的表（authentication、private、key、secret、password 等）")
|
||
print("=" * 60)
|
||
if os.path.exists('scan_exclude_config.json'):
|
||
print(f"额外排除配置: scan_exclude_config.json")
|
||
print()
|
||
|
||
configs = generate_config(conn, exclude_config)
|
||
|
||
conn.close()
|
||
|
||
# 输出 JSON 配置
|
||
print(f"\n\n生成 {len(configs)} 个配置项")
|
||
print("\n配置 JSON:")
|
||
print(json.dumps(configs, indent=2, ensure_ascii=False))
|
||
|
||
# 保存到文件
|
||
output_file = 'databases_config_new.json'
|
||
with open(output_file, 'w', encoding='utf-8') as f:
|
||
json.dump(configs, f, indent=2, ensure_ascii=False)
|
||
print(f"\n配置已保存到: {output_file}")
|
||
|
||
except Exception as e:
|
||
print(f"错误: {e}")
|
||
import traceback
|
||
traceback.print_exc()
|
||
|
||
if __name__ == '__main__':
|
||
main()
|
||
|