Nigel
/
RAG
0
0
Fork 2
Code Issues Pull Requests 1 Actions Packages Projects Releases 2 Wiki Activity

增加了chat和可视化配置界面

This commit is contained in:
zhangxunhui 2026-01-22 13:04:29 +08:00
parent b02b56af1a
commit 3ac4ad987c
20 changed files with 3699 additions and 2082 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.env.example
View File

@ -45,7 +45,6 @@ MYSQL_PORT=3306
MYSQL_USER=root
MYSQL_PASSWORD=your_password
MYSQL_DATABASE=forgeplus
MYSQL_CHARSET=utf8mb4
# 多数据库配置（可选）
# 方式1: 使用配置文件（推荐）
@ -77,7 +76,6 @@ MYSQL_CHARSET=utf8mb4
# "mysql_port": 3306,
# "mysql_user": "remote_user",
# "mysql_password": "remote_password",
# "mysql_charset": "utf8mb4",
# "comment": "使用独立的 MySQL 连接配置（不同的服务器）"
# }
# ]

801
api/main.py
View File

@ -14,11 +14,11 @@ from fastapi.responses import StreamingResponse, Response, FileResponse
from fastapi.staticfiles import StaticFiles
from fastapi.openapi.docs import get_swagger_ui_html, get_redoc_html
from pydantic import BaseModel, Field
from typing import Optional, List
from typing import Optional, List, Dict, Any
from loguru import logger
from config import settings
from rag import VectorStoreManager, RAGEngine, FileParser, DocumentProcessor
from sync_service import SyncService
from sync_service import SyncServiceManager
import requests
from datetime import datetime
import time
@ -40,7 +40,7 @@ import hashlib
# Global instances
vector_store_manager: Optional[VectorStoreManager] = None
rag_engine: Optional[RAGEngine] = None
sync_service: Optional[SyncService] = None
sync_manager: Optional[SyncServiceManager] = None
file_parser: Optional[FileParser] = None
document_processor: Optional[DocumentProcessor] = None
auto_sync_task = None # Keep reference to prevent garbage collection
@ -49,7 +49,7 @@ auto_sync_task = None # Keep reference to prevent garbage collection
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Lifespan context manager for startup and shutdown events"""
global vector_store_manager, rag_engine, sync_service, file_parser, document_processor, auto_sync_task
global vector_store_manager, rag_engine, sync_manager, file_parser, document_processor, auto_sync_task
# Startup
try:
@ -93,66 +93,26 @@ async def lifespan(app: FastAPI):
# This allows API to start immediately even if MySQL connection fails
async def init_sync_service():
"""Initialize sync service and start sync in background"""
global sync_service
global sync_manager
try:
logger.info("Initializing MySQL sync service in background...")
logger.info("Initializing sync services in background...")
# Run SyncService initialization in thread pool to avoid blocking event loop
# SyncService.__init__() contains synchronous MySQL connection checks
# Run SyncServiceManager initialization in thread pool to avoid blocking event loop
# SyncServiceManager.__init__() contains synchronous data source connection checks
loop = asyncio.get_event_loop()
sync_service = await loop.run_in_executor(None, SyncService)
sync_manager = await loop.run_in_executor(None, SyncServiceManager)
# Start initial sync in background and wait for it to complete before starting auto sync
# This ensures API starts immediately without blocking, but auto sync waits for initial sync
async def run_sync():
"""Run sync_all in background without blocking"""
try:
logger.info("Starting initial data sync in background...")
await sync_service.sync_all() # 全量同步 MySQL 数据到向量库
logger.info("✓ Initial sync completed")
return True
except Exception as e:
logger.error(f"Error during initial sync: {e}")
return False
# Create task for initial sync (don't await - allows API to start immediately)
initial_sync_task = asyncio.create_task(run_sync())
# Start auto sync if enabled
# Auto sync service will wait for initial sync to complete before starting incremental syncs
# 自动同步服务将在初始同步（全量同步）完成后开始增量同步
# Start all sync services in background
# This will start auto sync with recovery for each data source
if settings.AUTO_SYNC:
logger.info(f"Starting auto sync service (interval: {settings.SYNC_INTERVAL}s)...")
# Start auto sync in background - it will wait for initial sync to complete
async def start_auto_sync_after_init():
"""Wait for initial sync to complete, then start auto sync with error recovery"""
try:
# Wait for initial sync to complete (no timeout - wait indefinitely)
logger.info("Auto sync service waiting for initial sync to complete (no timeout, will wait until completion)...")
try:
initial_sync_success = await initial_sync_task
if initial_sync_success:
logger.info("✓ Initial sync completed successfully")
else:
logger.warning("Initial sync failed, but starting auto sync service anyway...")
except Exception as sync_error:
logger.error(f"Initial sync encountered an error: {sync_error}", exc_info=True)
logger.warning("Starting auto sync service despite initial sync error...")
# Start auto sync service with error recovery (it will skip initial sync since we already did it)
await sync_service.start_auto_sync_with_recovery(skip_initial_sync=True)
except Exception as e:
logger.error(f"Error starting auto sync service: {e}", exc_info=True)
# Start auto sync service in background (don't await - allows API to start immediately)
asyncio.create_task(start_auto_sync_after_init())
logger.info(f"Starting auto sync services (interval: {settings.SYNC_INTERVAL}s)...")
# Start all sync services in background
await sync_manager.start_all_sync_services()
else:
# If auto sync is disabled, just run initial sync in background
# Don't await it to avoid blocking
pass
logger.info("Auto sync is disabled, skipping auto sync service startup")
except Exception as e:
logger.error(f"Failed to initialize sync service: {e}")
logger.warning("API is still available, but MySQL sync is disabled. Please check MySQL connection.")
logger.error(f"Failed to initialize sync services: {e}")
logger.warning("API is still available, but sync services are disabled. Please check data source connections.")
# Don't raise - allow API to continue running
# Start sync service initialization in background (non-blocking)
@ -180,11 +140,9 @@ async def lifespan(app: FastAPI):
await auto_sync_task
except asyncio.CancelledError:
pass
if sync_service:
if hasattr(sync_service, 'stop_auto_sync'):
sync_service.stop_auto_sync()
if hasattr(sync_service, 'close'):
sync_service.close()
if sync_manager:
sync_manager.stop_all_sync_services()
sync_manager.close_all()
logger.info("RAG services shut down")
@ -336,6 +294,7 @@ class SyncRequest(BaseModel):
"""Manual sync request model"""
full_sync: bool = Field(False, description="Whether to perform full sync")
force: bool = Field(False, description="Whether to force re-processing of all documents (even if they exist)")
source_name: Optional[str] = Field(None, description="Specific data source to sync (optional, syncs all if not provided)")
class HealthResponse(BaseModel):
@ -1051,6 +1010,29 @@ async def retrieve(request: RetrieveRequest):
raise HTTPException(status_code=500, detail=f"Error retrieving documents: {str(e)}")
@app.delete("/documents/source/{source_name}")
async def delete_documents_by_source(source_name: str):
"""
Delete all documents from a specific data source
Args:
source_name: Name of the data source to delete documents from
Returns:
Deletion status
"""
if vector_store_manager is None:
raise HTTPException(status_code=503, detail="Vector store manager not initialized")
try:
# Delete documents by source name
vector_store_manager.delete_documents_by_source(source_name)
return {"status": "success", "message": f"Deleted all documents from {source_name}"}
except Exception as e:
logger.error(f"Error deleting documents by source: {e}")
raise HTTPException(status_code=500, detail=f"Error deleting documents: {str(e)}")
@app.post("/sync")
async def manual_sync(request: SyncRequest):
"""
@ -1062,17 +1044,47 @@ async def manual_sync(request: SyncRequest):
Returns:
Sync status
"""
if sync_service is None:
if sync_manager is None:
raise HTTPException(status_code=503, detail="Sync service not initialized")
try:
if request.full_sync:
await sync_service.sync_all(force=request.force)
message = "Full sync completed" + (" (forced re-processing)" if request.force else "")
return {"status": "success", "message": message}
if request.source_name:
# Sync specific data source
if hasattr(sync_manager, 'get_sync_service'): # Check if it's a SyncServiceManager
specific_sync_service = sync_manager.get_sync_service(request.source_name)
if not specific_sync_service:
raise HTTPException(status_code=404, detail=f"Sync service not found for source: {request.source_name}")
if request.full_sync:
await specific_sync_service.sync_all(force=request.force, is_manual=True)
message = f"Full sync completed for {request.source_name}" + (" (forced re-processing)" if request.force else "")
else:
await specific_sync_service.sync_incremental(is_manual=True)
message = f"Incremental sync completed for {request.source_name}"
else:
# Old SyncService that handles all sources
if request.full_sync:
await sync_manager.sync_all(force=request.force)
message = "Full sync completed for all sources" + (" (forced re-processing)" if request.force else "")
else:
await sync_manager.sync_incremental()
message = "Incremental sync completed for all sources"
else:
await sync_service.sync_incremental()
return {"status": "success", "message": "Incremental sync completed"}
# Sync all data sources
if hasattr(sync_manager, 'start_all_sync_services'):
# This is a SyncServiceManager, it starts auto sync services which do initial sync
await sync_manager.start_all_sync_services()
message = "Started synchronization for all data sources"
else:
# Old SyncService
if request.full_sync:
await sync_manager.sync_all(force=request.force)
message = "Full sync completed for all sources" + (" (forced re-processing)" if request.force else "")
else:
await sync_manager.sync_incremental()
message = "Incremental sync completed for all sources"
return {"status": "success", "message": message}
except Exception as e:
logger.error(f"Error during sync: {e}")
raise HTTPException(status_code=500, detail=f"Sync failed: {str(e)}")
@ -1314,45 +1326,286 @@ async def get_folder_configs():
List of all configurations with their IDs and details
"""
try:
config_dir = os.path.join(os.path.dirname(__file__), "..", "configs")
if not os.path.exists(config_dir):
os.makedirs(config_dir)
return {"configurations": []}
# Load configs from SQLite database
DATA_DIR = Path(__file__).parent.parent / "data"
DATA_DIR.mkdir(parents=True, exist_ok=True)
DB_PATH = DATA_DIR / "sessions.db"
configurations = []
config_files = [f for f in os.listdir(config_dir) if f.endswith(".json")]
conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor()
for config_file in config_files:
config_path = os.path.join(config_dir, config_file)
with open(config_path, "r", encoding="utf-8") as f:
config_data = json.load(f)
# 查询所有数据源配置
try:
cursor.execute('SELECT name, config, update_at FROM data_sources')
rows = cursor.fetchall()
# Handle both single config and array
if isinstance(config_data, list):
for idx, config in enumerate(config_data):
configurations = []
for row in rows:
name, config_json, update_at = row
try:
config_data = json.loads(config_json)
configurations.append({
"id": f"{os.path.splitext(config_file)[0]}-{idx}",
"type": config.get("type", "unknown"),
"config": config
"id": name,
"type": config_data.get("type", "unknown"),
"config": config_data,
"update_at": update_at
})
else:
configurations.append({
"id": os.path.splitext(config_file)[0],
"type": config_data.get("type", "unknown"),
"config": config_data
})
except Exception as e:
logger.error(f"Error parsing config from database: {name}, error: {e}")
except sqlite3.OperationalError as e:
# 表不存在的情况，返回空列表
logger.warning(f"SQLite table error: {e}. Returning empty config list.")
configurations = []
# 关闭数据库连接
conn.close()
return {"configurations": configurations}
except Exception as e:
logger.error(f"Error getting folder configs: {e}")
raise HTTPException(status_code=500, detail=f"Error getting configurations: {str(e)}")
# 返回空列表而不是错误响应
return {"configurations": []}
@app.post("/folder-configs")
async def create_config(config: Dict[str, Any]):
"""
Create a new configuration
Args:
config: Configuration data for the new config
Returns:
Created configuration with ID
"""
conn = None
try:
# Ensure config has required fields
if "type" not in config:
raise HTTPException(status_code=400, detail="Config type is required")
config_type = config["type"]
# Check for unique identifier based on config type
# 必须有数据源唯一标识才能创建配置
unique_id_parts = [config_type]
if config_type == "database":
# 数据库配置需要：主机、端口、数据库名、表名
if not config.get("mysql_host") or not config.get("mysql_port") or not config.get("database") or not config.get("table_name"):
raise HTTPException(status_code=400, detail="数据库配置必须包含主机、端口、数据库名和表名")
unique_id_parts.extend([
config["mysql_host"].lower(),
str(config["mysql_port"]),
config["database"].lower(),
config["table_name"].lower()
])
elif config_type == "local_folder":
# 本地文件夹配置需要：文件夹路径
if not config.get("folder_path"):
raise HTTPException(status_code=400, detail="本地文件夹配置必须包含文件夹路径")
# 替换路径中的特殊字符为下划线
folder_path = config["folder_path"].lower().replace("/", "_").replace(":", "_").replace("\\", "_")
# 去除开头的下划线
if folder_path.startswith("_"):
folder_path = folder_path[1:]
unique_id_parts.append(folder_path)
elif config_type == "remote_folder":
# 远程文件夹配置需要：主机、文件夹路径
if not config.get("host") or not config.get("folder_path"):
raise HTTPException(status_code=400, detail="远程文件夹配置必须包含主机和文件夹路径")
# 替换路径中的特殊字符为下划线
folder_path = config["folder_path"].lower().replace("/", "_").replace(":", "_").replace("\\", "_")
# 去除开头的下划线
if folder_path.startswith("_"):
folder_path = folder_path[1:]
unique_id_parts.extend([
config["host"].lower(),
folder_path
])
else:
raise HTTPException(status_code=400, detail=f"不支持的配置类型: {config_type}")
# Generate config ID based on unique identifier
config_id = "_".join(unique_id_parts)
# Save config to SQLite database
conn, cursor = get_db_connection()
# 检查是否存在同源配置
try:
# 先尝试查询现有配置
cursor.execute('SELECT name, config FROM data_sources')
existing_configs = cursor.fetchall()
for existing_name, existing_config_json in existing_configs:
existing_config_data = json.loads(existing_config_json)
# Check if it's the same type
if existing_config_data.get('type') == config_type:
if config_type == 'database':
# For database configs, same source means same host, port, and db name
if (existing_config_data.get('mysql_host') == config.get('mysql_host') and
existing_config_data.get('mysql_port') == config.get('mysql_port') and
existing_config_data.get('database') == config.get('database')):
raise HTTPException(
status_code=409,
detail=f"已存在相同源的数据库配置。如需调整，请点击配置列表中的配置并修改配置内容。"
)
elif config_type == 'local_folder':
# For local folder configs, same source means same folder path
if existing_config_data.get('folder_path') == config.get('folder_path'):
raise HTTPException(
status_code=409,
detail=f"已存在相同路径的本地文件夹配置。如需调整，请点击配置列表中的配置并修改配置内容。"
)
elif config_type == 'remote_folder':
# For remote folder configs, same source means same host and folder path
if (existing_config_data.get('host') == config.get('host') and
existing_config_data.get('folder_path') == config.get('folder_path')):
raise HTTPException(
status_code=409,
detail=f"已存在相同服务器和路径的远程文件夹配置。如需调整，请点击配置列表中的配置并修改配置内容。"
)
except sqlite3.OperationalError as e:
# 表不存在的情况，会在后面创建表
logger.warning(f"SQLite table error: {e}. This is expected if the table doesn't exist yet.")
# Check if a config with the same name already exists
cursor.execute('SELECT name FROM data_sources WHERE name = ?', (config_id,))
existing_config = cursor.fetchone()
if existing_config:
raise HTTPException(
status_code=409,
detail=f"配置 '{config['name']}' 已存在。如需调整，请点击配置列表中的配置并修改配置内容。"
)
# Insert new config into data_sources table
config_json = json.dumps(config, ensure_ascii=False, indent=2)
cursor.execute(
'INSERT INTO data_sources (name, config) VALUES (?, ?)',
(config_id, config_json)
)
conn.commit()
# Create sync service for this new data source
global sync_manager
if sync_manager is not None:
# Create appropriate data source config object
from config import BaseDataSourceConfig, DatabaseDataSourceConfig, LocalFolderDataSourceConfig, RemoteFolderDataSourceConfig
if config_type == "database":
source_config = DatabaseDataSourceConfig(
name=config_id,
database=config.get("database"),
table_name=config.get("table_name"),
id_column=config.get("id_column"),
content_column=config.get("content_column"),
file_column=config.get("file_column"),
title_column=config.get("title_column"),
metadata_columns=config.get("metadata_columns"),
content_separator=config.get("content_separator"),
updated_at_column=config.get("updated_at_column"),
mysql_host=config.get("mysql_host"),
mysql_port=config.get("mysql_port"),
mysql_user=config.get("mysql_user"),
mysql_password=config.get("mysql_password"),
file_source_type=config.get("file_source_type"),
file_system_base_path=config.get("file_system_base_path"),
scp_host=config.get("scp_host"),
scp_port=config.get("scp_port"),
scp_username=config.get("scp_username"),
scp_password=config.get("scp_password")
)
elif config_type == "local_folder":
source_config = LocalFolderDataSourceConfig(
name=config_id,
folder_path=config.get("folder_path"),
host=config.get("host"),
port=config.get("port"),
username=config.get("username"),
password=config.get("password"),
recursive=config.get("recursive", True),
ignore_patterns=config.get("ignore_patterns")
)
elif config_type == "remote_folder":
source_config = RemoteFolderDataSourceConfig(
name=config_id,
folder_path=config.get("folder_path"),
host=config.get("host"),
port=config.get("port"),
username=config.get("username"),
password=config.get("password"),
recursive=config.get("recursive", True),
ignore_patterns=config.get("ignore_patterns")
)
else:
logger.warning(f"Unknown config type: {config_type}")
# Create a base config as fallback
source_config = BaseDataSourceConfig(name=config_id, type=config_type)
# Create the sync service
sync_manager.create_or_update_sync_service(source_config)
# Return the created config
return {
"id": config_id,
"type": config_type,
"config": config
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Error creating config: {e}")
raise HTTPException(status_code=500, detail=f"Error creating configuration: {str(e)}")
finally:
if conn:
conn.close()
def get_db_connection():
"""
Get a SQLite database connection with data_sources table initialized
Returns:
tuple: (connection, cursor)
"""
DATA_DIR = Path(__file__).parent.parent / "data"
DATA_DIR.mkdir(parents=True, exist_ok=True)
DB_PATH = DATA_DIR / "sessions.db"
conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor()
# 检查并创建data_sources表（如果不存在）
try:
cursor.execute('''
CREATE TABLE IF NOT EXISTS data_sources (
name TEXT PRIMARY KEY,
config TEXT NOT NULL,
update_at TEXT NULL
)
''')
conn.commit()
except Exception as e:
logger.error(f"Error creating data_sources table: {e}")
conn.close()
raise HTTPException(status_code=500, detail=f"Error creating configuration table: {str(e)}")
return conn, cursor
@app.post("/folder-configs/remote")
async def create_remote_folder_config(config: Dict[str, Any]):
"""
Create a new remote folder configuration
Create a new remote folder configuration (deprecated, use /folder-configs instead)
Args:
config: Configuration data for the new remote folder
@ -1360,34 +1613,12 @@ async def create_remote_folder_config(config: Dict[str, Any]):
Returns:
Created configuration with ID
"""
try:
config_dir = os.path.join(os.path.dirname(__file__), "..", "configs")
if not os.path.exists(config_dir):
os.makedirs(config_dir)
# Generate unique config ID
config_id = f"remote_folder_{int(time.time())}"
config_file = f"{config_id}.json"
config_path = os.path.join(config_dir, config_file)
# Ensure config has required fields
if "type" not in config:
config["type"] = "remote_folder"
# Write config to file
with open(config_path, "w", encoding="utf-8") as f:
json.dump(config, f, ensure_ascii=False, indent=2)
# Return the created config
return {
"id": config_id,
"type": config["type"],
"config": config
}
except Exception as e:
logger.error(f"Error creating remote folder config: {e}")
raise HTTPException(status_code=500, detail=f"Error creating configuration: {str(e)}")
# Set type to remote_folder if not provided
if "type" not in config:
config["type"] = "remote_folder"
# Call the generic create_config function
return await create_config(config)
@app.put("/folder-configs/{config_id}")
@ -1402,26 +1633,140 @@ async def update_folder_config(config_id: str, config: Dict[str, Any]):
Returns:
Updated configuration
"""
conn = None
try:
config_dir = os.path.join(os.path.dirname(__file__), "..", "configs")
config_file = f"{config_id}.json"
config_path = os.path.join(config_dir, config_file)
conn, cursor = get_db_connection()
if not os.path.exists(config_path):
# Check if config exists
cursor.execute('SELECT * FROM data_sources WHERE name = ?', (config_id,))
existing_config = cursor.fetchone()
if not existing_config:
raise HTTPException(status_code=404, detail=f"Configuration with ID '{config_id}' not found")
# Ensure config has required fields
if "type" not in config:
config["type"] = "remote_folder"
raise HTTPException(status_code=400, detail="Configuration type is required")
# Update config file
with open(config_path, "w", encoding="utf-8") as f:
json.dump(config, f, ensure_ascii=False, indent=2)
# Generate new config ID based on the complete configuration information
config_type = config["type"]
new_config_id = None
# Return the updated config
# 根据不同类型的配置生成有意义的ID
if config_type == "database":
# 数据库配置：使用数据库名和表名生成ID
if not config.get("database"):
raise HTTPException(status_code=400, detail="Database name is required for database configuration")
if not config.get("table_name"):
raise HTTPException(status_code=400, detail="Table name is required for database configuration")
new_config_id = f"{config_type}_{config['database'].lower()}_{config['table_name'].lower()}"
elif config_type == "local_folder":
# 本地文件夹配置：使用文件夹路径生成ID
if not config.get("folder_path"):
raise HTTPException(status_code=400, detail="Folder path is required for local folder configuration")
# 替换路径中的特殊字符为下划线
folder_path = config["folder_path"].lower().replace("/", "_").replace(":", "_").replace("\\", "_")
# 去除开头的下划线
if folder_path.startswith("_"):
folder_path = folder_path[1:]
new_config_id = f"{config_type}_{folder_path}"
elif config_type == "remote_folder":
# 远程文件夹配置：使用主机和文件夹路径生成ID
# 端口不是唯一标识的必要元素，只用于连接配置
if not config.get("host"):
raise HTTPException(status_code=400, detail="Host is required for remote folder configuration")
if not config.get("folder_path"):
raise HTTPException(status_code=400, detail="Folder path is required for remote folder configuration")
# 替换路径中的特殊字符为下划线
folder_path = config["folder_path"].lower().replace("/", "_").replace(":", "_").replace("\\", "_")
# 去除开头的下划线
if folder_path.startswith("_"):
folder_path = folder_path[1:]
new_config_id = f"{config_type}_{config['host'].lower()}_{folder_path}"
# 如果无法生成新的有意义的ID，保留原来的ID
if not new_config_id:
new_config_id = config_id
# Update config in data_sources table
# Since name is primary key, we need to use INSERT OR REPLACE
config_json = json.dumps(config, ensure_ascii=False, indent=2)
# First, delete the old config if new ID is different
if new_config_id != config_id:
cursor.execute('DELETE FROM data_sources WHERE name = ?', (config_id,))
# Then insert the updated config with new ID, update_at value to null
cursor.execute(
'INSERT OR REPLACE INTO data_sources (name, config, update_at) VALUES (?, ?, ?)',
(new_config_id, config_json, None)
)
conn.commit()
# Update the sync service for this data source
global sync_manager
if sync_manager is not None:
# Create appropriate data source config object
from config import BaseDataSourceConfig, DatabaseDataSourceConfig, LocalFolderDataSourceConfig, RemoteFolderDataSourceConfig
if config_type == "database":
source_config = DatabaseDataSourceConfig(
name=new_config_id,
database=config.get("database"),
table_name=config.get("table_name"),
id_column=config.get("id_column"),
content_column=config.get("content_column"),
file_column=config.get("file_column"),
title_column=config.get("title_column"),
metadata_columns=config.get("metadata_columns"),
content_separator=config.get("content_separator"),
updated_at_column=config.get("updated_at_column"),
mysql_host=config.get("mysql_host"),
mysql_port=config.get("mysql_port"),
mysql_user=config.get("mysql_user"),
mysql_password=config.get("mysql_password"),
file_source_type=config.get("file_source_type"),
file_system_base_path=config.get("file_system_base_path"),
scp_host=config.get("scp_host"),
scp_port=config.get("scp_port"),
scp_username=config.get("scp_username"),
scp_password=config.get("scp_password")
)
elif config_type == "local_folder":
source_config = LocalFolderDataSourceConfig(
name=new_config_id,
folder_path=config.get("folder_path"),
host=config.get("host"),
port=config.get("port"),
username=config.get("username"),
password=config.get("password"),
recursive=config.get("recursive", True),
ignore_patterns=config.get("ignore_patterns")
)
elif config_type == "remote_folder":
source_config = RemoteFolderDataSourceConfig(
name=new_config_id,
folder_path=config.get("folder_path"),
host=config.get("host"),
port=config.get("port"),
username=config.get("username"),
password=config.get("password"),
recursive=config.get("recursive", True),
ignore_patterns=config.get("ignore_patterns")
)
else:
logger.warning(f"Unknown config type: {config_type}")
# Create a base config as fallback
source_config = BaseDataSourceConfig(name=new_config_id, type=config_type)
# Update the sync service
sync_manager.create_or_update_sync_service(source_config)
# Return the updated config with new ID
return {
"id": config_id,
"type": config["type"],
"id": new_config_id,
"type": config_type,
"config": config
}
@ -1430,6 +1775,9 @@ async def update_folder_config(config_id: str, config: Dict[str, Any]):
except Exception as e:
logger.error(f"Error updating folder config: {e}")
raise HTTPException(status_code=500, detail=f"Error updating configuration: {str(e)}")
finally:
if conn:
conn.close()
@app.delete("/folder-configs/{config_id}")
@ -1443,16 +1791,26 @@ async def delete_folder_config(config_id: str):
Returns:
Success message
"""
conn = None
try:
config_dir = os.path.join(os.path.dirname(__file__), "..", "configs")
config_file = f"{config_id}.json"
config_path = os.path.join(config_dir, config_file)
conn, cursor = get_db_connection()
if not os.path.exists(config_path):
# Check if config exists
cursor.execute('SELECT * FROM data_sources WHERE name = ?', (config_id,))
existing_config = cursor.fetchone()
if not existing_config:
raise HTTPException(status_code=404, detail=f"Configuration with ID '{config_id}' not found")
# Delete config file
os.remove(config_path)
# Delete config from data_sources table
cursor.execute('DELETE FROM data_sources WHERE name = ?', (config_id,))
conn.commit()
# Remove the sync service for this data source
global sync_manager
if sync_manager is not None:
sync_manager.remove_sync_service(config_id)
return {"status": "success", "message": f"Configuration '{config_id}' deleted successfully"}
@ -1461,6 +1819,173 @@ async def delete_folder_config(config_id: str):
except Exception as e:
logger.error(f"Error deleting folder config: {e}")
raise HTTPException(status_code=500, detail=f"Error deleting configuration: {str(e)}")
finally:
if conn:
conn.close()
# Database schema exploration endpoints
from pydantic import BaseModel
import mysql.connector
class DatabaseConnectionParams(BaseModel):
"""Database connection parameters"""
host: str = Field(..., description="Database host")
port: int = Field(default=3306, description="Database port")
username: str = Field(..., description="Database username")
password: str = Field(..., description="Database password")
class DatabaseParams(BaseModel):
"""Database parameters"""
host: str = Field(..., description="Database host")
port: int = Field(default=3306, description="Database port")
username: str = Field(..., description="Database username")
password: str = Field(..., description="Database password")
database: str = Field(..., description="Database name")
class TableParams(BaseModel):
"""Table parameters"""
host: str = Field(..., description="Database host")
port: int = Field(default=3306, description="Database port")
username: str = Field(..., description="Database username")
password: str = Field(..., description="Database password")
database: str = Field(..., description="Database name")
table_name: str = Field(..., description="Table name")
@app.post("/database/databases")
async def get_databases(params: DatabaseConnectionParams):
"""
Get list of databases from MySQL server
Args:
params: Database connection parameters
Returns:
List of available databases
"""
try:
# Direct database connection
connection = mysql.connector.connect(
host=params.host,
port=params.port,
user=params.username,
password=params.password
)
cursor = connection.cursor()
cursor.execute("SHOW DATABASES")
databases = []
for (database_name,) in cursor:
# Skip system databases
if database_name not in ['information_schema', 'mysql', 'performance_schema', 'sys']:
databases.append(database_name)
cursor.close()
connection.close()
return {"databases": databases}
except mysql.connector.Error as e:
logger.error(f"Database connection error: {e}")
raise HTTPException(status_code=500, detail=f"Database connection failed: {str(e)}")
except Exception as e:
logger.error(f"Error getting databases: {e}")
raise HTTPException(status_code=500, detail=f"Error getting databases: {str(e)}")
@app.post("/database/tables")
async def get_tables(params: DatabaseParams):
"""
Get list of tables from specified database
Args:
params: Database parameters including database name
Returns:
List of tables in the specified database
"""
try:
# Direct database connection
connection = mysql.connector.connect(
host=params.host,
port=params.port,
user=params.username,
password=params.password,
database=params.database
)
cursor = connection.cursor()
cursor.execute("SHOW TABLES")
tables = []
for (table_name,) in cursor:
tables.append(table_name)
cursor.close()
connection.close()
return {"tables": tables}
except mysql.connector.Error as e:
logger.error(f"Database connection error: {e}")
raise HTTPException(status_code=500, detail=f"Database connection failed: {str(e)}")
except Exception as e:
logger.error(f"Error getting tables: {e}")
raise HTTPException(status_code=500, detail=f"Error getting tables: {str(e)}")
@app.post("/database/table-structure")
async def get_table_structure(params: TableParams):
"""
Get structure of specified table
Args:
params: Table parameters including table name
Returns:
Table structure with column details
"""
try:
# Direct database connection
connection = mysql.connector.connect(
host=params.host,
port=params.port,
user=params.username,
password=params.password,
database=params.database
)
cursor = connection.cursor()
cursor.execute(f"DESCRIBE {params.table_name}")
columns = []
for (field, type, null, key, default, extra) in cursor:
columns.append({
"name": field,
"type": type,
"null": null,
"key": key,
"default": default,
"extra": extra
})
cursor.close()
connection.close()
return {"columns": columns}
except mysql.connector.Error as e:
logger.error(f"Database connection error: {e}")
raise HTTPException(status_code=500, detail=f"Database connection failed: {str(e)}")
except Exception as e:
logger.error(f"Error getting table structure: {e}")
raise HTTPException(status_code=500, detail=f"Error getting table structure: {str(e)}")

260
config.py
View File

@ -34,7 +34,6 @@ class DatabaseDataSourceConfig(BaseDataSourceConfig):
mysql_port: Optional[int] = None,
mysql_user: Optional[str] = None,
mysql_password: Optional[str] = None,
mysql_charset: Optional[str] = None,
# 文件源配置
file_source_type: Optional[str] = None, # 可选值: "api", "filesystem", "scp"
file_system_base_path: Optional[str] = None, # 文件系统基础路径
@ -67,7 +66,6 @@ class DatabaseDataSourceConfig(BaseDataSourceConfig):
self.mysql_port = mysql_port
self.mysql_user = mysql_user
self.mysql_password = mysql_password
self.mysql_charset = mysql_charset
# 文件源配置
self.file_source_type = file_source_type
@ -78,7 +76,6 @@ class DatabaseDataSourceConfig(BaseDataSourceConfig):
self.scp_port = scp_port
self.scp_username = scp_username
self.scp_password = scp_password
self.scp_key_path = scp_key_path
class LocalFolderDataSourceConfig(BaseDataSourceConfig):
@ -87,11 +84,19 @@ class LocalFolderDataSourceConfig(BaseDataSourceConfig):
self,
name: str,
folder_path: str,
host: str = "localhost",
port: int = 22,
username: Optional[str] = None,
password: Optional[str] = None,
recursive: bool = True,
ignore_patterns: Optional[List[str]] = None
):
super().__init__(name, "local_folder")
self.folder_path = folder_path # 本地文件夹路径
self.host = host # 主机地址
self.port = port # 主机端口
self.username = username # 主机用户名
self.password = password # 主机密码（可选）
self.recursive = recursive # 是否递归遍历子文件夹
self.ignore_patterns = ignore_patterns # 忽略的文件模式列表
@ -106,7 +111,6 @@ class RemoteFolderDataSourceConfig(BaseDataSourceConfig):
port: int,
username: str,
password: Optional[str] = None,
key_path: Optional[str] = None,
recursive: bool = True,
ignore_patterns: Optional[List[str]] = None
):
@ -116,7 +120,6 @@ class RemoteFolderDataSourceConfig(BaseDataSourceConfig):
self.port = port # 远程主机端口
self.username = username # 远程主机用户名
self.password = password # 远程主机密码（可选）
self.key_path = key_path # SSH密钥路径（可选）
self.recursive = recursive # 是否递归遍历子文件夹
self.ignore_patterns = ignore_patterns # 忽略的文件模式列表
@ -140,21 +143,6 @@ class Settings(BaseSettings):
# File Upload Settings
MAX_UPLOAD_SIZE_MB: int = 5 # Maximum file upload size in MB (default: 5MB)
# Multi-database configuration (JSON string or file path) - legacy
# Format: JSON array with database configs (same MySQL server, different databases)
# Each config only needs: name, database, table_name, id_column, content_column, etc.
# Connection info (host, port, user, password) will use MYSQL_* settings above
# Set this in .env file: MYSQL_DATABASES_CONFIG=./databases_config.json
MYSQL_DATABASES_CONFIG: Optional[str] = None
# Multi-data source configuration (JSON string or file path) - new
# Format: JSON array with data source configs of different types
# Each config needs: name, type (database, local_folder, remote_folder), and type-specific settings
# Set this in .env file: DATA_SOURCES_CONFIG=./data_sources_config.json
DATA_SOURCES_CONFIG: Optional[str] = None
# ChromaDB Settings
# Use HttpClient mode if CHROMA_SERVER_HOST is set, otherwise use PersistentClient
# For Docker deployment, set CHROMA_SERVER_HOST=localhost in .env
@ -213,175 +201,40 @@ class Settings(BaseSettings):
Get list of data source configurations
Returns:
List of BaseDataSourceConfig objects
List of BaseDataSourceConfig objects. Empty list if no configurations found.
"""
configs = []
try:
# Check if data sources config is provided
config_source = self.DATA_SOURCES_CONFIG
# Load configs from SQLite database only
import sqlite3
from pathlib import Path
if config_source:
# Try to parse as JSON string
if config_source.strip().startswith('[') or config_source.strip().startswith('{'):
data_sources = json.loads(config_source)
# Handle both array and single object
if isinstance(data_sources, dict):
data_sources = [data_sources]
DATA_DIR = Path(__file__).parent / "data"
DATA_DIR.mkdir(parents=True, exist_ok=True)
DB_PATH = DATA_DIR / "sessions.db"
try:
conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor()
# 查询所有数据源配置
try:
cursor.execute('SELECT name, config FROM data_sources')
rows = cursor.fetchall()
# Create data source objects based on type
for ds_config in data_sources:
source_type = ds_config.get('type', 'database')
if source_type == 'database':
# Create database data source
configs.append(DatabaseDataSourceConfig(
name=ds_config.get('name', ds_config.get('database', 'unknown')),
database=ds_config['database'], # Required field
table_name=ds_config.get('table_name', 'documents'), # Default to 'documents'
id_column=ds_config.get('id_column', 'id'), # Default to 'id'
content_column=ds_config.get('content_column', 'content'), # Default to 'content'
file_column=ds_config.get('file_column', None),
title_column=ds_config.get('title_column', 'title'), # Default to 'title'
metadata_columns=ds_config.get('metadata_columns', None),
content_separator=ds_config.get('content_separator', '\n'),
updated_at_column=ds_config.get('updated_at_column', None),
# MySQL connection info
mysql_host=ds_config.get('mysql_host', None),
mysql_port=ds_config.get('mysql_port', None),
mysql_user=ds_config.get('mysql_user', None),
mysql_password=ds_config.get('mysql_password', None),
mysql_charset=ds_config.get('mysql_charset', 'utf8mb4'), # Default to 'utf8mb4'
# 文件源配置
file_source_type=ds_config.get('file_source_type', None),
file_system_base_path=ds_config.get('file_system_base_path', None),
# SCP配置
scp_host=ds_config.get('scp_host', None),
scp_port=ds_config.get('scp_port', 22),
scp_username=ds_config.get('scp_username', None),
scp_password=ds_config.get('scp_password', None),
scp_key_path=ds_config.get('scp_key_path', None)
))
elif source_type == 'local_folder':
# Create local folder data source
configs.append(LocalFolderDataSourceConfig(
name=ds_config.get('name', 'local_folder'),
folder_path=ds_config.get('folder_path', '.'),
recursive=ds_config.get('recursive', True),
ignore_patterns=ds_config.get('ignore_patterns', None)
))
elif source_type == 'remote_folder':
# Create remote folder data source
configs.append(RemoteFolderDataSourceConfig(
name=ds_config.get('name', 'remote_folder'),
folder_path=ds_config.get('folder_path', '.'),
host=ds_config.get('host', ''),
port=ds_config.get('port', 22),
username=ds_config.get('username', ''),
password=ds_config.get('password', None),
key_path=ds_config.get('key_path', None),
recursive=ds_config.get('recursive', True),
ignore_patterns=ds_config.get('ignore_patterns', None)
))
else:
from loguru import logger
logger.warning(f"Unknown data source type: {source_type}, skipping")
else:
# Load all config files from configs directory
config_dir = os.path.join(os.path.dirname(__file__), "configs")
if os.path.exists(config_dir):
import glob
for config_file in glob.glob(os.path.join(config_dir, "*.json")):
for row in rows:
name, config_json = row
try:
with open(config_file, 'r', encoding='utf-8') as f:
data_sources = json.load(f)
# Handle both array and single object
if isinstance(data_sources, dict):
data_sources = [data_sources]
ds_config = json.loads(config_json)
# Create data source objects based on type
for ds_config in data_sources:
source_type = ds_config.get('type', 'database')
if source_type == 'database':
# Create database data source
configs.append(DatabaseDataSourceConfig(
name=ds_config.get('name', ds_config.get('database', 'unknown')),
database=ds_config['database'], # Required field
table_name=ds_config.get('table_name', 'documents'), # Default to 'documents'
id_column=ds_config.get('id_column', 'id'), # Default to 'id'
content_column=ds_config.get('content_column', 'content'), # Default to 'content'
file_column=ds_config.get('file_column', None),
title_column=ds_config.get('title_column', 'title'), # Default to 'title'
metadata_columns=ds_config.get('metadata_columns', None),
content_separator=ds_config.get('content_separator', '\n'),
updated_at_column=ds_config.get('updated_at_column', None),
# MySQL connection info
mysql_host=ds_config.get('mysql_host', None),
mysql_port=ds_config.get('mysql_port', None),
mysql_user=ds_config.get('mysql_user', None),
mysql_password=ds_config.get('mysql_password', None),
mysql_charset=ds_config.get('mysql_charset', 'utf8mb4'), # Default to 'utf8mb4'
# 文件源配置
file_source_type=ds_config.get('file_source_type', None),
file_system_base_path=ds_config.get('file_system_base_path', None),
# SCP配置
scp_host=ds_config.get('scp_host', None),
scp_port=ds_config.get('scp_port', 22),
scp_username=ds_config.get('scp_username', None),
scp_password=ds_config.get('scp_password', None),
scp_key_path=ds_config.get('scp_key_path', None)
))
elif source_type == 'local_folder':
# Create local folder data source
configs.append(LocalFolderDataSourceConfig(
name=ds_config.get('name', 'local_folder'),
folder_path=ds_config.get('folder_path', '.'),
recursive=ds_config.get('recursive', True),
ignore_patterns=ds_config.get('ignore_patterns', None)
))
elif source_type == 'remote_folder':
# Create remote folder data source
configs.append(RemoteFolderDataSourceConfig(
name=ds_config.get('name', 'remote_folder'),
folder_path=ds_config.get('folder_path', '.'),
host=ds_config.get('host', ''),
port=ds_config.get('port', 22),
username=ds_config.get('username', ''),
password=ds_config.get('password', None),
key_path=ds_config.get('key_path', None),
recursive=ds_config.get('recursive', True),
ignore_patterns=ds_config.get('ignore_patterns', None)
))
else:
from loguru import logger
logger.warning(f"Unknown data source type: {source_type}, skipping")
from loguru import logger
logger.info(f"Loaded config file: {config_file}")
except Exception as e:
from loguru import logger
logger.error(f"Error loading config file {config_file}: {e}")
else:
# Fallback to default database_config.json if configs directory doesn't exist
default_config_path = os.path.join(os.path.dirname(__file__), "database_config.json")
if os.path.exists(default_config_path):
with open(default_config_path, 'r', encoding='utf-8') as f:
data_sources = json.load(f)
# Handle both array and single object
if isinstance(data_sources, dict):
data_sources = [data_sources]
# Create data source objects based on type
for ds_config in data_sources:
source_type = ds_config.get('type', 'database')
if source_type == 'database':
# Create database data source
configs.append(DatabaseDataSourceConfig(
name=ds_config.get('name', ds_config.get('database', 'unknown')),
name=name, # 使用数据库表中的name列
database=ds_config['database'], # Required field
table_name=ds_config.get('table_name', 'documents'), # Default to 'documents'
id_column=ds_config.get('id_column', 'id'), # Default to 'id'
@ -396,7 +249,6 @@ class Settings(BaseSettings):
mysql_port=ds_config.get('mysql_port', None),
mysql_user=ds_config.get('mysql_user', None),
mysql_password=ds_config.get('mysql_password', None),
mysql_charset=ds_config.get('mysql_charset', 'utf8mb4'), # Default to 'utf8mb4'
# 文件源配置
file_source_type=ds_config.get('file_source_type', None),
file_system_base_path=ds_config.get('file_system_base_path', None),
@ -404,47 +256,63 @@ class Settings(BaseSettings):
scp_host=ds_config.get('scp_host', None),
scp_port=ds_config.get('scp_port', 22),
scp_username=ds_config.get('scp_username', None),
scp_password=ds_config.get('scp_password', None),
scp_key_path=ds_config.get('scp_key_path', None)
scp_password=ds_config.get('scp_password', None)
))
elif source_type == 'local_folder':
# Create local folder data source
configs.append(LocalFolderDataSourceConfig(
name=ds_config.get('name', 'local_folder'),
folder_path=ds_config.get('folder_path', '.'),
recursive=ds_config.get('recursive', True),
ignore_patterns=ds_config.get('ignore_patterns', None)
))
name=name, # 使用数据库表中的name列
folder_path=ds_config.get('folder_path', '.'),
host=ds_config.get('host', 'localhost'),
port=ds_config.get('port', 22),
username=ds_config.get('username', None),
password=ds_config.get('password', None),
recursive=ds_config.get('recursive', True),
ignore_patterns=ds_config.get('ignore_patterns', None)
))
elif source_type == 'remote_folder':
# Create remote folder data source
configs.append(RemoteFolderDataSourceConfig(
name=ds_config.get('name', 'remote_folder'),
name=name, # 使用数据库表中的name列
folder_path=ds_config.get('folder_path', '.'),
host=ds_config.get('host', ''),
port=ds_config.get('port', 22),
username=ds_config.get('username', ''),
password=ds_config.get('password', None),
key_path=ds_config.get('key_path', None),
recursive=ds_config.get('recursive', True),
ignore_patterns=ds_config.get('ignore_patterns', None)
))
else:
from loguru import logger
logger.warning(f"Unknown data source type: {source_type}, skipping")
else:
raise FileNotFoundError(f"Default config file not found: {default_config_path}")
from loguru import logger
logger.info(f"Loaded config from database: {name}")
except Exception as e:
from loguru import logger
logger.error(f"Error parsing config from database: {name}, error: {e}")
except sqlite3.OperationalError as e:
# 表不存在的情况，返回空列表
from loguru import logger
logger.warning(f"SQLite table error: {e}. Returning empty config list.")
except Exception as e:
# 其他数据库错误，返回空列表
from loguru import logger
logger.error(f"Error querying data sources from database: {e}")
# 关闭数据库连接
conn.close()
except Exception as e:
# 数据库连接失败，返回空列表
from loguru import logger
logger.error(f"Error connecting to SQLite database: {e}")
except Exception as e:
# 任何其他错误，返回空列表
from loguru import logger
logger.error(f"Error parsing data sources config: {e}")
raise ValueError(f"Failed to load data sources configuration: {e}") from e
# Ensure we have at least one data source
if not configs:
from loguru import logger
logger.error("No valid data sources found in configuration")
raise ValueError("No valid data sources found in configuration")
logger.error(f"Error in get_data_sources: {e}")
# 返回配置列表，即使为空
return configs
def get_database_configs(self) -> List[DatabaseDataSourceConfig]:

12
configs/host_folder.json
View File

@ -1,12 +0,0 @@
{
"type": "remote_folder",
"name": "host_folder",
"folder_path": "/home/zxh/programs/RAG/test_files",
"host": "localhost",
"port": 22,
"username": "zxh",
"password": "",
"key_path": "/home/zxh/.ssh/id_rsa",
"recursive": true,
"ignore_patterns": [".git", ".DS_Store"]
}

7
database/__init__.py
View File

@ -1,7 +0,0 @@
"""
Database module for RAG system
"""
from .sync import MySQLSync
__all__ = ["MySQLSync"]

907
database/sync.py
View File

@ -1,907 +0,0 @@
"""
MySQL to ChromaDB synchronization module
"""
import urllib
import pymysql
import requests
from typing import List, Dict, Optional, Tuple
from datetime import datetime
from loguru import logger
from config import DatabaseDataSourceConfig as DatabaseConfig, settings
from rag.file_parser import FileParser
class MySQLSync:
"""Handle synchronization between MySQL and ChromaDB"""
def __init__(self, db_config: DatabaseConfig, connection=None):
"""
Initialize MySQL sync with database configuration
Args:
db_config: DatabaseConfig object containing database table info
connection: Optional existing MySQL connection (for multi-database scenario)
"""
self.file_parser = FileParser()
self.db_config = db_config
self.connection = connection
if self.connection is None:
self._connect()
else:
# Use existing connection, just switch database
self._switch_database()
def _connect(self):
"""Establish MySQL connection using config-specific connection info"""
try:
# Use database-specific connection info only
if not all([self.db_config.mysql_host, self.db_config.mysql_port, self.db_config.mysql_user]):
missing_fields = []
if not self.db_config.mysql_host:
missing_fields.append("mysql_host")
if not self.db_config.mysql_port:
missing_fields.append("mysql_port")
if not self.db_config.mysql_user:
missing_fields.append("mysql_user")
raise ValueError(f"Missing required database connection fields for {self.db_config.name}: {', '.join(missing_fields)}")
host = self.db_config.mysql_host
port = self.db_config.mysql_port
user = self.db_config.mysql_user
password = self.db_config.mysql_password or "" # Password can be empty
charset = self.db_config.mysql_charset or "utf8mb4" # Charset has a reasonable default
self.connection = pymysql.connect(
host=host,
port=port,
user=user,
password=password,
database=self.db_config.database,
charset=charset,
cursorclass=pymysql.cursors.DictCursor
)
connection_info = f"{host}:{port}/{self.db_config.database}"
logger.info(f"Connected to MySQL database: {connection_info} (name: {self.db_config.name})")
except Exception as e:
logger.error(f"Failed to connect to MySQL database {self.db_config.name}: {e}")
raise
@staticmethod
def check_mysql_connection(host="localhost", port=3306, user="root", password="", charset="utf8mb4") -> Tuple[bool, str]:
"""
Check if MySQL server is accessible and connection can be established
Args:
host: MySQL host
port: MySQL port
user: MySQL username
password: MySQL password
charset: MySQL charset
Returns:
Tuple of (is_connected: bool, error_message: str)
If connected, error_message will be empty string
"""
temp_connection = None
try:
logger.info(f"Checking MySQL connection to {host}:{port}...")
temp_connection = pymysql.connect(
host=host,
port=port,
user=user,
password=password,
charset=charset,
cursorclass=pymysql.cursors.DictCursor,
connect_timeout=10 # 10 seconds timeout
)
# Test the connection by executing a simple query
with temp_connection.cursor() as cursor:
cursor.execute("SELECT VERSION() as version")
result = cursor.fetchone()
mysql_version = result.get('version', 'unknown') if result else 'unknown'
logger.info(f"✓ MySQL connection successful (version: {mysql_version})")
return True, ""
except pymysql.err.OperationalError as e:
# Safely get error code and message
error_code = e.args[0] if len(e.args) > 0 else None
error_msg = e.args[1] if len(e.args) > 1 else str(e)
if error_code == 2003:
error_message = f"Cannot connect to MySQL server at {host}:{port}. " \
f"Please check if MySQL server is running and accessible."
elif error_code == 1045:
error_message = f"Access denied for user '{user}'@{host}. " \
f"Please check MySQL username and password."
else:
error_message = f"MySQL connection error ({error_code}): {error_msg}"
logger.error(f"✗ MySQL connection failed: {error_message}")
return False, error_message
except Exception as e:
error_message = f"Unexpected error while checking MySQL connection: {str(e)}"
logger.error(f"✗ MySQL connection check failed: {error_message}")
return False, error_message
finally:
if temp_connection:
temp_connection.close()
@staticmethod
def check_database_exists(db_name: str, connection=None) -> bool:
"""
Check if a database exists
Args:
db_name: Database name to check
connection: Optional existing MySQL connection
Returns:
True if database exists, False otherwise
"""
temp_connection = None
try:
if connection is None:
# Create temporary connection without specifying database
temp_connection = pymysql.connect(
host="localhost",
port=3306,
user="root",
password="",
charset="utf8mb4",
cursorclass=pymysql.cursors.DictCursor
)
conn = temp_connection
else:
conn = connection
with conn.cursor() as cursor:
cursor.execute("SHOW DATABASES LIKE %s", (db_name,))
result = cursor.fetchone()
return result is not None
except Exception as e:
logger.error(f"Error checking database existence for {db_name}: {e}")
return False
finally:
if temp_connection:
temp_connection.close()
@staticmethod
def check_databases_exist(db_configs: List[DatabaseConfig]) -> Tuple[bool, List[str]]:
"""
Check if all configured databases exist
Each database config can use its own MySQL connection info
Args:
db_configs: List of DatabaseConfig objects
Returns:
Tuple of (all_exist: bool, missing_databases: List[str])
"""
if not db_configs:
return True, []
missing_databases = []
# Group database configs by connection info (host:port:user)
# This allows us to check multiple databases on the same server efficiently
connection_groups: Dict[Tuple[str, int, str], List[DatabaseConfig]] = {} # 使用DatabaseConfig类型
for db_config in db_configs:
# Use database-specific connection info only
if not all([db_config.mysql_host, db_config.mysql_port, db_config.mysql_user]):
missing_fields = []
if not db_config.mysql_host:
missing_fields.append("mysql_host")
if not db_config.mysql_port:
missing_fields.append("mysql_port")
if not db_config.mysql_user:
missing_fields.append("mysql_user")
raise ValueError(f"Missing required database connection fields for {db_config.name}: {', '.join(missing_fields)}")
host = db_config.mysql_host
port = db_config.mysql_port
user = db_config.mysql_user
connection_key = (host, port, user)
if connection_key not in connection_groups:
connection_groups[connection_key] = []
connection_groups[connection_key].append(db_config)
# Check databases for each connection group
for (host, port, user), configs in connection_groups.items():
temp_connection = None
try:
# Get password and charset from first config in group (or use defaults)
# Note: All configs in a group should use same password/charset for same host:port:user
first_config = configs[0]
password = first_config.mysql_password if first_config.mysql_password is not None else ""
charset = first_config.mysql_charset if first_config.mysql_charset is not None else "utf8mb4"
logger.info(f"Checking databases on MySQL server {host}:{port} (user: {user})...")
temp_connection = pymysql.connect(
host=host,
port=port,
user=user,
password=password,
charset=charset,
cursorclass=pymysql.cursors.DictCursor,
connect_timeout=10
)
# Get all existing databases on this server
with temp_connection.cursor() as cursor:
cursor.execute("SHOW DATABASES")
existing_databases = {row['Database'] for row in cursor.fetchall()}
# Check each configured database in this group
for db_config in configs:
if db_config.database not in existing_databases:
missing_databases.append(
f"{db_config.database} (name: {db_config.name}, server: {host}:{port})"
)
else:
logger.info(f"✓ Database {db_config.database} exists on {host}:{port}")
except Exception as e:
# If we can't connect to check, mark all databases in this group as missing
logger.error(f"Error checking databases on {host}:{port}: {e}")
for db_config in configs:
missing_databases.append(
f"{db_config.database} (name: {db_config.name}, server: {host}:{port}, error: {str(e)})"
)
finally:
if temp_connection:
temp_connection.close()
return len(missing_databases) == 0, missing_databases
def _switch_database(self):
"""Switch to the target database using existing connection"""
try:
with self.connection.cursor() as cursor:
cursor.execute(f"USE `{self.db_config.database}`")
logger.info(f"Switched to MySQL database: {self.db_config.database} (name: {self.db_config.name})")
except Exception as e:
logger.error(f"Failed to switch to MySQL database {self.db_config.name}: {e}")
raise
def close(self):
"""Close MySQL connection"""
if self.connection:
self.connection.close()
logger.info("MySQL connection closed")
def _ensure_connection(self):
"""Ensure MySQL connection is alive and reconnect if necessary"""
max_retries = 3
for attempt in range(max_retries):
try:
# Check if connection is None
if self.connection is None:
logger.warning(f"MySQL connection for {self.db_config.name} is None, reconnecting... (attempt {attempt + 1}/{max_retries})")
self._connect()
# Verify connection was created
if self.connection is None:
raise RuntimeError(f"Failed to create connection for {self.db_config.name}")
return
# Check if connection is open
if not hasattr(self.connection, 'open') or not self.connection.open:
logger.warning(f"MySQL connection for {self.db_config.name} is closed, reconnecting... (attempt {attempt + 1}/{max_retries})")
try:
self.connection.close()
except Exception:
pass
self.connection = None
self._connect()
# Verify connection was created
if self.connection is None:
raise RuntimeError(f"Failed to create connection for {self.db_config.name}")
return
# Test connection with ping
# Note: ping(reconnect=True) may not work if connection is in bad state
try:
self.connection.ping(reconnect=False)
except Exception:
# Ping failed, connection is bad, recreate it
logger.warning(f"Ping failed for {self.db_config.name}, recreating connection...")
try:
self.connection.close()
except Exception:
pass
self.connection = None
self._connect()
if self.connection is None:
raise RuntimeError(f"Failed to create connection for {self.db_config.name}")
return
return # Connection is good
except (pymysql.err.OperationalError, pymysql.err.InterfaceError, AttributeError) as e:
# Connection is dead or invalid, try to reconnect
logger.warning(f"MySQL connection for {self.db_config.name} is dead/invalid, reconnecting... (attempt {attempt + 1}/{max_retries}): {e}")
try:
if self.connection:
self.connection.close()
except Exception:
pass
self.connection = None
if attempt < max_retries - 1:
import time
time.sleep(0.5 * (attempt + 1)) # Exponential backoff
else:
# Last attempt, raise the error
raise
except Exception as e:
logger.error(f"Unexpected error checking MySQL connection for {self.db_config.name}: {e}")
# Set connection to None so next attempt will recreate it
try:
if self.connection:
self.connection.close()
except Exception:
pass
self.connection = None
if attempt < max_retries - 1:
import time
time.sleep(0.5 * (attempt + 1))
else:
raise
def fetch_all_documents(self) -> List[Dict]:
"""
Fetch all documents from MySQL table
Returns:
List of document dictionaries with database source info
"""
try:
# Ensure connection is alive before using it
self._ensure_connection()
# Execute query with retry mechanism for connection issues
max_retries = 3
results = None
for attempt in range(max_retries):
try:
# Ensure connection is still alive (may have been lost during previous attempt)
if attempt > 0:
self._ensure_connection()
else:
# Also ensure connection on first attempt (may have been set to None elsewhere)
self._ensure_connection()
# Double-check connection is not None before using
if self.connection is None:
logger.warning(f"Connection is None for {self.db_config.name} after _ensure_connection, retrying...")
if attempt < max_retries - 1:
import time
time.sleep(0.5 * (attempt + 1))
continue
else:
raise RuntimeError(f"Failed to establish connection for {self.db_config.name} after {max_retries} attempts")
# Ensure we're using the correct database
with self.connection.cursor() as cursor:
cursor.execute(f"USE `{self.db_config.database}`")
with self.connection.cursor() as cursor:
# Get actual columns in the table
cursor.execute(f"DESCRIBE `{self.db_config.table_name}`")
table_columns = {row['Field'] for row in cursor.fetchall()}
# Build query based on configured columns, only include columns that exist
columns = []
# Add ID column (required)
if self.db_config.id_column in table_columns:
columns.append(self.db_config.id_column)
else:
raise ValueError(f"ID column '{self.db_config.id_column}' does not exist in table {self.db_config.table_name}")
# 添加所有 content 列（支持多个列），只添加存在的列
for col in self.db_config.content_columns:
if col in table_columns:
columns.append(col)
else:
logger.warning(f"Content column '{col}' does not exist in table {self.db_config.table_name}, skipping")
if not columns or all(col == self.db_config.id_column for col in columns):
raise ValueError(f"No valid content columns found in table {self.db_config.table_name}")
# Add title column if configured and exists
if self.db_config.title_column and self.db_config.title_column in table_columns:
columns.append(self.db_config.title_column)
elif self.db_config.title_column:
logger.warning(f"Title column '{self.db_config.title_column}' does not exist in table {self.db_config.table_name}, skipping")
# Add metadata columns if specified, only add existing ones
if self.db_config.metadata_columns:
metadata_cols = [col.strip() for col in self.db_config.metadata_columns.split(",")]
for col in metadata_cols:
if col in table_columns:
columns.append(col)
else:
logger.warning(f"Metadata column '{col}' does not exist in table {self.db_config.table_name}, skipping")
query = f"SELECT {', '.join(columns)} FROM `{self.db_config.table_name}`"
cursor.execute(query)
results = cursor.fetchall()
break # Success, exit retry loop
except (pymysql.err.OperationalError, pymysql.err.InterfaceError) as e:
# Safely get error code
error_code = None
error_msg = str(e)
if hasattr(e, 'args') and e.args:
try:
error_code = e.args[0] if len(e.args) > 0 else None
error_msg = e.args[1] if len(e.args) > 1 else str(e)
except (IndexError, TypeError):
error_code = None
# Check for connection-related errors that should trigger reconnection
should_reconnect = False
if error_code == 2013: # Lost connection to MySQL server during query
should_reconnect = True
elif "Packet sequence number wrong" in error_msg or "Lost connection" in error_msg:
should_reconnect = True
if should_reconnect:
logger.warning(f"Connection error during query for {self.db_config.name} ({error_msg}), reconnecting and retrying... (attempt {attempt + 1}/{max_retries})")
# Close and recreate connection
try:
if self.connection:
self.connection.close()
except Exception:
pass
self.connection = None
if attempt < max_retries - 1:
import time
time.sleep(0.5 * (attempt + 1)) # Exponential backoff
continue
else:
logger.error(f"Failed to fetch documents after {max_retries} attempts: {e}")
raise
else:
# Other operational error, raise immediately
raise
except Exception as e:
# Non-connection errors, raise immediately
raise
if results is None:
raise RuntimeError(f"Failed to fetch documents from {self.db_config.name} after {max_retries} attempts")
logger.info(f"Fetched {len(results)} documents from MySQL database: {self.db_config.database} (name: {self.db_config.name})")
# 如果配置了文件字段，则需根据文件字段的值来获取文件内容
if self.db_config.file_column and self.db_config.file_column in self.db_config.content_columns:
# 文件处理成功的文档
processed_results = []
# 失败文档ID列表
failed_doc_ids = set()
logger.info(
f"MySQL database: {self.db_config.database} (table: {self.db_config.table_name}) has set file_column(value: {self.db_config.file_column}) field.")
for doc in results:
# 获取每个doc的文件字段值，即文件标识符
file_column_value = doc.get(self.db_config.file_column)
if not file_column_value or file_column_value is None:
processed_results.append(doc)
continue
file_identifiers = [file_identifier.strip() for file_identifier in file_column_value.split(",")]
try:
file_content_parts = []
for file_identifier in file_identifiers:
# 调用接口获取文件字节流及文件名
file_byte, file_name = self.fetch_file_bytes(file_identifier)
if not file_byte:
logger.warning(
f"文档ID:{doc.get(self.db_config.id_column, '未知')}，文件标识符：{file_identifier} 获取字节流失败")
failed_doc_ids.add(doc.get(self.db_config.id_column))
break
# 调用文件读取方法获取文本内容
file_text = self.file_parser.get_text_from_bytes(file_byte, file_name)
if file_text and file_text.strip():
file_content_parts.append(file_text.strip())
logger.info(f"成功读取文件内容: {file_identifier} (长度: {len(file_text)})")
else:
logger.warning(f"文件标识符 {file_identifier} 提取的文本内容为空")
failed_doc_ids.add(doc.get(self.db_config.id_column))
break
if file_content_parts:
file_content = "\n".join(file_content_parts)
# 将文件内容添加到文档中
doc[self.db_config.file_column] = file_content
logger.info(
f"文档 {doc.get(self.db_config.id_column, '未知')} 成功合并 {len(file_content_parts)} 个文件内容")
# 将处理成功的文档添加到新列表中
processed_results.append(doc)
else:
logger.warning(
f"文档 {doc.get(self.db_config.id_column, '未知')} 没有成功提取任何文件内容")
failed_doc_ids.add(doc.get(self.db_config.id_column))
except Exception as e:
logger.error(
f"处理文档({doc.get(self.db_config.id_column, '未知')}) 文件字段: {doc[self.db_config.file_column]} 时出错: {e}")
failed_doc_ids.add(doc.get(self.db_config.id_column))
continue # 继续下一次doc循环
# 记录失败文档统计信息
if failed_doc_ids:
failed_doc_ids_str = {str(doc_id) for doc_id in failed_doc_ids}
ids_str = ', '.join(sorted(failed_doc_ids_str))
logger.info(f"数据库配置[{self.db_config.name}]本次同步在处理文件过程中共有 {len(failed_doc_ids)} 个文档因文件处理失败被跳过: [{ids_str}]")
else:
logger.info(f"数据库配置[{self.db_config.name}]所有文档文件处理成功")
results = processed_results
# Add database source information to each document
if results:
for doc in results:
doc['_db_source'] = self.db_config.name
doc['_db_database'] = self.db_config.database
doc['_db_table'] = self.db_config.table_name
return results
except Exception as e:
logger.error(f"Error fetching documents from MySQL database {self.db_config.name}: {e}")
raise
def fetch_new_documents(self, last_sync_time: Optional[datetime] = None) -> List[Dict]:
"""
Fetch new or updated documents since last sync
Args:
last_sync_time: Last synchronization timestamp
Returns:
List of new/updated document dictionaries with database source info
"""
try:
# Ensure connection is alive before using it
self._ensure_connection()
# Ensure we're using the correct database
with self.connection.cursor() as cursor:
cursor.execute(f"USE `{self.db_config.database}`")
with self.connection.cursor() as cursor:
# Get actual columns in the table
cursor.execute(f"DESCRIBE `{self.db_config.table_name}`")
table_columns = {row['Field'] for row in cursor.fetchall()}
# Build query based on configured columns, only include columns that exist
columns = []
# Add ID column (required)
if self.db_config.id_column in table_columns:
columns.append(self.db_config.id_column)
else:
raise ValueError(f"ID column '{self.db_config.id_column}' does not exist in table {self.db_config.table_name}")
# 添加所有 content 列（支持多个列），只添加存在的列
for col in self.db_config.content_columns:
if col in table_columns:
columns.append(col)
else:
logger.warning(f"Content column '{col}' does not exist in table {self.db_config.table_name}, skipping")
if not columns or all(col == self.db_config.id_column for col in columns):
raise ValueError(f"No valid content columns found in table {self.db_config.table_name}")
# Add title column if configured and exists
if self.db_config.title_column and self.db_config.title_column in table_columns:
columns.append(self.db_config.title_column)
elif self.db_config.title_column:
logger.warning(f"Title column '{self.db_config.title_column}' does not exist in table {self.db_config.table_name}, skipping")
# Add metadata columns if specified, only add existing ones
if self.db_config.metadata_columns:
metadata_cols = [col.strip() for col in self.db_config.metadata_columns.split(",")]
for col in metadata_cols:
if col in table_columns:
columns.append(col)
else:
logger.warning(f"Metadata column '{col}' does not exist in table {self.db_config.table_name}, skipping")
query = f"SELECT {', '.join(columns)} FROM `{self.db_config.table_name}`"
# Use updated_at_column if configured and last_sync_time is provided
# Otherwise, fetch all documents (no incremental sync)
if last_sync_time and self.db_config.updated_at_column:
# Use configured updated_at column for incremental sync
# MySQL DATETIME only has second precision, so we need to ensure
# the comparison works correctly. Use >= to include records with same timestamp
# and format the datetime to match MySQL's DATETIME format
query += f" WHERE {self.db_config.updated_at_column} > %s"
# Format datetime to MySQL DATETIME format (YYYY-MM-DD HH:MM:SS)
# This ensures proper comparison with MySQL DATETIME type
mysql_datetime = last_sync_time.strftime('%Y-%m-%d %H:%M:%S')
cursor.execute(query, (mysql_datetime,))
logger.info(
f"Incremental sync using {self.db_config.updated_at_column}: "
f"last_sync_time={last_sync_time} (formatted: {mysql_datetime}), "
f"query: {query}"
)
else:
# No updated_at column configured or no last_sync_time, fetch all
if last_sync_time and not self.db_config.updated_at_column:
logger.warning(
f"No updated_at_column configured for {self.db_config.name}, "
"fetching all documents for incremental sync"
)
cursor.execute(query)
results = cursor.fetchall()
# 如果配置了文件字段，则需根据文件字段的值来获取文件内容
if self.db_config.file_column and self.db_config.file_column in self.db_config.content_columns:
# 文件处理成功的文档
processed_results = []
# 失败文档ID列表
failed_doc_ids = set()
logger.info(
f"MySQL database: {self.db_config.database} (table: {self.db_config.table_name}) has set file_column(value: {self.db_config.file_column}) field.")
for doc in results:
# 获取每个doc的文件字段值，即文件标识符
file_column_value = doc.get(self.db_config.file_column)
if not file_column_value or file_column_value is None:
processed_results.append(doc)
continue
file_identifiers = [file_identifier.strip() for file_identifier in file_column_value.split(",")]
try:
file_content_parts = []
for file_identifier in file_identifiers:
# 调用接口获取文件字节流及文件名
file_byte, file_name = self.fetch_file_bytes(file_identifier)
if not file_byte:
logger.warning(
f"文档ID:{doc.get(self.db_config.id_column, '未知')}，文件标识符：{file_identifier} 获取字节流失败")
failed_doc_ids.add(doc.get(self.db_config.id_column))
break
# 调用文件读取方法获取文本内容
file_text = self.file_parser.get_text_from_bytes(file_byte, file_name)
if file_text and file_text.strip():
file_content_parts.append(file_text.strip())
logger.info(f"成功读取文件内容: {file_identifier} (长度: {len(file_text)})")
else:
logger.warning(f"文件标识符 {file_identifier} 提取的文本内容为空")
failed_doc_ids.add(doc.get(self.db_config.id_column))
break
if file_content_parts:
file_content = "\n".join(file_content_parts)
# 将文件内容添加到文档中
doc[self.db_config.file_column] = file_content
logger.info(
f"文档 {doc.get(self.db_config.id_column, '未知')} 成功合并 {len(file_content_parts)} 个文件内容")
# 将处理成功的文档添加到新列表中
processed_results.append(doc)
else:
logger.warning(
f"文档 {doc.get(self.db_config.id_column, '未知')} 没有成功提取任何文件内容")
failed_doc_ids.add(doc.get(self.db_config.id_column))
except Exception as e:
logger.error(
f"处理文档({doc.get(self.db_config.id_column, '未知')}) 文件字段: {doc[self.db_config.file_column]} 时出错: {e}")
failed_doc_ids.add(doc.get(self.db_config.id_column))
continue # 继续下一次doc循环
# 记录失败文档统计信息
if failed_doc_ids:
failed_doc_ids_str = {str(doc_id) for doc_id in failed_doc_ids}
ids_str = ', '.join(sorted(failed_doc_ids_str))
logger.warning(
f"数据库配置[{self.db_config.name}]本次同步在处理文件过程中共有 {len(failed_doc_ids)} 个文档因文件处理失败被跳过: [{ids_str}]")
else:
logger.info(f"数据库配置[{self.db_config.name}]所有文档文件处理成功")
results = processed_results
# Add database source information to each document
if results:
for doc in results:
doc['_db_source'] = self.db_config.name
doc['_db_database'] = self.db_config.database
doc['_db_table'] = self.db_config.table_name
logger.info(f"Fetched {len(results)} new/updated documents from MySQL database: {self.db_config.database} (name: {self.db_config.name})")
return results
except Exception as e:
logger.error(f"Error fetching new documents from MySQL database {self.db_config.name}: {e}")
raise
def get_document_by_id(self, doc_id: str) -> Optional[Dict]:
"""
Get a single document by ID
Args:
doc_id: Document ID
Returns:
Document dictionary or None
"""
try:
# Ensure we're using the correct database
with self.connection.cursor() as cursor:
cursor.execute(f"USE `{self.db_config.database}`")
with self.connection.cursor() as cursor:
# Get actual columns in the table
cursor.execute(f"DESCRIBE `{self.db_config.table_name}`")
table_columns = {row['Field'] for row in cursor.fetchall()}
# Build query based on configured columns, only include columns that exist
columns = []
# Add ID column (required)
if self.db_config.id_column in table_columns:
columns.append(self.db_config.id_column)
else:
raise ValueError(f"ID column '{self.db_config.id_column}' does not exist in table {self.db_config.table_name}")
# 添加所有 content 列（支持多个列），只添加存在的列
for col in self.db_config.content_columns:
if col in table_columns:
columns.append(col)
else:
logger.warning(f"Content column '{col}' does not exist in table {self.db_config.table_name}, skipping")
if not columns or all(col == self.db_config.id_column for col in columns):
raise ValueError(f"No valid content columns found in table {self.db_config.table_name}")
# Add title column if configured and exists
if self.db_config.title_column and self.db_config.title_column in table_columns:
columns.append(self.db_config.title_column)
elif self.db_config.title_column:
logger.warning(f"Title column '{self.db_config.title_column}' does not exist in table {self.db_config.table_name}, skipping")
# Add metadata columns if specified, only add existing ones
if self.db_config.metadata_columns:
metadata_cols = [col.strip() for col in self.db_config.metadata_columns.split(",")]
for col in metadata_cols:
if col in table_columns:
columns.append(col)
else:
logger.warning(f"Metadata column '{col}' does not exist in table {self.db_config.table_name}, skipping")
query = f"SELECT {', '.join(columns)} FROM `{self.db_config.table_name}` WHERE {self.db_config.id_column} = %s"
cursor.execute(query, (doc_id,))
result = cursor.fetchone()
if result:
result['_db_source'] = self.db_config.name
result['_db_database'] = self.db_config.database
result['_db_table'] = self.db_config.table_name
return result
except Exception as e:
logger.error(f"Error fetching document {doc_id} from MySQL database {self.db_config.name}: {e}")
return None
# 获取文件字节流及文件名
def fetch_file_bytes(self, file_identifier: str) -> tuple[bytes, str]:
"""
获取文件流、文件名
Args:
file_identifier: 文件标识值
Returns:
tuple: (文件字节流, 文件名)
"""
# 获取文件源类型配置
file_source_type = self.db_config.file_source_type
if file_source_type == "filesystem":
# 从本地文件系统读取文件
import os
try:
if not self.db_config.file_system_base_path:
raise ValueError("file_system_base_path is required for filesystem source type")
file_path = os.path.join(self.db_config.file_system_base_path, file_identifier)
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
with open(file_path, 'rb') as f:
file_bytes = f.read()
file_name = os.path.basename(file_path)
logger.info(f"Successfully fetched file from local filesystem: {file_path} (size: {len(file_bytes)} bytes)")
return file_bytes, file_name
except Exception as e:
logger.error(f"Error fetching file from local filesystem: {e}")
raise
elif file_source_type == "scp":
# 从远程文件系统通过SCP读取文件
import os
try:
if not self.db_config.file_system_base_path:
raise ValueError("file_system_base_path is required for scp source type")
if not self.db_config.scp_host:
raise ValueError("scp_host is required for scp source type")
# 导入paramiko库
import paramiko
# 创建SSH客户端
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# 连接到远程主机
connect_kwargs = {
'hostname': self.db_config.scp_host,
'port': self.db_config.scp_port,
'username': self.db_config.scp_username
}
if self.db_config.scp_key_path:
connect_kwargs['key_filename'] = self.db_config.scp_key_path
elif self.db_config.scp_password:
connect_kwargs['password'] = self.db_config.scp_password
else:
raise ValueError("Either scp_key_path or scp_password must be provided for SCP")
ssh_client.connect(**connect_kwargs)
# 创建SCP客户端
scp_client = ssh_client.open_sftp()
# 构建远程文件路径
remote_file_path = os.path.join(self.db_config.file_system_base_path, file_identifier)
# 读取文件内容
with scp_client.open(remote_file_path, 'rb') as f:
file_bytes = f.read()
file_name = os.path.basename(remote_file_path)
logger.info(f"Successfully fetched file from remote filesystem via SCP: {remote_file_path} (size: {len(file_bytes)} bytes)")
# 关闭连接
scp_client.close()
ssh_client.close()
return file_bytes, file_name
except Exception as e:
logger.error(f"Error fetching file from remote filesystem via SCP: {e}")
raise
else: # api
# 从API获取文件
base_url = settings.FILE_DOWNLOAD_BASE_URL
url = f"{base_url}/{file_identifier}"
try:
# 发送GET请求获取文件流
response = requests.get(url, stream=True, timeout=30)
# 检查响应内容是否为文件
content_type = response.headers.get('Content-Type', '')
if 'application/json' in content_type:
# 如果是JSON响应，尝试解析错误信息
error_data = response.json()
error_msg = error_data.get('msg', '未知错误')
error_code = error_data.get('code', '未知')
raise ValueError(f"API返回错误: {error_msg} (代码: {error_code})")
response.raise_for_status() # 如果请求失败，抛出HTTPError异常
# 获取文件字节流
file_bytes = response.content
logger.info(f"Successfully fetch file bytes from API, file: {file_identifier} (size: {len(file_bytes)} bytes)")
# 从响应头中获取文件名
file_name = "unknown"
# 从 Content-Disposition 头获取文件名
content_disposition = response.headers.get('Content-Disposition', '')
if 'filename=' in content_disposition:
# 提取文件名，处理可能的引号和多个filename参数
import re
# match = re.search(r'filename=["\']?([^"\']+)["\']?', content_disposition)
match = re.search(r'filename=["\']?([^;"\']+)["\']?', content_disposition)
if match:
file_name = match.group(1)
# 处理URL编码的文件名
from urllib.parse import unquote
file_name = unquote(file_name)
logger.info(f"Successfully get filename: {file_name}")
return file_bytes, file_name
except requests.exceptions.RequestException as e:
logger.error(f"Error fetching file from API: {e}")
raise
except Exception as e:
logger.error(f"Unexpected error fetching file from API: {e}")
raise

3
docker-compose.yml
View File

@ -57,8 +57,6 @@ services:
env_file:
- .env
volumes:
# 挂载配置文件目录（支持动态增加配置）
- ./configs:/app/configs:rw
# 挂载日志目录（可选）
- ./logs:/app/logs
# 挂载本地 data 目录以持久化会话/SQLite 数据库
@ -87,7 +85,6 @@ services:
- MYSQL_USER=${MYSQL_USER:-root}
- MYSQL_PASSWORD=${MYSQL_PASSWORD:-}
- MYSQL_DATABASE=${MYSQL_DATABASE:-forgeplus}
- MYSQL_CHARSET=${MYSQL_CHARSET:-utf8mb4}
# 多数据库配置（可选）
- MYSQL_DATABASES_CONFIG=${MYSQL_DATABASES_CONFIG:-}

166
rag/file_parser.py
View File

@ -31,10 +31,10 @@ class FileParser:
def is_supported(self, filename: str) -> bool:
"""
Check if file format is supported
Check if the file format is supported by the parser
Args:
filename: File name or path
filename: Name or path of the file to check
Returns:
True if supported, False otherwise
@ -42,14 +42,44 @@ class FileParser:
ext = Path(filename).suffix.lower() # 获取文件类型并转为小写
return ext in self.SUPPORTED_EXTENSIONS
def parse_file(self, file_path: str, doc_id: Optional[str] = None, metadata: Optional[Dict] = None) -> List[Document]:
def _generate_doc_id(self, filename: str, host: Optional[str] = None) -> str:
"""
Generate a unique doc_id based on host address and filename
Args:
filename: Name of the file
host: Optional host address (if not provided, will use local machine IP)
Returns:
Unique doc_id generated from host address and filename
"""
# 使用传入的host参数（如果提供），否则获取本地机器IP地址
if host:
host_address = host
else:
import socket
try:
# 获取机器IP地址
hostname = socket.gethostname()
host_address = socket.gethostbyname(hostname)
except:
# 如果无法获取IP，使用默认值
host_address = 'unknown'
# 使用主机地址和完整文件路径生成doc_id，确保唯一性
# 替换路径中的特殊字符，避免生成无效的doc_id
sanitized_path = filename.replace('/', '_').replace('\\', '_').replace(':', '_').replace(' ', '_')
return f"{host_address}_{sanitized_path}"
def parse_file(self, file_path: str, doc_id: Optional[str] = None, metadata: Optional[Dict] = None, host: Optional[str] = None) -> List[Document]:
"""
Parse a file into LlamaIndex Documents
Args:
file_path: Path to the file
doc_id: Optional document ID (if not provided, will use filename)
doc_id: Optional document ID (if not provided, will use filename and host)
metadata: Optional metadata to add to documents
host: Optional host address (if not provided, will use local machine IP)
Returns:
List of LlamaIndex Document objects
@ -63,16 +93,47 @@ class FileParser:
if not self.is_supported(file_path):
raise ValueError(f"Unsupported file format: {ext}. Supported formats: {', '.join(self.SUPPORTED_EXTENSIONS)}")
# Use doc_id or generate from filename
# 生成基于主机地址和真实文件名的doc_id（如果未提供）
if not doc_id:
doc_id = Path(filename).stem
doc_id = self._generate_doc_id(filename, host)
try:
logger.info(f"Starting to parse file: {filename} (type: {ext})")
logger.info(f"Starting to parse file: {filename} (type: {ext})\n")
# 处理 .doc 文件：通过 soffice-service 转换为 .docx
if ext == '.doc':
logger.info(f"检测到 .doc 文件，开始转换为 .docx 格式: {file_path}")
# 读取 .doc 文件内容
with open(file_path, 'rb') as f:
file_bytes = f.read()
# 上传文件到 soffice-service 的 convert 接口
import requests
from config import settings
files = {'file': (filename, file_bytes, 'application/msword')}
soffice_url = f"http://{settings.SOFFICE_HOST}:{settings.SOFFICE_PORT}/convert"
response = requests.post(soffice_url, files=files, timeout=60)
try:
response.raise_for_status() # 检查 HTTP 错误
except requests.exceptions.HTTPError as e:
logger.error(f"转换请求失败: {e}")
raise
# 将转换后的 docx 内容保存到临时文件进行解析
import tempfile
with tempfile.NamedTemporaryFile(delete=False, suffix='.docx') as tmp_docx_file:
tmp_docx_file.write(response.content)
tmp_path = tmp_docx_file.name
# 使用转换后的临时文件路径
parse_path = tmp_path
cleanup_tmp = True
else:
# 对于其他文件格式，直接使用原始文件路径
parse_path = file_path
cleanup_tmp = False
# Use LlamaIndex's SimpleDirectoryReader for parsing
# It supports many formats out of the box
reader = SimpleDirectoryReader(
input_files=[file_path],
input_files=[parse_path],
filename_as_id=False # We'll set custom IDs
)
@ -131,14 +192,23 @@ class FileParser:
if not valid_documents:
raise ValueError(f"No valid documents extracted from {filename}. All documents are empty.")
logger.info(f"Successfully parsed file {filename}: {len(valid_documents)} valid document(s) (total: {len(documents)}, skipped empty: {len(documents) - len(valid_documents)})")
logger.info(f"Successfully parsed file {filename}: {len(valid_documents)} valid document(s) (total: {len(documents)}, skipped empty: {len(documents) - len(valid_documents)})\n")
return valid_documents
except Exception as e:
logger.error(f"Error parsing file {file_path}: {e}", exc_info=True)
raise
finally:
# 清理临时文件（如果有）
if 'cleanup_tmp' in locals() and cleanup_tmp:
try:
import os
os.unlink(tmp_path)
logger.debug(f"Cleaned up temporary file: {tmp_path}")
except Exception as e:
logger.warning(f"Failed to delete temporary file {tmp_path}: {e}")
def parse_file_content(self, content: bytes, filename: str, doc_id: Optional[str] = None, metadata: Optional[Dict] = None) -> List[Document]:
def parse_file_content(self, content: bytes, filename: str, doc_id: Optional[str] = None, metadata: Optional[Dict] = None, host: Optional[str] = None) -> List[Document]:
"""
Parse file content from bytes into LlamaIndex Documents
@ -147,6 +217,7 @@ class FileParser:
filename: Original filename (for format detection)
doc_id: Optional document ID
metadata: Optional metadata to add to documents
host: Optional host address (for remote files, use the remote host address)
Returns:
List of LlamaIndex Document objects
@ -163,6 +234,10 @@ class FileParser:
else:
ext = '.txt'
# 生成基于主机地址和真实文件名的doc_id（如果未提供）
if not doc_id:
doc_id = self._generate_doc_id(filename, host)
# Use tempfile to save content and parse
with tempfile.NamedTemporaryFile(delete=False, suffix=ext) as tmp_file:
tmp_file.write(content)
@ -180,75 +255,4 @@ class FileParser:
logger.warning(f"Failed to delete temporary file {tmp_path}: {e}")
def get_text_from_bytes(self, file_bytes: bytes, file_name: str) -> str:
"""
根据文件的字节流获取文件内的文本字符串
Args:
file_bytes: 文件字节流
file_name: 文件名
Returns:
文件内容的文本字符串
"""
import tempfile
# 获取文件类型
if file_name == "unknown":
ext = '.txt'
else:
ext = Path(file_name).suffix.lower()
# 初始化临时文件路径
tmp_path = None
try:
# 检查是否为.doc文件，如果是则调用soffice-service转换为.docx
if ext == '.doc':
logger.info(f"检测到.doc文件，开始转换为.docx格式: {file_name}")
# 上传文件到 soffice-service的 convert接口
files = {'file': (file_name, file_bytes, 'application/msword')}
soffice_url = f"http://{settings.SOFFICE_HOST}:{settings.SOFFICE_PORT}/convert"
response = requests.post(soffice_url, files=files, timeout=60)
try:
response.raise_for_status() # 检查 HTTP错误
except requests.exceptions.HTTPError as e:
logger.error(f"转换请求失败: {e}")
raise
# 将转换后的 docx 内容保存到临时文件进行解析
with tempfile.NamedTemporaryFile(delete=False, suffix='.docx') as tmp_docx_file:
tmp_docx_file.write(response.content)
tmp_path = tmp_docx_file.name
else:
# Use tempfile to save content and parse
with tempfile.NamedTemporaryFile(delete=False, suffix=ext) as tmp_file:
tmp_file.write(file_bytes)
tmp_path = tmp_file.name
# Parse the temporary file
documents = self.parse_file(tmp_path, doc_id=None)
# 提取所有非空文档的文本内容
text_parts = []
for i, doc in enumerate(documents):
if doc.text and doc.text.strip():
text_parts.append(doc.text.strip())
logger.debug(f"提取到文档 {i} 的文本内容，长度: {len(doc.text)}")
else:
logger.warning(f"文档 {i} 内容为空，已跳过")
if not text_parts:
raise ValueError("所有文档内容均为空，无法提取有效文本")
# 合并所有文本内容
full_text = "\n\n".join(text_parts)
logger.info(f"成功从文件(文件名：{file_name})字节流中提取文本，总长度: {len(full_text)} 字符")
return full_text
except Exception as e:
logger.error(f"解析文件(文件名：{file_name})字节流时出错: {e}", exc_info=True)
raise
finally:
# Clean up temporary file
if tmp_path is not None:
try:
os.unlink(tmp_path)
except Exception as e:
logger.warning(f"Failed to delete temporary file {tmp_path}: {e}")

402
scan_databases.py
View File

@ -1,402 +0,0 @@
#!/usr/bin/env python3
"""
扫描 MySQL 数据库，分析表结构并生成配置
自动识别包含内容列的表（include 模式）：
- 只考虑表中包含 text、longtext、mediumtext、tinytext 类型的列
- 或者表中包含 blob、longblob、mediumblob、tinyblob 类型的列
支持通过 scan_exclude_config.json 配置文件额外排除不需要的数据库和表
"""
import pymysql
import json
import os
import re
from typing import List, Dict, Any, Set
from config import settings
def load_exclude_config(config_path: str = 'scan_exclude_config.json') -> Dict[str, Any]:
"""
加载排除配置文件
Args:
config_path: 配置文件路径
Returns:
排除配置字典，包含 exclude_databases, exclude_databases_patterns,
和 exclude_tables_patterns（仅支持正则表达式模式）
"""
default_config = {
"exclude_databases": ["information_schema", "performance_schema", "mysql", "sys"],
"exclude_databases_patterns": [],
"exclude_tables_patterns": {}
}
if not os.path.exists(config_path):
print(f"排除配置文件不存在: {config_path}，使用默认配置")
return default_config
try:
with open(config_path, 'r', encoding='utf-8') as f:
config = json.load(f)
# 确保配置格式正确
if 'exclude_databases' not in config:
config['exclude_databases'] = default_config['exclude_databases']
if 'exclude_databases_patterns' not in config:
config['exclude_databases_patterns'] = []
if 'exclude_tables_patterns' not in config:
config['exclude_tables_patterns'] = {}
return config
except Exception as e:
print(f"加载排除配置文件失败: {e}，使用默认配置")
return default_config
def matches_pattern(name: str, patterns: List[str]) -> bool:
"""
检查名称是否匹配任何正则表达式模式
Args:
name: 要检查的名称
patterns: 正则表达式模式列表
Returns:
如果匹配任何模式返回 True，否则返回 False
"""
for pattern in patterns:
try:
if re.search(pattern, name):
return True
except re.error as e:
print(f"警告: 正则表达式模式 '{pattern}' 无效: {e}")
continue
return False
def get_mysql_connection(host="localhost", port=3306, user="root", password="", charset="utf8mb4"):
"""获取 MySQL 连接"""
return pymysql.connect(
host=host,
port=port,
user=user,
password=password,
charset=charset,
cursorclass=pymysql.cursors.DictCursor
)
def get_databases(conn, exclude_databases: Set[str] = None, exclude_patterns: List[str] = None) -> List[str]:
"""
获取所有数据库列表
Args:
conn: MySQL 连接
exclude_databases: 要排除的数据库集合（精确匹配）
exclude_patterns: 要排除的数据库正则表达式模式列表
Returns:
数据库列表
"""
if exclude_databases is None:
exclude_databases = set()
if exclude_patterns is None:
exclude_patterns = []
with conn.cursor() as cursor:
cursor.execute("SHOW DATABASES")
databases = [row['Database'] for row in cursor.fetchall()]
# 过滤掉排除的数据库
filtered_databases = []
for db in databases:
# 检查精确匹配
if db in exclude_databases:
continue
# 检查正则表达式匹配
if matches_pattern(db, exclude_patterns):
continue
filtered_databases.append(db)
return filtered_databases
def get_tables(conn, database: str) -> List[str]:
"""获取指定数据库的所有表"""
with conn.cursor() as cursor:
cursor.execute(f"USE `{database}`")
cursor.execute("SHOW TABLES")
# 表名在结果中的键名是 'Tables_in_{database}'
key = f"Tables_in_{database}"
return [row[key] for row in cursor.fetchall()]
def get_table_columns(conn, database: str, table: str) -> List[Dict[str, Any]]:
"""获取表的列信息"""
with conn.cursor() as cursor:
cursor.execute(f"USE `{database}`")
cursor.execute(f"DESCRIBE `{table}`")
return cursor.fetchall()
def should_include_table(table_name: str, columns: List[Dict[str, Any]]) -> bool:
"""
判断表是否应该被包含（仅根据列类型判断，并排除敏感信息表）
只考虑 text、longtext、mediumtext、tinytext 以及 blob 等类型的字段
排除包含敏感信息的表（authentication、private、key、secret、password 等）
Args:
table_name: 表名
columns: 表的列信息列表
Returns:
如果表应该被包含返回 True，否则返回 False
"""
# 排除敏感信息表（检查表名）
table_lower = table_name.lower()
sensitive_keywords = ['authentication', 'auth', 'private', 'key', 'secret', 'password',
'pwd', 'token', 'credential', 'credential', 'session', 'login',
'user_password', 'user_secret', 'api_key', 'access_key', 'secret_key']
# 检查表名是否包含敏感关键词
if any(keyword in table_lower for keyword in sensitive_keywords):
return False
# 检查列名是否包含敏感关键词
column_names = [col['Field'].lower() for col in columns]
if any(keyword in col_name for col_name in column_names for keyword in sensitive_keywords):
return False
column_types = [col['Type'].lower() for col in columns]
# 只检查是否有文本类型列（text、longtext、mediumtext、tinytext、blob、longblob、mediumblob、tinyblob）
text_types = ['text', 'longtext', 'mediumtext', 'tinytext', 'blob', 'longblob', 'mediumblob', 'tinyblob']
has_text_type = any(any(text_type in col_type for text_type in text_types) for col_type in column_types)
return has_text_type
def analyze_table_structure(columns: List[Dict[str, Any]]) -> Dict[str, Any]:
"""分析表结构，推断配置项"""
column_names = [col['Field'] for col in columns]
column_types = {col['Field']: col['Type'].lower() for col in columns}
# 查找 ID 列（通常是 id, *_id）
id_column = None
for col in column_names:
if col.lower() == 'id':
id_column = col
break
elif col.lower().endswith('_id'):
id_column = col
break
# 查找内容列（只考虑 text、longtext、mediumtext、tinytext 以及 blob 等类型）
text_types = ['text', 'longtext', 'mediumtext', 'tinytext', 'blob', 'longblob', 'mediumblob', 'tinyblob']
content_columns = []
for col in column_names:
col_type = column_types.get(col, '').lower()
# 只包含 text 和 blob 类型的列
if any(text_type in col_type for text_type in text_types):
if col != id_column and not col.lower().endswith('_id'):
content_columns.append(col)
# 查找标题列（可能是 title, subject, name）
title_column = None
title_keywords = ['title', 'subject', 'name']
for col in column_names:
if col.lower() in title_keywords:
title_column = col
break
# 查找更新时间列（可能是 updated_at, updated_on, modified_at, update_time）
updated_at_column = None
update_keywords = ['updated_at', 'updated_on', 'modified_at', 'update_time', 'updated_time']
for col in column_names:
if col.lower() in update_keywords:
updated_at_column = col
break
# 查找元数据列（author, category 等，不包括时间相关列）
metadata_columns = []
# 时间相关的关键词，这些不应该出现在 metadata_columns 中
time_keywords = ['created_at', 'created_on', 'created_time', 'create_time', 'create_by',
'updated_at', 'updated_on', 'updated_time', 'update_time', 'modified_at',
'created_unix', 'updated_unix']
# 元数据关键词（不包括时间相关）
metadata_keywords = ['author', 'category', 'tag', 'tags', 'status', 'type', 'priority', 'source']
for col in column_names:
col_lower = col.lower()
# 排除时间相关的列
if any(time_keyword in col_lower for time_keyword in time_keywords):
continue
# 排除以 created_ 或 create_ 开头的列（时间相关）
if col_lower.startswith('created_') or col_lower.startswith('create_'):
continue
# 排除以 updated_ 或 update_ 开头的列（时间相关）
if col_lower.startswith('updated_') or col_lower.startswith('update_'):
continue
# 排除以 modified_ 开头的列（时间相关）
if col_lower.startswith('modified_'):
continue
# 添加符合条件的元数据列
if col_lower in metadata_keywords:
metadata_columns.append(col)
return {
'id_column': id_column or 'id',
'content_columns': content_columns[:3] if content_columns else ['content'], # 最多取3个
'title_column': title_column,
'updated_at_column': updated_at_column,
'metadata_columns': metadata_columns[:5] if metadata_columns else None # 最多取5个
}
def generate_config(conn, exclude_config: Dict[str, Any] = None) -> List[Dict[str, Any]]:
"""
生成配置列表
Args:
conn: MySQL 连接
exclude_config: 排除配置字典
Returns:
配置列表
"""
if exclude_config is None:
exclude_config = {
"exclude_databases": [],
"exclude_databases_patterns": [],
"exclude_tables_patterns": {}
}
configs = []
exclude_databases = set(exclude_config.get('exclude_databases', []))
exclude_db_patterns = exclude_config.get('exclude_databases_patterns', [])
exclude_table_patterns = exclude_config.get('exclude_tables_patterns', {})
databases = get_databases(conn, exclude_databases, exclude_db_patterns)
print(f"找到 {len(databases)} 个数据库: {', '.join(databases)}")
if exclude_databases:
print(f"排除的数据库（精确匹配）: {', '.join(sorted(exclude_databases))}")
if exclude_db_patterns:
print(f"排除的数据库（正则表达式）: {', '.join(exclude_db_patterns)}")
for database in databases:
print(f"\n扫描数据库: {database}")
try:
tables = get_tables(conn, database)
# 获取该数据库要排除的表（正则表达式模式）
db_exclude_patterns = exclude_table_patterns.get(database, [])
if db_exclude_patterns:
print(f" 排除的表（正则表达式）: {', '.join(db_exclude_patterns)}")
# 过滤掉排除的表（仅使用正则表达式模式）
filtered_tables = []
for table in tables:
# 检查正则表达式匹配
if matches_pattern(table, db_exclude_patterns):
continue
filtered_tables.append(table)
print(f" 找到 {len(tables)} 个表，排除后 {len(filtered_tables)} 个表")
# 进一步过滤：只包含有内容列的表
included_tables = []
excluded_by_content = []
for table in filtered_tables:
try:
columns = get_table_columns(conn, database, table)
if should_include_table(table, columns):
included_tables.append(table)
else:
excluded_by_content.append(table)
except Exception as e:
print(f" 检查表 {table} 时出错: {e}")
continue
print(f" 自动识别包含内容列的表: {len(included_tables)} 个")
if excluded_by_content:
print(f" 排除无内容列的表: {len(excluded_by_content)} 个")
if len(excluded_by_content) <= 10:
print(f" {', '.join(excluded_by_content)}")
else:
print(f" {', '.join(excluded_by_content[:10])} ... (共 {len(excluded_by_content)} 个)")
for table in included_tables:
print(f" 分析表: {table}")
try:
columns = get_table_columns(conn, database, table)
analysis = analyze_table_structure(columns)
# 生成配置项
config = {
'name': f"{database}_{table}",
'database': database,
'table_name': table,
'id_column': analysis['id_column'],
'content_column': ','.join(analysis['content_columns']),
'title_column': analysis['title_column'],
'metadata_columns': ','.join(analysis['metadata_columns']) if analysis['metadata_columns'] else None,
'content_separator': '\n',
'updated_at_column': analysis['updated_at_column']
}
# 清理 None 值
config = {k: v for k, v in config.items() if v is not None}
configs.append(config)
print(f" ID列: {analysis['id_column']}")
print(f" 内容列: {', '.join(analysis['content_columns'])}")
if analysis['title_column']:
print(f" 标题列: {analysis['title_column']}")
if analysis['updated_at_column']:
print(f" 更新时间列: {analysis['updated_at_column']}")
if analysis['metadata_columns']:
print(f" 元数据列: {', '.join(analysis['metadata_columns'])}")
except Exception as e:
print(f" 错误: {e}")
continue
except Exception as e:
print(f" 错误: {e}")
continue
return configs
def main():
"""主函数"""
try:
# 加载排除配置
exclude_config = load_exclude_config()
conn = get_mysql_connection()
print("成功连接到 MySQL 服务器")
print("=" * 60)
print("扫描模式: 自动识别包含内容列的表（include 模式）")
print("识别条件:")
print(" - 表中包含 text、longtext、mediumtext、tinytext 类型的列")
print(" - 或者表中包含 blob、longblob、mediumblob、tinyblob 类型的列")
print(" - 自动排除包含敏感信息的表（authentication、private、key、secret、password 等）")
print("=" * 60)
if os.path.exists('scan_exclude_config.json'):
print(f"额外排除配置: scan_exclude_config.json")
print()
configs = generate_config(conn, exclude_config)
conn.close()
# 输出 JSON 配置
print(f"\n\n生成 {len(configs)} 个配置项")
print("\n配置 JSON:")
print(json.dumps(configs, indent=2, ensure_ascii=False))
# 保存到文件
output_file = 'databases_config_new.json'
with open(output_file, 'w', encoding='utf-8') as f:
json.dump(configs, f, indent=2, ensure_ascii=False)
print(f"\n配置已保存到: {output_file}")
except Exception as e:
print(f"错误: {e}")
import traceback
traceback.print_exc()
if __name__ == '__main__':
main()

14
static/config/index.html
View File

@ -4,8 +4,8 @@
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>RAG 配置管理</title>
<link rel="stylesheet" href="/static/config/style.css?v=20260118">
<script src="/static/config/script.js?v=20260118"></script>
<link rel="stylesheet" href="/static/config/style.css?v=202601212214">
<script src="/static/config/script.js?v=202601212214"></script>
</head>
<body>
<div class="config-container">
@ -13,7 +13,15 @@
<aside class="sidebar">
<div class="sidebar-header">
<h1>⚙️ 配置管理</h1>
<button class="add-config-btn" id="addConfigBtn">+</button>
<div class="add-config-container">
<label for="configTypeSelect" class="config-type-label">新增数据源：</label>
<select id="configTypeSelect">
<option value="">选择类型</option>
<option value="database">数据库</option>
<option value="local_folder">本地文件夹</option>
<option value="remote_folder">远程文件夹</option>
</select>
</div>
</div>
<div class="config-list" id="configList">

1210
static/config/script.js
View File

File diff suppressed because it is too large Load Diff

209
static/config/style.css
View File

@ -35,32 +35,106 @@ body {
padding: 20px;
border-bottom: 1px solid #e0e0e0;
display: flex;
justify-content: space-between;
flex-direction: column;
align-items: center;
gap: 15px;
background-color: white;
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
}
.sidebar-header h1 {
font-size: 1.2rem;
font-weight: 600;
color: #333;
}
.add-config-btn {
width: 32px;
height: 32px;
border-radius: 50%;
border: none;
background-color: #4a90e2;
color: white;
font-size: 1.2rem;
cursor: pointer;
font-size: 1.3rem;
font-weight: 700;
color: #2c3e50;
margin: 0;
display: flex;
align-items: center;
justify-content: center;
gap: 8px;
}
.add-config-btn:hover {
background-color: #357abd;
/* 改进添加配置容器的样式 */
.add-config-container {
width: 100%;
display: flex;
gap: 10px;
align-items: center;
}
/* 配置类型标签 */
.config-type-label {
font-weight: 600;
color: #2c3e50;
font-size: 0.95rem;
margin-right: 10px;
white-space: nowrap;
}
/* 美化下拉选择框 */
#configTypeSelect {
flex: 1;
padding: 10px 12px;
border: 2px solid #e1e8ed;
border-radius: 8px;
background-color: white;
color: #333;
font-size: 0.95rem;
cursor: pointer;
transition: all 0.2s ease;
outline: none;
}
#configTypeSelect:hover {
border-color: #4a90e2;
}
#configTypeSelect:focus {
border-color: #4a90e2;
box-shadow: 0 0 0 3px rgba(74, 144, 226, 0.1);
}
/* 美化添加按钮 - 已移除 */
/* 必填项星号样式 */
.required {
color: #e74c3c;
font-weight: bold;
margin-left: 2px;
}
/* 复选框组样式 */
.checkbox-group {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
gap: 10px;
margin-top: 8px;
background-color: #f9f9f9;
padding: 15px;
border-radius: 8px;
border: 1px solid #e0e0e0;
max-height: 200px;
overflow-y: auto;
}
.checkbox-group label {
display: flex;
align-items: center;
gap: 8px;
cursor: pointer;
font-weight: normal;
margin: 0;
padding: 5px 8px;
border-radius: 4px;
transition: all 0.2s ease;
}
.checkbox-group label:hover {
background-color: #e8f0fe;
}
.checkbox-group input[type="checkbox"] {
width: 16px;
height: 16px;
cursor: pointer;
}
.config-list {
@ -70,32 +144,117 @@ body {
}
.config-item {
padding: 12px;
padding: 15px;
margin-bottom: 8px;
background-color: #f9f9f9;
border-radius: 8px;
background-color: white;
border-radius: 10px;
cursor: pointer;
border-left: 3px solid transparent;
border-left: 4px solid transparent;
transition: all 0.2s ease;
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05);
border: 1px solid #e0e0e0;
}
.config-item:hover {
background-color: #f0f0f0;
background-color: #f8fbff;
border-left-color: #4a90e2;
transform: translateY(-1px);
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.08);
}
.config-item.active {
background-color: #e8f0fe;
border-left-color: #4a90e2;
box-shadow: 0 2px 6px rgba(74, 144, 226, 0.15);
}
.config-item-content {
display: flex;
align-items: flex-start;
gap: 12px;
}
.config-item-icon {
width: 36px;
height: 36px;
border-radius: 8px;
display: flex;
align-items: center;
justify-content: center;
font-size: 1.2rem;
flex-shrink: 0;
margin-top: 2px;
}
/* 不同类型配置的图标背景色 */
.config-item[data-type="database"] .config-item-icon {
background-color: #e3f2fd;
color: #1976d2;
}
.config-item[data-type="local_folder"] .config-item-icon {
background-color: #e8f5e9;
color: #388e3c;
}
.config-item[data-type="remote_folder"] .config-item-icon {
background-color: #fff3e0;
color: #f57c00;
}
.config-item-details {
flex: 1;
}
.config-item-title {
font-weight: 600;
margin-bottom: 4px;
margin-bottom: 6px;
color: #2c3e50;
font-size: 0.95rem;
word-break: break-all;
}
.config-item-type {
.config-item-info {
font-size: 0.85rem;
color: #666;
line-height: 1.4;
word-break: break-all;
}
.config-item-meta {
display: flex;
justify-content: space-between;
align-items: center;
margin-top: 8px;
}
.config-item-time {
font-size: 0.75rem;
color: #999;
}
.config-item-actions {
display: flex;
gap: 5px;
}
.config-item-action {
width: 24px;
height: 24px;
border-radius: 4px;
border: none;
background-color: transparent;
color: #999;
font-size: 0.8rem;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
transition: all 0.2s ease;
}
.config-item-action:hover {
background-color: #f0f0f0;
color: #666;
}

13
sync/__init__.py Normal file
View File

@ -0,0 +1,13 @@
"""Synchronization modules for all data sources"""
from .base_sync import BaseSync, get_sync_class
from .mysql_sync import MySQLSync
from .local_folder_sync import LocalFolderSync
from .remote_folder_sync import RemoteFolderSync
__all__ = [
'BaseSync',
'get_sync_class',
'MySQLSync',
'LocalFolderSync',
'RemoteFolderSync'
]

216
sync/base_sync.py Normal file
View File

@ -0,0 +1,216 @@
"""Base synchronization interface for all data sources"""
from abc import ABC, abstractmethod
from typing import List, Dict, Any, Set, Optional
from config import BaseDataSourceConfig
from rag.file_parser import FileParser
class BaseSync(ABC):
"""Abstract base class for all data source syncers"""
def __init__(self, config: BaseDataSourceConfig):
"""
Initialize sync with data source configuration
Args:
config: Configuration for the data source
"""
self.config = config
self.file_parser = FileParser()
def _get_file_bytes(self, file_path: str, source_type: str, source_config: Optional[Dict[str, Any]] = None) -> Optional[bytes]:
"""
Get file content as bytes based on source type
Args:
file_path: Path to the file
source_type: Type of file source (api, filesystem, scp)
source_config: Additional configuration for file source
Returns:
File content as bytes if successfully loaded, None otherwise
"""
try:
import requests
if source_type == 'api':
# 从 API 获取文件内容
from config import settings
url = f"{settings.FILE_DOWNLOAD_BASE_URL}?identifier={file_path}"
response = requests.get(url, timeout=30)
response.raise_for_status()
return response.content
elif source_type == 'filesystem':
# 从本地文件系统获取文件内容
base_path = source_config.get('file_system_base_path', '') if source_config else ''
full_path = f"{base_path}/{file_path}" if base_path else file_path
with open(full_path, 'rb') as f:
return f.read()
elif source_type == 'scp':
# 通过 SCP 获取文件内容
import paramiko
ssh_config = source_config or {}
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Connect to remote server
if ssh_config.get('scp_key_path'):
# Use SSH key for authentication
private_key = paramiko.RSAKey.from_private_key_file(ssh_config['scp_key_path'])
ssh.connect(
hostname=ssh_config.get('scp_host', ''),
port=ssh_config.get('scp_port', 22),
username=ssh_config.get('scp_username', ''),
pkey=private_key
)
else:
# Use password for authentication
ssh.connect(
hostname=ssh_config.get('scp_host', ''),
port=ssh_config.get('scp_port', 22),
username=ssh_config.get('scp_username', ''),
password=ssh_config.get('scp_password', '')
)
# Use SFTP to download file as bytes
with paramiko.SFTPClient.from_transport(ssh.get_transport()) as sftp:
with sftp.open(file_path, 'rb') as f:
content = f.read()
ssh.close()
return content
else:
from loguru import logger
logger.error(f"Unsupported file source type: {source_type}")
return None
except Exception as e:
from loguru import logger
logger.error(f"Error getting file bytes for {file_path}: {e}")
return None
def _parse_file_content(self, file_bytes: bytes, file_path: str, host: Optional[str] = None) -> Optional[str]:
"""
Parse file content using FileParser class
Args:
file_bytes: File content as bytes
file_path: Path to the file (for getting file extension)
host: Optional host address (for remote files, use the remote host address)
Returns:
Parsed file content if successfully parsed, None otherwise
"""
try:
# 使用 FileParser 解析文件内容
# parse_file_content 方法的参数顺序是 content, filename
documents = self.file_parser.parse_file_content(file_bytes, file_path, host=host)
if documents:
# 合并所有文档内容
return '\n\n'.join(doc.text for doc in documents if doc.text)
else:
from loguru import logger
logger.warning(f"No content extracted from {file_path}")
return None
except Exception as e:
from loguru import logger
logger.error(f"Error parsing file content for {file_path}: {e}")
return None
@abstractmethod
def fetch_all_documents(self) -> List[Dict[str, Any]]:
"""
Fetch all documents from the data source
Returns:
List of documents
"""
pass
@abstractmethod
def fetch_new_documents(self, last_sync_time=None, synced_doc_ids=None) -> List[Dict[str, Any]]:
"""
Fetch new/updated documents from the data source since last sync time
Args:
last_sync_time: Last synchronization time
synced_doc_ids: Set of document IDs that have already been synced
Returns:
List of new/updated documents
"""
pass
@abstractmethod
def get_synced_document_ids(self) -> Set[str]:
"""
Get IDs of all synced documents from the data source
Returns:
Set of document IDs
"""
pass
@staticmethod
@abstractmethod
def check_data_source_exists(config: BaseDataSourceConfig) -> bool:
"""
Check if the data source exists and is accessible
Args:
config: Data source configuration
Returns:
True if data source exists and is accessible, False otherwise
"""
pass
@staticmethod
def check_data_sources_exist(configs: List[BaseDataSourceConfig]) -> tuple[bool, List[str]]:
"""
Check if all data sources exist and are accessible
Args:
configs: List of data source configurations
Returns:
Tuple of (all_exist, missing_sources)
"""
missing = []
for config in configs:
# Get the appropriate sync class based on data source type
sync_class = get_sync_class(config.type)
if not sync_class.check_data_source_exists(config):
missing.append(config.name)
return len(missing) == 0, missing
def get_sync_class(source_type: str) -> type[BaseSync]:
"""
Get the appropriate sync class based on data source type
Args:
source_type: Type of data source (database, local_folder, remote_folder)
Returns:
Sync class corresponding to the data source type
Raises:
ValueError: If source type is not supported
"""
from sync.mysql_sync import MySQLSync
from sync.local_folder_sync import LocalFolderSync
from sync.remote_folder_sync import RemoteFolderSync
sync_classes = {
'database': MySQLSync, # Currently only MySQL, but can be extended
'local_folder': LocalFolderSync,
'remote_folder': RemoteFolderSync
}
if source_type not in sync_classes:
raise ValueError(f"Unsupported data source type: {source_type}")
return sync_classes[source_type]

291
sync/local_folder_sync.py Normal file
View File

@ -0,0 +1,291 @@
"""Local folder synchronization implementation"""
import os
import re
from typing import List, Dict, Any, Set
from datetime import datetime
from pathlib import Path
from loguru import logger
from config import BaseDataSourceConfig
from sync.base_sync import BaseSync
from rag.file_parser import FileParser
class LocalFolderSync(BaseSync):
"""Handle synchronization between local folder and ChromaDB"""
def __init__(self, config: BaseDataSourceConfig):
"""
Initialize local folder sync with configuration
Args:
config: Local folder configuration
"""
super().__init__(config)
self.file_parser = FileParser()
self._ssh_client = None
self._sftp_client = None
def fetch_all_documents(self) -> List[Dict[str, Any]]:
"""
Fetch all documents from the local folder
Returns:
List of documents
"""
return self._fetch_documents()
def fetch_new_documents(self, last_sync_time=None, synced_doc_ids=None) -> List[Dict[str, Any]]:
"""
Fetch new/updated documents from the local folder since last sync time
Args:
last_sync_time: Last synchronization time
synced_doc_ids: Set of document IDs that have already been synced
Returns:
List of new/updated documents
"""
return self._fetch_documents(last_sync_time, synced_doc_ids)
def get_synced_document_ids(self) -> Set[str]:
"""
Get IDs of all files in the local folder via SFTP
Returns:
Set of file paths (as document IDs)
"""
self._connect()
try:
files = self._get_all_files()
return set(files)
finally:
self._disconnect()
def _fetch_documents(self, last_sync_time=None, synced_doc_ids=None) -> List[Dict[str, Any]]:
"""
Internal method to fetch documents from local folder via SFTP
Args:
last_sync_time: Last synchronization time (for incremental sync)
synced_doc_ids: Set of document IDs that have already been synced
Returns:
List of documents
"""
documents = []
self._connect()
try:
files = self._get_all_files()
for file_path in files:
# Check if document has already been synced
if synced_doc_ids and file_path in synced_doc_ids:
# If file is already synced, check if it's been modified since last sync
if last_sync_time:
file_stat = self._sftp_client.stat(file_path)
file_mtime = datetime.fromtimestamp(file_stat.st_mtime)
# Skip if not modified since last sync
if file_mtime <= last_sync_time:
continue
# If file is not synced yet, always include it regardless of modification time
# This handles the case where files were added to the folder after last_sync_time but have older mtimes
# Parse file content
try:
# 检查文件扩展名是否在支持的列表中
file_ext = os.path.splitext(file_path)[1].lower()
if file_ext not in FileParser.SUPPORTED_EXTENSIONS:
logger.debug(f"Skipping unsupported file: {file_path}")
continue
# 使用 SFTP 获取文件内容
with self._sftp_client.open(file_path, 'rb') as f:
file_bytes = f.read()
# 使用 BaseSync 中的通用方法解析文件内容
content = self._parse_file_content(file_bytes, file_path) if file_bytes else f"[无法读取文件：{os.path.basename(file_path)}]"
file_stat = self._sftp_client.stat(file_path)
document = {
'id': file_path,
'title': os.path.basename(file_path),
'content': content,
'file_path': file_path,
'update_time': datetime.fromtimestamp(file_stat.st_mtime)
}
documents.append(document)
except Exception as e:
logger.error(f"Error processing file {file_path}: {e}")
finally:
self._disconnect()
return documents
def _connect(self):
"""
Connect to the host machine via SSH/SFTP
"""
import paramiko
self._ssh_client = paramiko.SSHClient()
self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Connect to SSH server
# 确保username不为None，否则paramiko会报错
username = self.config.username or ''
# 建立SSH连接的参数
ssh_params = {
'hostname': self.config.host,
'port': self.config.port or 22,
'username': username,
'password': self.config.password,
'timeout': 10,
'allow_agent': True, # 允许使用SSH代理
'look_for_keys': False # 禁用查找本地密钥文件
}
# 连接到SSH服务器
self._ssh_client.connect(**ssh_params)
# Create SFTP client
self._sftp_client = self._ssh_client.open_sftp()
def _disconnect(self):
"""
Disconnect from the host machine
"""
if self._sftp_client:
self._sftp_client.close()
self._sftp_client = None
if self._ssh_client:
self._ssh_client.close()
self._ssh_client = None
def _get_all_files(self) -> List[str]:
"""
Get all files in the local folder via SFTP
Returns:
List of file paths
"""
files = []
# _get_files_recursive is called from _fetch_documents which handles connection
self._get_files_recursive(self.config.folder_path, files)
return files
def _get_files_recursive(self, folder_path: str, files: List[str]):
"""
Recursively get all files in the local folder via SFTP
Args:
folder_path: Current folder path
files: List to store found files
"""
try:
items = self._sftp_client.listdir_attr(folder_path)
for item in items:
item_path = os.path.join(folder_path, item.filename)
if item.filename not in ('.', '..'):
if item.st_mode & 0o040000: # Check if it's a directory
if self.config.recursive:
self._get_files_recursive(item_path, files)
else:
# Check if file should be ignored
if not self._should_ignore_file(item_path):
files.append(item_path)
except Exception as e:
logger.error(f"Error listing local folder {folder_path}: {e}")
def _should_ignore_file(self, file_path: str) -> bool:
"""
Check if file should be ignored based on ignore patterns
Args:
file_path: File path to check
Returns:
True if file should be ignored, False otherwise
"""
if not hasattr(self.config, 'ignore_patterns') or not self.config.ignore_patterns:
return False
# Get relative path from folder root
relative_path = os.path.relpath(file_path, self.config.folder_path)
for pattern in self.config.ignore_patterns:
if self._match_pattern(relative_path, pattern):
return True
return False
def _match_pattern(self, path: str, pattern: str) -> bool:
"""
Match a path against a pattern (similar to .gitignore)
Args:
path: Path to match
pattern: Pattern to match against
Returns:
True if path matches pattern, False otherwise
"""
# Convert glob pattern to regex
regex_pattern = pattern
regex_pattern = regex_pattern.replace('.', r'\.')
regex_pattern = regex_pattern.replace('*', r'.*')
regex_pattern = regex_pattern.replace('?', r'.')
# Handle directory patterns
if pattern.endswith('/'):
regex_pattern = f'^{regex_pattern}.*$'
else:
regex_pattern = f'^{regex_pattern}$'
return bool(re.match(regex_pattern, path))
@staticmethod
def check_data_source_exists(config: BaseDataSourceConfig) -> bool:
"""
Check if the local folder exists and is accessible via SFTP
Args:
config: Local folder configuration
Returns:
True if local folder exists and is accessible, False otherwise
"""
import paramiko
ssh_client = None
sftp_client = None
try:
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Connect to SSH server - use SSH agent if available, otherwise password
username = config.username or ''
ssh_client.connect(
hostname=config.host,
port=config.port or 22,
username=username,
password=config.password,
timeout=10,
allow_agent=True, # 允许使用SSH代理
look_for_keys=False # 禁用查找本地密钥文件
)
sftp_client = ssh_client.open_sftp()
sftp_client.stat(config.folder_path)
return True
except Exception as e:
logger.error(f"Error checking local folder via SFTP: {e}")
return False
finally:
if sftp_client:
sftp_client.close()
if ssh_client:
ssh_client.close()

256
sync/mysql_sync.py Normal file
View File

@ -0,0 +1,256 @@
"""MySQL synchronization implementation"""
import urllib
import pymysql
import requests
from typing import List, Dict, Optional, Tuple, Any, Set
from datetime import datetime
from loguru import logger
from config import DatabaseDataSourceConfig as DatabaseConfig, settings
from rag.file_parser import FileParser
from sync.base_sync import BaseSync
class MySQLSync(BaseSync):
"""Handle synchronization between MySQL and ChromaDB"""
def __init__(self, db_config: DatabaseConfig, connection=None):
"""
Initialize MySQL sync with database configuration
Args:
db_config: DatabaseConfig object containing database table info
connection: Optional existing MySQL connection (for multi-database scenario)
"""
super().__init__(db_config)
self.db_config = self.config # Ensure self.db_config is always set
self.file_parser = FileParser()
self.connection = connection
if self.connection is None:
self._connect()
else:
self.db_config = db_config
def _connect(self):
"""Create MySQL connection"""
# Get connection parameters from config or use defaults
host = self.db_config.mysql_host or settings.MYSQL_HOST
port = self.db_config.mysql_port or settings.MYSQL_PORT
user = self.db_config.mysql_user or settings.MYSQL_USER
password = self.db_config.mysql_password or settings.MYSQL_PASSWORD
# Log connection info (without password)
logger.info(f"Connecting to MySQL database: {self.db_config.database}")
logger.debug(f"MySQL connection details: host={host}, port={port}, user={user}")
# Create connection
self.connection = pymysql.connect(
host=host,
port=port,
user=user,
password=password,
database=self.db_config.database
)
def fetch_all_documents(self) -> List[Dict[str, Any]]:
"""
Fetch all documents from the MySQL database
Returns:
List of documents
"""
return self._fetch_documents()
def fetch_new_documents(self, last_sync_time=None, synced_doc_ids=None) -> List[Dict[str, Any]]:
"""
Fetch new/updated documents from the MySQL database since last sync time
Args:
last_sync_time: Last synchronization time
synced_doc_ids: Set of document IDs that have already been synced (ignored for MySQL)
Returns:
List of new/updated documents
"""
return self._fetch_documents(last_sync_time)
def get_synced_document_ids(self) -> Set[str]:
"""
Get IDs of all documents in the MySQL database
Returns:
Set of document IDs
"""
cursor = self.connection.cursor(pymysql.cursors.DictCursor)
try:
query = f"SELECT {self.db_config.id_column} FROM {self.db_config.table_name}"
cursor.execute(query)
return {str(row[self.db_config.id_column]) for row in cursor.fetchall()}
finally:
cursor.close()
def _fetch_documents(self, last_sync_time=None) -> List[Dict[str, Any]]:
"""
Internal method to fetch documents from MySQL
Args:
last_sync_time: Last synchronization time (for incremental sync)
Returns:
List of documents
"""
cursor = self.connection.cursor(pymysql.cursors.DictCursor)
try:
# Build query with all columns
columns = [
self.db_config.id_column,
self.db_config.title_column,
self.db_config.content_column,
self.db_config.file_column,
self.db_config.updated_at_column
] if self.db_config.updated_at_column else [
self.db_config.id_column,
self.db_config.title_column,
self.db_config.content_column,
self.db_config.file_column
]
# Remove duplicates and None values
columns = list(set([col for col in columns if col]))
# Build the query
query = f"SELECT {', '.join(columns)} FROM {self.db_config.table_name}"
# Add incremental sync condition if applicable
params = []
if last_sync_time and self.db_config.updated_at_column:
query += f" WHERE {self.db_config.updated_at_column} > %s"
params.append(last_sync_time)
logger.debug(f"MySQL query: {query}, params: {params}")
cursor.execute(query, params)
# Parse results
documents = []
for row in cursor.fetchall():
# Handle file content if file_column is specified
if self.db_config.file_column and row.get(self.db_config.file_column):
# Extract file path from the file column
file_path = row[self.db_config.file_column]
# Load file content if file source is configured
if self.db_config.file_source_type:
file_content = self._load_file_content(file_path)
if file_content:
row[self.db_config.content_column] = file_content
documents.append(row)
return documents
finally:
cursor.close()
def _load_file_content(self, file_path: str) -> Optional[str]:
"""
Load file content using FileParser class
Args:
file_path: Path to the file
Returns:
File content if successfully loaded, None otherwise
"""
try:
if not self.db_config.file_source_type:
logger.debug(f"No file source type configured, skipping file loading for {file_path}")
return None
# 获取文件扩展名
import os
ext = os.path.splitext(file_path)[1].lower()
# 检查文件是否被支持
if ext not in FileParser.SUPPORTED_EXTENSIONS:
logger.debug(f"Unsupported file type {ext} for {file_path}, skipping...")
return None
# 准备文件源配置
source_config = {
'file_system_base_path': self.db_config.file_system_base_path,
'scp_host': self.db_config.scp_host,
'scp_port': self.db_config.scp_port,
'scp_username': self.db_config.scp_username,
'scp_password': self.db_config.scp_password,
'scp_key_path': self.db_config.scp_key_path
}
# 使用 BaseSync 中的通用方法获取文件字节
file_bytes = self._get_file_bytes(file_path, self.db_config.file_source_type, source_config)
if file_bytes:
# 对于 SCP 文件源，传递 SCP 主机地址
host = None
if self.db_config.file_source_type == 'scp' and self.db_config.scp_host:
host = self.db_config.scp_host
# 使用 BaseSync 中的通用方法解析文件内容
return self._parse_file_content(file_bytes, file_path, host=host)
except Exception as e:
logger.error(f"Error loading file content for {file_path}: {e}")
return None
return None
@staticmethod
def check_data_source_exists(config: DatabaseConfig) -> bool:
"""
Check if the MySQL database exists and is accessible
Args:
config: Database configuration
Returns:
True if database exists and is accessible, False otherwise
"""
temp_connection = None
try:
# Get connection parameters
host = config.mysql_host or settings.MYSQL_HOST
port = config.mysql_port or settings.MYSQL_PORT
user = config.mysql_user or settings.MYSQL_USER
password = config.mysql_password or settings.MYSQL_PASSWORD
# Create connection
temp_connection = pymysql.connect(
host=host,
port=port,
user=user,
password=password
)
# Check if database exists
cursor = temp_connection.cursor()
cursor.execute(f"SHOW DATABASES LIKE '{config.database}'")
result = cursor.fetchone()
cursor.close()
if not result:
logger.error(f"Database {config.database} does not exist")
return False
# Check if table exists
temp_connection.select_db(config.database)
cursor = temp_connection.cursor()
cursor.execute(f"SHOW TABLES LIKE '{config.table_name}'")
result = cursor.fetchone()
cursor.close()
if not result:
logger.error(f"Table {config.table_name} does not exist in database {config.database}")
return False
return True
except Exception as e:
logger.error(f"Error checking MySQL data source: {e}")
return False
finally:
if temp_connection:
temp_connection.close()

286
sync/remote_folder_sync.py Normal file
View File

@ -0,0 +1,286 @@
"""Remote folder synchronization implementation"""
import os
import re
from typing import List, Dict, Any, Set
from datetime import datetime
from pathlib import Path
from loguru import logger
from config import BaseDataSourceConfig
from sync.base_sync import BaseSync
from rag.file_parser import FileParser
class RemoteFolderSync(BaseSync):
"""Handle synchronization between remote folder (via SCP) and ChromaDB"""
def __init__(self, config: BaseDataSourceConfig):
"""
Initialize remote folder sync with configuration
Args:
config: Remote folder configuration
"""
super().__init__(config)
self.file_parser = FileParser()
self._ssh_client = None
self._sftp_client = None
def fetch_all_documents(self) -> List[Dict[str, Any]]:
"""
Fetch all documents from the remote folder
Returns:
List of documents
"""
return self._fetch_documents()
def fetch_new_documents(self, last_sync_time=None, synced_doc_ids=None) -> List[Dict[str, Any]]:
"""
Fetch new/updated documents from the remote folder since last sync time
Args:
last_sync_time: Last synchronization time
synced_doc_ids: Set of document IDs that have already been synced
Returns:
List of new/updated documents
"""
return self._fetch_documents(last_sync_time, synced_doc_ids)
def get_synced_document_ids(self) -> Set[str]:
"""
Get IDs of all files in the remote folder
Returns:
Set of file paths (as document IDs)
"""
files = self._get_all_files()
return {str(file) for file in files}
def _fetch_documents(self, last_sync_time=None, synced_doc_ids=None) -> List[Dict[str, Any]]:
"""
Internal method to fetch documents from remote folder via SFTP
Args:
last_sync_time: Last synchronization time (for incremental sync)
synced_doc_ids: Set of document IDs that have already been synced
Returns:
List of documents
"""
documents = []
self._connect()
try:
files = self._get_all_files()
for file_path in files:
# Check if document has already been synced
if synced_doc_ids and file_path in synced_doc_ids:
# If file is already synced, check if it's been modified since last sync
if last_sync_time:
file_stat = self._sftp_client.stat(file_path)
file_mtime = datetime.fromtimestamp(file_stat.st_mtime)
# Skip if not modified since last sync
if file_mtime <= last_sync_time:
continue
# If file is not synced yet, always include it regardless of modification time
# This handles the case where files were added to the folder after last_sync_time but have older mtimes
# Parse file content
try:
# 使用 BaseSync 中的通用方法获取文件字节
# 对于远程文件，我们需要先通过 SFTP 获取文件内容
with self._sftp_client.open(file_path, 'rb') as f:
file_bytes = f.read()
# 检查文件扩展名是否在支持的列表中
if Path(file_path).suffix.lower() not in FileParser.SUPPORTED_EXTENSIONS:
logger.debug(f"Skipping unsupported file: {file_path}")
continue
# 使用 BaseSync 中的通用方法解析文件内容，传递远程主机地址
content = self._parse_file_content(file_bytes, file_path, host=self.config.host) if file_bytes else f"[无法读取文件：{Path(file_path).name}]"
document = {
'id': str(file_path),
'title': Path(file_path).name,
'content': content,
'file_path': str(file_path),
'update_time': datetime.fromtimestamp(self._sftp_client.stat(file_path).st_mtime)
}
documents.append(document)
except Exception as e:
logger.error(f"Error processing file {file_path}: {e}")
finally:
self._disconnect()
return documents
def _get_all_files(self) -> List[str]:
"""
Get all files in the remote folder
Returns:
List of file paths
"""
files = []
self._get_files_recursive(self.config.folder_path, files)
return files
def _get_files_recursive(self, folder_path: str, files: List[str]):
"""
Recursively get all files in the remote folder
Args:
folder_path: Current folder path
files: List to store found files
"""
try:
items = self._sftp_client.listdir_attr(folder_path)
for item in items:
item_path = os.path.join(folder_path, item.filename)
if item.filename not in ('.', '..'):
if item.st_mode & 0o040000: # Check if it's a directory
if self.config.recursive:
self._get_files_recursive(item_path, files)
else:
# Check if file should be ignored
if not self._should_ignore_file(item_path):
files.append(item_path)
except Exception as e:
logger.error(f"Error listing remote folder {folder_path}: {e}")
def _should_ignore_file(self, file_path: str) -> bool:
"""
Check if file should be ignored based on ignore patterns
Args:
file_path: File path to check
Returns:
True if file should be ignored, False otherwise
"""
if not hasattr(self.config, 'ignore_patterns') or not self.config.ignore_patterns:
return False
# Get relative path from folder root
relative_path = os.path.relpath(file_path, self.config.folder_path)
for pattern in self.config.ignore_patterns:
if self._match_pattern(relative_path, pattern):
return True
return False
def _match_pattern(self, path: str, pattern: str) -> bool:
"""
Match a path against a pattern (similar to .gitignore)
Args:
path: Path to match
pattern: Pattern to match against
Returns:
True if path matches pattern, False otherwise
"""
# Convert glob pattern to regex
regex_pattern = pattern
regex_pattern = regex_pattern.replace('.', r'\.')
regex_pattern = regex_pattern.replace('*', r'.*')
regex_pattern = regex_pattern.replace('?', r'.')
# Handle directory patterns
if pattern.endswith('/'):
regex_pattern = f'^{regex_pattern}.*$'
else:
regex_pattern = f'^{regex_pattern}$'
return bool(re.match(regex_pattern, path))
def _connect(self):
"""
Connect to the remote server via SSH/SFTP
"""
import paramiko
self._ssh_client = paramiko.SSHClient()
self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Connect to SSH server
# 确保username不为None，否则paramiko会报错
username = self.config.username or ''
# 使用密码认证或SSH代理认证
self._ssh_client.connect(
hostname=self.config.host,
port=self.config.port or 22,
username=username,
password=self.config.password,
timeout=10,
allow_agent=True, # 允许使用SSH代理
look_for_keys=False # 禁用查找本地密钥文件
)
# Create SFTP client
self._sftp_client = self._ssh_client.open_sftp()
def _disconnect(self):
"""
Disconnect from the remote server
"""
if self._sftp_client:
self._sftp_client.close()
self._sftp_client = None
if self._ssh_client:
self._ssh_client.close()
self._ssh_client = None
@staticmethod
def check_data_source_exists(config: BaseDataSourceConfig) -> bool:
"""
Check if the remote folder exists and is accessible
Args:
config: Remote folder configuration
Returns:
True if remote folder exists and is accessible, False otherwise
"""
import paramiko
ssh_client = None
sftp_client = None
try:
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Connect to SSH server - use SSH agent if available, otherwise password
username = config.username or ''
ssh_client.connect(
hostname=config.host,
port=config.port or 22,
username=username,
password=config.password,
timeout=10,
allow_agent=True, # 允许使用SSH代理
look_for_keys=False # 禁用查找本地密钥文件
)
# Create SFTP client and check folder exists
sftp_client = ssh_client.open_sftp()
sftp_client.stat(config.folder_path)
return True
except Exception as e:
logger.error(f"Error checking remote folder: {e}")
return False
finally:
if sftp_client:
sftp_client.close()
if ssh_client:
ssh_client.close()

480
sync_service.py
View File

@ -1,71 +1,70 @@
"""
Background service for syncing MySQL data to ChromaDB
Background service for syncing data from various sources to ChromaDB
"""
import asyncio
from datetime import datetime
from typing import Set, List, Dict
from typing import Set, List, Dict, Any
from loguru import logger
from config import settings, DatabaseDataSourceConfig as DatabaseConfig
from database import MySQLSync
from config import settings, BaseDataSourceConfig
from sync.base_sync import BaseSync, get_sync_class
from rag import VectorStoreManager, DocumentProcessor
class SyncService:
"""Service for synchronizing MySQL data to ChromaDB"""
"""Service for synchronizing data from various sources (database, local_folder, remote_folder) to ChromaDB"""
def __init__(self):
self.db_configs = settings.get_database_configs()
def __init__(self, source_config: BaseDataSourceConfig):
self.source_config = source_config
self.source_name = source_config.name
# Check if all configured databases exist before proceeding
# This will check each database using its own connection info (if specified)
logger.info("Checking if all configured databases exist...")
all_exist, missing_databases = MySQLSync.check_databases_exist(self.db_configs)
# Check if the data source exists before proceeding
logger.info(f"Checking if data source exists: {self.source_name}")
all_exist, missing_sources = BaseSync.check_data_sources_exist([self.source_config])
if not all_exist:
error_msg = (
f"Error: The following databases do not exist or cannot be accessed:\n"
f" {', '.join(missing_databases)}\n"
f"Error: Data source {self.source_name} does not exist or cannot be accessed\n"
f"Please check:\n"
f" 1. All databases exist on their respective MySQL servers\n"
f" 2. MySQL servers are running and accessible\n"
f" 3. MySQL usernames and passwords are correct\n"
f" 4. Firewall rules allow connections to MySQL ports"
f" 1. Data source exists and is accessible\n"
f" 2. Connection details (host, port, credentials) are correct\n"
f" 3. Network connectivity is available\n"
f" 4. Firewall rules allow connections"
)
logger.error(error_msg)
raise RuntimeError(error_msg)
logger.info(f"✓ All {len(self.db_configs)} configured database(s) exist")
logger.info(f"✓ Data source {self.source_name} exists")
self.mysql_syncs: Dict[str, MySQLSync] = {}
self.vector_store_manager = VectorStoreManager()
self.document_processor = DocumentProcessor()
self.last_sync_times: Dict[str, datetime] = {}
self.synced_doc_ids: Dict[str, Set[str]] = {}
self._running = False
self._sync_in_progress = False # Flag to prevent concurrent syncs
self._auto_sync_task = None # Reference to auto sync task to prevent multiple instances
# Initialize MySQL connection - create separate connection for each database
# This avoids connection state issues when running in thread pool
for db_config in self.db_configs:
try:
# Create separate connection for each database to avoid connection state conflicts
# when running in thread pool (connections are not thread-safe)
mysql_sync = MySQLSync(db_config)
self.mysql_syncs[db_config.name] = mysql_sync
self.last_sync_times[db_config.name] = None
self.synced_doc_ids[db_config.name] = set()
except Exception as e:
logger.error(f"Failed to initialize MySQL sync for {db_config.name}: {e}")
raise
# Initialize syncer for this data source
try:
# Get the appropriate sync class based on data source type
sync_class = get_sync_class(self.source_config.type)
# Create a new syncer instance for this data source
self.syncer = sync_class(self.source_config)
# Initialize sync tracking data
self.last_sync_time = None
self.synced_doc_ids = set()
logger.info(f"Initialized {self.source_config.type} syncer for {self.source_name}")
except Exception as e:
logger.error(f"Failed to initialize sync for {self.source_name}: {e}")
raise
async def sync_all(self, force: bool = False):
async def sync_all(self, force: bool = False, is_manual: bool = False):
"""
Sync all documents from all MySQL databases to ChromaDB
Sync all documents from this data source to ChromaDB
Args:
force: If True, re-process all documents even if they exist (default: False)
is_manual: If True, this sync was triggered manually, ignore _running flag (default: False)
"""
# Prevent concurrent syncs
if self._sync_in_progress:
@ -75,7 +74,7 @@ class SyncService:
self._sync_in_progress = True
sync_start_time = datetime.now()
try:
logger.info(f"[同步] 开始全量同步: {len(self.db_configs)} 个数据库")
logger.info(f"[同步] 开始全量同步: {self.source_name}")
# Run all synchronous operations in thread pool to avoid blocking event loop
import asyncio
@ -91,23 +90,29 @@ class SyncService:
all_chunked_docs = [] # 存储所有分块后的文档
total_docs = 0
skipped_docs_count = 0
processed_databases = 0
# Sync from each database
for db_index, db_config in enumerate(self.db_configs, 1):
try:
mysql_sync = self.mysql_syncs[db_config.name]
mysql_docs = mysql_sync.fetch_all_documents()
if not mysql_docs:
processed_databases += 1
continue
# Sync from this data source
try:
# 检查服务运行状态：
# - 自动同步：必须检查_running标志，确保服务没有被停止
# - 手动同步：忽略_running标志，允许在服务停止后执行
if not is_manual and not self._running:
logger.info(f"Sync interrupted: {self.source_name}")
return all_chunked_docs, total_docs, skipped_docs_count # 返回结果，不退出程序
documents = self.syncer.fetch_all_documents()
if not documents:
return all_chunked_docs, total_docs, skipped_docs_count
# Database-specific processing for content columns
if self.source_config.type == 'database':
# 从配置中获取内容列
content_columns = db_config.content_columns
content_columns = self.source_config.content_columns
# 获取每个数据源对应 ChromaDB metadata 中的 “content_column”
db_content_column_str = self.vector_store_manager.get_specific_db_source_metadata(db_config.database+'_'+db_config.table_name)
source_identifier = f"{self.source_config.database}_{self.source_config.table_name}"
db_content_column_str = self.vector_store_manager.get_specific_db_source_metadata(source_identifier)
if db_content_column_str:
existing_db_content_columns = [col.strip() for col in db_content_column_str.split(",")]
# 获取 existing_doc_content_columns 与 content_columns 差异项
@ -118,54 +123,90 @@ class SyncService:
# Filter out documents that already exist (if not forcing)
db_skipped_count = 0
if not force:
id_column = db_config.id_column
id_column = self.source_config.id_column
# 若 diff_columns为空，说明无新增列，则需过滤ChromaDB中已存在的doc
if not diff_columns:
new_mysql_docs = []
for doc in mysql_docs:
new_documents = []
for doc in documents:
# 检查服务运行状态：仅在非手动同步时检查
if not is_manual and not self._running:
logger.info(f"Sync interrupted during document filtering: {self.source_name}")
return all_chunked_docs, total_docs, skipped_docs_count # 返回结果，不退出程序
doc_id = str(doc.get(id_column, ""))
# 生成唯一文档标识符（数据库标识名称_表名_文档ID）{db_source}_{table_name}_{id}
unique_doc_id = f"{db_config.name}_{db_config.table_name}_{doc_id}"
unique_doc_id = f"{self.source_config.name}_{self.source_config.table_name}_{doc_id}"
if unique_doc_id not in existing_doc_ids:
new_mysql_docs.append(doc)
new_documents.append(doc)
else:
db_skipped_count += 1
skipped_docs_count += 1
if not new_mysql_docs:
self.synced_doc_ids[db_config.name] = {
str(doc.get(id_column)) for doc in mysql_docs
if not new_documents:
self.synced_doc_ids = {
str(doc.get(id_column)) for doc in documents
}
self.last_sync_times[db_config.name] = datetime.now()
processed_databases += 1
continue
mysql_docs = new_mysql_docs
self.last_sync_time = datetime.now()
return all_chunked_docs, total_docs, skipped_docs_count
documents = new_documents
elif db_content_column_str:
self.vector_store_manager.delete_documents_by_source(db_config.database+'_'+db_config.table_name)
self.vector_store_manager.delete_documents_by_source(source_identifier)
# Process and chunk documents
documents = self.document_processor.process_documents(mysql_docs, db_config)
chunked_docs = self.document_processor.chunk_documents(documents)
processed_docs = self.document_processor.process_documents(documents, self.source_config)
chunked_docs = self.document_processor.chunk_documents(processed_docs)
all_chunked_docs.extend(chunked_docs)
# Update synced document IDs for this database
id_column = db_config.id_column
self.synced_doc_ids[db_config.name] = {
str(doc.get(id_column)) for doc in mysql_docs
# Update synced document IDs
id_column = self.source_config.id_column
self.synced_doc_ids = {
str(doc.get(id_column)) for doc in documents
}
self.last_sync_times[db_config.name] = datetime.now()
total_docs += len(mysql_docs)
processed_databases += 1
else:
# Non-database sources (local/remote folder)
if not force:
new_documents = []
for doc in documents:
# 检查服务运行状态：仅在非手动同步时检查
if not is_manual and not self._running:
logger.info(f"Sync interrupted during document filtering: {self.source_name}")
return all_chunked_docs, total_docs, skipped_docs_count # 返回结果，不退出程序
doc_id = doc.get('id', str(doc.get('file_path', '')))
if doc_id not in existing_doc_ids:
new_documents.append(doc)
else:
skipped_docs_count += 1
if not new_documents:
self.synced_doc_ids = {
doc.get('id', str(doc.get('file_path', ''))) for doc in documents
}
self.last_sync_time = datetime.now()
return all_chunked_docs, total_docs, skipped_docs_count
documents = new_documents
except Exception as e:
logger.error(f"[同步] 数据库 {db_config.name} 同步失败: {e}")
processed_databases += 1
continue
# Process and chunk documents
processed_docs = self.document_processor.process_documents(documents, self.source_config)
chunked_docs = self.document_processor.chunk_documents(processed_docs)
all_chunked_docs.extend(chunked_docs)
return all_chunked_docs, total_docs, skipped_docs_count, processed_databases
# Update synced document IDs
self.synced_doc_ids = {
doc.get('id', str(doc.get('file_path', ''))) for doc in documents
}
self.last_sync_time = datetime.now()
total_docs += len(documents)
except Exception as e:
logger.error(f"[同步] 数据源 {self.source_name} 同步失败: {e}")
raise
return all_chunked_docs, total_docs, skipped_docs_count
# Run synchronous work in thread pool
all_chunked_docs, total_docs, skipped_docs_count, processed_databases = await loop.run_in_executor(None, sync_work)
all_chunked_docs, total_docs, skipped_docs_count = await loop.run_in_executor(None, sync_work)
# Add all documents to vector store (run in thread pool to avoid blocking event loop)
if all_chunked_docs:
@ -178,7 +219,7 @@ class SyncService:
sync_duration = (datetime.now() - sync_start_time).total_seconds()
logger.info(
f"[同步] 完成: {processed_databases}/{len(self.db_configs)} 个数据库, "
f"[同步] 完成: {self.source_name}, "
f"{total_docs} 个新文档, {len(all_chunked_docs)} 个分块"
+ (f", 跳过 {skipped_docs_count} 个已存在" if skipped_docs_count > 0 else "")
+ f", 耗时 {sync_duration:.1f}秒"
@ -186,26 +227,31 @@ class SyncService:
else:
sync_duration = (datetime.now() - sync_start_time).total_seconds()
if skipped_docs_count > 0:
logger.info(f"[同步] 完成: 所有 {skipped_docs_count} 个文档已存在, 耗时 {sync_duration:.1f}秒")
logger.info(f"[同步] 完成: {self.source_name}, 所有 {skipped_docs_count} 个文档已存在, 耗时 {sync_duration:.1f}秒")
else:
logger.warning(f"[同步] 没有文档需要同步, 耗时 {sync_duration:.1f}秒")
logger.warning(f"[同步] 没有文档需要同步: {self.source_name}, 耗时 {sync_duration:.1f}秒")
except Exception as e:
sync_duration = (datetime.now() - sync_start_time).total_seconds()
logger.error(f"[同步进度] ✗ 同步过程中出错 (耗时: {sync_duration:.1f} 秒): {e}")
logger.error(f"[同步进度] ✗ {self.source_name} 同步过程中出错 (耗时: {sync_duration:.1f} 秒): {e}")
raise
finally:
self._sync_in_progress = False
async def sync_incremental(self):
"""Sync only new/updated documents from MySQL to ChromaDB"""
async def sync_incremental(self, is_manual: bool = False):
"""
Sync only new/updated documents from this data source to ChromaDB
Args:
is_manual: If True, this sync was triggered manually, ignore _running flag (default: False)
"""
# Prevent concurrent syncs
if self._sync_in_progress:
logger.warning("Sync already in progress, skipping incremental sync")
return
self._sync_in_progress = True
try:
logger.info(f"Starting incremental sync from {len(self.db_configs)} MySQL database(s) to ChromaDB")
logger.info(f"Starting incremental sync from data source: {self.source_name}")
# Run all synchronous operations in thread pool
import asyncio
@ -216,36 +262,42 @@ class SyncService:
all_chunked_docs = []
total_docs = 0
# Sync from each database
for db_config in self.db_configs:
try:
mysql_sync = self.mysql_syncs[db_config.name]
last_sync_time = self.last_sync_times.get(db_config.name)
# Fetch new documents
mysql_docs = mysql_sync.fetch_new_documents(last_sync_time)
if not mysql_docs:
logger.debug(f"No new documents in database: {db_config.name}")
continue
# Process and chunk documents
documents = self.document_processor.process_documents(mysql_docs, db_config)
chunked_docs = self.document_processor.chunk_documents(documents)
all_chunked_docs.extend(chunked_docs)
# Sync from this data source
try:
# 检查服务运行状态：仅在非手动同步时检查
if not is_manual and not self._running:
logger.info(f"Incremental sync interrupted: {self.source_name}")
return all_chunked_docs, total_docs # 返回结果，不退出程序
# Fetch new documents, passing synced_doc_ids to determine what needs syncing
new_documents = self.syncer.fetch_new_documents(self.last_sync_time, self.synced_doc_ids)
if not new_documents:
logger.debug(f"No new documents in data source: {self.source_name}")
return all_chunked_docs, total_docs
# Process and chunk documents
processed_docs = self.document_processor.process_documents(new_documents, self.source_config)
chunked_docs = self.document_processor.chunk_documents(processed_docs)
all_chunked_docs.extend(chunked_docs)
# Update synced document IDs for this database
id_column = db_config.id_column
new_doc_ids = {str(doc.get(id_column)) for doc in mysql_docs}
self.synced_doc_ids[db_config.name].update(new_doc_ids)
self.last_sync_times[db_config.name] = datetime.now()
total_docs += len(mysql_docs)
logger.info(f"Incremental sync: {len(chunked_docs)} chunks from {len(mysql_docs)} documents in database: {db_config.name}")
except Exception as e:
logger.error(f"Error during incremental sync for database {db_config.name}: {e}")
# Continue with other databases
continue
# Update synced document IDs for this data source
if self.source_config.type == 'database':
id_column = self.source_config.id_column
new_doc_ids = {str(doc.get(id_column)) for doc in new_documents}
self.synced_doc_ids.update(new_doc_ids)
else:
# Use file path or id as document ID for folder sources
new_doc_ids = {doc.get('id', str(doc.get('file_path', ''))) for doc in new_documents}
self.synced_doc_ids.update(new_doc_ids)
self.last_sync_time = datetime.now()
total_docs += len(new_documents)
logger.info(f"Incremental sync: {len(chunked_docs)} chunks from {len(new_documents)} documents in {self.source_config.type}: {self.source_name}")
except Exception as e:
logger.error(f"Error during incremental sync for data source {self.source_name}: {e}")
raise
return all_chunked_docs, total_docs
@ -261,18 +313,18 @@ class SyncService:
all_chunked_docs,
False # skip_existing，不跳过已存在的文档，默认是更新了内容
)
logger.info(f"Incremental sync completed: {len(all_chunked_docs)} chunks from {total_docs} documents across {len(self.db_configs)} database(s)")
logger.info(f"Incremental sync completed: {len(all_chunked_docs)} chunks from {total_docs} documents in {self.source_name}")
else:
logger.info("No new documents to sync")
logger.info(f"No new documents to sync in {self.source_name}")
except Exception as e:
logger.error(f"Error during incremental sync: {e}")
logger.error(f"Error during incremental sync for {self.source_name}: {e}")
raise
finally:
self._sync_in_progress = False
async def start_auto_sync_with_recovery(self, skip_initial_sync: bool = False):
"""
Start automatic periodic sync in background with error recovery.
Start automatic periodic sync in background with error recovery for this data source.
If the sync service stops due to an error, it will automatically restart.
This method runs continuously until stop_auto_sync() is called.
@ -281,17 +333,17 @@ class SyncService:
Use this when initial sync is already done elsewhere.
"""
if not settings.AUTO_SYNC:
logger.info("Auto sync is disabled")
logger.info(f"Auto sync is disabled for data source: {self.source_name}")
return
# Prevent multiple auto sync service instances
if self._running:
logger.warning("Auto sync service is already running, skipping duplicate start request")
logger.warning(f"Auto sync service is already running for data source: {self.source_name}, skipping duplicate start request")
return
# Check if there's an existing auto sync task still running
if self._auto_sync_task is not None and not self._auto_sync_task.done():
logger.warning("Auto sync task is still running, skipping duplicate start request")
logger.warning(f"Auto sync task is still running for data source: {self.source_name}, skipping duplicate start request")
return
max_restart_attempts = 10 # Maximum number of restart attempts
@ -299,7 +351,7 @@ class SyncService:
restart_count = 0
self._running = True
logger.info(f"Auto sync service with error recovery started (interval: {settings.SYNC_INTERVAL}s)")
logger.info(f"Auto sync service with error recovery started for {self.source_name} (interval: {settings.SYNC_INTERVAL}s)")
while self._running and restart_count < max_restart_attempts:
try:
@ -309,36 +361,36 @@ class SyncService:
)
await self._auto_sync_task
# If we reach here, the loop exited normally (not due to error)
logger.info("Auto sync loop exited normally")
logger.info(f"Auto sync loop exited normally for {self.source_name}")
break
except Exception as e:
restart_count += 1
logger.error(
f"Auto sync service stopped due to error (restart attempt {restart_count}/{max_restart_attempts}): {e}",
f"Auto sync service stopped due to error for {self.source_name} (restart attempt {restart_count}/{max_restart_attempts}): {e}",
exc_info=True
)
if restart_count >= max_restart_attempts:
logger.error(f"Auto sync service failed after {max_restart_attempts} restart attempts. Stopping auto sync.")
logger.error(f"Auto sync service failed after {max_restart_attempts} restart attempts for {self.source_name}. Stopping auto sync.")
self._running = False
break
if not self._running:
logger.info("Auto sync service stop requested, not restarting")
logger.info(f"Auto sync service stop requested for {self.source_name}, not restarting")
break
logger.info(f"Waiting {restart_delay}s before restarting auto sync service...")
logger.info(f"Waiting {restart_delay}s before restarting auto sync service for {self.source_name}...")
await asyncio.sleep(restart_delay)
logger.info(f"Restarting auto sync service (attempt {restart_count + 1}/{max_restart_attempts})...")
logger.info(f"Restarting auto sync service for {self.source_name} (attempt {restart_count + 1}/{max_restart_attempts})...")
# Reset skip_initial_sync after first attempt (only skip on first start)
skip_initial_sync = False
logger.info("Auto sync service with recovery stopped")
logger.info(f"Auto sync service with recovery stopped for {self.source_name}")
self._running = False
async def start_auto_sync(self, skip_initial_sync: bool = False):
"""
Start automatic periodic sync in background.
Start automatic periodic sync in background for this data source.
This method runs continuously until stop_auto_sync() is called.
For production use, prefer start_auto_sync_with_recovery() which includes error recovery.
@ -346,38 +398,39 @@ class SyncService:
skip_initial_sync: If True, skip the initial sync_all() call.
Use this when initial sync is already done elsewhere.
"""
logger.info(f"Starting auto sync for data source: {self.source_name}")
await self._run_auto_sync_loop(skip_initial_sync)
async def _run_auto_sync_loop(self, skip_initial_sync: bool = False):
"""
Internal method that runs the auto sync loop.
Internal method that runs the auto sync loop for this data source.
This is separated so it can be called by both start_auto_sync and start_auto_sync_with_recovery.
Args:
skip_initial_sync: If True, skip the initial sync_all() call.
"""
if not settings.AUTO_SYNC:
logger.info("Auto sync is disabled")
logger.info(f"Auto sync is disabled for {self.source_name}")
return
logger.info(f"Auto sync loop started (interval: {settings.SYNC_INTERVAL}s)")
logger.info(f"Auto sync loop started for {self.source_name} (interval: {settings.SYNC_INTERVAL}s)")
# Initial sync (only if not skipped)
if not skip_initial_sync:
logger.info("Performing initial sync in auto sync service...")
logger.info(f"Performing initial sync for {self.source_name} in auto sync service...")
await self.sync_all()
else:
logger.info("Skipping initial sync in auto sync service (already done elsewhere)")
logger.info(f"Skipping initial sync for {self.source_name} in auto sync service (already done elsewhere)")
# Periodic incremental sync
sync_count = 0
last_sync_start_time = None
while self._running:
try:
logger.info(f"Auto sync waiting {settings.SYNC_INTERVAL}s before next sync (count: {sync_count})...")
logger.info(f"Auto sync waiting {settings.SYNC_INTERVAL}s before next sync for {self.source_name} (count: {sync_count})...")
await asyncio.sleep(settings.SYNC_INTERVAL)
if not self._running:
logger.info("Auto sync stopped, exiting loop")
logger.info(f"Auto sync stopped for {self.source_name}, exiting loop")
break
# Check if another sync is in progress (e.g., initial sync or previous incremental sync still running)
@ -385,91 +438,166 @@ class SyncService:
wait_interval = 10 # Check every 10 seconds
waited_time = 0
while self._sync_in_progress:
logger.info(f"Another sync is in progress, waiting... (waited {waited_time}s, will wait until completion)")
logger.info(f"Another sync is in progress for {self.source_name}, waiting... (waited {waited_time}s, will wait until completion)")
await asyncio.sleep(wait_interval)
waited_time += wait_interval
# Log warning if waiting for a very long time (for monitoring purposes)
if waited_time % 300 == 0: # Every 5 minutes
logger.info(f"Still waiting for sync to complete... (waited {waited_time}s / {waited_time // 60} minutes)")
logger.info(f"Still waiting for sync to complete for {self.source_name}... (waited {waited_time}s / {waited_time // 60} minutes)")
# If last sync started more than 2 hours ago and is still running, log warning
if last_sync_start_time is not None:
time_since_last_sync = (datetime.now() - last_sync_start_time).total_seconds()
if time_since_last_sync > 7200: # 2 hours
logger.warning(f"Last incremental sync has been running for {time_since_last_sync / 3600:.1f} hours, this might indicate a slow sync. Continuing to wait...")
logger.warning(f"Last incremental sync for {self.source_name} has been running for {time_since_last_sync / 3600:.1f} hours, this might indicate a slow sync. Continuing to wait...")
if waited_time > 0:
logger.info(f"Previous sync completed, waited {waited_time}s / {waited_time // 60} minutes")
logger.info(f"Previous sync completed for {self.source_name}, waited {waited_time}s / {waited_time // 60} minutes")
if not self._running:
logger.info("Auto sync stopped during wait, exiting loop")
logger.info(f"Auto sync stopped during wait for {self.source_name}, exiting loop")
break
# Record sync start time for monitoring
last_sync_start_time = datetime.now()
sync_count += 1
logger.info(f"Running periodic incremental sync #{sync_count}...")
logger.info(f"Running periodic incremental sync #{sync_count} for {self.source_name}...")
# Execute incremental sync - it will check _sync_in_progress internally
try:
await self.sync_incremental()
logger.info(f"Incremental sync #{sync_count} completed successfully")
logger.info(f"Incremental sync #{sync_count} completed successfully for {self.source_name}")
except Exception as sync_error:
logger.error(f"Incremental sync #{sync_count} failed: {sync_error}", exc_info=True)
logger.error(f"Incremental sync #{sync_count} failed for {self.source_name}: {sync_error}", exc_info=True)
# Reset sync in progress flag if it was set (in case of unexpected error)
if self._sync_in_progress:
logger.warning("Resetting _sync_in_progress flag due to error in incremental sync")
logger.warning(f"Resetting _sync_in_progress flag for {self.source_name} due to error in incremental sync")
self._sync_in_progress = False
# Continue to next cycle even if this sync failed
logger.info("Continuing to next sync cycle despite error...")
logger.info(f"Continuing to next sync cycle for {self.source_name} despite error...")
continue
except asyncio.CancelledError:
logger.info("Auto sync task was cancelled")
logger.info(f"Auto sync task was cancelled for {self.source_name}")
break
except Exception as e:
logger.error(f"Unexpected error in auto sync loop: {e}", exc_info=True)
logger.error(f"Unexpected error in auto sync loop for {self.source_name}: {e}", exc_info=True)
# Reset sync in progress flag if it was set (in case of unexpected error)
if self._sync_in_progress:
logger.warning("Resetting _sync_in_progress flag due to unexpected error in auto sync loop")
logger.warning(f"Resetting _sync_in_progress flag for {self.source_name} due to unexpected error in auto sync loop")
self._sync_in_progress = False
# Continue running even if there's an error
logger.info("Continuing auto sync loop despite error...")
logger.info(f"Continuing auto sync loop for {self.source_name} despite error...")
continue
logger.info(f"Auto sync loop stopped (total syncs: {sync_count})")
logger.info(f"Auto sync loop stopped for {self.source_name} (total syncs: {sync_count})")
def stop_auto_sync(self):
"""Stop automatic sync"""
"""Stop automatic sync for this data source"""
if not self._running:
logger.info("Auto sync service is not running")
logger.info(f"Auto sync service is not running for {self.source_name}")
return
self._running = False
logger.info("Auto sync service stop requested")
logger.info(f"Auto sync service stop requested for {self.source_name}")
# Wait for current sync to complete (if any) before stopping
# This prevents stopping in the middle of a sync operation
if self._sync_in_progress:
logger.info("Waiting for current sync to complete before stopping auto sync service...")
import time
max_wait = 300 # Wait up to 5 minutes
waited = 0
while self._sync_in_progress and waited < max_wait:
time.sleep(2)
waited += 2
if self._sync_in_progress:
logger.warning(f"Sync still in progress after waiting {waited}s, forcing stop")
else:
logger.info("Current sync completed, auto sync service stopped")
else:
logger.info("Auto sync service stopped")
# 不再等待当前同步完成，直接设置标志位并返回
# 同步操作内部会定期检查self._running标志
logger.info(f"Auto sync service stopped for {self.source_name}, current sync will be interrupted if running")
def close(self):
"""Close MySQL connections"""
"""Close connections for this data source"""
self.stop_auto_sync()
for mysql_sync in self.mysql_syncs.values():
if hasattr(mysql_sync, 'close'):
mysql_sync.close()
if hasattr(self.syncer, 'close'):
self.syncer.close()
class SyncServiceManager:
"""Manager for multiple SyncService instances, one per data source"""
def __init__(self):
self.data_sources = settings.get_data_sources()
self.sync_services: Dict[str, SyncService] = {}
# Initialize sync services for each data source
for source_config in self.data_sources:
try:
sync_service = SyncService(source_config)
self.sync_services[source_config.name] = sync_service
logger.info(f"Created SyncService for {source_config.name}")
except Exception as e:
logger.error(f"Failed to create SyncService for {source_config.name}: {e}")
async def start_all_sync_services(self):
"""Start auto sync with recovery for all data sources"""
tasks = []
for sync_service in self.sync_services.values():
# Start each sync service in background
task = asyncio.create_task(sync_service.start_auto_sync_with_recovery())
tasks.append(task)
# Wait for all tasks to complete (they should run indefinitely until stopped)
await asyncio.gather(*tasks, return_exceptions=True)
def stop_all_sync_services(self):
"""Stop all sync services"""
for sync_service in self.sync_services.values():
sync_service.stop_auto_sync()
logger.info("All sync services stopped")
def close_all(self):
"""Close all sync services"""
for sync_service in self.sync_services.values():
sync_service.close()
logger.info("All sync services closed")
def get_sync_service(self, source_name: str) -> SyncService:
"""Get a SyncService instance for a specific data source"""
return self.sync_services.get(source_name)
def create_or_update_sync_service(self, source_config: BaseDataSourceConfig):
"""Create or update a SyncService for a data source"""
source_name = source_config.name
# Stop and remove existing sync service if it exists
if source_name in self.sync_services:
self.sync_services[source_name].close()
del self.sync_services[source_name]
logger.info(f"Removed existing SyncService for {source_name}")
# Create new sync service
try:
sync_service = SyncService(source_config)
self.sync_services[source_name] = sync_service
logger.info(f"Created new SyncService for {source_name}")
return sync_service
except Exception as e:
logger.error(f"Failed to create SyncService for {source_name}: {e}")
raise
def remove_sync_service(self, source_name: str):
"""Remove a SyncService for a data source"""
if source_name in self.sync_services:
self.sync_services[source_name].close()
del self.sync_services[source_name]
logger.info(f"Removed SyncService for {source_name}")
def main():
"""Main function to run the sync service"""
import asyncio
# Create sync service manager and start all sync services
sync_manager = SyncServiceManager()
try:
asyncio.run(sync_manager.start_all_sync_services())
except KeyboardInterrupt:
sync_manager.stop_all_sync_services()
sync_manager.close_all()
print("Sync service manager stopped by user")
if __name__ == "__main__":
main()

58
test_sync_configurations.py
View File

@ -125,12 +125,45 @@ async def test_sync(configurations, sync_type: str = "all", force: bool = False)
loader = ConfigurationLoader()
# Define database path outside try block for cleanup
import sqlite3
from pathlib import Path
import json
DATA_DIR = Path(__file__).parent / "data"
DATA_DIR.mkdir(parents=True, exist_ok=True)
DB_PATH = DATA_DIR / "sessions.db"
try:
# Create test configuration file
config_path = loader.create_test_config_file(configurations)
# Write test configurations to SQLite database
conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor()
# Create settings with the test configuration
settings = Settings(DATA_SOURCES_CONFIG=config_path)
# Create table if not exists
cursor.execute('''
CREATE TABLE IF NOT EXISTS data_sources (
name TEXT PRIMARY KEY,
config TEXT,
update_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
# Insert/update test configurations
for config in configurations:
config_json = json.dumps(config, ensure_ascii=False, indent=2)
cursor.execute('''
INSERT OR REPLACE INTO data_sources (name, config, update_at)
VALUES (?, ?, CURRENT_TIMESTAMP)
''', (config['name'], config_json))
conn.commit()
conn.close()
print(f"Inserted {len(configurations)} test configurations into SQLite database")
# Initialize settings normally (will read from SQLite)
from config import Settings
settings = Settings()
# Initialize sync service
print("\nInitializing SyncService...")
@ -174,9 +207,20 @@ async def test_sync(configurations, sync_type: str = "all", force: bool = False)
return False
finally:
# Clean up
if 'config_path' in locals() and os.path.exists(config_path):
os.remove(config_path)
# Clean up: remove test configurations from SQLite database
try:
conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor()
# Delete test configurations
for config in configurations:
cursor.execute('DELETE FROM data_sources WHERE name = ?', (config['name'],))
conn.commit()
conn.close()
print(f"Removed {len(configurations)} test configurations from SQLite database")
except Exception as e:
print(f"Error during cleanup: {e}")
async def main():

188
uv.lock
View File

@ -617,6 +617,15 @@ wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" },
]
[[package]]
name = "docx2txt"
version = "0.9"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ea/07/4486a038624e885e227fe79111914c01f55aa70a51920ff1a7f2bd216d10/docx2txt-0.9.tar.gz", hash = "sha256:18013f6229b14909028b19aa7bf4f8f3d6e4632d7b089ab29f7f0a4d1f660e28", size = 3613, upload-time = "2025-03-24T20:59:25.21Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/d6/51/756e71bec48ece0ecc2a10e921ef2756e197dcb7e478f2b43673b6683902/docx2txt-0.9-py3-none-any.whl", hash = "sha256:e3718c0653fd6f2fcf4b51b02a61452ad1c38a4c163bcf0a6fd9486cd38f529a", size = 4025, upload-time = "2025-03-24T20:59:24.394Z" },
]
[[package]]
name = "durationpy"
version = "0.10"
@ -631,7 +640,7 @@ name = "exceptiongroup"
version = "1.3.1"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }
dependencies = [
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" }
wheels = [
@ -1640,6 +1649,139 @@ wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/03/0a/4f6fed21aa246c6b49b561ca55facacc2a44b87d65b8b92362a8e99ba202/loguru-0.7.2-py3-none-any.whl", hash = "sha256:003d71e3d3ed35f0f8984898359d65b79e5b21943f78af86aa5491210429b8eb", size = 62549, upload-time = "2023-09-11T15:24:35.016Z" },
]
[[package]]
name = "lxml"
version = "6.0.2"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/aa/88/262177de60548e5a2bfc46ad28232c9e9cbde697bd94132aeb80364675cb/lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62", size = 4073426, upload-time = "2025-09-22T04:04:59.287Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/db/8a/f8192a08237ef2fb1b19733f709db88a4c43bc8ab8357f01cb41a27e7f6a/lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388", size = 8590589, upload-time = "2025-09-22T04:00:10.51Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/12/64/27bcd07ae17ff5e5536e8d88f4c7d581b48963817a13de11f3ac3329bfa2/lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153", size = 4629671, upload-time = "2025-09-22T04:00:15.411Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/02/5a/a7d53b3291c324e0b6e48f3c797be63836cc52156ddf8f33cd72aac78866/lxml-6.0.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f952dacaa552f3bb8834908dddd500ba7d508e6ea6eb8c52eb2d28f48ca06a31", size = 4999961, upload-time = "2025-09-22T04:00:17.619Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/f5/55/d465e9b89df1761674d8672bb3e4ae2c47033b01ec243964b6e334c6743f/lxml-6.0.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:71695772df6acea9f3c0e59e44ba8ac50c4f125217e84aab21074a1a55e7e5c9", size = 5157087, upload-time = "2025-09-22T04:00:19.868Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/62/38/3073cd7e3e8dfc3ba3c3a139e33bee3a82de2bfb0925714351ad3d255c13/lxml-6.0.2-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f68764f35fd78d7c4cc4ef209a184c38b65440378013d24b8aecd327c3e0c8", size = 5067620, upload-time = "2025-09-22T04:00:21.877Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/4a/d3/1e001588c5e2205637b08985597827d3827dbaaece16348c8822bfe61c29/lxml-6.0.2-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:058027e261afed589eddcfe530fcc6f3402d7fd7e89bfd0532df82ebc1563dba", size = 5406664, upload-time = "2025-09-22T04:00:23.714Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/20/cf/cab09478699b003857ed6ebfe95e9fb9fa3d3c25f1353b905c9b73cfb624/lxml-6.0.2-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8ffaeec5dfea5881d4c9d8913a32d10cfe3923495386106e4a24d45300ef79c", size = 5289397, upload-time = "2025-09-22T04:00:25.544Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/a3/84/02a2d0c38ac9a8b9f9e5e1bbd3f24b3f426044ad618b552e9549ee91bd63/lxml-6.0.2-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:f2e3b1a6bb38de0bc713edd4d612969dd250ca8b724be8d460001a387507021c", size = 4772178, upload-time = "2025-09-22T04:00:27.602Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/56/87/e1ceadcc031ec4aa605fe95476892d0b0ba3b7f8c7dcdf88fdeff59a9c86/lxml-6.0.2-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d6690ec5ec1cce0385cb20896b16be35247ac8c2046e493d03232f1c2414d321", size = 5358148, upload-time = "2025-09-22T04:00:29.323Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/fe/13/5bb6cf42bb228353fd4ac5f162c6a84fd68a4d6f67c1031c8cf97e131fc6/lxml-6.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f2a50c3c1d11cad0ebebbac357a97b26aa79d2bcaf46f256551152aa85d3a4d1", size = 5112035, upload-time = "2025-09-22T04:00:31.061Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/e4/e2/ea0498552102e59834e297c5c6dff8d8ded3db72ed5e8aad77871476f073/lxml-6.0.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:3efe1b21c7801ffa29a1112fab3b0f643628c30472d507f39544fd48e9549e34", size = 4799111, upload-time = "2025-09-22T04:00:33.11Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/6a/9e/8de42b52a73abb8af86c66c969b3b4c2a96567b6ac74637c037d2e3baa60/lxml-6.0.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:59c45e125140b2c4b33920d21d83681940ca29f0b83f8629ea1a2196dc8cfe6a", size = 5351662, upload-time = "2025-09-22T04:00:35.237Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/28/a2/de776a573dfb15114509a37351937c367530865edb10a90189d0b4b9b70a/lxml-6.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:452b899faa64f1805943ec1c0c9ebeaece01a1af83e130b69cdefeda180bb42c", size = 5314973, upload-time = "2025-09-22T04:00:37.086Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/50/a0/3ae1b1f8964c271b5eec91db2043cf8c6c0bce101ebb2a633b51b044db6c/lxml-6.0.2-cp310-cp310-win32.whl", hash = "sha256:1e786a464c191ca43b133906c6903a7e4d56bef376b75d97ccbb8ec5cf1f0a4b", size = 3611953, upload-time = "2025-09-22T04:00:39.224Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/d1/70/bd42491f0634aad41bdfc1e46f5cff98825fb6185688dc82baa35d509f1a/lxml-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:dacf3c64ef3f7440e3167aa4b49aa9e0fb99e0aa4f9ff03795640bf94531bcb0", size = 4032695, upload-time = "2025-09-22T04:00:41.402Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/d2/d0/05c6a72299f54c2c561a6c6cbb2f512e047fca20ea97a05e57931f194ac4/lxml-6.0.2-cp310-cp310-win_arm64.whl", hash = "sha256:45f93e6f75123f88d7f0cfd90f2d05f441b808562bf0bc01070a00f53f5028b5", size = 3680051, upload-time = "2025-09-22T04:00:43.525Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/77/d5/becbe1e2569b474a23f0c672ead8a29ac50b2dc1d5b9de184831bda8d14c/lxml-6.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:13e35cbc684aadf05d8711a5d1b5857c92e5e580efa9a0d2be197199c8def607", size = 8634365, upload-time = "2025-09-22T04:00:45.672Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/28/66/1ced58f12e804644426b85d0bb8a4478ca77bc1761455da310505f1a3526/lxml-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b1675e096e17c6fe9c0e8c81434f5736c0739ff9ac6123c87c2d452f48fc938", size = 4650793, upload-time = "2025-09-22T04:00:47.783Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/11/84/549098ffea39dfd167e3f174b4ce983d0eed61f9d8d25b7bf2a57c3247fc/lxml-6.0.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac6e5811ae2870953390452e3476694196f98d447573234592d30488147404d", size = 4944362, upload-time = "2025-09-22T04:00:49.845Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/ac/bd/f207f16abf9749d2037453d56b643a7471d8fde855a231a12d1e095c4f01/lxml-6.0.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5aa0fc67ae19d7a64c3fe725dc9a1bb11f80e01f78289d05c6f62545affec438", size = 5083152, upload-time = "2025-09-22T04:00:51.709Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/15/ae/bd813e87d8941d52ad5b65071b1affb48da01c4ed3c9c99e40abb266fbff/lxml-6.0.2-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de496365750cc472b4e7902a485d3f152ecf57bd3ba03ddd5578ed8ceb4c5964", size = 5023539, upload-time = "2025-09-22T04:00:53.593Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/02/cd/9bfef16bd1d874fbe0cb51afb00329540f30a3283beb9f0780adbb7eec03/lxml-6.0.2-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:200069a593c5e40b8f6fc0d84d86d970ba43138c3e68619ffa234bc9bb806a4d", size = 5344853, upload-time = "2025-09-22T04:00:55.524Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/b8/89/ea8f91594bc5dbb879734d35a6f2b0ad50605d7fb419de2b63d4211765cc/lxml-6.0.2-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d2de809c2ee3b888b59f995625385f74629707c9355e0ff856445cdcae682b7", size = 5225133, upload-time = "2025-09-22T04:00:57.269Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/b9/37/9c735274f5dbec726b2db99b98a43950395ba3d4a1043083dba2ad814170/lxml-6.0.2-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:b2c3da8d93cf5db60e8858c17684c47d01fee6405e554fb55018dd85fc23b178", size = 4677944, upload-time = "2025-09-22T04:00:59.052Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/20/28/7dfe1ba3475d8bfca3878365075abe002e05d40dfaaeb7ec01b4c587d533/lxml-6.0.2-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:442de7530296ef5e188373a1ea5789a46ce90c4847e597856570439621d9c553", size = 5284535, upload-time = "2025-09-22T04:01:01.335Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/e7/cf/5f14bc0de763498fc29510e3532bf2b4b3a1c1d5d0dff2e900c16ba021ef/lxml-6.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2593c77efde7bfea7f6389f1ab249b15ed4aa5bc5cb5131faa3b843c429fbedb", size = 5067343, upload-time = "2025-09-22T04:01:03.13Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/1c/b0/bb8275ab5472f32b28cfbbcc6db7c9d092482d3439ca279d8d6fa02f7025/lxml-6.0.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3e3cb08855967a20f553ff32d147e14329b3ae70ced6edc2f282b94afbc74b2a", size = 4725419, upload-time = "2025-09-22T04:01:05.013Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/25/4c/7c222753bc72edca3b99dbadba1b064209bc8ed4ad448af990e60dcce462/lxml-6.0.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2ed6c667fcbb8c19c6791bbf40b7268ef8ddf5a96940ba9404b9f9a304832f6c", size = 5275008, upload-time = "2025-09-22T04:01:07.327Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/6c/8c/478a0dc6b6ed661451379447cdbec77c05741a75736d97e5b2b729687828/lxml-6.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b8f18914faec94132e5b91e69d76a5c1d7b0c73e2489ea8929c4aaa10b76bbf7", size = 5248906, upload-time = "2025-09-22T04:01:09.452Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/2d/d9/5be3a6ab2784cdf9accb0703b65e1b64fcdd9311c9f007630c7db0cfcce1/lxml-6.0.2-cp311-cp311-win32.whl", hash = "sha256:6605c604e6daa9e0d7f0a2137bdc47a2e93b59c60a65466353e37f8272f47c46", size = 3610357, upload-time = "2025-09-22T04:01:11.102Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/e2/7d/ca6fb13349b473d5732fb0ee3eec8f6c80fc0688e76b7d79c1008481bf1f/lxml-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e5867f2651016a3afd8dd2c8238baa66f1e2802f44bc17e236f547ace6647078", size = 4036583, upload-time = "2025-09-22T04:01:12.766Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/ab/a2/51363b5ecd3eab46563645f3a2c3836a2fc67d01a1b87c5017040f39f567/lxml-6.0.2-cp311-cp311-win_arm64.whl", hash = "sha256:4197fb2534ee05fd3e7afaab5d8bfd6c2e186f65ea7f9cd6a82809c887bd1285", size = 3680591, upload-time = "2025-09-22T04:01:14.874Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/f3/c8/8ff2bc6b920c84355146cd1ab7d181bc543b89241cfb1ebee824a7c81457/lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456", size = 8661887, upload-time = "2025-09-22T04:01:17.265Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/37/6f/9aae1008083bb501ef63284220ce81638332f9ccbfa53765b2b7502203cf/lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924", size = 4667818, upload-time = "2025-09-22T04:01:19.688Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/f1/ca/31fb37f99f37f1536c133476674c10b577e409c0a624384147653e38baf2/lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f", size = 4950807, upload-time = "2025-09-22T04:01:21.487Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/da/87/f6cb9442e4bada8aab5ae7e1046264f62fdbeaa6e3f6211b93f4c0dd97f1/lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534", size = 5109179, upload-time = "2025-09-22T04:01:23.32Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/c8/20/a7760713e65888db79bbae4f6146a6ae5c04e4a204a3c48896c408cd6ed2/lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564", size = 5023044, upload-time = "2025-09-22T04:01:25.118Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/a2/b0/7e64e0460fcb36471899f75831509098f3fd7cd02a3833ac517433cb4f8f/lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f", size = 5359685, upload-time = "2025-09-22T04:01:27.398Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/b9/e1/e5df362e9ca4e2f48ed6411bd4b3a0ae737cc842e96877f5bf9428055ab4/lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0", size = 5654127, upload-time = "2025-09-22T04:01:29.629Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/c6/d1/232b3309a02d60f11e71857778bfcd4acbdb86c07db8260caf7d008b08f8/lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192", size = 5253958, upload-time = "2025-09-22T04:01:31.535Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/35/35/d955a070994725c4f7d80583a96cab9c107c57a125b20bb5f708fe941011/lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0", size = 4711541, upload-time = "2025-09-22T04:01:33.801Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/1e/be/667d17363b38a78c4bd63cfd4b4632029fd68d2c2dc81f25ce9eb5224dd5/lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092", size = 5267426, upload-time = "2025-09-22T04:01:35.639Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/ea/47/62c70aa4a1c26569bc958c9ca86af2bb4e1f614e8c04fb2989833874f7ae/lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f", size = 5064917, upload-time = "2025-09-22T04:01:37.448Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/bd/55/6ceddaca353ebd0f1908ef712c597f8570cc9c58130dbb89903198e441fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8", size = 4788795, upload-time = "2025-09-22T04:01:39.165Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/cf/e8/fd63e15da5e3fd4c2146f8bbb3c14e94ab850589beab88e547b2dbce22e1/lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f", size = 5676759, upload-time = "2025-09-22T04:01:41.506Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/76/47/b3ec58dc5c374697f5ba37412cd2728f427d056315d124dd4b61da381877/lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6", size = 5255666, upload-time = "2025-09-22T04:01:43.363Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/19/93/03ba725df4c3d72afd9596eef4a37a837ce8e4806010569bedfcd2cb68fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322", size = 5277989, upload-time = "2025-09-22T04:01:45.215Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/c6/80/c06de80bfce881d0ad738576f243911fccf992687ae09fd80b734712b39c/lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849", size = 3611456, upload-time = "2025-09-22T04:01:48.243Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/f7/d7/0cdfb6c3e30893463fb3d1e52bc5f5f99684a03c29a0b6b605cfae879cd5/lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f", size = 4011793, upload-time = "2025-09-22T04:01:50.042Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/ea/7b/93c73c67db235931527301ed3785f849c78991e2e34f3fd9a6663ffda4c5/lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6", size = 3672836, upload-time = "2025-09-22T04:01:52.145Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/53/fd/4e8f0540608977aea078bf6d79f128e0e2c2bba8af1acf775c30baa70460/lxml-6.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9b33d21594afab46f37ae58dfadd06636f154923c4e8a4d754b0127554eb2e77", size = 8648494, upload-time = "2025-09-22T04:01:54.242Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/5d/f4/2a94a3d3dfd6c6b433501b8d470a1960a20ecce93245cf2db1706adf6c19/lxml-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c8963287d7a4c5c9a432ff487c52e9c5618667179c18a204bdedb27310f022f", size = 4661146, upload-time = "2025-09-22T04:01:56.282Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/25/2e/4efa677fa6b322013035d38016f6ae859d06cac67437ca7dc708a6af7028/lxml-6.0.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1941354d92699fb5ffe6ed7b32f9649e43c2feb4b97205f75866f7d21aa91452", size = 4946932, upload-time = "2025-09-22T04:01:58.989Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/ce/0f/526e78a6d38d109fdbaa5049c62e1d32fdd70c75fb61c4eadf3045d3d124/lxml-6.0.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb2f6ca0ae2d983ded09357b84af659c954722bbf04dea98030064996d156048", size = 5100060, upload-time = "2025-09-22T04:02:00.812Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/81/76/99de58d81fa702cc0ea7edae4f4640416c2062813a00ff24bd70ac1d9c9b/lxml-6.0.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb2a12d704f180a902d7fa778c6d71f36ceb7b0d317f34cdc76a5d05aa1dd1df", size = 5019000, upload-time = "2025-09-22T04:02:02.671Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/b5/35/9e57d25482bc9a9882cb0037fdb9cc18f4b79d85df94fa9d2a89562f1d25/lxml-6.0.2-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6ec0e3f745021bfed19c456647f0298d60a24c9ff86d9d051f52b509663feeb1", size = 5348496, upload-time = "2025-09-22T04:02:04.904Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/a6/8e/cb99bd0b83ccc3e8f0f528e9aa1f7a9965dfec08c617070c5db8d63a87ce/lxml-6.0.2-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:846ae9a12d54e368933b9759052d6206a9e8b250291109c48e350c1f1f49d916", size = 5643779, upload-time = "2025-09-22T04:02:06.689Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/d0/34/9e591954939276bb679b73773836c6684c22e56d05980e31d52a9a8deb18/lxml-6.0.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef9266d2aa545d7374938fb5c484531ef5a2ec7f2d573e62f8ce722c735685fd", size = 5244072, upload-time = "2025-09-22T04:02:08.587Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/8d/27/b29ff065f9aaca443ee377aff699714fcbffb371b4fce5ac4ca759e436d5/lxml-6.0.2-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:4077b7c79f31755df33b795dc12119cb557a0106bfdab0d2c2d97bd3cf3dffa6", size = 4718675, upload-time = "2025-09-22T04:02:10.783Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/2b/9f/f756f9c2cd27caa1a6ef8c32ae47aadea697f5c2c6d07b0dae133c244fbe/lxml-6.0.2-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a7c5d5e5f1081955358533be077166ee97ed2571d6a66bdba6ec2f609a715d1a", size = 5255171, upload-time = "2025-09-22T04:02:12.631Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/61/46/bb85ea42d2cb1bd8395484fd72f38e3389611aa496ac7772da9205bbda0e/lxml-6.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f8d0cbd0674ee89863a523e6994ac25fd5be9c8486acfc3e5ccea679bad2679", size = 5057175, upload-time = "2025-09-22T04:02:14.718Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/95/0c/443fc476dcc8e41577f0af70458c50fe299a97bb6b7505bb1ae09aa7f9ac/lxml-6.0.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2cbcbf6d6e924c28f04a43f3b6f6e272312a090f269eff68a2982e13e5d57659", size = 4785688, upload-time = "2025-09-22T04:02:16.957Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/48/78/6ef0b359d45bb9697bc5a626e1992fa5d27aa3f8004b137b2314793b50a0/lxml-6.0.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dfb874cfa53340009af6bdd7e54ebc0d21012a60a4e65d927c2e477112e63484", size = 5660655, upload-time = "2025-09-22T04:02:18.815Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/ff/ea/e1d33808f386bc1339d08c0dcada6e4712d4ed8e93fcad5f057070b7988a/lxml-6.0.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb8dae0b6b8b7f9e96c26fdd8121522ce5de9bb5538010870bd538683d30e9a2", size = 5247695, upload-time = "2025-09-22T04:02:20.593Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/4f/47/eba75dfd8183673725255247a603b4ad606f4ae657b60c6c145b381697da/lxml-6.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:358d9adae670b63e95bc59747c72f4dc97c9ec58881d4627fe0120da0f90d314", size = 5269841, upload-time = "2025-09-22T04:02:22.489Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/76/04/5c5e2b8577bc936e219becb2e98cdb1aca14a4921a12995b9d0c523502ae/lxml-6.0.2-cp313-cp313-win32.whl", hash = "sha256:e8cd2415f372e7e5a789d743d133ae474290a90b9023197fd78f32e2dc6873e2", size = 3610700, upload-time = "2025-09-22T04:02:24.465Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/fe/0a/4643ccc6bb8b143e9f9640aa54e38255f9d3b45feb2cbe7ae2ca47e8782e/lxml-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:b30d46379644fbfc3ab81f8f82ae4de55179414651f110a1514f0b1f8f6cb2d7", size = 4010347, upload-time = "2025-09-22T04:02:26.286Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/31/ef/dcf1d29c3f530577f61e5fe2f1bd72929acf779953668a8a47a479ae6f26/lxml-6.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:13dcecc9946dca97b11b7c40d29fba63b55ab4170d3c0cf8c0c164343b9bfdcf", size = 3671248, upload-time = "2025-09-22T04:02:27.918Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/03/15/d4a377b385ab693ce97b472fe0c77c2b16ec79590e688b3ccc71fba19884/lxml-6.0.2-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:b0c732aa23de8f8aec23f4b580d1e52905ef468afb4abeafd3fec77042abb6fe", size = 8659801, upload-time = "2025-09-22T04:02:30.113Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/c8/e8/c128e37589463668794d503afaeb003987373c5f94d667124ffd8078bbd9/lxml-6.0.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4468e3b83e10e0317a89a33d28f7aeba1caa4d1a6fd457d115dd4ffe90c5931d", size = 4659403, upload-time = "2025-09-22T04:02:32.119Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/00/ce/74903904339decdf7da7847bb5741fc98a5451b42fc419a86c0c13d26fe2/lxml-6.0.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:abd44571493973bad4598a3be7e1d807ed45aa2adaf7ab92ab7c62609569b17d", size = 4966974, upload-time = "2025-09-22T04:02:34.155Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/1f/d3/131dec79ce61c5567fecf82515bd9bc36395df42501b50f7f7f3bd065df0/lxml-6.0.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:370cd78d5855cfbffd57c422851f7d3864e6ae72d0da615fca4dad8c45d375a5", size = 5102953, upload-time = "2025-09-22T04:02:36.054Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/3a/ea/a43ba9bb750d4ffdd885f2cd333572f5bb900cd2408b67fdda07e85978a0/lxml-6.0.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:901e3b4219fa04ef766885fb40fa516a71662a4c61b80c94d25336b4934b71c0", size = 5055054, upload-time = "2025-09-22T04:02:38.154Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/60/23/6885b451636ae286c34628f70a7ed1fcc759f8d9ad382d132e1c8d3d9bfd/lxml-6.0.2-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:a4bf42d2e4cf52c28cc1812d62426b9503cdb0c87a6de81442626aa7d69707ba", size = 5352421, upload-time = "2025-09-22T04:02:40.413Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/48/5b/fc2ddfc94ddbe3eebb8e9af6e3fd65e2feba4967f6a4e9683875c394c2d8/lxml-6.0.2-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2c7fdaa4d7c3d886a42534adec7cfac73860b89b4e5298752f60aa5984641a0", size = 5673684, upload-time = "2025-09-22T04:02:42.288Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/29/9c/47293c58cc91769130fbf85531280e8cc7868f7fbb6d92f4670071b9cb3e/lxml-6.0.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98a5e1660dc7de2200b00d53fa00bcd3c35a3608c305d45a7bbcaf29fa16e83d", size = 5252463, upload-time = "2025-09-22T04:02:44.165Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/9b/da/ba6eceb830c762b48e711ded880d7e3e89fc6c7323e587c36540b6b23c6b/lxml-6.0.2-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:dc051506c30b609238d79eda75ee9cab3e520570ec8219844a72a46020901e37", size = 4698437, upload-time = "2025-09-22T04:02:46.524Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/a5/24/7be3f82cb7990b89118d944b619e53c656c97dc89c28cfb143fdb7cd6f4d/lxml-6.0.2-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8799481bbdd212470d17513a54d568f44416db01250f49449647b5ab5b5dccb9", size = 5269890, upload-time = "2025-09-22T04:02:48.812Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/1b/bd/dcfb9ea1e16c665efd7538fc5d5c34071276ce9220e234217682e7d2c4a5/lxml-6.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9261bb77c2dab42f3ecd9103951aeca2c40277701eb7e912c545c1b16e0e4917", size = 5097185, upload-time = "2025-09-22T04:02:50.746Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/21/04/a60b0ff9314736316f28316b694bccbbabe100f8483ad83852d77fc7468e/lxml-6.0.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:65ac4a01aba353cfa6d5725b95d7aed6356ddc0a3cd734de00124d285b04b64f", size = 4745895, upload-time = "2025-09-22T04:02:52.968Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/d6/bd/7d54bd1846e5a310d9c715921c5faa71cf5c0853372adf78aee70c8d7aa2/lxml-6.0.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:b22a07cbb82fea98f8a2fd814f3d1811ff9ed76d0fc6abc84eb21527596e7cc8", size = 5695246, upload-time = "2025-09-22T04:02:54.798Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/fd/32/5643d6ab947bc371da21323acb2a6e603cedbe71cb4c99c8254289ab6f4e/lxml-6.0.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d759cdd7f3e055d6bc8d9bec3ad905227b2e4c785dc16c372eb5b5e83123f48a", size = 5260797, upload-time = "2025-09-22T04:02:57.058Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/33/da/34c1ec4cff1eea7d0b4cd44af8411806ed943141804ac9c5d565302afb78/lxml-6.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:945da35a48d193d27c188037a05fec5492937f66fb1958c24fc761fb9d40d43c", size = 5277404, upload-time = "2025-09-22T04:02:58.966Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/82/57/4eca3e31e54dc89e2c3507e1cd411074a17565fa5ffc437c4ae0a00d439e/lxml-6.0.2-cp314-cp314-win32.whl", hash = "sha256:be3aaa60da67e6153eb15715cc2e19091af5dc75faef8b8a585aea372507384b", size = 3670072, upload-time = "2025-09-22T04:03:38.05Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/e3/e0/c96cf13eccd20c9421ba910304dae0f619724dcf1702864fd59dd386404d/lxml-6.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:fa25afbadead523f7001caf0c2382afd272c315a033a7b06336da2637d92d6ed", size = 4080617, upload-time = "2025-09-22T04:03:39.835Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/d5/5d/b3f03e22b3d38d6f188ef044900a9b29b2fe0aebb94625ce9fe244011d34/lxml-6.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:063eccf89df5b24e361b123e257e437f9e9878f425ee9aae3144c77faf6da6d8", size = 3754930, upload-time = "2025-09-22T04:03:41.565Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/5e/5c/42c2c4c03554580708fc738d13414801f340c04c3eff90d8d2d227145275/lxml-6.0.2-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:6162a86d86893d63084faaf4ff937b3daea233e3682fb4474db07395794fa80d", size = 8910380, upload-time = "2025-09-22T04:03:01.645Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/bf/4f/12df843e3e10d18d468a7557058f8d3733e8b6e12401f30b1ef29360740f/lxml-6.0.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:414aaa94e974e23a3e92e7ca5b97d10c0cf37b6481f50911032c69eeb3991bba", size = 4775632, upload-time = "2025-09-22T04:03:03.814Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/e4/0c/9dc31e6c2d0d418483cbcb469d1f5a582a1cd00a1f4081953d44051f3c50/lxml-6.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48461bd21625458dd01e14e2c38dd0aea69addc3c4f960c30d9f59d7f93be601", size = 4975171, upload-time = "2025-09-22T04:03:05.651Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/e7/2b/9b870c6ca24c841bdd887504808f0417aa9d8d564114689266f19ddf29c8/lxml-6.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:25fcc59afc57d527cfc78a58f40ab4c9b8fd096a9a3f964d2781ffb6eb33f4ed", size = 5110109, upload-time = "2025-09-22T04:03:07.452Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/bf/0c/4f5f2a4dd319a178912751564471355d9019e220c20d7db3fb8307ed8582/lxml-6.0.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5179c60288204e6ddde3f774a93350177e08876eaf3ab78aa3a3649d43eb7d37", size = 5041061, upload-time = "2025-09-22T04:03:09.297Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/12/64/554eed290365267671fe001a20d72d14f468ae4e6acef1e179b039436967/lxml-6.0.2-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:967aab75434de148ec80597b75062d8123cadf2943fb4281f385141e18b21338", size = 5306233, upload-time = "2025-09-22T04:03:11.651Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/7a/31/1d748aa275e71802ad9722df32a7a35034246b42c0ecdd8235412c3396ef/lxml-6.0.2-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d100fcc8930d697c6561156c6810ab4a508fb264c8b6779e6e61e2ed5e7558f9", size = 5604739, upload-time = "2025-09-22T04:03:13.592Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/8f/41/2c11916bcac09ed561adccacceaedd2bf0e0b25b297ea92aab99fd03d0fa/lxml-6.0.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ca59e7e13e5981175b8b3e4ab84d7da57993eeff53c07764dcebda0d0e64ecd", size = 5225119, upload-time = "2025-09-22T04:03:15.408Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/99/05/4e5c2873d8f17aa018e6afde417c80cc5d0c33be4854cce3ef5670c49367/lxml-6.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:957448ac63a42e2e49531b9d6c0fa449a1970dbc32467aaad46f11545be9af1d", size = 4633665, upload-time = "2025-09-22T04:03:17.262Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/0f/c9/dcc2da1bebd6275cdc723b515f93edf548b82f36a5458cca3578bc899332/lxml-6.0.2-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b7fc49c37f1786284b12af63152fe1d0990722497e2d5817acfe7a877522f9a9", size = 5234997, upload-time = "2025-09-22T04:03:19.14Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/9c/e2/5172e4e7468afca64a37b81dba152fc5d90e30f9c83c7c3213d6a02a5ce4/lxml-6.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e19e0643cc936a22e837f79d01a550678da8377d7d801a14487c10c34ee49c7e", size = 5090957, upload-time = "2025-09-22T04:03:21.436Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/a5/b3/15461fd3e5cd4ddcb7938b87fc20b14ab113b92312fc97afe65cd7c85de1/lxml-6.0.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1db01e5cf14345628e0cbe71067204db658e2fb8e51e7f33631f5f4735fefd8d", size = 4764372, upload-time = "2025-09-22T04:03:23.27Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/05/33/f310b987c8bf9e61c4dd8e8035c416bd3230098f5e3cfa69fc4232de7059/lxml-6.0.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:875c6b5ab39ad5291588aed6925fac99d0097af0dd62f33c7b43736043d4a2ec", size = 5634653, upload-time = "2025-09-22T04:03:25.767Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/70/ff/51c80e75e0bc9382158133bdcf4e339b5886c6ee2418b5199b3f1a61ed6d/lxml-6.0.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cdcbed9ad19da81c480dfd6dd161886db6096083c9938ead313d94b30aadf272", size = 5233795, upload-time = "2025-09-22T04:03:27.62Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/56/4d/4856e897df0d588789dd844dbed9d91782c4ef0b327f96ce53c807e13128/lxml-6.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:80dadc234ebc532e09be1975ff538d154a7fa61ea5031c03d25178855544728f", size = 5257023, upload-time = "2025-09-22T04:03:30.056Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/0f/85/86766dfebfa87bea0ab78e9ff7a4b4b45225df4b4d3b8cc3c03c5cd68464/lxml-6.0.2-cp314-cp314t-win32.whl", hash = "sha256:da08e7bb297b04e893d91087df19638dc7a6bb858a954b0cc2b9f5053c922312", size = 3911420, upload-time = "2025-09-22T04:03:32.198Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/fe/1a/b248b355834c8e32614650b8008c69ffeb0ceb149c793961dd8c0b991bb3/lxml-6.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:252a22982dca42f6155125ac76d3432e548a7625d56f5a273ee78a5057216eca", size = 4406837, upload-time = "2025-09-22T04:03:34.027Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/92/aa/df863bcc39c5e0946263454aba394de8a9084dbaff8ad143846b0d844739/lxml-6.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:bb4c1847b303835d89d785a18801a883436cdfd5dc3d62947f9c49e24f0f5a2c", size = 3822205, upload-time = "2025-09-22T04:03:36.249Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/e7/9c/780c9a8fce3f04690b374f72f41306866b0400b9d0fdf3e17aaa37887eed/lxml-6.0.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e748d4cf8fef2526bb2a589a417eba0c8674e29ffcb570ce2ceca44f1e567bf6", size = 3939264, upload-time = "2025-09-22T04:04:32.892Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/f5/5a/1ab260c00adf645d8bf7dec7f920f744b032f69130c681302821d5debea6/lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4ddb1049fa0579d0cbd00503ad8c58b9ab34d1254c77bc6a5576d96ec7853dba", size = 4216435, upload-time = "2025-09-22T04:04:34.907Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/f2/37/565f3b3d7ffede22874b6d86be1a1763d00f4ea9fc5b9b6ccb11e4ec8612/lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cb233f9c95f83707dae461b12b720c1af9c28c2d19208e1be03387222151daf5", size = 4325913, upload-time = "2025-09-22T04:04:37.205Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/22/ec/f3a1b169b2fb9d03467e2e3c0c752ea30e993be440a068b125fc7dd248b0/lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc456d04db0515ce3320d714a1eac7a97774ff0849e7718b492d957da4631dd4", size = 4269357, upload-time = "2025-09-22T04:04:39.322Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/77/a2/585a28fe3e67daa1cf2f06f34490d556d121c25d500b10082a7db96e3bcd/lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2613e67de13d619fd283d58bda40bff0ee07739f624ffee8b13b631abf33083d", size = 4412295, upload-time = "2025-09-22T04:04:41.647Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/7b/d9/a57dd8bcebd7c69386c20263830d4fa72d27e6b72a229ef7a48e88952d9a/lxml-6.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:24a8e756c982c001ca8d59e87c80c4d9dcd4d9b44a4cbeb8d9be4482c514d41d", size = 3516913, upload-time = "2025-09-22T04:04:43.602Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/0b/11/29d08bc103a62c0eba8016e7ed5aeebbf1e4312e83b0b1648dd203b0e87d/lxml-6.0.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1c06035eafa8404b5cf475bb37a9f6088b0aca288d4ccc9d69389750d5543700", size = 3949829, upload-time = "2025-09-22T04:04:45.608Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/12/b3/52ab9a3b31e5ab8238da241baa19eec44d2ab426532441ee607165aebb52/lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c7d13103045de1bdd6fe5d61802565f1a3537d70cd3abf596aa0af62761921ee", size = 4226277, upload-time = "2025-09-22T04:04:47.754Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/a0/33/1eaf780c1baad88224611df13b1c2a9dfa460b526cacfe769103ff50d845/lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a3c150a95fbe5ac91de323aa756219ef9cf7fde5a3f00e2281e30f33fa5fa4f", size = 4330433, upload-time = "2025-09-22T04:04:49.907Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/7a/c1/27428a2ff348e994ab4f8777d3a0ad510b6b92d37718e5887d2da99952a2/lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60fa43be34f78bebb27812ed90f1925ec99560b0fa1decdb7d12b84d857d31e9", size = 4272119, upload-time = "2025-09-22T04:04:51.801Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/f0/d0/3020fa12bcec4ab62f97aab026d57c2f0cfd480a558758d9ca233bb6a79d/lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21c73b476d3cfe836be731225ec3421fa2f048d84f6df6a8e70433dff1376d5a", size = 4417314, upload-time = "2025-09-22T04:04:55.024Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/6c/77/d7f491cbc05303ac6801651aabeb262d43f319288c1ea96c66b1d2692ff3/lxml-6.0.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:27220da5be049e936c3aca06f174e8827ca6445a4353a1995584311487fc4e3e", size = 3518768, upload-time = "2025-09-22T04:04:57.097Z" },
]
[[package]]
name = "markdown"
version = "3.10"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7d/ab/7dd27d9d863b3376fcf23a5a13cb5d024aed1db46f963f1b5735ae43b3be/markdown-3.10.tar.gz", hash = "sha256:37062d4f2aa4b2b6b32aefb80faa300f82cc790cb949a35b8caede34f2b68c0e", size = 364931, upload-time = "2025-11-03T19:51:15.007Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/70/81/54e3ce63502cd085a0c556652a4e1b919c45a446bd1e5300e10c44c8c521/markdown-3.10-py3-none-any.whl", hash = "sha256:b5b99d6951e2e4948d939255596523444c0e677c669700b1d17aa4a8a464cb7c", size = 107678, upload-time = "2025-11-03T19:51:13.887Z" },
]
[[package]]
name = "markdown-it-py"
version = "4.0.0"
@ -3050,6 +3192,21 @@ wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" },
]
[[package]]
name = "pymupdf"
version = "1.26.7"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/48/d6/09b28f027b510838559f7748807192149c419b30cb90e6d5f0cf916dc9dc/pymupdf-1.26.7.tar.gz", hash = "sha256:71add8bdc8eb1aaa207c69a13400693f06ad9b927bea976f5d5ab9df0bb489c3", size = 84327033, upload-time = "2025-12-11T21:48:50.694Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/94/35/cd74cea1787b2247702ef8522186bdef32e9cb30a099e6bb864627ef6045/pymupdf-1.26.7-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:07085718dfdae5ab83b05eb5eb397f863bcc538fe05135318a01ea353e7a1353", size = 23179369, upload-time = "2025-12-11T21:47:21.587Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/72/74/448b6172927c829c6a3fba80078d7b0a016ebbe2c9ee528821f5ea21677a/pymupdf-1.26.7-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:31aa9c8377ea1eea02934b92f4dcf79fb2abba0bf41f8a46d64c3e31546a3c02", size = 22470101, upload-time = "2025-12-11T21:47:37.105Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/65/e7/47af26f3ac76be7ac3dd4d6cc7ee105948a8355d774e5ca39857bf91c11c/pymupdf-1.26.7-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e419b609996434a14a80fa060adec72c434a1cca6a511ec54db9841bc5d51b3c", size = 23502486, upload-time = "2025-12-12T09:51:25.824Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/2a/6b/3de1714d734ff949be1e90a22375d0598d3540b22ae73eb85c2d7d1f36a9/pymupdf-1.26.7-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:69dfc78f206a96e5b3ac22741263ebab945fdf51f0dbe7c5757c3511b23d9d72", size = 24115727, upload-time = "2025-12-11T21:47:51.274Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/62/9b/f86224847949577a523be2207315ae0fd3155b5d909cd66c274d095349a3/pymupdf-1.26.7-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1d5106f46e1ca0d64d46bd51892372a4f82076bdc14a9678d33d630702abca36", size = 24324386, upload-time = "2025-12-12T14:58:45.483Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/85/8e/a117d39092ca645fde8b903f4a941d9aa75b370a67b4f1f435f56393dc5a/pymupdf-1.26.7-cp310-abi3-win32.whl", hash = "sha256:7c9645b6f5452629c747690190350213d3e5bbdb6b2eca227d82702b327f6eee", size = 17203888, upload-time = "2025-12-12T13:59:57.613Z" },
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/dd/c3/d0047678146c294469c33bae167c8ace337deafb736b0bf97b9bc481aa65/pymupdf-1.26.7-cp310-abi3-win_amd64.whl", hash = "sha256:425b1befe40d41b72eb0fe211711c7ae334db5eb60307e9dd09066ed060cceba", size = 18405952, upload-time = "2025-12-11T21:48:02.947Z" },
]
[[package]]
name = "pymysql"
version = "1.1.0"
@ -3139,6 +3296,19 @@ wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
]
[[package]]
name = "python-docx"
version = "1.2.0"
source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple/" }
dependencies = [
{ name = "lxml" },
{ name = "typing-extensions" },
]
sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a9/f7/eddfe33871520adab45aaa1a71f0402a2252050c14c7e3009446c8f4701c/python_docx-1.2.0.tar.gz", hash = "sha256:7bc9d7b7d8a69c9c02ca09216118c86552704edc23bac179283f2e38f86220ce", size = 5723256, upload-time = "2025-06-16T20:46:27.921Z" }
wheels = [
{ url = "https://pypi.tuna.tsinghua.edu.cn/packages/d0/00/1e03a4989fa5795da308cd774f05b704ace555a70f9bf9d3be057b680bcf/python_docx-1.2.0-py3-none-any.whl", hash = "sha256:3fd478f3250fbbbfd3b94fe1e985955737c145627498896a8a6bf81f4baf66c7", size = 252987, upload-time = "2025-06-16T20:46:22.506Z" },
]
[[package]]
name = "python-dotenv"
version = "1.0.0"
@ -3236,7 +3406,9 @@ version = "1.0.0"
source = { editable = "." }
dependencies = [
{ name = "aiofiles" },
{ name = "beautifulsoup4" },
{ name = "chromadb" },
{ name = "docx2txt" },
{ name = "fastapi" },
{ name = "httpx" },
{ name = "llama-index" },
@ -3244,9 +3416,15 @@ dependencies = [
{ name = "llama-index-llms-ollama" },
{ name = "llama-index-vector-stores-chroma" },
{ name = "loguru" },
{ name = "lxml" },
{ name = "markdown" },
{ name = "pandas" },
{ name = "pydantic" },
{ name = "pydantic-settings" },
{ name = "pymupdf" },
{ name = "pymysql" },
{ name = "pypdf" },
{ name = "python-docx" },
{ name = "python-dotenv" },
{ name = "sqlalchemy" },
{ name = "uvicorn", extra = ["standard"] },
@ -3273,8 +3451,10 @@ dev = [
[package.metadata]
requires-dist = [
{ name = "aiofiles", specifier = "==23.2.1" },
{ name = "beautifulsoup4", specifier = ">=4.12.0" },
{ name = "black", marker = "extra == 'dev'", specifier = ">=23.0.0" },
{ name = "chromadb", specifier = ">=0.5.17" },
{ name = "docx2txt", specifier = ">=0.8" },
{ name = "fastapi", specifier = "==0.104.1" },
{ name = "httpx", specifier = ">=0.27.0" },
{ name = "llama-index", specifier = "==0.14.8" },
@ -3282,12 +3462,18 @@ requires-dist = [
{ name = "llama-index-llms-ollama", specifier = ">=0.1.0" },
{ name = "llama-index-vector-stores-chroma", specifier = ">=0.4.2" },
{ name = "loguru", specifier = "==0.7.2" },
{ name = "lxml", specifier = ">=4.9.0" },
{ name = "markdown", specifier = ">=3.5.0" },
{ name = "mypy", marker = "extra == 'dev'", specifier = ">=1.0.0" },
{ name = "pandas", specifier = ">=2.0.0" },
{ name = "pydantic", specifier = ">=2.8.0" },
{ name = "pydantic-settings", specifier = ">=2.1.0" },
{ name = "pymupdf", specifier = ">=1.23.0" },
{ name = "pymysql", specifier = "==1.1.0" },
{ name = "pypdf", specifier = ">=3.0.0" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" },
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.21.0" },
{ name = "python-docx", specifier = ">=1.0.0" },
{ name = "python-dotenv", specifier = "==1.0.0" },
{ name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" },
{ name = "sqlalchemy", specifier = ">=2.0.40" },