Compare commits
10 Commits
master
...
feature/ve
| Author | SHA1 | Date |
|---|---|---|
xmy
|
f5771b34a5 | |
|
xmy
|
4fdc3ef758 | |
|
linlin
|
b374116286 | |
|
linlin
|
77c81640f5 | |
|
linlin
|
b44b7dec8b | |
|
linlin
|
31c3f4bc08 | |
|
linlin
|
6c14f69dac | |
 zhangxunhui
|
34481b92fb | |
 Nigel
|
d8b49609bb | |
|
zhangxunhui
|
84614e708d |
|
|
@ -16,7 +16,7 @@ API_VERSION=1.0.0
|
|||
MAX_UPLOAD_SIZE_MB=5
|
||||
|
||||
# LibreOffice soffice service port (used by docker/soffice service)
|
||||
SOFFICE_HOST=localhost
|
||||
SOFFICE_HOST=rag-soffice #localhost
|
||||
SOFFICE_PORT=8003
|
||||
|
||||
# ============================================
|
||||
|
|
@ -25,15 +25,15 @@ SOFFICE_PORT=8003
|
|||
# CHROMA_SERVER_HOST: ChromaDB 服务器地址
|
||||
# - 使用 host 网络模式: localhost
|
||||
# - 远程服务器: 192.168.1.100 或 chromadb.example.com
|
||||
CHROMA_SERVER_HOST=localhost
|
||||
CHROMA_SERVER_PORT=8002
|
||||
CHROMA_SERVER_HOST=rag-chromadb #localhost
|
||||
CHROMA_SERVER_PORT=8000 #8002
|
||||
CHROMA_COLLECTION_NAME=rag_collection
|
||||
|
||||
# ============================================
|
||||
# Ollama 配置
|
||||
# ============================================
|
||||
# OLLAMA_BASE_URL: Ollama 服务地址
|
||||
OLLAMA_BASE_URL=http://localhost:11434
|
||||
OLLAMA_BASE_URL=http://host.docker.internal:11434
|
||||
OLLAMA_MODEL=qwen3:8b
|
||||
OLLAMA_EMBEDDING_MODEL=qwen3-embedding:0.6b
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
# ============================================
|
||||
# RAG API 环境变量配置文件
|
||||
# ============================================
|
||||
# 复制此文件为 .env 并根据实际情况修改
|
||||
# 所有配置都可以通过此文件统一管理，方便不同机器之间移植
|
||||
# docker-compose.yml 会自动读取此文件中的配置
|
||||
|
||||
# ============================================
|
||||
# API 配置
|
||||
# ============================================
|
||||
API_HOST=0.0.0.0
|
||||
API_PORT=8001
|
||||
API_TITLE=RAG API
|
||||
API_VERSION=1.0.0
|
||||
# 文件上传大小限制（单位：MB，默认：5MB）
|
||||
MAX_UPLOAD_SIZE_MB=5
|
||||
|
||||
# LibreOffice soffice service port (used by docker/soffice service)
|
||||
SOFFICE_HOST=rag-soffice #localhost
|
||||
SOFFICE_PORT=8003
|
||||
|
||||
# ============================================
|
||||
# ChromaDB 配置
|
||||
# ============================================
|
||||
# CHROMA_SERVER_HOST: ChromaDB 服务器地址
|
||||
# - 使用 host 网络模式: localhost
|
||||
# - 远程服务器: 192.168.1.100 或 chromadb.example.com
|
||||
CHROMA_SERVER_HOST=rag-chromadb #localhost
|
||||
CHROMA_SERVER_PORT=8000 #8002
|
||||
CHROMA_COLLECTION_NAME=rag_collection
|
||||
|
||||
# ============================================
|
||||
# Ollama 配置
|
||||
# ============================================
|
||||
# OLLAMA_BASE_URL: Ollama 服务地址
|
||||
OLLAMA_BASE_URL=http://host.docker.internal:11434
|
||||
OLLAMA_MODEL=qwen3:8b
|
||||
OLLAMA_EMBEDDING_MODEL=qwen3-embedding:0.6b
|
||||
|
||||
# ============================================
|
||||
# RAG 配置
|
||||
# ============================================
|
||||
EMBEDDING_DIMENSION=768
|
||||
CHUNK_SIZE=1024
|
||||
CHUNK_OVERLAP=200
|
||||
TOP_K=5
|
||||
|
||||
# ============================================
|
||||
# 同步配置
|
||||
# ============================================
|
||||
SYNC_INTERVAL=300
|
||||
AUTO_SYNC=true
|
||||
|
||||
# ============================================
|
||||
# NLTK 配置
|
||||
# 推荐：如果你已在仓库内保存了 NLTK 数据（离线使用），可以设置为相对路径。例如:
|
||||
# ./nltk_data/
|
||||
# 或者使用本地绝对路径: /path/to/nltk_data
|
||||
# ============================================
|
||||
NLTK_DATA=./nltk_data/
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
# ============================================
|
||||
# RAG API 环境变量配置文件
|
||||
# ============================================
|
||||
# 复制此文件为 .env 并根据实际情况修改
|
||||
# 所有配置都可以通过此文件统一管理，方便不同机器之间移植
|
||||
# docker-compose.yml 会自动读取此文件中的配置
|
||||
|
||||
# ============================================
|
||||
# API 配置
|
||||
# ============================================
|
||||
API_HOST=0.0.0.0
|
||||
API_PORT=8001
|
||||
API_TITLE=RAG API
|
||||
API_VERSION=1.0.0
|
||||
# 文件上传大小限制（单位：MB，默认：5MB）
|
||||
MAX_UPLOAD_SIZE_MB=5
|
||||
|
||||
# LibreOffice soffice service port (used by docker/soffice service)
|
||||
# SOFFICE_HOST=rag-soffice #localhost
|
||||
SOFFICE_PORT=8003
|
||||
|
||||
# ============================================
|
||||
# ChromaDB 配置
|
||||
# ============================================
|
||||
# CHROMA_SERVER_HOST: ChromaDB 服务器地址
|
||||
# - 使用 host 网络模式: localhost
|
||||
# - 远程服务器: 192.168.1.100 或 chromadb.example.com
|
||||
# CHROMA_SERVER_HOST=rag-chromadb #localhost
|
||||
CHROMA_SERVER_PORT=8000
|
||||
CHROMA_COLLECTION_NAME=rag_collection
|
||||
|
||||
# ============================================
|
||||
# Ollama 配置
|
||||
# ============================================
|
||||
# OLLAMA_BASE_URL: Ollama 服务地址
|
||||
# OLLAMA_BASE_URL=http://host.docker.internal:11434 #http://localhost:11434
|
||||
OLLAMA_MODEL=qwen3:8b
|
||||
OLLAMA_EMBEDDING_MODEL=qwen3-embedding:0.6b
|
||||
|
||||
# ============================================
|
||||
# RAG 配置
|
||||
# ============================================
|
||||
EMBEDDING_DIMENSION=768
|
||||
CHUNK_SIZE=1024
|
||||
CHUNK_OVERLAP=200
|
||||
TOP_K=5
|
||||
|
||||
# ============================================
|
||||
# 同步配置
|
||||
# ============================================
|
||||
SYNC_INTERVAL=300
|
||||
AUTO_SYNC=true
|
||||
|
||||
# ============================================
|
||||
# NLTK 配置
|
||||
# 推荐：如果你已在仓库内保存了 NLTK 数据（离线使用），可以设置为相对路径。例如:
|
||||
# ./nltk_data/
|
||||
# 或者使用本地绝对路径: /path/to/nltk_data
|
||||
# ============================================
|
||||
NLTK_DATA=./nltk_data/
|
||||
|
|
@ -40,7 +40,7 @@ llamaindex/
|
|||
!.vscode/extensions.json
|
||||
!.vscode/tasks.json
|
||||
# Ignore user-specific VSCode files
|
||||
.vscode/launch.json
|
||||
!.vscode/launch.json
|
||||
.vscode/*.code-workspace
|
||||
|
||||
# JetBrains IDEs
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
// 使用 IntelliSense 了解相关属性。
|
||||
// 悬停以查看现有属性的描述。
|
||||
// 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Python: RAG Main",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/main.py", // 指向api/main.py
|
||||
"cwd": "${workspaceFolder}", // 关键：工作目录设为项目根目录（S:\research\RAG）
|
||||
"console": "integratedTerminal",
|
||||
"justMyCode": false
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -29,6 +29,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||
libssl-dev \
|
||||
libcrypto++-dev \
|
||||
libgmp-dev \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 配置 pip 镜像源（加速 Python 包安装）
|
||||
|
|
@ -40,7 +41,7 @@ COPY requirements.txt /app/
|
|||
|
||||
# 安装 Python 依赖
|
||||
RUN pip install --upgrade pip && \
|
||||
pip install -r requirements.txt
|
||||
pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
|
||||
# 复制项目文件
|
||||
COPY . /app/
|
||||
|
|
|
|||
46
README.md
46
README.md
|
|
@ -5,13 +5,14 @@
|
|||
## 功能特性
|
||||
|
||||
- 🔍 **智能检索**: 使用 LlamaIndex 和 ChromaDB 实现高效的向量检索
|
||||
- 💾 **数据同步**: 自动同步 MySQL 数据库数据到 ChromaDB 向量库
|
||||
- 💾 **数据同步**: 自动同步 MySQL 数据库、本地/远程文件夹和 Git 代码库数据到 ChromaDB 向量库
|
||||
- 🌊 **流式输出**: 基于 FastAPI 的流式响应，支持实时对话
|
||||
- 🤖 **本地 LLM**: 集成 Ollama 本地部署的大模型
|
||||
- ⚡ **高并发**: 支持多用户同时访问
|
||||
- 🔄 **自动同步**: 支持定时自动同步和手动触发同步
|
||||
- 🐳 **Docker 部署**: 使用 Docker Compose 一键部署
|
||||
- ⚙️ **统一配置**: 所有配置统一在 `.env` 文件中管理，方便不同机器之间移植
|
||||
- 🧑💻 **Git 集成**: 支持 Git 代码库的自动同步和检索，包括连接测试和分支管理
|
||||
|
||||
## 快速开始
|
||||
|
||||
|
|
@ -29,7 +30,7 @@
|
|||
# 检查 Ollama 是否运行
|
||||
curl http://localhost:11434/api/tags
|
||||
|
||||
# 下载所需的模型（如果未下载）
|
||||
# 下载所需的模型（如果未下载）（可以使用更小的模型）
|
||||
ollama pull qwen3:235b # LLM模型，用于文本生成
|
||||
ollama pull qwen3-embedding:8b # Embedding模型，用于向量化
|
||||
```
|
||||
|
|
@ -168,7 +169,20 @@ ollama pull qwen3-embedding:8b # Embedding模型，用于向量化
|
|||
- 点击"测试SSH连接"，检查 SSH 连接是否成功。
|
||||
- 点击右上角"保存"按钮，保存文件夹配置
|
||||
|
||||
3. 更新数据源配置
|
||||
3. **Git代码库类型 (git)**
|
||||
|
||||
- 点击"新增数据源"-"选择类型"-"Git代码库"
|
||||
|
||||
- Git仓库配置
|
||||
- Git仓库URL（必填）：Git代码库的URL地址
|
||||
- 分支（必填）：要同步的Git分支（默认main）
|
||||
- 协议：选择https或ssh协议
|
||||
- HTTPS Token：如果使用https协议，填写访问令牌
|
||||
- SSH密钥：如果使用ssh协议，填写SSH私钥
|
||||
- 点击"测试Git连接"，检查 Git 连接是否成功。
|
||||
- 点击右上角"保存"按钮，保存Git仓库配置
|
||||
|
||||
4. 更新数据源配置
|
||||
- 点击左侧数据源列表中的数据源
|
||||
- 修改配置后点击保存，后台会自动删除原来同步的数据并重新同步
|
||||
|
||||
|
|
@ -313,6 +327,27 @@ docker-compose restart rag-api
|
|||
- 查看同步服务日志: `docker-compose logs rag-api | grep sync`
|
||||
- 手动触发同步: 在配置管理界面中点击"同步"按钮
|
||||
|
||||
### 8. Git连接失败
|
||||
|
||||
**错误**: `Git连接失败` 或 `Failed to connect to Git repository`
|
||||
|
||||
**解决**:
|
||||
- 确保 Git 仓库 URL 正确
|
||||
- 检查网络连接是否正常
|
||||
- 验证 Git 凭证（HTTPS Token 或 SSH 密钥）是否有效
|
||||
- 确保目标 Git 仓库存在且可访问
|
||||
- 查看详细错误信息: `docker-compose logs rag-api | grep git`
|
||||
|
||||
### 9. Git同步失败
|
||||
|
||||
**错误**: `Git同步失败` 或 `Failed to sync Git repository`
|
||||
|
||||
**解决**:
|
||||
- 检查 Git 仓库是否有访问权限
|
||||
- 验证本地磁盘空间是否充足
|
||||
- 查看同步服务日志获取详细错误信息: `docker-compose logs rag-api | grep sync`
|
||||
- 尝试手动触发同步: 在配置管理界面中点击"同步"按钮
|
||||
|
||||
## 性能优化建议
|
||||
|
||||
### 1. 调整配置参数
|
||||
|
|
@ -356,6 +391,9 @@ docker-compose restart rag-api
|
|||
**启动步骤**：
|
||||
|
||||
```bash
|
||||
# 0. 确保 .env 文件中的host配置准确
|
||||
cp .env.zkxlocal .env
|
||||
|
||||
# 1. 创建虚拟环境
|
||||
uv venv --python 3.13.9
|
||||
source .venv/bin/activate # Windows: venv\Scripts\activate
|
||||
|
|
@ -377,4 +415,6 @@ curl http://localhost:8003/health
|
|||
|
||||
# 7. 启动 RAG API 服务
|
||||
python main.py
|
||||
|
||||
# 8. 如要调试，使用.vscode/launch.json 启动调试会话
|
||||
```
|
||||
|
|
|
|||
93
api/main.py
93
api/main.py
|
|
@ -77,6 +77,7 @@ async def lifespan(app: FastAPI):
|
|||
logger.error(f"Failed to initialize FileParser: {e}")
|
||||
raise
|
||||
|
||||
|
||||
|
||||
logger.info("✓ Core RAG services initialized")
|
||||
|
||||
|
|
@ -281,6 +282,9 @@ class RetrieveResponse(BaseModel):
|
|||
count: int = Field(..., description="Number of documents retrieved")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
class SyncRequest(BaseModel):
|
||||
"""Manual sync request model"""
|
||||
full_sync: bool = Field(False, description="Whether to perform full sync")
|
||||
|
|
@ -919,7 +923,7 @@ async def retrieve(request: RetrieveRequest):
|
|||
try:
|
||||
|
||||
# Get retriever (works even if collection is empty, will return empty results)
|
||||
retriever = vector_store_manager.get_retriever(top_k=request.top_k or settings.TOP_K)
|
||||
retriever = vector_store_manager.get_retriever(top_k=request.top_k or settings.TOP_K, query=request.query)
|
||||
|
||||
# Retrieve documents (run in thread pool since retriever.retrieve() is synchronous)
|
||||
def retrieve_docs():
|
||||
|
|
@ -972,6 +976,9 @@ async def retrieve(request: RetrieveRequest):
|
|||
raise HTTPException(status_code=500, detail=f"Error retrieving documents: {str(e)}")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@app.delete("/documents/source/{source_name}")
|
||||
async def delete_documents_by_source(source_name: str):
|
||||
"""
|
||||
|
|
@ -1365,6 +1372,18 @@ async def create_config(config: Dict[str, Any]):
|
|||
config["host"].lower(),
|
||||
folder_path
|
||||
])
|
||||
elif config_type == "git":
|
||||
# Git配置需要：仓库URL
|
||||
if not config.get("git_url"):
|
||||
raise HTTPException(status_code=400, detail="Git配置必须包含仓库URL")
|
||||
# 添加Git仓库URL到唯一标识符
|
||||
# 替换URL中的特殊字符为下划线
|
||||
git_url = config["git_url"].lower().replace("/", "_").replace(":", "_").replace(".", "_")
|
||||
# 截取URL的一部分作为唯一标识
|
||||
git_url_part = git_url[:100] # 限制长度
|
||||
unique_id_parts.extend([
|
||||
git_url_part
|
||||
])
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail=f"不支持的配置类型: {config_type}")
|
||||
|
||||
|
|
@ -1403,6 +1422,13 @@ async def create_config(config: Dict[str, Any]):
|
|||
status_code=409,
|
||||
detail=f"已存在相同服务器和路径的文件夹配置。如需调整，请点击配置列表中的配置并修改配置内容。"
|
||||
)
|
||||
elif config_type == 'git':
|
||||
# For git configs, same source means same git url
|
||||
if existing_config_data.get('git_url') == config.get('git_url'):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"已存在相同Git仓库的配置。如需调整，请点击配置列表中的配置并修改配置内容。"
|
||||
)
|
||||
except sqlite3.OperationalError as e:
|
||||
# 表不存在的情况，会在后面创建表
|
||||
logger.warning(f"SQLite table error: {e}. This is expected if the table doesn't exist yet.")
|
||||
|
|
@ -1433,7 +1459,7 @@ async def create_config(config: Dict[str, Any]):
|
|||
global sync_manager
|
||||
if sync_manager is not None:
|
||||
# Create appropriate data source config object
|
||||
from config import BaseDataSourceConfig, DatabaseDataSourceConfig, FolderDataSourceConfig
|
||||
from config import BaseDataSourceConfig, DatabaseDataSourceConfig, FolderDataSourceConfig, GitDataSourceConfig
|
||||
|
||||
if config_type == "database":
|
||||
source_config = DatabaseDataSourceConfig(
|
||||
|
|
@ -1470,6 +1496,20 @@ async def create_config(config: Dict[str, Any]):
|
|||
recursive=config.get("recursive", True),
|
||||
ignore_patterns=config.get("ignore_patterns")
|
||||
)
|
||||
elif config_type == "git":
|
||||
source_config = GitDataSourceConfig(
|
||||
name=config_id,
|
||||
git_url=config.get("git_url"),
|
||||
branch=config.get("branch", "main"),
|
||||
protocol=config.get("protocol", "https"),
|
||||
ssh_key=config.get("ssh_key"),
|
||||
https_token=config.get("https_token"),
|
||||
local_repo_path=config.get("local_repo_path"),
|
||||
poll_interval=config.get("poll_interval", 300),
|
||||
support_lang=config.get("support_lang"),
|
||||
latest_commit_id=config.get("latest_commit_id"),
|
||||
last_sync_time=config.get("last_sync_time")
|
||||
)
|
||||
else:
|
||||
logger.warning(f"Unknown config type: {config_type}")
|
||||
# Create a base config as fallback
|
||||
|
|
@ -1540,6 +1580,55 @@ async def test_folder_connection(connection_data: Dict[str, Any]):
|
|||
raise HTTPException(status_code=500, detail=f"SSH连接失败: {str(e)}")
|
||||
|
||||
|
||||
@app.post("/git/test-connection")
|
||||
async def test_git_connection(connection_data: Dict[str, Any]):
|
||||
"""
|
||||
Test Git connection for Git repository configuration
|
||||
|
||||
Args:
|
||||
connection_data: Connection data including git_url, protocol, branch, https_token, ssh_key
|
||||
|
||||
Returns:
|
||||
Success message if connection is successful
|
||||
"""
|
||||
try:
|
||||
git_url = connection_data.get("git_url")
|
||||
protocol = connection_data.get("protocol", "https")
|
||||
branch = connection_data.get("branch", "main")
|
||||
https_token = connection_data.get("https_token")
|
||||
ssh_key = connection_data.get("ssh_key")
|
||||
|
||||
if not git_url:
|
||||
raise HTTPException(status_code=400, detail="Git仓库URL是必填项")
|
||||
|
||||
# Import GitTool here to avoid circular imports
|
||||
from utils.git_tool import GitTool
|
||||
|
||||
# Create a temporary GitTool instance to test connection
|
||||
git_tool = GitTool(
|
||||
git_url=git_url,
|
||||
branch=branch,
|
||||
protocol=protocol,
|
||||
https_token=https_token,
|
||||
ssh_key=ssh_key,
|
||||
local_repo_path=None # 测试连接不需要本地路径
|
||||
)
|
||||
|
||||
# Try to test connection
|
||||
success = git_tool.test_connection()
|
||||
|
||||
if success:
|
||||
return {"message": "Git连接成功！"}
|
||||
else:
|
||||
raise HTTPException(status_code=500, detail="Git连接失败")
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error testing Git connection: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Git连接失败: {str(e)}")
|
||||
|
||||
|
||||
@app.post("/folder-configs/remote")
|
||||
async def create_remote_folder_config(config: Dict[str, Any]):
|
||||
"""
|
||||
|
|
|
|||
52
config.py
52
config.py
|
|
@ -111,6 +111,35 @@ class FolderDataSourceConfig(BaseDataSourceConfig):
|
|||
self.ignore_patterns = ignore_patterns # 忽略的文件模式列表
|
||||
|
||||
|
||||
class GitDataSourceConfig(BaseDataSourceConfig):
|
||||
"""Git data source configuration"""
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
git_url: str,
|
||||
branch: str = "main",
|
||||
protocol: str = "https", # https 或 ssh
|
||||
ssh_key: Optional[str] = None, # SSH私钥
|
||||
https_token: Optional[str] = None, # HTTPS令牌
|
||||
local_repo_path: Optional[str] = None, # 本地存储路径
|
||||
poll_interval: int = 300, # 轮询间隔（秒）
|
||||
support_lang: Optional[List[str]] = None, # 支持的编程语言
|
||||
latest_commit_id: Optional[str] = None, # 最新commit ID
|
||||
last_sync_time: Optional[str] = None # 最后同步时间
|
||||
):
|
||||
super().__init__(name, "git")
|
||||
self.git_url = git_url # Git仓库地址
|
||||
self.branch = branch # 分支名称
|
||||
self.protocol = protocol # 协议类型
|
||||
self.ssh_key = ssh_key # SSH私钥（加密存储）
|
||||
self.https_token = https_token # HTTPS令牌（加密存储）
|
||||
self.local_repo_path = local_repo_path # 本地存储路径
|
||||
self.poll_interval = poll_interval # 轮询间隔
|
||||
self.support_lang = support_lang # 支持的编程语言
|
||||
self.latest_commit_id = latest_commit_id # 最新commit ID
|
||||
self.last_sync_time = last_sync_time # 最后同步时间
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""
|
||||
Application settings
|
||||
|
|
@ -148,7 +177,7 @@ class Settings(BaseSettings):
|
|||
|
||||
# RAG Settings
|
||||
EMBEDDING_DIMENSION: int = 768
|
||||
CHUNK_SIZE: int = 1024
|
||||
CHUNK_SIZE: int = 4000
|
||||
CHUNK_OVERLAP: int = 200
|
||||
TOP_K: int = 5 # Number of documents to retrieve
|
||||
|
||||
|
|
@ -176,6 +205,12 @@ class Settings(BaseSettings):
|
|||
SOFFICE_HOST: str = "127.0.0.1"
|
||||
SOFFICE_PORT: int = 8003
|
||||
|
||||
# Git 相关配置
|
||||
GIT_LOCAL_STORAGE_ROOT: str = "./git_repos" # Git仓库本地存储根目录
|
||||
GIT_DEFAULT_BRANCH: str = "main" # 默认分支
|
||||
GIT_POLL_INTERVAL: int = 300 # 默认轮询间隔（秒）
|
||||
GIT_MAX_REPO_SIZE_MB: int = 500 # 最大仓库大小（MB）
|
||||
|
||||
# Pydantic v2 configuration
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
|
|
@ -260,6 +295,21 @@ class Settings(BaseSettings):
|
|||
recursive=ds_config.get('recursive', True),
|
||||
ignore_patterns=ds_config.get('ignore_patterns', None)
|
||||
))
|
||||
elif source_type == 'git':
|
||||
# Create git data source
|
||||
configs.append(GitDataSourceConfig(
|
||||
name=name, # 使用数据库表中的name列
|
||||
git_url=ds_config.get('git_url'),
|
||||
branch=ds_config.get('branch', 'main'),
|
||||
protocol=ds_config.get('protocol', 'https'),
|
||||
ssh_key=ds_config.get('ssh_key'),
|
||||
https_token=ds_config.get('https_token'),
|
||||
local_repo_path=ds_config.get('local_repo_path'),
|
||||
poll_interval=ds_config.get('poll_interval', 300),
|
||||
support_lang=ds_config.get('support_lang'),
|
||||
latest_commit_id=ds_config.get('latest_commit_id'),
|
||||
last_sync_time=ds_config.get('last_sync_time')
|
||||
))
|
||||
else:
|
||||
from loguru import logger
|
||||
logger.warning(f"Unknown data source type: {source_type}, skipping")
|
||||
|
|
|
|||
70
db_utils.py
70
db_utils.py
|
|
@ -2,6 +2,7 @@
|
|||
Database utilities for RAG system
|
||||
"""
|
||||
import sqlite3
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Tuple, Optional
|
||||
|
|
@ -90,6 +91,75 @@ def update_data_source_update_at(source_name: str, update_at: datetime) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def add_git_datasource(user_id: str, repo_config: dict):
|
||||
"""
|
||||
新增Git仓库配置
|
||||
|
||||
Args:
|
||||
user_id: 用户ID
|
||||
repo_config: 仓库配置
|
||||
"""
|
||||
conn, cursor = get_db_connection()
|
||||
try:
|
||||
cursor.execute("""
|
||||
INSERT INTO datasource (user_id, name, type, git_config, create_time)
|
||||
VALUES (?, ?, 'git', ?, datetime('now'))
|
||||
""", (user_id, repo_config["name"], json.dumps(repo_config)))
|
||||
conn.commit()
|
||||
logger.info(f"新增Git数据源: {repo_config['name']}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def update_git_sync_status(user_id: str, repo_id: str, sync_status: dict):
|
||||
"""
|
||||
更新Git仓库同步状态
|
||||
|
||||
Args:
|
||||
user_id: 用户ID
|
||||
repo_id: 仓库ID
|
||||
sync_status: 同步状态
|
||||
"""
|
||||
conn, cursor = get_db_connection()
|
||||
try:
|
||||
cursor.execute("""
|
||||
UPDATE datasource SET git_config = json_set(git_config, '$.latest_commit_id', ?, '$.last_sync_time', ?)
|
||||
WHERE user_id = ? AND id = ?
|
||||
""", (sync_status["latest_commit_id"], sync_status["last_sync_time"], user_id, repo_id))
|
||||
conn.commit()
|
||||
logger.info(f"更新Git同步状态: {repo_id}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def update_git_repo_config(user_id: str, repo_id: str, config: dict):
|
||||
"""
|
||||
更新Git仓库配置
|
||||
|
||||
Args:
|
||||
user_id: 用户ID
|
||||
repo_id: 仓库ID
|
||||
config: 配置信息
|
||||
"""
|
||||
conn, cursor = get_db_connection()
|
||||
try:
|
||||
# 获取当前配置
|
||||
cursor.execute("SELECT git_config FROM datasource WHERE user_id = ? AND id = ?", (user_id, repo_id))
|
||||
result = cursor.fetchone()
|
||||
if result:
|
||||
current_config = json.loads(result[0])
|
||||
# 更新配置
|
||||
current_config.update(config)
|
||||
cursor.execute("""
|
||||
UPDATE datasource SET git_config = ?
|
||||
WHERE user_id = ? AND id = ?
|
||||
""", (json.dumps(current_config), user_id, repo_id))
|
||||
conn.commit()
|
||||
logger.info(f"更新Git仓库配置: {repo_id}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def init_session_db():
|
||||
"""
|
||||
Initialize session database with users and sessions tables
|
||||
|
|
|
|||
|
|
@ -52,7 +52,10 @@ services:
|
|||
container_name: rag-api
|
||||
# 使用宿主机网络模式，可以直接访问宿主机上的服务（Ollama、MySQL 等）
|
||||
# 注意：使用 host 网络模式时，不能使用 ports 映射，容器直接使用宿主机的网络
|
||||
network_mode: host
|
||||
# network_mode: host # 注释/删除host网络模式（Windows下无效)
|
||||
# 添加端口映射（Windows下开发）
|
||||
ports:
|
||||
- "${API_PORT:-8001}:8001"
|
||||
# 自动读取 .env 文件（如果存在）
|
||||
env_file:
|
||||
- .env
|
||||
|
|
@ -96,7 +99,7 @@ services:
|
|||
|
||||
# RAG 配置
|
||||
- EMBEDDING_DIMENSION=${EMBEDDING_DIMENSION:-768}
|
||||
- CHUNK_SIZE=${CHUNK_SIZE:-1024}
|
||||
- CHUNK_SIZE=${CHUNK_SIZE:-4000}
|
||||
- CHUNK_OVERLAP=${CHUNK_OVERLAP:-200}
|
||||
- TOP_K=${TOP_K:-5}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ RUN apt-get update \
|
|||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
RUN pip install --no-cache-dir -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
|
||||
COPY docker/soffice/app.py /app/app.py
|
||||
COPY docker/soffice/README.md /app/README.md
|
||||
|
|
|
|||
|
|
@ -0,0 +1,828 @@
|
|||
# RAG智能问答助手——Git 代码库二次开发
|
||||
|
||||
# 新增Git代码库作为数据源的二次开发实现细节
|
||||
|
||||
本次二次开发核心是在原仓库**MySQL/达梦数据库/文件夹**三种数据源基础上，新增**Git代码库**数据源类型，需适配**代码拉取-解析切片-向量化存储-增量同步-代码专属检索/生成**全流程。以下结合原仓库的代码架构、文件结构，按**工程结构调整、核心模块开发、前后端适配、配置/部署修改、核心方法实现**五个维度，给出可落地的实现细节，完全复用原仓库的LlamaIndex/ChromaDB/Ollama基础能力，仅做针对性扩展。
|
||||
|
||||
## 一、原仓库核心架构复用与工程结构调整
|
||||
|
||||
原仓库已实现**通用同步基类、ChromaDB向量操作、FastAPI接口、前端配置管理**等基础能力，本次开发仅需**新增Git相关模块、扩展原有基类/接口、适配代码场景的解析/检索逻辑**，不改动原核心代码，保证兼容性。
|
||||
|
||||
### 1. 原仓库核心复用模块
|
||||
|
||||
|原仓库模块/文件|复用功能|扩展点|
|
||||
|---|---|---|
|
||||
|`sync/base_sync.py`|同步基类、通用向量化、ChromaDB基础写入|新增**代码库专属的抽象方法**（如`git_clone`/`detect_git_update`），让GitSync子类实现|
|
||||
|`config.py`|全局环境变量读取、配置管理|新增Git代码库相关的全局配置（本地存储根目录、默认分支等）|
|
||||
|`main.py`|FastAPI接口、流式响应、路由注册|新增Git数据源的配置路由、Git仓库手动同步路由|
|
||||
|`static/config/`|前端数据源配置界面|新增Git类型的配置表单（仓库地址、协议、SSH密钥等）|
|
||||
|`db_utils.py`|数据库工具、配置持久化|新增Git仓库同步配置的存储逻辑（最后同步commit ID、分支、轮询频率等）|
|
||||
|原ChromaDB操作逻辑|向量增删改查、embedding配对|重构**存储结构**，适配函数级代码的元数据/业务数据（如函数唯一ID、仓库名、分支等）|
|
||||
### 2. 新增/修改的文件结构
|
||||
|
||||
在原仓库基础上新增**Git同步、代码解析**专属模块，修改少量核心文件，新增文件如下（按目录分类）：
|
||||
|
||||
```Plain Text
|
||||
|
||||
# 核心同步模块新增
|
||||
sync/
|
||||
├── git_sync.py # Git代码库同步子类，继承BaseSync，实现代码拉取/增量同步/函数解析
|
||||
└── ast_parser.py # 代码AST解析工具类，实现跨语言函数级切片（核心）
|
||||
|
||||
# 前端配置界面新增（Git配置表单）
|
||||
static/config/
|
||||
├── js/git_config.js # Git配置的前端逻辑（凭证验证、仓库地址解析）
|
||||
└── components/
|
||||
└── git-form.html # Git数据源配置的HTML组件（嵌入原config/index.html）
|
||||
|
||||
# 工具类新增
|
||||
utils/
|
||||
├── git_tool.py # Git命令封装工具类（clone/fetch/merge/日志解析，封装subprocess执行Git命令）
|
||||
└── func_id_generator.py # 函数全局唯一ID生成工具类（按用户/仓库/分支/文件/函数生成）
|
||||
|
||||
# 原文件修改（仅扩展，不改动原有逻辑）
|
||||
sync/base_sync.py # 扩展基类，新增代码向量化专属方法
|
||||
config.py # 新增Git相关全局配置
|
||||
main.py # 新增Git数据源路由
|
||||
db_utils.py # 新增Git同步配置持久化
|
||||
Dockerfile # 安装git命令（容器内需要执行Git操作）
|
||||
.env.example # 新增Git相关环境变量配置项
|
||||
```
|
||||
|
||||
## 二、核心模块开发（按技术流程拆解）
|
||||
|
||||
按**代码拉取与存储→代码解析与向量化→增量同步→代码专属检索/生成**的技术流程，结合原仓库代码实现核心功能，每个环节均给出**原仓库对接点+代码实现思路**。
|
||||
|
||||
### 阶段1：代码拉取与存储（Git仓库专属）
|
||||
|
||||
核心实现**用户Git配置验证、多协议克隆、本地结构化存储**，封装为`git_tool.py`工具类，在`git_sync.py`中调用，复用原仓库的**数据源配置管理**能力。
|
||||
|
||||
#### 1. 原仓库对接点
|
||||
|
||||
- 前端配置界面提交的Git配置信息（仓库地址、协议、SSH私钥/HTTPS令牌、分支），通过原仓库的`/api/config/datasource`路由接收，新增`type: git`标识，与`mysql/folder`区分；
|
||||
|
||||
- 配置信息通过`db_utils.py`持久化到原仓库的配置库（SQLite/MySQL），新增`git_config`字段存储Git专属配置（commit ID、存储路径、轮询间隔等）。
|
||||
|
||||
#### 2. 核心实现细节
|
||||
|
||||
##### （1）Git工具类封装（`utils/git_tool.py`）
|
||||
|
||||
封装所有Git原生命令，避免硬编码，处理**SSH/HTTPS/git**多协议，实现**克隆、远程更新检测、增量拉取、文件变更解析**等核心功能，示例核心方法：
|
||||
|
||||
```Python
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
from config import settings # 原仓库的全局配置
|
||||
|
||||
class GitTool:
|
||||
def __init__(self, user_id: str, repo_id: str, git_config: dict):
|
||||
self.user_id = user_id
|
||||
self.repo_id = repo_id
|
||||
self.git_url = git_config["git_url"] # 用户配置的Git仓库地址
|
||||
self.branch = git_config.get("branch", settings.GIT_DEFAULT_BRANCH)
|
||||
self.ssh_key = git_config.get("ssh_key") # SSH私钥（base64加密存储）
|
||||
# 本地结构化存储路径（按文档规范：根目录/用户ID/仓库ID）
|
||||
self.local_repo_path = os.path.join(settings.GIT_LOCAL_STORAGE_ROOT, user_id, repo_id)
|
||||
self._init_git_env() # 初始化Git环境（SSH密钥配置）
|
||||
|
||||
# 初始化Git SSH环境（核心，解决容器内SSH密钥验证）
|
||||
def _init_git_env(self):
|
||||
if self.ssh_key:
|
||||
# 解密SSH私钥，写入临时文件，配置Git SSH
|
||||
os.environ["GIT_SSH_COMMAND"] = f"ssh -i /tmp/ssh_key_{self.user_id} -o StrictHostKeyChecking=no"
|
||||
with open(f"/tmp/ssh_key_{self.user_id}", "w") as f:
|
||||
f.write(self.ssh_key)
|
||||
os.chmod(f"/tmp/ssh_key_{self.user_id}", 0o600)
|
||||
|
||||
# 克隆Git仓库（适配多协议，复用文档的git clone命令）
|
||||
def clone_repo(self) -> bool:
|
||||
if not os.path.exists(self.local_repo_path):
|
||||
os.makedirs(os.path.dirname(self.local_repo_path), exist_ok=True)
|
||||
# 执行git clone --depth=none --single-branch --branch <分支> <地址> <本地路径>
|
||||
cmd = [
|
||||
"git", "clone", "--depth=none", "--single-branch",
|
||||
"--branch", self.branch, self.git_url, self.local_repo_path
|
||||
]
|
||||
res = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if res.returncode != 0:
|
||||
raise Exception(f"Git克隆失败: {res.stderr}")
|
||||
# 克隆后校验（git fsck + 语言检测）
|
||||
self._check_repo_integrity()
|
||||
return True
|
||||
return False
|
||||
|
||||
# 仓库完整性校验（git fsck）+ 支持的编程语言检测
|
||||
def _check_repo_integrity(self):
|
||||
# 执行git fsck
|
||||
subprocess.run(["git", "fsck"], cwd=self.local_repo_path, check=True)
|
||||
# 扫描文件类型，记录支持的编程语言（如.py/.java/.go），存入配置库
|
||||
from utils.lang_detect import detect_support_lang # 简单的文件后缀检测工具
|
||||
support_lang = detect_support_lang(self.local_repo_path)
|
||||
from db_utils import update_git_repo_config
|
||||
update_git_repo_config(self.user_id, self.repo_id, {"support_lang": support_lang})
|
||||
|
||||
# 远程更新检测（对比本地/远程commit ID，复用文档逻辑）
|
||||
def detect_remote_update(self) -> tuple[bool, str, str]:
|
||||
# 拉取远程commit记录（仅拉取，不拉取文件）
|
||||
subprocess.run(["git", "fetch", "origin", f"{self.branch}:{self.branch}"], cwd=self.local_repo_path, check=True)
|
||||
# 获取本地/远程commit ID
|
||||
local_commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=self.local_repo_path, capture_output=True, text=True).stdout.strip()
|
||||
remote_commit = subprocess.run(["git", "rev-parse", f"origin/{self.branch}"], cwd=self.local_repo_path, capture_output=True, text=True).stdout.strip()
|
||||
return local_commit != remote_commit, local_commit, remote_commit
|
||||
|
||||
# 增量拉取代码+解析文件变更（新增/修改/删除）
|
||||
def incremental_pull(self) -> dict:
|
||||
# 快进合并到远程最新版本
|
||||
subprocess.run(["git", "merge", "--ff-only", f"origin/{self.branch}"], cwd=self.local_repo_path, check=True)
|
||||
# 提取增量commit的文件变更
|
||||
delta_commits = subprocess.run(["git", "log", "--pretty=format:%H", f"{self.local_commit}..{self.remote_commit}"], cwd=self.local_repo_path, capture_output=True, text=True).stdout.split()
|
||||
# 解析文件变更为ADD/MODIFY/DELETE
|
||||
delta_files = self._parse_delta_files(delta_commits)
|
||||
return delta_files
|
||||
|
||||
# 解析文件变更集（复用文档的parse_delta_files.py逻辑）
|
||||
def _parse_delta_files(self, delta_commits: list) -> dict:
|
||||
add_files, modify_files, delete_files = [], [], []
|
||||
for commit in delta_commits:
|
||||
# git show --name-status 获取文件变更
|
||||
res = subprocess.run(["git", "show", "--name-status", commit], cwd=self.local_repo_path, capture_output=True, text=True).stdout
|
||||
for line in res.splitlines():
|
||||
if not line: continue
|
||||
status, file_path = line.split("\t", 1)
|
||||
file_path = os.path.join(self.local_repo_path, file_path)
|
||||
if status == "A": add_files.append(file_path)
|
||||
elif status == "M": modify_files.append(file_path)
|
||||
elif status == "D": delete_files.append(file_path)
|
||||
# 去重并返回
|
||||
return {
|
||||
"ADD": list(set(add_files)),
|
||||
"MODIFY": list(set(modify_files)),
|
||||
"DELETE": list(set(delete_files))
|
||||
}
|
||||
```
|
||||
|
||||
##### （2）Git配置持久化（修改`db_utils.py`）
|
||||
|
||||
原仓库已实现数据源配置的持久化，新增**Git仓库专属配置字段**，存储：
|
||||
|
||||
- 仓库基础配置：`git_url`/`protocol`/`branch`/`ssh_key`（AES256加密）/`https_token`（加密）；
|
||||
|
||||
- 同步状态配置：`local_repo_path`/`latest_commit_id`/`support_lang`/`poll_interval`（轮询间隔）/`last_sync_time`；
|
||||
|
||||
- 示例新增方法：
|
||||
|
||||
```Python
|
||||
|
||||
# 新增Git仓库配置存储
|
||||
def add_git_datasource(user_id: str, repo_config: dict):
|
||||
# 原仓库的配置表新增git_config字段，存储json格式的配置
|
||||
conn = get_sqlite_conn() # 原仓库的SQLite连接方法
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
INSERT INTO datasource (user_id, name, type, git_config, create_time)
|
||||
VALUES (?, ?, 'git', ?, datetime('now'))
|
||||
""", (user_id, repo_config["name"], json.dumps(repo_config)))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# 更新Git仓库同步状态（最后同步commit ID、时间）
|
||||
def update_git_sync_status(user_id: str, repo_id: str, sync_status: dict):
|
||||
conn = get_sqlite_conn()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
UPDATE datasource SET git_config = json_set(git_config, '$.latest_commit_id', ?, '$.last_sync_time', ?)
|
||||
WHERE user_id = ? AND id = ?
|
||||
""", (sync_status["latest_commit_id"], sync_status["last_sync_time"], user_id, repo_id))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
```
|
||||
|
||||
### 阶段2：代码解析与向量化（代码场景核心改造）
|
||||
|
||||
核心实现**函数级AST切片、LLM标准化生成函数描述、ChromaDB函数级存储**，是本次开发的**核心改造点**，需扩展原仓库的`base_sync.py`向量化逻辑，新增`ast_parser.py`和函数ID生成工具。
|
||||
|
||||
#### 1. 原仓库对接点
|
||||
|
||||
- 复用原仓库的**Ollama向量化能力**（`qwen3-embedding:8b`），仅修改向量化的**源数据**（从普通文本→LLM生成的函数描述）；
|
||||
|
||||
- 复用原仓库的ChromaDB基础操作（`add/delete/query`），**重构ChromaDB的存储结构**，适配函数级代码的元数据/业务数据；
|
||||
|
||||
- 继承`sync/base_sync.py`的`BaseSync`基类，实现`extract_data`（函数切片）、`vectorize_data`（函数描述向量化）、`save_to_chroma`（函数级存入）方法。
|
||||
|
||||
#### 2. 核心实现细节
|
||||
|
||||
##### （1）AST函数级切片（`sync/ast_parser.py`）
|
||||
|
||||
摒弃正则，采用**编程语言专属AST解析库**，实现跨语言函数提取，输出**标准化函数字典**（复用文档的格式），示例核心方法：
|
||||
|
||||
```Python
|
||||
|
||||
import ast
|
||||
import libcst # Python AST解析，支持代码修改
|
||||
from typing import List, Dict
|
||||
|
||||
class ASTParser:
|
||||
def __init__(self, file_path: str, lang: str):
|
||||
self.file_path = file_path
|
||||
self.lang = lang # 编程语言（python/java/go等）
|
||||
self.func_list: List[Dict] = [] # 提取的函数列表
|
||||
|
||||
# 统一入口：根据语言调用对应解析方法
|
||||
def parse_functions(self) -> List[Dict]:
|
||||
if not os.path.exists(self.file_path):
|
||||
raise Exception(f"文件不存在: {self.file_path}")
|
||||
with open(self.file_path, "r", encoding="utf-8") as f:
|
||||
self.code = f.read()
|
||||
# 按语言解析
|
||||
if self.lang == "python":
|
||||
self._parse_python()
|
||||
# 后续扩展java/go，此处先实现Python
|
||||
return self.func_list
|
||||
|
||||
# Python函数解析（基于ast+libcst）
|
||||
def _parse_python(self):
|
||||
try:
|
||||
tree = ast.parse(self.code)
|
||||
for node in ast.walk(tree):
|
||||
# 提取函数定义（普通函数/类方法）
|
||||
if isinstance(node, ast.FunctionDef) or isinstance(node, ast.AsyncFunctionDef):
|
||||
func_info = self._extract_python_func_info(node)
|
||||
self.func_list.append(func_info)
|
||||
except SyntaxError as e:
|
||||
raise Exception(f"Python代码语法错误: {e}")
|
||||
|
||||
# 提取Python函数的标准化信息
|
||||
def _extract_python_func_info(self, node) -> Dict:
|
||||
# 提取函数名、参数、返回值、函数体等
|
||||
func_name = node.name
|
||||
params = [arg.arg for arg in node.args.args] # 简化参数提取，可扩展类型注解
|
||||
return_type = ast.unparse(node.returns) if node.returns else "None"
|
||||
# 提取函数体代码
|
||||
func_body = libcst.parse_module(self.code).code_for_node(node)
|
||||
return {
|
||||
"file_path": self.file_path,
|
||||
"func_name": func_name,
|
||||
"params": params,
|
||||
"return_type": return_type,
|
||||
"func_body": func_body,
|
||||
"class_name": None # 类方法需额外解析，此处简化
|
||||
}
|
||||
```
|
||||
|
||||
##### （2）函数全局唯一ID生成（`utils/func_id_generator.py`）
|
||||
|
||||
为每个函数生成**全局唯一ID**（核心，用于增量同步时精准定位ChromaDB条目），复用文档的ID格式：`用户ID_仓库ID_分支_文件相对路径_类名_函数名`：
|
||||
|
||||
```Python
|
||||
|
||||
import os
|
||||
from config import settings
|
||||
|
||||
def generate_func_unique_id(user_id: str, repo_id: str, branch: str, file_path: str, class_name: str, func_name: str) -> str:
|
||||
# 将本地绝对路径转为仓库根目录的相对路径
|
||||
local_repo_root = os.path.join(settings.GIT_LOCAL_STORAGE_ROOT, user_id, repo_id)
|
||||
rel_file_path = os.path.relpath(file_path, local_repo_root).replace(os.sep, "_")
|
||||
# 类名为None则拼接空字符串
|
||||
class_name = class_name if class_name else "None"
|
||||
# 生成唯一ID
|
||||
unique_id = f"{user_id}_{repo_id}_{branch}_{rel_file_path}_{class_name}_{func_name}"
|
||||
# 替换特殊字符，避免ChromaDB主键冲突
|
||||
unique_id = unique_id.replace("/", "_").replace("\\", "_").replace(":", "_")
|
||||
return unique_id
|
||||
```
|
||||
|
||||
##### （3）LLM生成标准化函数描述（修改`sync/base_sync.py`）
|
||||
|
||||
复用原仓库的Ollama LLM调用能力（`qwen3:235b`），**新增标准化Prompt模板**（复用文档），为每个函数生成描述，示例方法：
|
||||
|
||||
```Python
|
||||
|
||||
# 在sync/base_sync.py的BaseSync类中新增方法
|
||||
def generate_func_desc(self, func_info: Dict) -> str:
|
||||
"""调用LLM生成标准化函数描述"""
|
||||
# 文档中的标准化Prompt模板
|
||||
prompt = f"""
|
||||
### 任务要求
|
||||
你是资深程序员,需要为给定的代码函数生成**简洁､准确､结构化的自然语言描述**,用于代码语义检索,严格遵循以下规则:
|
||||
1. 描述仅包含｢函数功能+入参作用+返回值意义｣,无额外冗余内容;
|
||||
2. 语言为中文,字数控制在50-100字;
|
||||
3. 若为类中的方法,需体现方法与类的关联;
|
||||
4. 不添加代码､注释､表情,仅纯自然语言描述｡
|
||||
### 待描述函数信息
|
||||
文件路径:{func_info['file_path']}
|
||||
所属类:{func_info['class_name']}
|
||||
函数名:{func_info['func_name']}
|
||||
参数:{func_info['params']}
|
||||
返回值类型:{func_info['return_type']}
|
||||
函数代码:
|
||||
{func_info['func_body']}
|
||||
### 输出示例
|
||||
示例1(全局函数):该函数为工具函数,接收两个整数类型的参数a和b,实现两数相加的功能,返回相加后的整数结果｡
|
||||
### 请输出你的描述
|
||||
""".strip()
|
||||
# 调用原仓库的Ollama LLM调用方法
|
||||
from utils.ollama_client import call_ollama # 原仓库的Ollama客户端
|
||||
desc = call_ollama(prompt, model=self.ollama_model)
|
||||
return desc.strip()
|
||||
```
|
||||
|
||||
##### （4）ChromaDB函数级存储（重构`base_sync.py`的`save_to_chroma`，贴合检索需求）
|
||||
|
||||
核心贴合你的需求：**按func_desc检索、返回对应func_body**，改造核心是明确「func_desc向量化生成embedding（检索核心）+ 业务数据关联存储（返回func_body依据）」，复用原仓库ChromaDB客户端，仅重构入参结构，确保检索时通过func_desc匹配，精准返回对应func_body，具体改造如下：
|
||||
|
||||
**核心改造原仓库的ChromaDB存储结构**，按文档要求设计**向量字段+元数据字段+业务数据字段**，复用原仓库的ChromaDB客户端，仅修改入参结构，示例方法：
|
||||
|
||||
```Python
|
||||
|
||||
# 重构sync/base_sync.py的save_to_chroma方法，完全贴合「按func_desc检索、返回func_body」需求
|
||||
def save_to_chroma(self, func_data_list: List[Dict], embeddings: List[List[float]]):
|
||||
"""
|
||||
核心设计：
|
||||
1. 检索核心：embeddings仅基于func_desc生成（与检索逻辑完全对齐）
|
||||
2. 关联存储：将func_body及关键信息存入metadatas（结构化存储，便于检索后直接提取）
|
||||
3. 检索匹配：documents仅存func_desc（确保检索时仅匹配函数描述，提升精准度）
|
||||
"""
|
||||
# 初始化ChromaDB客户端（复用原仓库配置，不做修改）
|
||||
import chromadb
|
||||
client = chromadb.HttpClient(host=settings.CHROMA_SERVER_HOST, port=settings.CHROMA_SERVER_PORT)
|
||||
# 按用户隔离集合（复用原仓库多用户隔离逻辑，避免数据冲突）
|
||||
collection = client.get_or_create_collection(name=f"code_rag_{self.user_id}")
|
||||
|
||||
# 构造ChromaDB入参（核心重构，贴合需求）
|
||||
# 1. 唯一ID：沿用函数全局唯一ID，用于精准定位和增量更新（复用原生成逻辑）
|
||||
ids = [func["func_unique_id"] for func in func_data_list]
|
||||
# 2. 元数据（核心关联存储）：存入func_body及关键信息，作为检索后返回func_body的直接依据
|
||||
metadatas = [
|
||||
{
|
||||
"user_id": self.user_id,
|
||||
"repo_id": self.repo_id,
|
||||
"branch": self.branch,
|
||||
"file_path": func["file_path"],
|
||||
"func_name": func["func_name"],
|
||||
"func_body": func["func_body"], # 关键：存储func_body，检索后直接提取返回
|
||||
"latest_commit_id": self.latest_commit_id
|
||||
} for func in func_data_list
|
||||
]
|
||||
# 3. 检索匹配字段：仅存func_desc（确保检索时，仅基于函数描述进行向量匹配，提升精准度）
|
||||
documents = [func["func_desc"] for func in func_data_list]
|
||||
# 4. 向量核心：embeddings仅基于func_desc生成（与documents完全对应，检索核心）
|
||||
# （注：embeddings由外部传入，对应vectorize_data方法中func_desc的向量化结果）
|
||||
|
||||
# 批量存入ChromaDB（复用原仓库批量操作逻辑，不做修改）
|
||||
collection.add(
|
||||
ids=ids,
|
||||
embeddings=embeddings,
|
||||
metadatas=metadatas,
|
||||
documents=documents
|
||||
)
|
||||
# 补充：检索逻辑对应调整（后续检索时，通过func_desc生成embedding查询，从metadatas提取func_body）
|
||||
# 此处提前预留检索逻辑适配说明，确保存储与检索闭环
|
||||
```
|
||||
|
||||
##### （5）GitSync子类实现（`sync/git_sync.py`）
|
||||
|
||||
继承原仓库的`BaseSync`基类，整合**Git拉取、AST切片、LLM描述、向量化、ChromaDB存储**全流程，实现基类的抽象方法：
|
||||
|
||||
```Python
|
||||
|
||||
from sync.base_sync import BaseSync
|
||||
from utils.git_tool import GitTool
|
||||
from sync.ast_parser import ASTParser
|
||||
from utils.func_id_generator import generate_func_unique_id
|
||||
from config import settings
|
||||
|
||||
class GitSync(BaseSync):
|
||||
def __init__(self, user_id: str, repo_id: str, git_config: dict):
|
||||
super().__init__(user_id)
|
||||
self.repo_id = repo_id
|
||||
self.git_config = git_config
|
||||
self.branch = git_config.get("branch", settings.GIT_DEFAULT_BRANCH)
|
||||
self.git_tool = GitTool(user_id, repo_id, git_config)
|
||||
self.latest_commit_id = git_config.get("latest_commit_id")
|
||||
|
||||
# 实现基类的extract_data：拉取代码+AST函数切片
|
||||
def extract_data(self) -> List[Dict]:
|
||||
# 1. 克隆/拉取代码
|
||||
self.git_tool.clone_repo()
|
||||
# 2. 获取仓库支持的编程语言
|
||||
support_lang = self.git_config.get("support_lang", ["python"])
|
||||
# 3. 遍历仓库文件，AST切片提取函数
|
||||
func_data_list = []
|
||||
for root, _, files in os.walk(self.git_tool.local_repo_path):
|
||||
for file in files:
|
||||
file_path = os.path.join(root, file)
|
||||
# 匹配支持的编程语言
|
||||
lang = self._get_file_lang(file_path)
|
||||
if lang not in support_lang:
|
||||
continue
|
||||
# 4. AST解析函数
|
||||
ast_parser = ASTParser(file_path, lang)
|
||||
func_list = ast_parser.parse_functions()
|
||||
# 5. 为每个函数生成唯一ID+LLM描述
|
||||
for func in func_list:
|
||||
func["func_unique_id"] = generate_func_unique_id(
|
||||
self.user_id, self.repo_id, self.branch,
|
||||
file_path, func["class_name"], func["func_name"]
|
||||
)
|
||||
func["func_desc"] = self.generate_func_desc(func) # 调用基类的LLM描述方法
|
||||
func["latest_commit_id"] = self.latest_commit_id
|
||||
func_data_list.append(func)
|
||||
return func_data_list
|
||||
|
||||
# 实现基类的vectorize_data：函数描述向量化（复用原仓库Ollama）
|
||||
def vectorize_data(self, func_data_list: List[Dict]) -> List[List[float]]:
|
||||
func_descs = [func["func_desc"] for func in func_data_list]
|
||||
# 调用原仓库的向量化方法（复用qwen3-embedding:8b）
|
||||
return self._ollama_embedding(func_descs)
|
||||
|
||||
# 实现基类的run_sync：整合全流程
|
||||
def run_sync(self):
|
||||
# 1. 提取函数数据
|
||||
func_data_list = self.extract_data()
|
||||
if not func_data_list:
|
||||
return "无函数数据可同步"
|
||||
# 2. 函数描述向量化
|
||||
embeddings = self.vectorize_data(func_data_list)
|
||||
# 3. 存入ChromaDB
|
||||
self.save_to_chroma(func_data_list, embeddings)
|
||||
# 4. 更新同步状态（最后commit ID）
|
||||
from db_utils import update_git_sync_status
|
||||
update_git_sync_status(self.user_id, self.repo_id, {
|
||||
"latest_commit_id": self.latest_commit_id,
|
||||
"last_sync_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
})
|
||||
return f"同步成功，共处理{len(func_data_list)}个函数"
|
||||
|
||||
# 辅助方法：根据文件后缀判断编程语言
|
||||
def _get_file_lang(self, file_path: str) -> str:
|
||||
suffix = os.path.splitext(file_path)[1].lower()
|
||||
lang_map = {".py": "python", ".java": "java", ".go": "go", ".js": "javascript"}
|
||||
return lang_map.get(suffix, "unknown")
|
||||
```
|
||||
|
||||
### 阶段3：代码增量同步（Git仓库核心亮点）
|
||||
|
||||
核心实现**定期轮询、Git增量拉取、函数级增删改识别、ChromaDB精准增量更新**，复用原仓库的**定时同步框架**（`sync_service.py`），在`git_sync.py`中新增增量同步方法，**全程避免全量解析/向量化**。
|
||||
|
||||
#### 1. 原仓库对接点
|
||||
|
||||
- 复用原仓库的**定时同步能力**（`SYNC_INTERVAL`/`AUTO_SYNC`），为Git数据源新增**自定义轮询间隔**（用户可配置）；
|
||||
|
||||
- 复用原仓库的**手动同步路由**，新增`/api/sync/git`路由，支持手动触发Git仓库增量同步；
|
||||
|
||||
- 基于ChromaDB的`delete`+`add`实现增量更新（原仓库已支持ChromaDB的增删操作）。
|
||||
|
||||
#### 2. 核心实现细节（在`git_sync.py`中新增增量同步方法）
|
||||
|
||||
ps. 「 2. 增量拉取代码，解析文件变更集（ADD/MODIFY/DELETE）」更细粒度的处理： 新增一个path_change_files，识别仅「路径变、内容不变」的文件，在增量更新时，仅更新元数据，复用原有向量和业务数据。
|
||||
|
||||
```Python
|
||||
|
||||
# 在GitSync类中新增增量同步方法
|
||||
def run_incremental_sync(self) -> str:
|
||||
"""Git仓库增量同步：检测更新→增量拉取→函数级变更→ChromaDB增量更新"""
|
||||
# 1. 检测远程更新
|
||||
has_update, local_commit, remote_commit = self.git_tool.detect_remote_update()
|
||||
if not has_update:
|
||||
return "无远程更新，无需同步"
|
||||
self.latest_commit_id = remote_commit # 更新为最新commit ID
|
||||
|
||||
# 2. 增量拉取代码，解析文件变更集（ADD/MODIFY/DELETE）
|
||||
delta_files = self.git_tool.incremental_pull(local_commit, remote_commit)
|
||||
add_files, modify_files, delete_files = delta_files["ADD"], delta_files["MODIFY"], delta_files["DELETE"]
|
||||
|
||||
# 3. 函数级增删改识别（核心）
|
||||
func_change = self._detect_func_change(add_files, modify_files, delete_files)
|
||||
add_funcs, modify_funcs, delete_func_ids = func_change["ADD"], func_change["MODIFY"], func_change["DELETE"]
|
||||
|
||||
# 4. ChromaDB增量更新（复用文档的先删后加逻辑）
|
||||
self._chroma_incremental_update(add_funcs, modify_funcs, delete_func_ids)
|
||||
|
||||
# 5. 更新同步状态
|
||||
from db_utils import update_git_sync_status
|
||||
update_git_sync_status(self.user_id, self.repo_id, {
|
||||
"latest_commit_id": remote_commit,
|
||||
"last_sync_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
})
|
||||
return f"增量同步成功：新增{len(add_funcs)}个函数，修改{len(modify_funcs)}个函数，删除{len(delete_func_ids)}个函数"
|
||||
|
||||
# 函数级增删改识别：文件变更→函数变更
|
||||
def _detect_func_change(self, add_files: list, modify_files: list, delete_files: list) -> dict:
|
||||
add_funcs, modify_funcs = [], []
|
||||
# 初始化ChromaDB客户端，获取当前仓库的函数数据
|
||||
client = chromadb.HttpClient(host=settings.CHROMA_SERVER_HOST, port=settings.CHROMA_SERVER_PORT)
|
||||
collection = client.get_collection(name=f"code_rag_{self.user_id}")
|
||||
# 过滤当前仓库的所有函数ID
|
||||
repo_funcs = collection.query(
|
||||
where={"$and": [{"user_id": self.user_id}, {"repo_id": self.repo_id}]},
|
||||
ids_only=True
|
||||
)
|
||||
repo_func_ids = set(repo_funcs["ids"])
|
||||
|
||||
# 处理新增/修改文件：重解析→对比函数ID
|
||||
all_change_files = add_files + modify_files
|
||||
support_lang = self.git_config.get("support_lang", ["python"])
|
||||
for file in all_change_files:
|
||||
lang = self._get_file_lang(file)
|
||||
if lang not in support_lang:
|
||||
continue
|
||||
# AST解析最新函数
|
||||
ast_parser = ASTParser(file, lang)
|
||||
latest_funcs = ast_parser.parse_functions()
|
||||
# 生成函数唯一ID
|
||||
for func in latest_funcs:
|
||||
func["func_unique_id"] = generate_func_unique_id(
|
||||
self.user_id, self.repo_id, self.branch,
|
||||
file, func["class_name"], func["func_name"]
|
||||
)
|
||||
func["func_desc"] = self.generate_func_desc(func)
|
||||
func["latest_commit_id"] = self.latest_commit_id
|
||||
# 新增函数：ID不在仓库函数ID中
|
||||
if func["func_unique_id"] not in repo_func_ids:
|
||||
add_funcs.append(func)
|
||||
# 修改函数：ID存在，代码不一致
|
||||
else:
|
||||
modify_funcs.append(func)
|
||||
|
||||
# 处理删除文件：过滤ChromaDB中该文件的所有函数ID
|
||||
delete_func_ids = []
|
||||
for file in delete_files:
|
||||
rel_file_path = os.path.relpath(file, self.git_tool.local_repo_path).replace(os.sep, "_")
|
||||
# 按文件路径过滤函数ID
|
||||
del_funcs = collection.query(
|
||||
where={"$and": [{"user_id": self.user_id}, {"repo_id": self.repo_id}, {"file_path": file}]},
|
||||
ids_only=True
|
||||
)
|
||||
delete_func_ids.extend(del_funcs["ids"])
|
||||
|
||||
return {"ADD": add_funcs, "MODIFY": modify_funcs, "DELETE": list(set(delete_func_ids))}
|
||||
|
||||
# ChromaDB增量更新：删→增（修改函数先删后加）
|
||||
def _chroma_incremental_update(self, add_funcs: list, modify_funcs: list, delete_func_ids: list):
|
||||
client = chromadb.HttpClient(host=settings.CHROMA_SERVER_HOST, port=settings.CHROMA_SERVER_PORT)
|
||||
collection = client.get_collection(name=f"code_rag_{self.user_id}")
|
||||
|
||||
# 1. 删除函数：批量删除
|
||||
if delete_func_ids:
|
||||
collection.delete(ids=delete_func_ids)
|
||||
|
||||
# 2. 新增函数：向量化+批量添加
|
||||
if add_funcs:
|
||||
embeddings = self.vectorize_data(add_funcs)
|
||||
self.save_to_chroma(add_funcs, embeddings)
|
||||
|
||||
# 3. 修改函数：先删后加（ChromaDB无更新API）
|
||||
if modify_funcs:
|
||||
# 删除旧版本
|
||||
old_func_ids = [func["func_unique_id"] for func in modify_funcs]
|
||||
collection.delete(ids=old_func_ids)
|
||||
# 添加新版本
|
||||
embeddings = self.vectorize_data(modify_funcs)
|
||||
self.save_to_chroma(modify_funcs, embeddings)
|
||||
```
|
||||
|
||||
#### 3. 定时同步整合（修改`sync_service.py`）
|
||||
|
||||
原仓库的`sync_service.py`实现了定时同步的核心逻辑，新增**Git数据源的同步调度**，在`sync_service.py`的main函数里启动所有的同步（`GitSync`/`MySQLSync`/`FolderSync`）：
|
||||
|
||||
### 阶段4：代码专属检索与生成（查询阶段改造）
|
||||
|
||||
核心实现**代码意图识别、查询优化、ChromaDB元数据过滤、代码专属Prompt生成**，修改原仓库的`main.py`中`/api/chat/stream`接口逻辑，复用原仓库的**流式响应**能力；同时确保检索函数（code_retrieve）与`git_sync.py`存储逻辑完全适配，形成「存储-检索」闭环。
|
||||
|
||||
#### 1. 原仓库对接点
|
||||
|
||||
- 复用原仓库的**Ollama生成能力**和**流式响应逻辑**，仅修改**检索逻辑**和**Prompt模板**；
|
||||
|
||||
- 复用原仓库的ChromaDB`query`方法，新增**元数据过滤条件**（user_id/repo_id/branch）；
|
||||
|
||||
- 前端聊天界面新增**代码仓库/分支选择器**，传递仓库/分支参数到后端。
|
||||
|
||||
#### 2. 核心实现细节（修改`main.py`的聊天接口）
|
||||
|
||||
```Python
|
||||
|
||||
# 新增：代码专属检索（ChromaDB元数据过滤+向量匹配）- 已适配git_sync.py存储逻辑
|
||||
def code_retrieve(user_id: str, repo_id: str, branch: str, query: str, top_k: int) -> list:
|
||||
"""ChromaDB检索代码函数
|
||||
核心逻辑：通过用户查询生成embedding，匹配存储的func_desc向量，从metadatas中提取func_body（贴合存储逻辑）
|
||||
适配性说明（与git_sync.py存储逻辑对应）：
|
||||
1. 向量匹配：与git_sync.py中vectorize_data方法一致，均调用BaseSync._ollama_embedding生成embedding，确保检索与存储的向量逻辑统一；
|
||||
2. 元数据过滤：where条件（user_id/repo_id/branch），与git_sync.py.save_to_chroma存入的metadatas字段完全对应，确保数据隔离精准；
|
||||
3. 数据提取：从metadatas提取func_body/file_path/func_name，均为git_sync.py中明确存入的字段，无字段缺失；
|
||||
4. 集合命名：collection命名（code_rag_{user_id}），与git_sync.py中存储时的集合命名规则完全一致，避免集合错乱。
|
||||
返回：包含func_body及溯源信息的列表，供后续生成回答使用
|
||||
"""
|
||||
import chromadb
|
||||
client = chromadb.HttpClient(host=settings.CHROMA_SERVER_HOST, port=settings.CHROMA_SERVER_PORT)
|
||||
collection = client.get_collection(name=f"code_rag_{self.user_id}")
|
||||
# 调用原仓库的向量化方法，生成查询embedding（与git_sync.py中func_desc向量化逻辑完全一致）
|
||||
from sync.base_sync import BaseSync
|
||||
embedding = BaseSync(user_id)._ollama_embedding([query])[0]
|
||||
# 元数据过滤：仅检索当前用户/仓库/分支，与git_sync.py存入的metadatas字段精准对应
|
||||
results = collection.query(
|
||||
query_embeddings=[embedding], # 基于用户查询生成的embedding，匹配git_sync.py存储的func_desc向量
|
||||
n_results=top_k,
|
||||
where={
|
||||
"$and": [
|
||||
{"user_id": user_id},
|
||||
{"repo_id": repo_id},
|
||||
{"branch": branch}
|
||||
]
|
||||
},
|
||||
include=["metadatas"] # 明确指定获取metadatas，对应git_sync.py中存储func_body的核心位置，无需额外获取documents
|
||||
)
|
||||
# 从metadatas中提取func_body，字段与git_sync.py存入的metadatas完全匹配，确保能精准提取
|
||||
retrieved_funcs = []
|
||||
for metadata in results["metadatas"][0]: # results["metadatas"]是二维列表，外层对应查询次数，内层对应top-k结果
|
||||
func_info = f"文件路径：{metadata['file_path']}
|
||||
函数名：{metadata['func_name']}
|
||||
函数代码：{metadata['func_body']}"
|
||||
retrieved_funcs.append(func_info)
|
||||
return retrieved_funcs # 返回提取的func_body列表，替代原有的documents列表，与git_sync.py存储逻辑闭环
|
||||
|
||||
# 适配性补充说明（与git_sync.py存储逻辑强关联）
|
||||
def retrieve_storage_compatibility_check() -> bool:
|
||||
"""校验code_retrieve与git_sync.py存储逻辑的适配性，可用于启动时自检"""
|
||||
# 1. 校验集合命名规则一致（确保检索与存储使用同一集合）
|
||||
from sync.git_sync import GitSync
|
||||
mock_sync = GitSync(user_id="test", repo_id="test_repo", git_config={})
|
||||
sync_collection_name = f"code_rag_{mock_sync.user_id}"
|
||||
retrieve_collection_name = f"code_rag_test"
|
||||
if sync_collection_name != retrieve_collection_name:
|
||||
raise Exception("适配异常：code_retrieve与git_sync.py的ChromaDB集合命名规则不一致")
|
||||
|
||||
# 2. 校验元数据字段一致（确保检索时提取的字段，均在git_sync.py中已存入）
|
||||
sync_metadata_fields = ["user_id", "repo_id", "branch", "file_path", "func_name", "func_body"]
|
||||
retrieve_extract_fields = ["file_path", "func_name", "func_body"]
|
||||
for field in retrieve_extract_fields:
|
||||
if field not in sync_metadata_fields:
|
||||
raise Exception(f"适配异常：code_retrieve提取的{field}字段，未在git_sync.py存储逻辑中定义")
|
||||
|
||||
# 3. 校验向量化逻辑一致（确保检索与存储的embedding生成方法统一）
|
||||
sync_embedding_logic = "基于func_desc调用BaseSync._ollama_embedding"
|
||||
retrieve_embedding_logic = "基于用户query调用BaseSync._ollama_embedding"
|
||||
if not sync_embedding_logic.split("调用")[1] == retrieve_embedding_logic.split("调用")[1]:
|
||||
raise Exception("适配异常：code_retrieve与git_sync.py的向量化方法不一致")
|
||||
|
||||
return True
|
||||
|
||||
# 新增：构建代码专属Prompt
|
||||
def build_code_prompt(optimized_query: str, retrieved_funcs: list) -> str:
|
||||
"""复用文档的代码回答Prompt模板，适配新的retrieved_funcs（func_body列表）"""
|
||||
retrieved_context = "\n".join([f"{i+1}. {func}" for i, func in enumerate(retrieved_funcs)])
|
||||
prompt = f"""
|
||||
### 角色
|
||||
你是资深程序员,负责解答用户关于指定代码仓库的技术问题,回答必须严格基于提供的代码上下文,不得编造代码/信息｡
|
||||
### 核心规则
|
||||
1. 回答需｢先给出核心结论,再补充详细解释｣,逻辑清晰;
|
||||
2. 若询问函数功能,需结合函数代码说明功能､参数作用､返回值意义;
|
||||
3. 若询问实现逻辑,需逐行/分模块解析代码的执行流程;
|
||||
4. 若询问使用方式,需给出具体的调用示例(基于函数参数);
|
||||
5. 若提供的代码中无相关答案,明确告知｢未检索到相关代码,无法解答｣,不做猜测;
|
||||
6. 代码相关的回答需附带｢所属文件路径+函数名｣,方便用户溯源｡
|
||||
### 检索到的相关代码（共{len(retrieved_funcs)}个，含完整函数体）
|
||||
{retrieved_context}
|
||||
### 用户问题
|
||||
{optimized_query}
|
||||
### 请输出你的回答
|
||||
""".strip()
|
||||
return prompt
|
||||
```
|
||||
|
||||
## 三、前后端适配（新增Git配置+代码聊天界面）
|
||||
|
||||
### 1. 后端接口扩展（修改`main.py`）
|
||||
|
||||
在原仓库的数据源配置路由中，**新增Git类型的配置支持**，无需新增独立路由，仅在入参中判断`type: git`，示例：
|
||||
|
||||
```Python
|
||||
|
||||
# 修改/api/config/datasource的POST接口
|
||||
@router.post("/config/datasource")
|
||||
async def add_datasource(ds_config: DataSourceConfig):
|
||||
if ds_config.type == "git":
|
||||
from db_utils import add_git_datasource
|
||||
add_git_datasource(ds_config.user_id, ds_config.git_config)
|
||||
return {"code": 200, "msg": "Git数据源配置成功"}
|
||||
elif ds_config.type == "mysql":
|
||||
# 原仓库的MySQL配置逻辑
|
||||
elif ds_config.type == "folder":
|
||||
# 原仓库的文件夹配置逻辑
|
||||
```
|
||||
|
||||
新增**Git仓库手动同步路由**：
|
||||
|
||||
```Python
|
||||
|
||||
# 新增Git手动同步路由
|
||||
@router.post("/sync/git")
|
||||
async def sync_git(user_id: str, repo_id: str):
|
||||
from db_utils import get_git_datasource
|
||||
git_config = get_git_datasource(user_id, repo_id)
|
||||
git_sync = GitSync(user_id, repo_id, git_config)
|
||||
res = git_sync.run_incremental_sync()
|
||||
return {"code": 200, "msg": res}
|
||||
```
|
||||
|
||||
### 2. 前端适配（修改`static/config/`和`static/chat/`）
|
||||
|
||||
#### （1）配置界面：新增Git配置表单
|
||||
|
||||
在`static/config/index.html`中嵌入`git-form.html`组件，实现**Git仓库地址、协议选择、SSH密钥/HTTPS令牌、分支、轮询间隔**的配置，新增：
|
||||
|
||||
- 协议选择（SSH/HTTPS/git），动态显示对应的凭证输入框（SSH私钥/HTTPS令牌）；
|
||||
|
||||
- **Git配置验证按钮**：调用`/api/config/verify/git`接口，验证仓库地址和凭证的有效性；
|
||||
|
||||
- 分支输入框，默认填充`main/master`。
|
||||
|
||||
#### （2）聊天界面：
|
||||
|
||||
无需特别改动。
|
||||
|
||||
暂时对代码问答和基于其他知识库的问答不做区分。先简单粗暴把相关代码搜出来即可，让模型判断是否在回答中用搜出来的代码进行增强。
|
||||
|
||||
- 知识库检索：按照现在处理，即只是后台无差别检索。
|
||||
|
||||
- 围绕知识库回答：按照现在处理，即“根据参考消息x”。
|
||||
|
||||
## 四、配置与部署修改
|
||||
|
||||
### 1. 环境变量配置（修改`.env.example`）
|
||||
|
||||
新增Git代码库相关的全局配置项，所有配置通过`config.py`读取，示例：
|
||||
|
||||
```TOML
|
||||
|
||||
# Git代码库配置
|
||||
GIT_LOCAL_STORAGE_ROOT=/opt/rag-code-repo # 本地Git仓库存储根目录
|
||||
GIT_DEFAULT_BRANCH=main # 默认克隆分支
|
||||
GIT_DEFAULT_POLL_INTERVAL=300 # 默认轮询间隔（秒）
|
||||
CODE_TOP_K=5 # 代码检索top-k值
|
||||
LIGHT_LLM_MODEL=qwen2:0.5b # 代码意图识别的轻量LLM模型
|
||||
# SSH密钥加密配置
|
||||
AES_KEY=xxxxxxxxxxxxxxxx # AES256加密密钥（用于加密SSH/HTTPS凭证）
|
||||
```
|
||||
|
||||
### 2. Docker部署修改（修改`Dockerfile`和`docker-compose.yml`）
|
||||
|
||||
原仓库的Docker容器内需要执行Git命令，因此**修改Dockerfile安装git**：
|
||||
|
||||
```Dockerfile
|
||||
|
||||
# 原仓库的Dockerfile新增
|
||||
RUN apt-get update && apt-get install -y git && apt-get clean
|
||||
# 新建SSH临时目录
|
||||
RUN mkdir -p /tmp/ssh && chmod 777 /tmp/ssh
|
||||
```
|
||||
|
||||
`docker-compose.yml`中**挂载Git本地存储目录**，实现数据持久化：
|
||||
|
||||
```YAML
|
||||
|
||||
# 新增卷挂载
|
||||
volumes:
|
||||
- ./chroma_db_data:/chroma_db_data
|
||||
- ./data:/data
|
||||
- ./rag-code-repo:/opt/rag-code-repo # Git仓库存储目录
|
||||
```
|
||||
|
||||
## 五、异常处理与性能优化（补充）
|
||||
|
||||
### 1. 核心异常处理（新增到`git_tool.py`和`git_sync.py`）
|
||||
|
||||
按文档要求，处理Git操作的常见异常，示例：
|
||||
|
||||
- **SSH密钥验证失败**：捕获`subprocess.CalledProcessError`，返回凭证无效提示；
|
||||
|
||||
- **Git强制推送**：检测到版本分叉时，执行`git fetch --force`，触发全量同步；
|
||||
|
||||
- **网络故障**：采用**指数退避重试**机制，重试3次失败后暂停轮询；
|
||||
|
||||
- **ChromaDB写入失败**：捕获ChromaDB的API异常，回滚操作，记录告警日志。
|
||||
|
||||
### 2. 性能优化（复用文档建议）
|
||||
|
||||
- **异步分批次解析**：大仓库首次解析时，按文件分片，借助Celery实现异步解析；
|
||||
|
||||
- **Redis缓存**：将高频检索的函数向量/代码缓存到Redis，减少ChromaDB查询压力；
|
||||
|
||||
- **解析白名单**：支持用户配置需要解析的目录（如`src/`），忽略`node_modules/`/`dist/`等无效目录；
|
||||
|
||||
- **LLM描述缓存**：函数代码未变更时，复用原有LLM描述，避免重复调用。
|
||||
|
||||
## 六、二次开发后整体流程验证
|
||||
|
||||
1. **前端配置**：用户新增Git数据源，填写仓库地址、SSH密钥、分支，点击验证并保存；
|
||||
|
||||
2. **首次同步**：系统自动克隆Git仓库，AST切片提取函数，LLM生成描述，向量化后存入ChromaDB；
|
||||
|
||||
3. **增量同步**：定时轮询远程Git仓库，检测到新commit后，增量拉取代码，识别函数级增删改，精准更新ChromaDB；
|
||||
|
||||
4. **代码查询**：用户在代码聊天界面提问，系统识别代码意图，优化查询后检索ChromaDB，基于代码生成精准回答并流式返回。
|
||||
|
||||
本次开发完全**复用原仓库的核心架构和能力**，仅做Git代码库的专属扩展，保证了代码的兼容性和可维护性，同时实现了文档中要求的**企业级代码RAG核心能力**。
|
||||
|
||||
|
||||
> （注：文档部分内容可能由 AI 生成）
|
||||
|
|
@ -0,0 +1,201 @@
|
|||
# 代码检索模块性能调优文档
|
||||
|
||||
## 1. 存储结构设计
|
||||
|
||||
### 1.1 核心设计原则
|
||||
```
|
||||
向量化文本: func_desc (函数描述，语义丰富)
|
||||
元数据存储: func_body (完整函数代码，供问答使用)
|
||||
```
|
||||
|
||||
### 1.2 存储格式
|
||||
```python
|
||||
Document(
|
||||
text=func_desc, # 用于 embedding 的文本
|
||||
metadata={
|
||||
"func_body": "...", # 完整函数代码（核心）
|
||||
"func_name": "...",
|
||||
"file_path": "...",
|
||||
"repo_id": "...",
|
||||
# ... 其他元数据
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
## 2. 检索性能优化
|
||||
|
||||
### 2.1 Top-K 参数调优
|
||||
|
||||
| 场景 | 推荐 top_k | 说明 |
|
||||
|------|-----------|------|
|
||||
| 精确查找特定函数 | 3-5 | 减少噪声，提高精确度 |
|
||||
| 探索性查询 | 10-15 | 获取更多上下文 |
|
||||
| 复杂问题 | 20+ | 需要多个函数协作回答 |
|
||||
|
||||
### 2.2 元数据过滤
|
||||
|
||||
**优势**：在向量检索前过滤，减少计算量
|
||||
|
||||
```python
|
||||
# 按仓库过滤
|
||||
results = code_retrieve(query, repo_id="git_https___gitee_com_xxx")
|
||||
|
||||
# 按函数名精确查找
|
||||
results = code_retrieve(query, func_name="push")
|
||||
```
|
||||
|
||||
**性能提升**：
|
||||
- 无过滤：全库扫描 O(N)
|
||||
- 有过滤：仅扫描子集 O(M)，M << N
|
||||
|
||||
### 2.3 Embedding 模型选择
|
||||
|
||||
| 模型 | 维度 | 速度 | 适用场景 |
|
||||
|------|------|------|----------|
|
||||
| nomic-embed-text | 768 | 快 | 通用代码检索 |
|
||||
| qwen3-embedding | 768 | 中等 | 中文代码理解 |
|
||||
| text-embedding-3 | 1536 | 慢 | 高精度需求 |
|
||||
|
||||
## 3. 检索质量优化
|
||||
|
||||
### 3.1 函数描述生成策略
|
||||
|
||||
当前实现：
|
||||
```python
|
||||
def generate_func_desc(func_info):
|
||||
parts = []
|
||||
# 1. 函数类型和名称
|
||||
if func_info.get("class_name"):
|
||||
parts.append(f"{func_info['class_name']}类的{func_info['func_name']}方法")
|
||||
else:
|
||||
parts.append(f"{func_info['func_name']}函数")
|
||||
|
||||
# 2. 参数信息
|
||||
if params:
|
||||
parts.append(f"接收参数: {', '.join(param_str)}")
|
||||
|
||||
# 3. 返回值
|
||||
if return_type:
|
||||
parts.append(f"返回类型: {return_type}")
|
||||
|
||||
# 4. 文档字符串
|
||||
if docstring:
|
||||
parts.append(f"功能描述: {docstring}")
|
||||
|
||||
return ". ".join(parts)
|
||||
```
|
||||
|
||||
**优化建议**：
|
||||
1. **添加函数调用关系**："调用 xx 函数"
|
||||
2. **添加代码复杂度**："包含 xx 行代码"
|
||||
3. **添加关键 API**："使用 requests 库"
|
||||
|
||||
### 3.2 相似度阈值
|
||||
|
||||
```python
|
||||
# 过滤低质量结果
|
||||
MIN_SIMILARITY_SCORE = 0.5
|
||||
|
||||
results = [r for r in results if r['score'] >= MIN_SIMILARITY_SCORE]
|
||||
```
|
||||
|
||||
### 3.3 结果重排序
|
||||
|
||||
```python
|
||||
def rerank_results(results, query):
|
||||
"""
|
||||
基于额外特征重排序：
|
||||
1. 函数名匹配度
|
||||
2. 文档字符串相关性
|
||||
3. 代码长度适中度
|
||||
"""
|
||||
for r in results:
|
||||
# 函数名完全匹配加分
|
||||
if query.lower() in r['func_name'].lower():
|
||||
r['score'] += 0.2
|
||||
|
||||
# 有文档字符串加分
|
||||
if r['docstring']:
|
||||
r['score'] += 0.1
|
||||
|
||||
return sorted(results, key=lambda x: x['score'], reverse=True)
|
||||
```
|
||||
|
||||
## 4. 增量同步优化
|
||||
|
||||
### 4.1 变更检测
|
||||
|
||||
```python
|
||||
def detect_changes(repo_path, last_sync_time):
|
||||
"""
|
||||
只检测变更的文件，避免全量扫描
|
||||
"""
|
||||
changed_files = []
|
||||
for file in get_tracked_files(repo_path):
|
||||
if get_file_mtime(file) > last_sync_time:
|
||||
changed_files.append(file)
|
||||
return changed_files
|
||||
```
|
||||
|
||||
### 4.2 增量更新策略
|
||||
|
||||
1. **新增函数**：直接添加
|
||||
2. **修改函数**：更新 embedding 和 metadata
|
||||
3. **删除函数**：从向量库删除
|
||||
|
||||
## 5. 性能监控指标
|
||||
|
||||
### 5.1 关键指标
|
||||
|
||||
| 指标 | 目标值 | 监控方式 |
|
||||
|------|--------|----------|
|
||||
| 检索延迟 | < 500ms | 日志记录 |
|
||||
| 准确率@5 | > 80% | 人工评估 |
|
||||
| 覆盖率 | > 95% | 自动化测试 |
|
||||
| 存储空间 | < 10GB | 系统监控 |
|
||||
|
||||
### 5.2 日志记录
|
||||
|
||||
```python
|
||||
logger.info(f"[CodeRetrieve] 查询: '{query}', 找到 {len(results)} 个结果, 耗时 {elapsed_time:.2f}s")
|
||||
```
|
||||
|
||||
## 6. 最佳实践
|
||||
|
||||
### 6.1 查询优化
|
||||
|
||||
**好的查询**：
|
||||
- "push 函数的功能是什么"
|
||||
- "如何处理 GitHub issue"
|
||||
- "webhook.py 中的 run 方法"
|
||||
|
||||
**差的查询**：
|
||||
- "代码"（太宽泛）
|
||||
- "问题"（无针对性）
|
||||
- "怎么写"（不明确）
|
||||
|
||||
### 6.2 索引维护
|
||||
|
||||
1. **定期重建索引**：每月一次
|
||||
2. **清理孤立文档**：每周检查
|
||||
3. **监控存储增长**：设置告警
|
||||
|
||||
## 7. 故障排查
|
||||
|
||||
### 7.1 检索不到结果
|
||||
|
||||
1. 检查向量库是否为空
|
||||
2. 检查 embedding 模型是否正常
|
||||
3. 检查查询文本是否有效
|
||||
|
||||
### 7.2 结果质量差
|
||||
|
||||
1. 检查 func_desc 生成质量
|
||||
2. 调整 top_k 参数
|
||||
3. 考虑更换 embedding 模型
|
||||
|
||||
### 7.3 性能下降
|
||||
|
||||
1. 检查 ChromaDB 连接
|
||||
2. 检查 embedding 服务负载
|
||||
3. 考虑增加缓存层
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
1. 没找到id。
|
||||
错因：字典用的字段错了。
|
||||
解法：debug找到纠正。
|
||||
2. metadata类型错误。
|
||||
self.collection.add(
|
||||
ids=valid_batch_ids,
|
||||
embeddings=valid_batch_embeddings,
|
||||
documents=valid_batch_texts,
|
||||
metadatas=valid_batch_metadatas
|
||||
)
|
||||
Traceback (most recent call last):
|
||||
File "<string>", line 1, in <module>
|
||||
File "s:\research\RAG\.venv\Lib\site-packages\chromadb\api\models\Collection.py", line 106, in add
|
||||
self._client._add(
|
||||
~~~~~~~~~~~~~~~~~^
|
||||
collection_id=self.id,
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
...<6 lines>...
|
||||
database=self.database,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
)
|
||||
^
|
||||
File "s:\research\RAG\.venv\Lib\site-packages\chromadb\api\rust.py", line 452, in _add
|
||||
return self.bindings.add(
|
||||
~~~~~~~~~~~~~~~~~^
|
||||
ids,
|
||||
^^^^
|
||||
...<6 lines>...
|
||||
database,
|
||||
^^^^^^^^^
|
||||
)
|
||||
^
|
||||
TypeError: argument 'metadatas': Cannot convert Python object to MetadataValue
|
||||
错因：是因为valid_batch_metadatas 中包含了 None 值，而 ChromaDB 不允许 None 值作为元数据。
|
||||
解法：将 None 值设置为 "None" 字符串。
|
||||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit c22e97f255c92b61ad6014ef74ec662d9f5f6325
|
||||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 18ae6d9bcefae1a1b2d6d98ef002cdb9e823e834
|
||||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 31c3f4bc08e848cb34760d3cdf9065d385ecca14
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -174,7 +174,7 @@ class RAGEngine:
|
|||
except:
|
||||
pass
|
||||
|
||||
async def query_stream(self, query: str, history: str, top_k: Optional[int] = None) -> AsyncIterator[str]:
|
||||
async def query_stream(self, query: str, history: str, top_k: Optional[int] = None, filters: Optional[dict] = None) -> AsyncIterator[str]:
|
||||
"""
|
||||
Query the RAG system and stream the response
|
||||
|
||||
|
|
@ -182,20 +182,25 @@ class RAGEngine:
|
|||
query: User query string
|
||||
history: Chat history string
|
||||
top_k: Number of documents to retrieve (optional)
|
||||
filters: Metadata filters for pre-filtering documents (optional)
|
||||
|
||||
Yields:
|
||||
Response text chunks
|
||||
"""
|
||||
try:
|
||||
# Create query engine with streaming mode
|
||||
retriever = self.vector_store_manager.get_retriever(top_k=top_k or settings.TOP_K)
|
||||
retriever = self.vector_store_manager.get_retriever(top_k=top_k or settings.TOP_K, filters=filters)
|
||||
# query index
|
||||
retrieved_nodes = await retriever.aretrieve(query)
|
||||
|
||||
# 2. 构建上下文
|
||||
context_parts = []
|
||||
for i, node in enumerate(retrieved_nodes[:top_k or settings.TOP_K], 1): # 限制数量
|
||||
text = node.text if hasattr(node, 'text') else str(node)
|
||||
# 优先使用metadata中的func_body字段
|
||||
if hasattr(node, 'metadata') and 'func_body' in node.metadata:
|
||||
text = node.metadata['func_body']
|
||||
else:
|
||||
text = node.text if hasattr(node, 'text') else str(node)
|
||||
# 清理和截断
|
||||
text = text.strip()
|
||||
if len(text) > 400:
|
||||
|
|
@ -233,7 +238,7 @@ class RAGEngine:
|
|||
except Exception as e:
|
||||
logger.error(f"Error in RAG query: {e}")
|
||||
|
||||
async def query(self, query: str, history: str, top_k: Optional[int] = None) -> str:
|
||||
async def query(self, query: str, history: str, top_k: Optional[int] = None, filters: Optional[dict] = None) -> str:
|
||||
"""
|
||||
Query the RAG system and return complete response
|
||||
|
||||
|
|
@ -241,20 +246,25 @@ class RAGEngine:
|
|||
query: User query string
|
||||
history: Chat history string
|
||||
top_k: Number of documents to retrieve (optional)
|
||||
filters: Metadata filters for pre-filtering documents (optional)
|
||||
|
||||
Returns:
|
||||
Complete response string
|
||||
"""
|
||||
try:
|
||||
# Create query engine with streaming mode
|
||||
retriever = self.vector_store_manager.get_retriever(top_k=top_k or settings.TOP_K)
|
||||
retriever = self.vector_store_manager.get_retriever(top_k=top_k or settings.TOP_K, filters=filters)
|
||||
# query index
|
||||
retrieved_nodes = await retriever.aretrieve(query)
|
||||
|
||||
# 2. 构建上下文
|
||||
context_parts = []
|
||||
for i, node in enumerate(retrieved_nodes[:top_k or settings.TOP_K], 1): # 限制数量
|
||||
text = node.text if hasattr(node, 'text') else str(node)
|
||||
# 优先使用metadata中的func_body字段
|
||||
if hasattr(node, 'metadata') and 'func_body' in node.metadata:
|
||||
text = node.metadata['func_body']
|
||||
else:
|
||||
text = node.text if hasattr(node, 'text') else str(node)
|
||||
# 清理和截断
|
||||
text = text.strip()
|
||||
if len(text) > 400:
|
||||
|
|
|
|||
|
|
@ -695,10 +695,17 @@ class VectorStoreManager:
|
|||
logger.error(f"写入剩余文档到 ChromaDB 失败: {write_error}")
|
||||
|
||||
total_elapsed = time.time() - start_time
|
||||
logger.info(
|
||||
f"[Embedding进度] ✓ 单个生成完成: 成功写入 {total_added}/{total_count} 个文档 "
|
||||
f"(耗时: {total_elapsed:.1f}秒, 平均: {total_elapsed/total_added*1000:.1f}ms/个, 跳过: {skipped_count}个)"
|
||||
)
|
||||
if total_added > 0:
|
||||
avg_time = total_elapsed/total_added*1000
|
||||
logger.info(
|
||||
f"[Embedding进度] ✓ 单个生成完成: 成功写入 {total_added}/{total_count} 个文档 "
|
||||
f"(耗时: {total_elapsed:.1f}秒, 平均: {avg_time:.1f}ms/个, 跳过: {skipped_count}个)"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"[Embedding进度] ✓ 单个生成完成: 成功写入 {total_added}/{total_count} 个文档 "
|
||||
f"(耗时: {total_elapsed:.1f}秒, 跳过: {skipped_count}个)"
|
||||
)
|
||||
|
||||
if total_added == 0:
|
||||
raise ValueError(f"Failed to add any valid documents to ChromaDB: {e}")
|
||||
|
|
@ -815,12 +822,64 @@ class VectorStoreManager:
|
|||
logger.warning(f"Error checking document count: {e}")
|
||||
return False
|
||||
|
||||
def get_retriever(self, top_k: int = None):
|
||||
def extract_filters_from_query(self, query: str) -> dict:
|
||||
"""
|
||||
从查询中提取元数据筛选条件
|
||||
|
||||
Args:
|
||||
query: 用户查询字符串
|
||||
|
||||
Returns:
|
||||
提取的元数据筛选条件字典
|
||||
"""
|
||||
import re
|
||||
filters = {}
|
||||
|
||||
# 提取文件路径筛选条件
|
||||
# 模式1：直接的文件路径（以常见代码文件扩展名结尾）
|
||||
file_path_pattern = r'\b([\w\-/\\]+\.(?:py|java|go|js|ts|jsx|tsx|c|cpp|h|hpp|cs|rs|php|rb|swift|kt|scala))\b'
|
||||
file_path_matches = re.findall(file_path_pattern, query)
|
||||
if file_path_matches:
|
||||
filters['file_path'] = file_path_matches[0]
|
||||
|
||||
# 提取函数名筛选条件
|
||||
# 模式："函数名" + "函数" 或 "方法"
|
||||
func_name_pattern = r'(\w+)\s*(?:函数|方法)'
|
||||
func_name_matches = re.findall(func_name_pattern, query)
|
||||
if func_name_matches:
|
||||
filters['func_name'] = func_name_matches[0]
|
||||
|
||||
# 提取类名筛选条件
|
||||
# 模式："类" + "类名"
|
||||
class_name_pattern = r'类[:：]\s*(\w+)'
|
||||
class_name_matches = re.findall(class_name_pattern, query)
|
||||
if class_name_matches:
|
||||
filters['class_name'] = class_name_matches[0]
|
||||
|
||||
# 提取仓库ID筛选条件
|
||||
# 模式："仓库" + "名称"
|
||||
repo_id_pattern = r'仓库[:：]\s*(\w+)'
|
||||
repo_id_matches = re.findall(repo_id_pattern, query)
|
||||
if repo_id_matches:
|
||||
filters['repo_id'] = repo_id_matches[0]
|
||||
|
||||
# 提取语言筛选条件
|
||||
# 模式："语言" + "语言名称"
|
||||
lang_pattern = r'语言[:：]\s*(\w+)'
|
||||
lang_matches = re.findall(lang_pattern, query)
|
||||
if lang_matches:
|
||||
filters['lang'] = lang_matches[0]
|
||||
|
||||
return filters
|
||||
|
||||
def get_retriever(self, top_k: int = None, filters: dict = None, query: str = None):
|
||||
"""
|
||||
Get a retriever for querying the vector store
|
||||
|
||||
Args:
|
||||
top_k: Number of documents to retrieve (defaults to settings.TOP_K)
|
||||
filters: Metadata filters for retrieval, e.g., {"repo_id": "test_repo", "lang": "python"}
|
||||
query: User query string to extract filters from
|
||||
|
||||
Returns:
|
||||
VectorStoreRetriever instance
|
||||
|
|
@ -834,7 +893,25 @@ class VectorStoreManager:
|
|||
if top_k is None:
|
||||
top_k = settings.TOP_K
|
||||
|
||||
retriever = self.index.as_retriever(similarity_top_k=top_k)
|
||||
# 从查询中提取筛选条件
|
||||
extracted_filters = {}
|
||||
if query:
|
||||
extracted_filters = self.extract_filters_from_query(query)
|
||||
if extracted_filters:
|
||||
print(f"从查询中提取的筛选条件: {extracted_filters}")
|
||||
|
||||
# 合并筛选条件
|
||||
final_filters = {}
|
||||
if filters:
|
||||
final_filters.update(filters)
|
||||
if extracted_filters:
|
||||
final_filters.update(extracted_filters)
|
||||
|
||||
# Create retriever with optional filters
|
||||
retriever = self.index.as_retriever(
|
||||
similarity_top_k=top_k,
|
||||
filter=final_filters if final_filters else None
|
||||
)
|
||||
return retriever
|
||||
|
||||
def reset(self):
|
||||
|
|
@ -877,3 +954,61 @@ class VectorStoreManager:
|
|||
except Exception as e:
|
||||
logger.error(f"获取db_source({target_db_source}) metadata中content_column失败: {e}")
|
||||
return ''
|
||||
|
||||
def list_documents(self, limit: int = 100, filters: dict = None) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
列出向量库中的文档
|
||||
|
||||
Args:
|
||||
limit: 限制返回的文档数量
|
||||
filters: 元数据过滤条件，如 {"repo_id": "test_repo", "lang": "python"}
|
||||
|
||||
Returns:
|
||||
文档列表，每个文档包含id、文本和元数据
|
||||
"""
|
||||
try:
|
||||
# 获取文档
|
||||
results = self.collection.get(
|
||||
where=filters,
|
||||
include=['documents', 'metadatas'],
|
||||
limit=limit
|
||||
)
|
||||
|
||||
documents = []
|
||||
ids = results.get('ids', [])
|
||||
texts = results.get('documents', [])
|
||||
metadatas = results.get('metadatas', [])
|
||||
|
||||
for i, doc_id in enumerate(ids):
|
||||
doc = {
|
||||
'id': doc_id,
|
||||
'text': texts[i] if i < len(texts) else '',
|
||||
'metadata': metadatas[i] if i < len(metadatas) else {}
|
||||
}
|
||||
documents.append(doc)
|
||||
|
||||
logger.info(f"列出了 {len(documents)} 个文档")
|
||||
return documents
|
||||
except Exception as e:
|
||||
logger.error(f"列出文档失败: {e}")
|
||||
return []
|
||||
|
||||
def get_collection_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取向量库统计信息
|
||||
|
||||
Returns:
|
||||
统计信息，包括文档数量、向量维度等
|
||||
"""
|
||||
try:
|
||||
count = self.collection.count()
|
||||
stats = {
|
||||
'document_count': count,
|
||||
'collection_name': settings.CHROMA_COLLECTION_NAME,
|
||||
'chroma_db_path': settings.CHROMA_DB_PATH if not settings.CHROMA_SERVER_HOST else f"{settings.CHROMA_SERVER_HOST}:{settings.CHROMA_SERVER_PORT}"
|
||||
}
|
||||
logger.info(f"向量库统计: {stats}")
|
||||
return stats
|
||||
except Exception as e:
|
||||
logger.error(f"获取统计信息失败: {e}")
|
||||
return {}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
<option value="">选择类型</option>
|
||||
<option value="database">数据库</option>
|
||||
<option value="folder">文件夹</option>
|
||||
<option value="git">Git代码库</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -75,6 +76,7 @@
|
|||
<select id="configType" name="type" required>
|
||||
<option value="database">数据库 (database)</option>
|
||||
<option value="folder">文件夹 (folder)</option>
|
||||
<option value="git">Git代码库 (git)</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-group" id="dbTypeGroup" style="display: none;">
|
||||
|
|
|
|||
|
|
@ -224,6 +224,7 @@ function generateConfigForm(config) {
|
|||
<select id="formType" disabled>
|
||||
<option value="database" ${config.type === 'database' ? 'selected' : ''}>数据库 (database)</option>
|
||||
<option value="folder" ${config.type === 'folder' ? 'selected' : ''}>文件夹 (folder)</option>
|
||||
<option value="git" ${config.type === 'git' ? 'selected' : ''}>Git代码库 (git)</option>
|
||||
</select>
|
||||
</div>
|
||||
`;
|
||||
|
|
@ -285,6 +286,48 @@ function generateConfigForm(config) {
|
|||
bindFolderEvents();
|
||||
}
|
||||
|
||||
// Git代码库配置
|
||||
if (config.type === 'git') {
|
||||
const gitSection = document.createElement('div');
|
||||
gitSection.innerHTML = `
|
||||
<h3 class="section-title">Git代码库配置</h3>
|
||||
<div class="form-group">
|
||||
<label for="formGitUrl">Git仓库地址 <span class="required">*</span></label>
|
||||
<input type="text" id="formGitUrl" value="${config.git_url || ''}" required>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="formBranch">分支名称</label>
|
||||
<input type="text" id="formBranch" value="${config.branch || 'main'}">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="formProtocol">协议类型</label>
|
||||
<select id="formProtocol">
|
||||
<option value="https" ${config.protocol === 'https' ? 'selected' : ''}>HTTPS</option>
|
||||
<option value="ssh" ${config.protocol === 'ssh' ? 'selected' : ''}>SSH</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-group" id="httpsTokenGroup">
|
||||
<label for="formHttpsToken">HTTPS令牌</label>
|
||||
<input type="password" id="formHttpsToken" value="${config.https_token || ''}">
|
||||
</div>
|
||||
<div class="form-group" id="sshKeyGroup" style="display: none;">
|
||||
<label for="formSshKey">SSH私钥</label>
|
||||
<textarea id="formSshKey" rows="10" value="${config.ssh_key || ''}">${config.ssh_key || ''}</textarea>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="formPollInterval">轮询间隔（秒）</label>
|
||||
<input type="number" id="formPollInterval" value="${config.poll_interval || 300}" min="60" max="3600">
|
||||
</div>
|
||||
<div class="form-actions">
|
||||
<button type="button" id="testGitConnectionBtn" class="btn secondary" style="margin-top: 10px;">🔌 测试Git连接</button>
|
||||
</div>
|
||||
`;
|
||||
formElement.appendChild(gitSection);
|
||||
|
||||
// 绑定事件
|
||||
bindGitEvents();
|
||||
}
|
||||
|
||||
if (config.type === 'database') {
|
||||
// 数据库连接配置（放在前面，方便先测试连接）
|
||||
const connectionSection = document.createElement('div');
|
||||
|
|
@ -1085,6 +1128,16 @@ async function handleAddConfigDirectly() {
|
|||
username: '',
|
||||
password: ''
|
||||
};
|
||||
} else if (configType === 'git') {
|
||||
tempConfig = {
|
||||
type: configType,
|
||||
git_url: '',
|
||||
branch: 'main',
|
||||
protocol: 'https',
|
||||
https_token: '',
|
||||
ssh_key: '',
|
||||
poll_interval: 300
|
||||
};
|
||||
} else {
|
||||
alert('不支持的配置类型');
|
||||
return;
|
||||
|
|
@ -1217,12 +1270,17 @@ async function saveConfig() {
|
|||
// 文件夹：folder_主机_文件夹路径（替换特殊字符）
|
||||
const folderName = formData.folder_path ? formData.folder_path.replace(/[\\/:*?"<>|]/g, '_') : 'unknown';
|
||||
generatedName = `folder_${formData.host || 'unknown'}_${folderName}`;
|
||||
} else if (formData.type === 'git') {
|
||||
// Git配置：git_仓库地址（替换特殊字符）
|
||||
const repoName = formData.git_url ? formData.git_url.replace(/[\\/:*?"<>|]/g, '_') : 'unknown';
|
||||
generatedName = `git_${repoName}_${formData.branch || 'main'}`;
|
||||
} else {
|
||||
// 不支持的配置类型
|
||||
alert('不支持的配置类型');
|
||||
return;
|
||||
}
|
||||
|
||||
formData.name = generatedName;
|
||||
}
|
||||
|
||||
// 根据不同类型检查特定字段
|
||||
|
|
@ -1243,6 +1301,10 @@ async function saveConfig() {
|
|||
if (!formData.port) missingFields.push('端口');
|
||||
if (!formData.username) missingFields.push('用户名');
|
||||
if (!formData.password) missingFields.push('密码');
|
||||
} else if (formData.type === 'git') {
|
||||
if (!formData.git_url) missingFields.push('Git仓库地址');
|
||||
if (!formData.branch) missingFields.push('分支名称');
|
||||
if (!formData.protocol) missingFields.push('协议类型');
|
||||
}
|
||||
|
||||
// 如果有缺失的字段，提示用户
|
||||
|
|
@ -1417,6 +1479,16 @@ function collectFormData() {
|
|||
|
||||
}
|
||||
|
||||
// Git配置
|
||||
if (formData.type === 'git') {
|
||||
formData.git_url = document.getElementById('formGitUrl').value;
|
||||
formData.branch = document.getElementById('formBranch').value;
|
||||
formData.protocol = document.getElementById('formProtocol').value;
|
||||
formData.https_token = document.getElementById('formHttpsToken').value;
|
||||
formData.ssh_key = document.getElementById('formSshKey').value;
|
||||
formData.poll_interval = parseInt(document.getElementById('formPollInterval').value);
|
||||
}
|
||||
|
||||
return formData;
|
||||
}
|
||||
|
||||
|
|
@ -1468,6 +1540,81 @@ async function confirmDeleteConfig() {
|
|||
}
|
||||
}
|
||||
|
||||
// Git事件绑定函数
|
||||
function bindGitEvents() {
|
||||
// 协议切换逻辑
|
||||
const protocolSelect = document.getElementById('formProtocol');
|
||||
const httpsTokenGroup = document.getElementById('httpsTokenGroup');
|
||||
const sshKeyGroup = document.getElementById('sshKeyGroup');
|
||||
|
||||
protocolSelect.addEventListener('change', function() {
|
||||
const protocol = this.value;
|
||||
if (protocol === 'https') {
|
||||
httpsTokenGroup.style.display = 'block';
|
||||
sshKeyGroup.style.display = 'none';
|
||||
} else if (protocol === 'ssh') {
|
||||
httpsTokenGroup.style.display = 'none';
|
||||
sshKeyGroup.style.display = 'block';
|
||||
}
|
||||
});
|
||||
|
||||
// 触发一次change事件，确保初始状态正确
|
||||
protocolSelect.dispatchEvent(new Event('change'));
|
||||
|
||||
// 测试Git连接
|
||||
document.getElementById('testGitConnectionBtn')?.addEventListener('click', async () => {
|
||||
try {
|
||||
const gitUrl = document.getElementById('formGitUrl').value;
|
||||
const branch = document.getElementById('formBranch').value;
|
||||
const protocol = document.getElementById('formProtocol').value;
|
||||
const httpsToken = document.getElementById('formHttpsToken').value;
|
||||
const sshKey = document.getElementById('formSshKey').value;
|
||||
|
||||
if (!gitUrl) {
|
||||
alert('请填写Git仓库地址');
|
||||
return;
|
||||
}
|
||||
|
||||
// 禁用按钮
|
||||
const btn = document.getElementById('testGitConnectionBtn');
|
||||
const originalText = btn.textContent;
|
||||
btn.textContent = '🔌 测试中...';
|
||||
btn.disabled = true;
|
||||
|
||||
// 发送请求
|
||||
const response = await fetch('/git/test-connection', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
git_url: gitUrl,
|
||||
branch: branch,
|
||||
protocol: protocol,
|
||||
https_token: httpsToken,
|
||||
ssh_key: sshKey
|
||||
})
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const result = await response.json();
|
||||
alert('Git连接测试成功！');
|
||||
} else {
|
||||
const errorData = await response.json();
|
||||
throw new Error(errorData.detail || '连接失败');
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
alert('Git连接测试失败: ' + error.message);
|
||||
} finally {
|
||||
// 恢复按钮
|
||||
const btn = document.getElementById('testGitConnectionBtn');
|
||||
btn.textContent = '🔌 测试Git连接';
|
||||
btn.disabled = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// 为文件夹配置添加事件绑定
|
||||
function bindFolderEvents() {
|
||||
// 测试SSH连接
|
||||
|
|
|
|||
|
|
@ -0,0 +1,235 @@
|
|||
"""
|
||||
代码AST解析工具类
|
||||
实现跨语言函数级切片
|
||||
"""
|
||||
import ast
|
||||
import os
|
||||
from typing import List, Dict, Optional
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class ASTParser:
|
||||
def __init__(self, file_path: str, lang: str):
|
||||
"""
|
||||
初始化AST解析器
|
||||
|
||||
Args:
|
||||
file_path: 文件路径
|
||||
lang: 编程语言
|
||||
"""
|
||||
self.file_path = file_path
|
||||
self.lang = lang
|
||||
self.func_list: List[Dict] = [] # 提取的函数列表
|
||||
|
||||
def parse_functions(self) -> List[Dict]:
|
||||
"""
|
||||
统一入口：根据语言调用对应解析方法
|
||||
|
||||
Returns:
|
||||
List[Dict]: 函数信息列表
|
||||
"""
|
||||
if not os.path.exists(self.file_path):
|
||||
raise Exception(f"文件不存在: {self.file_path}")
|
||||
|
||||
# 读取文件内容
|
||||
try:
|
||||
with open(self.file_path, "r", encoding="utf-8") as f:
|
||||
self.code = f.read()
|
||||
except Exception as e:
|
||||
logger.error(f"读取文件失败: {e}")
|
||||
raise
|
||||
|
||||
# 按语言解析
|
||||
if self.lang == "python":
|
||||
self._parse_python()
|
||||
elif self.lang == "java":
|
||||
self._parse_java()
|
||||
elif self.lang == "go":
|
||||
self._parse_go()
|
||||
elif self.lang == "javascript" or self.lang == "typescript":
|
||||
self._parse_javascript()
|
||||
else:
|
||||
logger.warning(f"暂不支持的编程语言: {self.lang}")
|
||||
|
||||
logger.info(f"解析文件 {self.file_path}，提取到 {len(self.func_list)} 个函数")
|
||||
return self.func_list
|
||||
|
||||
def _parse_python(self):
|
||||
"""
|
||||
解析Python代码
|
||||
"""
|
||||
try:
|
||||
tree = ast.parse(self.code)
|
||||
for node in ast.walk(tree):
|
||||
# 提取函数定义（普通函数/类方法/异步函数）
|
||||
if isinstance(node, ast.FunctionDef) or isinstance(node, ast.AsyncFunctionDef):
|
||||
func_info = self._extract_python_func_info(node)
|
||||
self.func_list.append(func_info)
|
||||
except SyntaxError as e:
|
||||
logger.error(f"Python代码语法错误: {e}")
|
||||
raise Exception(f"Python代码语法错误: {e}")
|
||||
|
||||
def _extract_python_func_info(self, node) -> Dict:
|
||||
"""
|
||||
提取Python函数的标准化信息
|
||||
|
||||
Args:
|
||||
node: AST节点
|
||||
|
||||
Returns:
|
||||
Dict: 函数信息
|
||||
"""
|
||||
# 提取函数名
|
||||
func_name = node.name
|
||||
|
||||
# 提取参数
|
||||
params = []
|
||||
for arg in node.args.args:
|
||||
param_info = {
|
||||
"name": arg.arg,
|
||||
"type": None
|
||||
}
|
||||
# 提取类型注解
|
||||
if arg.annotation:
|
||||
try:
|
||||
param_info["type"] = ast.unparse(arg.annotation)
|
||||
except Exception:
|
||||
pass
|
||||
params.append(param_info)
|
||||
|
||||
# 提取返回值类型
|
||||
return_type = None
|
||||
if node.returns:
|
||||
try:
|
||||
return_type = ast.unparse(node.returns)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 提取函数体代码
|
||||
func_body = self._get_func_body(node)
|
||||
|
||||
# 提取所属类名
|
||||
class_name = None
|
||||
parent = node
|
||||
while hasattr(parent, "parent"):
|
||||
parent = parent.parent
|
||||
if isinstance(parent, ast.ClassDef):
|
||||
class_name = parent.name
|
||||
break
|
||||
|
||||
# 提取函数文档字符串
|
||||
docstring = ast.get_docstring(node)
|
||||
|
||||
return {
|
||||
"file_path": self.file_path,
|
||||
"lang": "python",
|
||||
"func_name": func_name,
|
||||
"class_name": class_name,
|
||||
"params": params,
|
||||
"return_type": return_type,
|
||||
"func_body": func_body,
|
||||
"docstring": docstring,
|
||||
"start_line": node.lineno,
|
||||
"end_line": node.end_lineno
|
||||
}
|
||||
|
||||
def _parse_java(self):
|
||||
"""
|
||||
解析Java代码
|
||||
注意：这里使用简单的正则解析，实际项目中建议使用专业的Java解析库
|
||||
"""
|
||||
logger.warning("Java解析功能暂未完全实现，使用简单的正则解析")
|
||||
# TODO: 实现Java代码的AST解析
|
||||
|
||||
def _parse_go(self):
|
||||
"""
|
||||
解析Go代码
|
||||
注意：这里使用简单的正则解析，实际项目中建议使用专业的Go解析库
|
||||
"""
|
||||
logger.warning("Go解析功能暂未完全实现，使用简单的正则解析")
|
||||
# TODO: 实现Go代码的AST解析
|
||||
|
||||
def _parse_javascript(self):
|
||||
"""
|
||||
解析JavaScript/TypeScript代码
|
||||
注意：这里使用简单的正则解析，实际项目中建议使用专业的JS解析库
|
||||
"""
|
||||
logger.warning("JavaScript解析功能暂未完全实现，使用简单的正则解析")
|
||||
# TODO: 实现JavaScript代码的AST解析
|
||||
|
||||
def _get_func_body(self, node) -> str:
|
||||
"""
|
||||
获取函数体代码
|
||||
|
||||
Args:
|
||||
node: AST节点
|
||||
|
||||
Returns:
|
||||
str: 函数体代码
|
||||
"""
|
||||
try:
|
||||
# 使用ast.unparse获取函数体代码
|
||||
return ast.unparse(node)
|
||||
except Exception:
|
||||
# 降级方案：根据行号提取代码
|
||||
lines = self.code.splitlines()
|
||||
start_line = node.lineno - 1 # 转换为0-based索引
|
||||
end_line = node.end_lineno # 转换为0-based索引
|
||||
if start_line >= 0 and end_line <= len(lines):
|
||||
return "\n".join(lines[start_line:end_line])
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def detect_language(file_path: str) -> Optional[str]:
|
||||
"""
|
||||
根据文件扩展名检测编程语言
|
||||
|
||||
Args:
|
||||
file_path: 文件路径
|
||||
|
||||
Returns:
|
||||
Optional[str]: 编程语言
|
||||
"""
|
||||
ext = os.path.splitext(file_path)[1].lower()
|
||||
|
||||
lang_map = {
|
||||
".py": "python",
|
||||
".java": "java",
|
||||
".go": "go",
|
||||
".js": "javascript",
|
||||
".ts": "typescript",
|
||||
".jsx": "javascript",
|
||||
".tsx": "typescript",
|
||||
".c": "c",
|
||||
".cpp": "cpp",
|
||||
".h": "c",
|
||||
".hpp": "cpp",
|
||||
".cs": "csharp",
|
||||
".rs": "rust",
|
||||
".php": "php",
|
||||
".rb": "ruby",
|
||||
".swift": "swift",
|
||||
".kt": "kotlin",
|
||||
".scala": "scala"
|
||||
}
|
||||
|
||||
return lang_map.get(ext)
|
||||
|
||||
@staticmethod
|
||||
def parse_file(file_path: str) -> List[Dict]:
|
||||
"""
|
||||
静态方法：解析文件
|
||||
|
||||
Args:
|
||||
file_path: 文件路径
|
||||
|
||||
Returns:
|
||||
List[Dict]: 函数信息列表
|
||||
"""
|
||||
lang = ASTParser.detect_language(file_path)
|
||||
if not lang:
|
||||
logger.warning(f"无法检测文件类型: {file_path}")
|
||||
return []
|
||||
|
||||
parser = ASTParser(file_path, lang)
|
||||
return parser.parse_functions()
|
||||
|
|
@ -105,19 +105,20 @@ class BaseSync(ABC):
|
|||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def doc_to_llamaindex_doc(self, doc: Dict) -> 'Document':
|
||||
"""
|
||||
Convert data source document to LlamaIndex Document
|
||||
|
||||
Args:
|
||||
doc: Document from the data source
|
||||
|
||||
Returns:
|
||||
LlamaIndex Document object
|
||||
"""
|
||||
pass
|
||||
|
||||
# @abstractmethod
|
||||
# def doc_to_llamaindex_doc(self, doc: Dict) -> 'Document':
|
||||
# """
|
||||
# Convert data source document to LlamaIndex Document
|
||||
#
|
||||
# Args:
|
||||
# doc: Document from the data source
|
||||
#
|
||||
# Returns:
|
||||
# LlamaIndex Document object
|
||||
# """
|
||||
# pass
|
||||
|
||||
|
||||
def process_documents(self, docs: List[Dict]) -> List['Document']:
|
||||
"""
|
||||
Process multiple documents into LlamaIndex Documents
|
||||
|
|
@ -134,7 +135,7 @@ class BaseSync(ABC):
|
|||
for doc in docs:
|
||||
try:
|
||||
llamaindex_doc = self.doc_to_llamaindex_doc(doc)
|
||||
if len(llamaindex_doc.text.strip()) >= 100:
|
||||
if len(llamaindex_doc.text.strip()) >= 20:
|
||||
documents.append(llamaindex_doc)
|
||||
else:
|
||||
logger.warning(f"跳过过短文档 (id: {llamaindex_doc.id_})，内容长度: {len(llamaindex_doc.text)} 字符")
|
||||
|
|
@ -164,7 +165,7 @@ class BaseSync(ABC):
|
|||
from config import settings
|
||||
|
||||
node_parser = SentenceSplitter(
|
||||
chunk_size=settings.CHUNK_SIZE,
|
||||
chunk_size=settings.CHUNK_SIZE, #NOTE: 从settings中获取，默认1024
|
||||
chunk_overlap=settings.CHUNK_OVERLAP
|
||||
)
|
||||
|
||||
|
|
@ -285,7 +286,7 @@ def get_sync_class(source_type: str, db_type: str = 'mysql') -> type[BaseSync]:
|
|||
Get the appropriate sync class based on data source type
|
||||
|
||||
Args:
|
||||
source_type: Type of data source (database, folder)
|
||||
source_type: Type of data source (database, folder, git)
|
||||
db_type: Type of database (mysql, dameng, etc.) - only used when source_type is 'database'
|
||||
|
||||
Returns:
|
||||
|
|
@ -297,6 +298,7 @@ def get_sync_class(source_type: str, db_type: str = 'mysql') -> type[BaseSync]:
|
|||
from sync.mysql_sync import MySQLSync
|
||||
from sync.folder_sync import FolderSync
|
||||
from sync.dameng_sync import DaMengSync
|
||||
from sync.git_sync import GitSync
|
||||
|
||||
if source_type == 'database':
|
||||
# 根据数据库类型选择相应的同步类
|
||||
|
|
@ -307,5 +309,7 @@ def get_sync_class(source_type: str, db_type: str = 'mysql') -> type[BaseSync]:
|
|||
return MySQLSync
|
||||
elif source_type == 'folder':
|
||||
return FolderSync
|
||||
elif source_type == 'git':
|
||||
return GitSync
|
||||
else:
|
||||
raise ValueError(f"Unsupported data source type: {source_type}")
|
||||
|
|
|
|||
|
|
@ -260,6 +260,9 @@ class FolderSync(BaseSync):
|
|||
|
||||
# Create SFTP client
|
||||
self._sftp_client = self._ssh_client.open_sftp()
|
||||
|
||||
# 检测服务器操作系统类型，确定路径分隔符
|
||||
self._detect_server_os()
|
||||
except paramiko.AuthenticationException:
|
||||
raise Exception(f"SSH connection failed: Authentication failed for user {username} on {self.config.host}")
|
||||
except paramiko.SSHException as ssh_error:
|
||||
|
|
@ -267,6 +270,32 @@ class FolderSync(BaseSync):
|
|||
except Exception as e:
|
||||
raise Exception(f"Connection failed: {str(e)}")
|
||||
|
||||
def _detect_server_os(self):
|
||||
"""
|
||||
检测服务器操作系统类型，确定路径分隔符
|
||||
"""
|
||||
try:
|
||||
# 尝试执行uname命令（Linux/Unix系统）
|
||||
stdin, stdout, stderr = self._ssh_client.exec_command('uname')
|
||||
output = stdout.read().decode('utf-8').strip().lower()
|
||||
|
||||
if output:
|
||||
# Linux/Unix系统
|
||||
self._path_sep = '/'
|
||||
else:
|
||||
# 尝试执行ver命令（Windows系统）
|
||||
stdin, stdout, stderr = self._ssh_client.exec_command('ver')
|
||||
output = stdout.read().decode('utf-8').strip().lower()
|
||||
if 'windows' in output:
|
||||
# Windows系统
|
||||
self._path_sep = '\\'
|
||||
else:
|
||||
# 默认使用正斜杠（大多数SFTP服务器支持）
|
||||
self._path_sep = '/'
|
||||
except Exception:
|
||||
# 如果命令执行失败，默认使用正斜杠
|
||||
self._path_sep = '/'
|
||||
|
||||
def _disconnect(self):
|
||||
"""
|
||||
Disconnect from the server
|
||||
|
|
@ -309,7 +338,8 @@ class FolderSync(BaseSync):
|
|||
items = self._sftp_client.listdir_attr(folder_path)
|
||||
|
||||
for item in items:
|
||||
item_path = os.path.join(folder_path, item.filename)
|
||||
# 使用检测到的路径分隔符连接路径
|
||||
item_path = f"{folder_path}{getattr(self, '_path_sep', '/')}{item.filename}"
|
||||
|
||||
if item.filename not in ('.', '..'):
|
||||
if item.st_mode & 0o040000: # Check if it's a directory
|
||||
|
|
@ -335,8 +365,8 @@ class FolderSync(BaseSync):
|
|||
if not hasattr(self.config, 'ignore_patterns') or not self.config.ignore_patterns:
|
||||
return False
|
||||
|
||||
# Get relative path from folder root
|
||||
relative_path = os.path.relpath(file_path, self.config.folder_path)
|
||||
# Get relative path from folder root and ensure forward slashes for pattern matching
|
||||
relative_path = os.path.relpath(file_path, self.config.folder_path).replace('\\', '/')
|
||||
|
||||
for pattern in self.config.ignore_patterns:
|
||||
if self._match_pattern(relative_path, pattern):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,363 @@
|
|||
"""
|
||||
Git代码库同步子类
|
||||
继承BaseSync，实现代码拉取/增量同步/函数解析
|
||||
"""
|
||||
import os
|
||||
from typing import List, Dict, Any, Set, Optional
|
||||
from datetime import datetime
|
||||
from loguru import logger
|
||||
from config import BaseDataSourceConfig, GitDataSourceConfig, settings
|
||||
from sync.base_sync import BaseSync
|
||||
from sync.ast_parser import ASTParser
|
||||
from utils.git_tool import GitTool
|
||||
from utils.func_id_generator import generate_func_unique_id
|
||||
|
||||
|
||||
class GitSync(BaseSync):
|
||||
def __init__(self, config: GitDataSourceConfig, vector_store_manager=None):
|
||||
"""
|
||||
初始化Git同步器
|
||||
|
||||
Args:
|
||||
config: Git数据源配置
|
||||
vector_store_manager: 向量存储管理器
|
||||
"""
|
||||
super().__init__(config, vector_store_manager)
|
||||
self.config = config
|
||||
# 初始化Git工具
|
||||
self.git_tool = GitTool(
|
||||
user_id="default", # 暂时使用默认用户ID
|
||||
repo_id=config.name,
|
||||
git_config={
|
||||
"git_url": config.git_url,
|
||||
"branch": config.branch,
|
||||
"ssh_key": config.ssh_key,
|
||||
"https_token": config.https_token,
|
||||
"local_repo_path": config.local_repo_path
|
||||
}
|
||||
)
|
||||
|
||||
def fetch_all_documents(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
获取所有文档（函数）
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 函数信息列表
|
||||
"""
|
||||
# 克隆仓库
|
||||
self.git_tool.clone_repo()
|
||||
|
||||
# 扫描仓库文件
|
||||
func_list = []
|
||||
support_lang = self.git_tool._detect_support_lang()
|
||||
|
||||
for root, dirs, files in os.walk(self.git_tool.local_repo_path):
|
||||
# 跳过.git目录
|
||||
if ".git" in dirs:
|
||||
dirs.remove(".git")
|
||||
|
||||
for file in files:
|
||||
file_path = os.path.join(root, file)
|
||||
# 检测文件语言
|
||||
lang = ASTParser.detect_language(file_path)
|
||||
if lang and lang in support_lang:
|
||||
# 解析文件中的函数
|
||||
parser = ASTParser(file_path, lang)
|
||||
try:
|
||||
functions = parser.parse_functions()
|
||||
# 为每个函数生成doc_id并设置到字典中
|
||||
for func in functions:
|
||||
# 生成唯一的文档ID
|
||||
func_id = generate_func_unique_id(
|
||||
user_id="default",
|
||||
repo_id=self.config.name,
|
||||
branch=self.config.branch,
|
||||
file_path=func["file_path"],
|
||||
class_name=func.get("class_name"),
|
||||
func_name=func["func_name"]
|
||||
)
|
||||
func['id'] = func_id
|
||||
func_list.extend(functions)
|
||||
except Exception as e:
|
||||
logger.error(f"解析文件失败 {file_path}: {e}")
|
||||
|
||||
logger.info(f"获取到 {len(func_list)} 个函数")
|
||||
return func_list
|
||||
|
||||
|
||||
|
||||
def doc_to_llamaindex_doc(self, doc: Dict) -> 'Document':
|
||||
"""
|
||||
转换函数信息为LlamaIndex Document
|
||||
|
||||
Args:
|
||||
doc: 函数信息
|
||||
|
||||
Returns:
|
||||
Document: LlamaIndex Document对象
|
||||
"""
|
||||
from llama_index.core import Document
|
||||
|
||||
# 生成函数唯一ID
|
||||
func_id = doc.get('id')
|
||||
|
||||
# 生成函数描述
|
||||
func_desc = self.generate_func_desc(doc)
|
||||
|
||||
# 获取函数体
|
||||
func_body = doc.get("func_body", "")
|
||||
|
||||
|
||||
# 创建Document对象
|
||||
document = Document(
|
||||
text=func_desc, # 使用描述作为文本
|
||||
id_=func_id,
|
||||
metadata={
|
||||
"func_id": func_id,
|
||||
"func_name": doc["func_name"],
|
||||
"class_name": doc.get("class_name") if doc.get("class_name")!=None else "None",
|
||||
"file_path": doc["file_path"],
|
||||
"lang": doc["lang"],
|
||||
"params": len(doc.get("params", [])), # 只存储参数数量，不存储完整参数列表
|
||||
"return_type": doc.get("return_type") if doc.get("return_type")!=None else "None",
|
||||
"docstring": (doc.get("docstring") or "")[:200], # 进一步限制文档字符串长度
|
||||
"start_line": doc.get("start_line"),
|
||||
"end_line": doc.get("end_line"),
|
||||
"repo_id": self.config.name,
|
||||
"branch": self.config.branch,
|
||||
"func_body": func_body[:1000] # 限制函数体长度，避免metadata过长
|
||||
}
|
||||
)
|
||||
|
||||
return document
|
||||
|
||||
def generate_func_desc(self, func_info: Dict) -> str:
|
||||
"""
|
||||
生成函数描述
|
||||
|
||||
Args:
|
||||
func_info: 函数信息
|
||||
|
||||
Returns:
|
||||
str: 函数描述
|
||||
"""
|
||||
# 构建函数描述
|
||||
parts = []
|
||||
|
||||
# 函数类型
|
||||
if func_info.get("class_name"):
|
||||
parts.append(f"{func_info['class_name']}类的{func_info['func_name']}方法")
|
||||
else:
|
||||
parts.append(f"{func_info['func_name']}函数")
|
||||
|
||||
# 参数信息
|
||||
params = func_info.get("params", [])
|
||||
if params:
|
||||
param_str = []
|
||||
for param in params:
|
||||
if param.get("type"):
|
||||
param_str.append(f"{param['name']}: {param['type']}")
|
||||
else:
|
||||
param_str.append(param['name'])
|
||||
parts.append(f"接收参数: {', '.join(param_str)}")
|
||||
|
||||
# 返回值信息
|
||||
return_type = func_info.get("return_type")
|
||||
if return_type:
|
||||
parts.append(f"返回类型: {return_type}")
|
||||
|
||||
# 文档字符串
|
||||
docstring = func_info.get("docstring")
|
||||
if docstring:
|
||||
parts.append(f"功能描述: {docstring.strip()}")
|
||||
else:
|
||||
# 如果没有文档字符串，使用本地大模型根据函数体生成描述
|
||||
parts.append(f"功能描述: {self._generate_docstring_from_body(func_info.get('func_body', ''))}")
|
||||
|
||||
return ". ".join(parts)
|
||||
|
||||
def _generate_docstring_from_body(self, func_body: str) -> str:
|
||||
"""
|
||||
使用本地大模型根据函数体生成文档字符串
|
||||
|
||||
Args:
|
||||
func_body: 函数体代码
|
||||
|
||||
Returns:
|
||||
str: 生成的文档字符串
|
||||
"""
|
||||
if not func_body:
|
||||
return "无文档字符串"
|
||||
|
||||
try:
|
||||
from config import settings
|
||||
from llama_index.llms.ollama import Ollama
|
||||
|
||||
# 初始化Ollama LLM
|
||||
llm = Ollama(
|
||||
model=settings.OLLAMA_MODEL,
|
||||
base_url=settings.OLLAMA_BASE_URL,
|
||||
temperature=0.3, # 降低温度，生成更确定的结果
|
||||
request_timeout=300.0
|
||||
)
|
||||
|
||||
# 构建提示词
|
||||
prompt = f"""
|
||||
请为以下函数生成简洁的文档字符串，描述其功能、参数和返回值：
|
||||
|
||||
{func_body}
|
||||
|
||||
要求：
|
||||
1. 语言简洁明了，不超过100字
|
||||
2. 只返回文档字符串内容，不要包含其他内容
|
||||
3. 重点描述函数的核心功能
|
||||
"""
|
||||
|
||||
# 生成文档字符串
|
||||
response = llm.complete(prompt)
|
||||
generated_docstring = response.text.strip()
|
||||
|
||||
# 限制长度
|
||||
if len(generated_docstring) > 200:
|
||||
generated_docstring = generated_docstring[:200] + "..."
|
||||
|
||||
logger.debug(f"生成的文档字符串: {generated_docstring}")
|
||||
return generated_docstring
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"生成文档字符串失败: {e}")
|
||||
# 降级方案：返回基于函数名的简单描述
|
||||
return "执行相关操作的函数"
|
||||
|
||||
def fetch_new_documents(self, last_sync_time=None) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
获取新文档（增量同步）
|
||||
|
||||
Args:
|
||||
last_sync_time: 上次同步时间
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 新函数信息列表
|
||||
"""
|
||||
# 检测远程更新
|
||||
has_update, local_commit, remote_commit = self.git_tool.detect_remote_update()
|
||||
|
||||
if not has_update:
|
||||
logger.info("Git仓库无更新")
|
||||
return []
|
||||
|
||||
# 增量拉取
|
||||
delta_files = self.git_tool.incremental_pull(local_commit, remote_commit)
|
||||
|
||||
# 解析新增/修改的文件
|
||||
func_list = []
|
||||
for file_path in delta_files.get("ADD", []) + delta_files.get("MODIFY", []):
|
||||
lang = ASTParser.detect_language(file_path)
|
||||
if lang:
|
||||
parser = ASTParser(file_path, lang)
|
||||
try:
|
||||
functions = parser.parse_functions()
|
||||
# 为每个函数生成doc_id并设置到字典中
|
||||
for func in functions:
|
||||
# 生成唯一的文档ID
|
||||
func_id = generate_func_unique_id(
|
||||
user_id="default",
|
||||
repo_id=self.config.name,
|
||||
branch=self.config.branch,
|
||||
file_path=func["file_path"],
|
||||
class_name=func.get("class_name"),
|
||||
func_name=func["func_name"]
|
||||
)
|
||||
func['id'] = func_id
|
||||
func_list.extend(functions)
|
||||
except Exception as e:
|
||||
logger.error(f"解析文件失败 {file_path}: {e}")
|
||||
|
||||
logger.info(f"增量同步获取到 {len(func_list)} 个函数")
|
||||
return func_list
|
||||
|
||||
def get_synced_document_ids(self) -> Set[str]:
|
||||
"""
|
||||
获取已同步的文档ID
|
||||
|
||||
Returns:
|
||||
Set[str]: 文档ID集合
|
||||
"""
|
||||
# 从向量存储中获取已同步的函数ID
|
||||
if not self.vector_store_manager:
|
||||
return set()
|
||||
|
||||
try:
|
||||
# 获取所有已存在的文档ID
|
||||
all_doc_ids = self.vector_store_manager.get_existing_doc_ids()
|
||||
|
||||
# 过滤出与当前Git仓库相关的文档ID
|
||||
synced_ids = set()
|
||||
|
||||
# 获取所有文档的元数据，用于过滤
|
||||
results = self.vector_store_manager.collection.get(include=['metadatas'])
|
||||
metadatas = results.get('metadatas', [])
|
||||
ids = results.get('ids', [])
|
||||
|
||||
for doc_id, metadata in zip(ids, metadatas):
|
||||
if metadata and metadata.get('repo_id') == self.config.name:
|
||||
synced_ids.add(doc_id)
|
||||
|
||||
logger.info(f"获取到 {len(synced_ids)} 个已同步的Git函数ID")
|
||||
return synced_ids
|
||||
except Exception as e:
|
||||
logger.error(f"获取已同步文档ID失败: {e}")
|
||||
return set()
|
||||
|
||||
def generate_doc_id(self, identifier: str) -> str:
|
||||
"""
|
||||
生成唯一的文档ID
|
||||
|
||||
Args:
|
||||
identifier: 文档的唯一标识符（文件路径等）
|
||||
|
||||
Returns:
|
||||
str: 唯一的文档ID
|
||||
"""
|
||||
from utils.func_id_generator import generate_func_unique_id
|
||||
|
||||
# 对于Git数据源，使用函数唯一ID生成器
|
||||
# 假设identifier是文件路径
|
||||
return generate_func_unique_id(
|
||||
user_id="default",
|
||||
repo_id=self.config.name,
|
||||
branch=self.config.branch,
|
||||
file_path=identifier,
|
||||
class_name="",
|
||||
func_name=identifier.split('/')[-1].split('.')[0]
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def check_data_source_exists(config: BaseDataSourceConfig) -> bool:
|
||||
"""
|
||||
检查数据源是否存在
|
||||
|
||||
Args:
|
||||
config: 数据源配置
|
||||
|
||||
Returns:
|
||||
bool: 是否存在
|
||||
"""
|
||||
try:
|
||||
# 尝试克隆仓库
|
||||
git_tool = GitTool(
|
||||
user_id="default",
|
||||
repo_id=config.name,
|
||||
git_config={
|
||||
"git_url": config.git_url,
|
||||
"branch": config.branch,
|
||||
"ssh_key": config.ssh_key,
|
||||
"https_token": config.https_token
|
||||
}
|
||||
)
|
||||
git_tool.clone_repo()
|
||||
logger.info(f"Git数据源检查成功: {config.name}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Git数据源检查失败: {e}")
|
||||
return False
|
||||
|
|
@ -197,6 +197,35 @@ class SyncService:
|
|||
chunked_docs = self.syncer.chunk_documents(processed_docs)
|
||||
all_chunked_docs.extend(chunked_docs)
|
||||
|
||||
# Update last sync time
|
||||
self.last_sync_time = datetime.now()
|
||||
elif self.source_config.type == "git":
|
||||
if not force:
|
||||
new_documents = []
|
||||
for doc in documents:
|
||||
# 检查服务运行状态：仅在非手动同步时检查
|
||||
if not is_manual and not self._running:
|
||||
logger.info(f"Sync interrupted during document filtering: {self.source_name}")
|
||||
return all_chunked_docs, total_docs, skipped_docs_count # 返回结果，不退出程序
|
||||
|
||||
doc_id = doc.get('id')
|
||||
if not self.vector_store_manager.document_exists(doc_id):
|
||||
new_documents.append(doc)
|
||||
else:
|
||||
skipped_docs_count += 1
|
||||
|
||||
if not new_documents:
|
||||
self.last_sync_time = datetime.now()
|
||||
return all_chunked_docs, total_docs, skipped_docs_count
|
||||
documents = new_documents
|
||||
|
||||
# Process and chunk documents
|
||||
processed_docs = self.syncer.process_documents(documents)
|
||||
chunked_docs = self.syncer.chunk_documents(processed_docs)
|
||||
all_chunked_docs.extend(chunked_docs)
|
||||
|
||||
|
||||
|
||||
# Update last sync time
|
||||
self.last_sync_time = datetime.now()
|
||||
else:
|
||||
|
|
@ -358,7 +387,7 @@ class SyncService:
|
|||
return
|
||||
|
||||
max_restart_attempts = 10 # Maximum number of restart attempts
|
||||
restart_delay = 60 # Wait 60 seconds before restarting after an error
|
||||
restart_delay = settings.SYNC_INTERVAL # Wait 60 seconds before restarting after an error
|
||||
restart_count = 0
|
||||
|
||||
self._running = True
|
||||
|
|
@ -432,7 +461,7 @@ class SyncService:
|
|||
|
||||
# Check if another sync is in progress (e.g., initial sync or previous incremental sync still running)
|
||||
# Wait for it to complete before starting incremental sync (no timeout - wait indefinitely)
|
||||
wait_interval = 10 # Check every 10 seconds
|
||||
wait_interval = settings.SYNC_INTERVAL # Check every 10 seconds
|
||||
waited_time = 0
|
||||
while self._sync_in_progress:
|
||||
logger.info(f"Another sync is in progress for {self.source_name}, waiting... (waited {waited_time}s, will wait until completion)")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,116 @@
|
|||
"""
|
||||
函数全局唯一ID生成工具类
|
||||
按用户/仓库/分支/文件/函数生成唯一ID
|
||||
"""
|
||||
import os
|
||||
from typing import Optional, Dict
|
||||
from config import settings
|
||||
|
||||
|
||||
def generate_func_unique_id(
|
||||
user_id: str,
|
||||
repo_id: str,
|
||||
branch: str,
|
||||
file_path: str,
|
||||
class_name: Optional[str],
|
||||
func_name: str
|
||||
) -> str:
|
||||
"""
|
||||
生成函数全局唯一ID
|
||||
|
||||
Args:
|
||||
user_id: 用户ID
|
||||
repo_id: 仓库ID
|
||||
branch: 分支名
|
||||
file_path: 文件路径
|
||||
class_name: 类名
|
||||
func_name: 函数名
|
||||
|
||||
Returns:
|
||||
str: 函数唯一ID
|
||||
"""
|
||||
# 类名为None则使用空字符串
|
||||
class_name = class_name if class_name else "None"
|
||||
|
||||
# 直接使用文件路径的绝对路径部分，确保唯一性
|
||||
# 替换路径分隔符为下划线
|
||||
file_path = file_path.split(os.sep)[3:]
|
||||
file_path = "_".join(file_path)
|
||||
normalized_file_path = file_path.replace(os.sep, "_")
|
||||
|
||||
# 生成唯一ID
|
||||
unique_id = f"{user_id}_{repo_id}_{branch}_{normalized_file_path}_{class_name}_{func_name}"
|
||||
|
||||
# 替换特殊字符，避免ChromaDB主键冲突
|
||||
unique_id = unique_id.replace("/", "_").replace("\\", "_").replace(":", "_").replace(" ", "_")
|
||||
|
||||
return unique_id
|
||||
|
||||
|
||||
def parse_func_unique_id(func_id: str) -> Dict[str, str]:
|
||||
"""
|
||||
解析函数唯一ID
|
||||
|
||||
Args:
|
||||
func_id: 函数唯一ID
|
||||
|
||||
Returns:
|
||||
Dict[str, str]: 解析后的信息
|
||||
"""
|
||||
parts = func_id.split("_")
|
||||
if len(parts) < 6:
|
||||
raise Exception(f"无效的函数ID格式: {func_id}")
|
||||
|
||||
# 解析各部分
|
||||
user_id = parts[0]
|
||||
repo_id = parts[1]
|
||||
branch = parts[2]
|
||||
|
||||
# 解析文件路径（可能包含下划线）
|
||||
# 从第3个部分开始，到倒数第2个部分结束
|
||||
file_path_parts = parts[3:-2]
|
||||
file_path = "_".join(file_path_parts).replace("_", os.sep)
|
||||
|
||||
class_name = parts[-2]
|
||||
if class_name == "None":
|
||||
class_name = None
|
||||
|
||||
func_name = parts[-1]
|
||||
|
||||
return {
|
||||
"user_id": user_id,
|
||||
"repo_id": repo_id,
|
||||
"branch": branch,
|
||||
"file_path": file_path,
|
||||
"class_name": class_name,
|
||||
"func_name": func_name
|
||||
}
|
||||
|
||||
|
||||
def get_repo_path_from_func_id(func_id: str) -> str:
|
||||
"""
|
||||
从函数ID获取仓库路径
|
||||
|
||||
Args:
|
||||
func_id: 函数唯一ID
|
||||
|
||||
Returns:
|
||||
str: 仓库路径
|
||||
"""
|
||||
info = parse_func_unique_id(func_id)
|
||||
return os.path.join(settings.GIT_LOCAL_STORAGE_ROOT, info["user_id"], info["repo_id"])
|
||||
|
||||
|
||||
def get_file_path_from_func_id(func_id: str) -> str:
|
||||
"""
|
||||
从函数ID获取文件路径
|
||||
|
||||
Args:
|
||||
func_id: 函数唯一ID
|
||||
|
||||
Returns:
|
||||
str: 文件路径
|
||||
"""
|
||||
info = parse_func_unique_id(func_id)
|
||||
repo_path = get_repo_path_from_func_id(func_id)
|
||||
return os.path.join(repo_path, info["file_path"])
|
||||
|
|
@ -0,0 +1,323 @@
|
|||
"""
|
||||
Git命令封装工具类
|
||||
实现Git仓库的克隆、更新检测、增量拉取等功能
|
||||
"""
|
||||
import subprocess
|
||||
import os
|
||||
from typing import Tuple, Dict, List
|
||||
from loguru import logger
|
||||
from config import settings
|
||||
|
||||
|
||||
class GitTool:
|
||||
def __init__(self, user_id: str = "test", repo_id: str = "test", git_config: dict = None,
|
||||
git_url: str = None, branch: str = None, protocol: str = None,
|
||||
https_token: str = None, ssh_key: str = None, local_repo_path: str = None):
|
||||
"""
|
||||
初始化Git工具类
|
||||
|
||||
Args:
|
||||
user_id: 用户ID
|
||||
repo_id: 仓库ID
|
||||
git_config: Git配置信息
|
||||
git_url: Git仓库URL（直接参数，优先级高于git_config）
|
||||
branch: Git分支（直接参数，优先级高于git_config）
|
||||
protocol: Git协议（直接参数，优先级高于git_config）
|
||||
https_token: HTTPS令牌（直接参数，优先级高于git_config）
|
||||
ssh_key: SSH密钥（直接参数，优先级高于git_config）
|
||||
local_repo_path: 本地仓库路径（直接参数，优先级高于git_config）
|
||||
"""
|
||||
self.user_id = user_id
|
||||
self.repo_id = repo_id
|
||||
|
||||
# 优先使用直接参数，如果没有则使用git_config
|
||||
if git_config:
|
||||
self.git_url = git_url or git_config.get("git_url")
|
||||
self.branch = branch or git_config.get("branch", settings.GIT_DEFAULT_BRANCH)
|
||||
self.protocol = protocol or git_config.get("protocol", "https")
|
||||
self.ssh_key = ssh_key or git_config.get("ssh_key")
|
||||
self.https_token = https_token or git_config.get("https_token")
|
||||
# 本地结构化存储路径
|
||||
self.local_repo_path = local_repo_path or git_config.get("local_repo_path")
|
||||
else:
|
||||
self.git_url = git_url
|
||||
self.branch = branch or settings.GIT_DEFAULT_BRANCH
|
||||
self.protocol = protocol or "https"
|
||||
self.ssh_key = ssh_key
|
||||
self.https_token = https_token
|
||||
self.local_repo_path = local_repo_path
|
||||
|
||||
if not self.local_repo_path:
|
||||
self.local_repo_path = os.path.join(settings.GIT_LOCAL_STORAGE_ROOT, user_id, repo_id)
|
||||
# 初始化Git环境
|
||||
self._init_git_env()
|
||||
|
||||
def _init_git_env(self):
|
||||
"""
|
||||
初始化Git环境（SSH密钥配置）
|
||||
"""
|
||||
if self.ssh_key:
|
||||
# 解密SSH私钥，写入临时文件，配置Git SSH
|
||||
ssh_key_path = f"/tmp/ssh_key_{self.user_id}_{self.repo_id}"
|
||||
with open(ssh_key_path, "w") as f:
|
||||
f.write(self.ssh_key)
|
||||
os.chmod(ssh_key_path, 0o600)
|
||||
os.environ["GIT_SSH_COMMAND"] = f"ssh -i {ssh_key_path} -o StrictHostKeyChecking=no"
|
||||
|
||||
def clone_repo(self) -> bool:
|
||||
"""
|
||||
克隆Git仓库
|
||||
|
||||
Returns:
|
||||
bool: 是否成功克隆
|
||||
"""
|
||||
if not os.path.exists(self.local_repo_path):
|
||||
os.makedirs(os.path.dirname(self.local_repo_path), exist_ok=True)
|
||||
# 执行git clone命令
|
||||
cmd = [
|
||||
"git", "clone", "--single-branch",
|
||||
"--branch", self.branch, self.git_url, self.local_repo_path
|
||||
]
|
||||
logger.info(f"执行Git克隆命令: {' '.join(cmd)}")
|
||||
res = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if res.returncode != 0:
|
||||
logger.error(f"Git克隆失败: {res.stderr}")
|
||||
raise Exception(f"Git克隆失败: {res.stderr}")
|
||||
# 克隆后校验
|
||||
self._check_repo_integrity()
|
||||
logger.info(f"Git仓库克隆成功: {self.local_repo_path}")
|
||||
return True
|
||||
logger.info(f"Git仓库已存在: {self.local_repo_path}")
|
||||
return False
|
||||
|
||||
def _check_repo_integrity(self):
|
||||
"""
|
||||
仓库完整性校验（git fsck）+ 支持的编程语言检测
|
||||
"""
|
||||
# 执行git fsck
|
||||
try:
|
||||
subprocess.run(["git", "fsck"], cwd=self.local_repo_path, check=True, capture_output=True, text=True)
|
||||
logger.info(f"Git仓库完整性校验成功: {self.local_repo_path}")
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning(f"Git仓库完整性校验失败: {e.stderr}")
|
||||
|
||||
# 扫描文件类型，记录支持的编程语言
|
||||
support_lang = self._detect_support_lang()
|
||||
logger.info(f"检测到支持的编程语言: {support_lang}")
|
||||
return support_lang
|
||||
|
||||
def _detect_support_lang(self) -> List[str]:
|
||||
"""
|
||||
检测仓库支持的编程语言
|
||||
|
||||
Returns:
|
||||
List[str]: 支持的编程语言列表
|
||||
"""
|
||||
lang_extensions = {
|
||||
"python": [".py"],
|
||||
"java": [".java"],
|
||||
"go": [".go"],
|
||||
"javascript": [".js", ".jsx"],
|
||||
"typescript": [".ts", ".tsx"],
|
||||
"c": [".c", ".h"],
|
||||
"cpp": [".cpp", ".hpp", ".cc"],
|
||||
"csharp": [".cs"],
|
||||
"rust": [".rs"],
|
||||
"php": [".php"],
|
||||
"ruby": [".rb"],
|
||||
"swift": [".swift"],
|
||||
"kotlin": [".kt"],
|
||||
"scala": [".scala"]
|
||||
}
|
||||
|
||||
support_lang = []
|
||||
for root, dirs, files in os.walk(self.local_repo_path):
|
||||
# 跳过.git目录
|
||||
if ".git" in dirs:
|
||||
dirs.remove(".git")
|
||||
# 跳过其他常见的非代码目录
|
||||
dirs_to_skip = ["node_modules", "venv", "dist", "build", "__pycache__"]
|
||||
dirs[:] = [d for d in dirs if d not in dirs_to_skip]
|
||||
|
||||
for file in files:
|
||||
for lang, extensions in lang_extensions.items():
|
||||
if any(file.endswith(ext) for ext in extensions):
|
||||
if lang not in support_lang:
|
||||
support_lang.append(lang)
|
||||
break
|
||||
|
||||
return support_lang
|
||||
|
||||
def detect_remote_update(self) -> Tuple[bool, str, str]:
|
||||
"""
|
||||
远程更新检测
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str, str]: (是否有更新, 本地commit ID, 远程commit ID)
|
||||
"""
|
||||
# 确保仓库存在
|
||||
if not os.path.exists(self.local_repo_path):
|
||||
raise Exception(f"Git仓库不存在: {self.local_repo_path}")
|
||||
|
||||
# 拉取远程commit记录
|
||||
try:
|
||||
# 先尝试 fetch 到远程跟踪分支，避免与本地检出分支冲突
|
||||
subprocess.run(["git", "fetch", "origin", self.branch],
|
||||
cwd=self.local_repo_path, check=True, capture_output=True, text=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"Git fetch失败: {e.stderr}")
|
||||
raise
|
||||
|
||||
# 获取本地/远程commit ID
|
||||
local_commit = subprocess.run(["git", "rev-parse", "HEAD"],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True).stdout.strip()
|
||||
remote_commit = subprocess.run(["git", "rev-parse", f"origin/{self.branch}"],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
has_update = local_commit != remote_commit
|
||||
logger.info(f"Git更新检测: 本地={local_commit[:7]}, 远程={remote_commit[:7]}, 有更新={has_update}")
|
||||
return has_update, local_commit, remote_commit
|
||||
|
||||
def incremental_pull(self, local_commit: str, remote_commit: str) -> Dict[str, List[str]]:
|
||||
"""
|
||||
增量拉取代码+解析文件变更
|
||||
|
||||
Args:
|
||||
local_commit: 本地commit ID
|
||||
remote_commit: 远程commit ID
|
||||
|
||||
Returns:
|
||||
Dict[str, List[str]]: 文件变更集
|
||||
"""
|
||||
# 快进合并到远程最新版本
|
||||
try:
|
||||
subprocess.run(["git", "merge", "--ff-only", f"origin/{self.branch}"],
|
||||
cwd=self.local_repo_path, check=True, capture_output=True, text=True)
|
||||
logger.info(f"Git快进合并成功: {self.branch}")
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"Git合并失败: {e.stderr}")
|
||||
raise
|
||||
|
||||
# 提取增量commit的文件变更
|
||||
delta_commits = subprocess.run(["git", "log", "--pretty=format:%H", f"{local_commit}..{remote_commit}"],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True).stdout.split()
|
||||
|
||||
# 解析文件变更为ADD/MODIFY/DELETE
|
||||
delta_files = self._parse_delta_files(delta_commits)
|
||||
logger.info(f"Git增量变更: ADD={len(delta_files['ADD'])}, MODIFY={len(delta_files['MODIFY'])}, DELETE={len(delta_files['DELETE'])}")
|
||||
return delta_files
|
||||
|
||||
def _parse_delta_files(self, delta_commits: List[str]) -> Dict[str, List[str]]:
|
||||
"""
|
||||
解析文件变更集
|
||||
|
||||
Args:
|
||||
delta_commits: 增量commit列表
|
||||
|
||||
Returns:
|
||||
Dict[str, List[str]]: 文件变更集
|
||||
"""
|
||||
add_files, modify_files, delete_files = [], [], []
|
||||
|
||||
for commit in delta_commits:
|
||||
# git show --name-status 获取文件变更
|
||||
res = subprocess.run(["git", "show", "--name-status", commit],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True).stdout
|
||||
|
||||
for line in res.splitlines():
|
||||
if not line:
|
||||
continue
|
||||
# 解析状态和文件路径
|
||||
if "\t" in line:
|
||||
status, file_path = line.split("\t", 1)
|
||||
full_path = os.path.join(self.local_repo_path, file_path)
|
||||
if status == "A":
|
||||
add_files.append(full_path)
|
||||
elif status == "M":
|
||||
modify_files.append(full_path)
|
||||
elif status == "D":
|
||||
delete_files.append(full_path)
|
||||
|
||||
# 去重并返回
|
||||
return {
|
||||
"ADD": list(set(add_files)),
|
||||
"MODIFY": list(set(modify_files)),
|
||||
"DELETE": list(set(delete_files))
|
||||
}
|
||||
|
||||
def get_current_commit(self) -> str:
|
||||
"""
|
||||
获取当前commit ID
|
||||
|
||||
Returns:
|
||||
str: 当前commit ID
|
||||
"""
|
||||
if not os.path.exists(self.local_repo_path):
|
||||
raise Exception(f"Git仓库不存在: {self.local_repo_path}")
|
||||
|
||||
commit_id = subprocess.run(["git", "rev-parse", "HEAD"],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True).stdout.strip()
|
||||
return commit_id
|
||||
|
||||
def get_repo_info(self) -> Dict[str, str]:
|
||||
"""
|
||||
获取仓库信息
|
||||
|
||||
Returns:
|
||||
Dict[str, str]: 仓库信息
|
||||
"""
|
||||
if not os.path.exists(self.local_repo_path):
|
||||
raise Exception(f"Git仓库不存在: {self.local_repo_path}")
|
||||
|
||||
# 获取仓库URL
|
||||
remote_url = subprocess.run(["git", "config", "--get", "remote.origin.url"],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
# 获取当前分支
|
||||
current_branch = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
# 获取当前commit
|
||||
current_commit = self.get_current_commit()
|
||||
|
||||
return {
|
||||
"remote_url": remote_url,
|
||||
"current_branch": current_branch,
|
||||
"current_commit": current_commit,
|
||||
"local_path": self.local_repo_path
|
||||
}
|
||||
|
||||
def test_connection(self) -> bool:
|
||||
"""
|
||||
测试Git连接
|
||||
|
||||
Returns:
|
||||
bool: 是否连接成功
|
||||
"""
|
||||
if not self.git_url:
|
||||
raise Exception("Git仓库URL未设置")
|
||||
|
||||
logger.info(f"测试Git连接: {self.git_url}")
|
||||
|
||||
# 尝试执行git ls-remote命令来测试连接
|
||||
try:
|
||||
cmd = ["git", "ls-remote", "--heads", self.git_url, f"refs/heads/{self.branch}"]
|
||||
logger.info(f"执行Git连接测试命令: {' '.join(cmd)}")
|
||||
|
||||
res = subprocess.run(cmd, capture_output=True, text=True)
|
||||
|
||||
if res.returncode == 0:
|
||||
# 检查输出是否包含预期的分支信息
|
||||
if self.branch in res.stdout:
|
||||
logger.info("Git连接测试成功！")
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"Git连接测试失败：分支 {self.branch} 不存在")
|
||||
return False
|
||||
else:
|
||||
logger.error(f"Git连接测试失败: {res.stderr}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Git连接测试异常: {e}")
|
||||
return False
|
||||
Loading…
Reference in New Issue