Compare commits
60 Commits
| Author | SHA1 | Date |
|---|---|---|
|
|
ad11367497 | |
|
|
c0f0c318e5 | |
|
|
f3b5c9409f | |
|
|
99f88f134b | |
|
|
b948a0c4f9 | |
|
|
f8a514b30d | |
|
|
7005ff2c66 | |
|
|
68941e9494 | |
|
|
2fd1e2f156 | |
|
|
aa26622c6b | |
|
|
65ac248216 | |
|
|
066b96d45d | |
|
|
e008cd76ee | |
|
|
6dfc217ba6 | |
|
|
31e5a61f4a | |
|
|
0a839523fd | |
|
|
ecfed9d339 | |
|
|
8794cb2d2c | |
|
|
99115692e5 | |
|
|
ced7021e16 | |
|
|
392d9ba90b | |
|
|
2646103d12 | |
|
|
80f024c22d | |
|
|
24cc595653 | |
|
|
e72938b15b | |
|
|
16e230252f | |
|
|
abb03e7d6e | |
|
|
c569c713d6 | |
|
|
65c63513f2 | |
|
|
226a10b9e7 | |
|
|
84dd8a0902 | |
|
|
62f3472571 | |
|
|
019f5f497a | |
|
|
914d5d6fc6 | |
|
|
aa25140362 | |
|
|
5618996595 | |
|
|
651d2a03d0 | |
|
|
922ca5e431 | |
|
|
efb04ae14b | |
|
|
fad17804e0 | |
|
|
349a2ce4c9 | |
|
|
3f9580be9b | |
|
|
2ac7827e76 | |
|
|
53a6bd5cdd | |
|
|
f5771b34a5 | |
|
|
4fdc3ef758 | |
|
|
b75679d1cf | |
|
|
3599304ef7 | |
|
|
daec870ea9 | |
|
|
3fca89d7a8 | |
|
|
b374116286 | |
|
|
3eb9ddee7a | |
|
|
86bbb1d248 | |
|
|
77c81640f5 | |
|
|
b44b7dec8b | |
|
|
31c3f4bc08 | |
|
|
6c14f69dac | |
|
|
34481b92fb | |
|
|
d8b49609bb | |
|
|
84614e708d |
32
.env.example
32
.env.example
|
|
@ -16,7 +16,7 @@ API_VERSION=1.0.0
|
|||
MAX_UPLOAD_SIZE_MB=5
|
||||
|
||||
# LibreOffice soffice service port (used by docker/soffice service)
|
||||
SOFFICE_HOST=localhost
|
||||
SOFFICE_HOST=rag-soffice #localhost
|
||||
SOFFICE_PORT=8003
|
||||
|
||||
# ============================================
|
||||
|
|
@ -25,34 +25,20 @@ SOFFICE_PORT=8003
|
|||
# CHROMA_SERVER_HOST: ChromaDB 服务器地址
|
||||
# - 使用 host 网络模式: localhost
|
||||
# - 远程服务器: 192.168.1.100 或 chromadb.example.com
|
||||
CHROMA_SERVER_HOST=localhost
|
||||
CHROMA_SERVER_PORT=8002
|
||||
CHROMA_SERVER_HOST=rag-chromadb #localhost
|
||||
CHROMA_SERVER_PORT=8000 #8002
|
||||
CHROMA_COLLECTION_NAME=rag_collection
|
||||
|
||||
# ============================================
|
||||
# LLM 配置 (用于文本生成)
|
||||
# Ollama 配置
|
||||
# ============================================
|
||||
# LLM provider: ollama, openai 等
|
||||
LLM_PROVIDER=ollama
|
||||
LLM_BASE_URL=http://localhost:11434/v1
|
||||
LLM_MODEL=qwen3:8b
|
||||
LLM_API_KEY= # 如使用openai等需要API Key的服务
|
||||
|
||||
# ============================================
|
||||
# Embedding 配置 (用于向量检索)
|
||||
# ============================================
|
||||
# Embedding provider: ollama, openai 等
|
||||
EMBEDDING_PROVIDER=ollama
|
||||
EMBEDDING_BASE_URL=http://localhost:11434
|
||||
EMBEDDING_MODEL=qwen3-embedding:0.6b
|
||||
EMBEDDING_API_KEY= # 如使用openai等需要API Key的服务
|
||||
|
||||
# ============================================
|
||||
# 兼容旧版本配置 (已废弃,仍可用但推荐使用上面的配置)
|
||||
# ============================================
|
||||
OLLAMA_BASE_URL=http://localhost:11434
|
||||
# OLLAMA_BASE_URL: Ollama 服务地址
|
||||
OLLAMA_BASE_URL=http://host.docker.internal:11434
|
||||
OLLAMA_MODEL=qwen3:8b
|
||||
OLLAMA_EMBEDDING_MODEL=qwen3-embedding:0.6b
|
||||
# OLLAMA_SMALL_MODEL: 小型模型,用于代码问题判断(硬编码在 rag_engine.py 中)
|
||||
# 如需更改,请修改 rag/rag_engine.py 中的 model 参数
|
||||
# OLLAMA_SMALL_MODEL=qwen3:0.6b
|
||||
|
||||
# ============================================
|
||||
# RAG 配置
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
# ============================================
|
||||
# RAG API 环境变量配置文件
|
||||
# ============================================
|
||||
# 复制此文件为 .env 并根据实际情况修改
|
||||
# 所有配置都可以通过此文件统一管理,方便不同机器之间移植
|
||||
# docker-compose.yml 会自动读取此文件中的配置
|
||||
|
||||
# ============================================
|
||||
# API 配置
|
||||
# ============================================
|
||||
API_HOST=0.0.0.0
|
||||
API_PORT=8001
|
||||
API_TITLE=RAG API
|
||||
API_VERSION=1.0.0
|
||||
# 文件上传大小限制(单位:MB,默认:5MB)
|
||||
MAX_UPLOAD_SIZE_MB=5
|
||||
|
||||
# LibreOffice soffice service port (used by docker/soffice service)
|
||||
SOFFICE_HOST=rag-soffice #localhost
|
||||
SOFFICE_PORT=8003
|
||||
|
||||
# ============================================
|
||||
# ChromaDB 配置
|
||||
# ============================================
|
||||
# CHROMA_SERVER_HOST: ChromaDB 服务器地址
|
||||
# - 使用 host 网络模式: localhost
|
||||
# - 远程服务器: 192.168.1.100 或 chromadb.example.com
|
||||
CHROMA_SERVER_HOST=rag-chromadb #localhost
|
||||
CHROMA_SERVER_PORT=8000 #8002
|
||||
CHROMA_COLLECTION_NAME=rag_collection
|
||||
|
||||
# ============================================
|
||||
# Ollama 配置
|
||||
# ============================================
|
||||
# OLLAMA_BASE_URL: Ollama 服务地址
|
||||
OLLAMA_BASE_URL=http://host.docker.internal:11434
|
||||
OLLAMA_MODEL=qwen3:8b
|
||||
OLLAMA_EMBEDDING_MODEL=qwen3-embedding:0.6b
|
||||
|
||||
# ============================================
|
||||
# RAG 配置
|
||||
# ============================================
|
||||
EMBEDDING_DIMENSION=768
|
||||
CHUNK_SIZE=1024
|
||||
CHUNK_OVERLAP=200
|
||||
TOP_K=5
|
||||
|
||||
# ============================================
|
||||
# 同步配置
|
||||
# ============================================
|
||||
SYNC_INTERVAL=300
|
||||
AUTO_SYNC=true
|
||||
|
||||
# ============================================
|
||||
# NLTK 配置
|
||||
# 推荐:如果你已在仓库内保存了 NLTK 数据(离线使用),可以设置为相对路径。例如:
|
||||
# ./nltk_data/
|
||||
# 或者使用本地绝对路径: /path/to/nltk_data
|
||||
# ============================================
|
||||
NLTK_DATA=./nltk_data/
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
# ============================================
|
||||
# RAG API 环境变量配置文件
|
||||
# ============================================
|
||||
# 复制此文件为 .env 并根据实际情况修改
|
||||
# 所有配置都可以通过此文件统一管理,方便不同机器之间移植
|
||||
# docker-compose.yml 会自动读取此文件中的配置
|
||||
|
||||
# ============================================
|
||||
# API 配置
|
||||
# ============================================
|
||||
API_HOST=0.0.0.0
|
||||
API_PORT=8001
|
||||
API_TITLE=RAG API
|
||||
API_VERSION=1.0.0
|
||||
# 文件上传大小限制(单位:MB,默认:5MB)
|
||||
MAX_UPLOAD_SIZE_MB=5
|
||||
|
||||
# LibreOffice soffice service port (used by docker/soffice service)
|
||||
# SOFFICE_HOST=rag-soffice #localhost
|
||||
SOFFICE_PORT=8003
|
||||
|
||||
# ============================================
|
||||
# ChromaDB 配置
|
||||
# ============================================
|
||||
# CHROMA_SERVER_HOST: ChromaDB 服务器地址
|
||||
# - 使用 host 网络模式: localhost
|
||||
# - 远程服务器: 192.168.1.100 或 chromadb.example.com
|
||||
# CHROMA_SERVER_HOST=rag-chromadb #localhost
|
||||
CHROMA_SERVER_PORT=8000
|
||||
CHROMA_COLLECTION_NAME=rag_collection
|
||||
|
||||
# ============================================
|
||||
# Ollama 配置
|
||||
# ============================================
|
||||
# OLLAMA_BASE_URL: Ollama 服务地址
|
||||
# OLLAMA_BASE_URL=http://host.docker.internal:11434 #http://localhost:11434
|
||||
OLLAMA_MODEL=qwen3:8b
|
||||
OLLAMA_EMBEDDING_MODEL=qwen3-embedding:0.6b
|
||||
|
||||
# ============================================
|
||||
# RAG 配置
|
||||
# ============================================
|
||||
EMBEDDING_DIMENSION=768
|
||||
CHUNK_SIZE=1024
|
||||
CHUNK_OVERLAP=200
|
||||
TOP_K=5
|
||||
|
||||
# ============================================
|
||||
# 同步配置
|
||||
# ============================================
|
||||
SYNC_INTERVAL=300
|
||||
AUTO_SYNC=true
|
||||
|
||||
# ============================================
|
||||
# NLTK 配置
|
||||
# 推荐:如果你已在仓库内保存了 NLTK 数据(离线使用),可以设置为相对路径。例如:
|
||||
# ./nltk_data/
|
||||
# 或者使用本地绝对路径: /path/to/nltk_data
|
||||
# ============================================
|
||||
NLTK_DATA=./nltk_data/
|
||||
|
|
@ -40,7 +40,7 @@ llamaindex/
|
|||
!.vscode/extensions.json
|
||||
!.vscode/tasks.json
|
||||
# Ignore user-specific VSCode files
|
||||
.vscode/launch.json
|
||||
!.vscode/launch.json
|
||||
.vscode/*.code-workspace
|
||||
|
||||
# JetBrains IDEs
|
||||
|
|
@ -142,3 +142,9 @@ docker-images-export/
|
|||
|
||||
# requirements.txt should be uploaded
|
||||
!requirements.txt
|
||||
|
||||
test-git-server/
|
||||
git_repos/
|
||||
|
||||
logs/
|
||||
docs/
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
// 使用 IntelliSense 了解相关属性。
|
||||
// 悬停以查看现有属性的描述。
|
||||
// 欲了解更多信息,请访问: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Python: RAG Main",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/main.py", // 指向api/main.py
|
||||
"cwd": "${workspaceFolder}", // 关键:工作目录设为项目根目录(S:\research\RAG)
|
||||
"console": "integratedTerminal",
|
||||
"justMyCode": false
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -20,6 +20,8 @@
|
|||
"terminal.integrated.env.linux": {
|
||||
"PYTHONPATH": "${workspaceFolder}"
|
||||
},
|
||||
"basedpyright.disableLanguageServices": true
|
||||
"basedpyright.disableLanguageServices": true,
|
||||
"python-envs.defaultEnvManager": "ms-python.python:conda",
|
||||
"python-envs.defaultPackageManager": "ms-python.python:conda"
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||
libssl-dev \
|
||||
libcrypto++-dev \
|
||||
libgmp-dev \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 配置 pip 镜像源(加速 Python 包安装)
|
||||
|
|
@ -40,7 +41,7 @@ COPY requirements.txt /app/
|
|||
|
||||
# 安装 Python 依赖
|
||||
RUN pip install --upgrade pip && \
|
||||
pip install -r requirements.txt
|
||||
pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
|
||||
# 复制项目文件
|
||||
COPY . /app/
|
||||
|
|
|
|||
83
README.md
83
README.md
|
|
@ -5,13 +5,14 @@
|
|||
## 功能特性
|
||||
|
||||
- 🔍 **智能检索**: 使用 LlamaIndex 和 ChromaDB 实现高效的向量检索
|
||||
- 💾 **数据同步**: 自动同步 MySQL 数据库数据到 ChromaDB 向量库
|
||||
- 💾 **数据同步**: 自动同步 MySQL 数据库、本地/远程文件夹和 Git 代码库数据到 ChromaDB 向量库
|
||||
- 🌊 **流式输出**: 基于 FastAPI 的流式响应,支持实时对话
|
||||
- 🤖 **本地 LLM**: 集成 Ollama 本地部署的大模型
|
||||
- ⚡ **高并发**: 支持多用户同时访问
|
||||
- 🔄 **自动同步**: 支持定时自动同步和手动触发同步
|
||||
- 🐳 **Docker 部署**: 使用 Docker Compose 一键部署
|
||||
- ⚙️ **统一配置**: 所有配置统一在 `.env` 文件中管理,方便不同机器之间移植
|
||||
- 🧑💻 **Git 集成**: 支持 Git 代码库的自动同步和检索,包括连接测试和分支管理
|
||||
|
||||
## 快速开始
|
||||
|
||||
|
|
@ -20,8 +21,8 @@
|
|||
1. **Docker** 和 **Docker Compose** 已安装
|
||||
2. **MySQL** 数据库已安装并运行(可在主机或远程服务器)
|
||||
3. **Ollama** 服务已安装并运行(可在主机或远程服务器),且已下载以下模型(可以替换为其他模型):
|
||||
- `qwen3:235b` - LLM模型(用于文本生成)
|
||||
- `qwen3-embedding:8b` - Embedding模型(用于向量化)
|
||||
- `qwen3:8b` - LLM模型(用于文本生成)
|
||||
- `qwen3-embedding:0.6b` - Embedding模型(用于向量化)
|
||||
|
||||
### 检查 Ollama
|
||||
|
||||
|
|
@ -29,9 +30,9 @@
|
|||
# 检查 Ollama 是否运行
|
||||
curl http://localhost:11434/api/tags
|
||||
|
||||
# 下载所需的模型(如果未下载)
|
||||
ollama pull qwen3:235b # LLM模型,用于文本生成
|
||||
ollama pull qwen3-embedding:8b # Embedding模型,用于向量化
|
||||
# 下载所需的模型(如果未下载)(可以使用更小的模型)
|
||||
ollama pull qwen3:8b # LLM模型,用于文本生成
|
||||
ollama pull qwen3-embedding:0.6b # Embedding模型,用于向量化
|
||||
```
|
||||
|
||||
### 安装步骤
|
||||
|
|
@ -58,8 +59,13 @@ ollama pull qwen3-embedding:8b # Embedding模型,用于向量化
|
|||
|
||||
# Ollama 配置(使用 host 网络模式)
|
||||
OLLAMA_BASE_URL=http://localhost:11434
|
||||
OLLAMA_MODEL=qwen3:235b
|
||||
OLLAMA_EMBEDDING_MODEL=qwen3-embedding:8b
|
||||
# 注:代码默认使用较小模型以提高性能
|
||||
# 可以根据实际硬件情况选择合适的模型
|
||||
OLLAMA_MODEL=qwen3:8b # 较小模型,适合一般硬件
|
||||
OLLAMA_EMBEDDING_MODEL=qwen3-embedding:0.6b # 较小嵌入模型
|
||||
# 可选:使用更大模型以获得更好效果(需要更强硬件)
|
||||
# OLLAMA_MODEL=qwen3:235b
|
||||
# OLLAMA_EMBEDDING_MODEL=qwen3-embedding:8b
|
||||
```
|
||||
|
||||
**RAG服务配置**:
|
||||
|
|
@ -84,7 +90,7 @@ ollama pull qwen3-embedding:8b # Embedding模型,用于向量化
|
|||
|
||||
4. **访问配置管理界面**:
|
||||
服务启动后,可以通过以下方式访问配置管理界面:
|
||||
- **配置管理界面**: http://localhost:8001/config (端口可通过 API_PORT 配置)
|
||||
- **配置管理界面**: http://localhost:8001/config 或 http://localhost:8001/static/config/index.html (端口可通过 API_PORT 配置)
|
||||
- **智能问答界面**: http://localhost:8001/chat (端口可通过 API_PORT 配置)
|
||||
- **API 文档**: http://localhost:8001/docs
|
||||
|
||||
|
|
@ -168,7 +174,36 @@ ollama pull qwen3-embedding:8b # Embedding模型,用于向量化
|
|||
- 点击"测试SSH连接",检查 SSH 连接是否成功。
|
||||
- 点击右上角"保存"按钮,保存文件夹配置
|
||||
|
||||
3. 更新数据源配置
|
||||
3. **Git代码库类型 (git)**
|
||||
|
||||
- 点击"新增数据源"-"选择类型"-"Git代码库"
|
||||
- Git代码库支持Git单仓库和Git服务器两种配置模式
|
||||
|
||||
- 模式1:Git仓库配置
|
||||
- Git仓库URL(必填):Git代码库的URL地址
|
||||
- 分支(必填):要同步的Git分支(默认main)
|
||||
- 协议:选择https或ssh协议
|
||||
- HTTPS Token:如果使用https协议,填写访问令牌
|
||||
- SSH密钥:如果使用ssh协议,填写SSH私钥
|
||||
- 点击"测试Git连接",检查 Git 连接是否成功。
|
||||
<img src="README_imgs/代码库-单个配置界面.png" alt="代码库-单个配置界面" width="600">
|
||||
|
||||
- 模式2:Git服务器模式配置
|
||||
- 模式选择:选择"服务器模式"启用多仓库管理
|
||||
- Git服务器主机(必填):Git服务器的主机地址
|
||||
- Git服务器端口(必填):Git服务器的端口(默认9418)
|
||||
- Git服务器用户名:SSH协议下的用户名
|
||||
- Git服务器密码:SSH协议下的密码
|
||||
- 仓库配置:添加多个仓库配置,每个仓库包含
|
||||
- 仓库名称:仓库的唯一标识
|
||||
- 仓库路径:仓库在服务器上的路径
|
||||
- 分支:要同步的Git分支(默认main)
|
||||
- 点击"测试Git连接",检查 Git 服务器连接是否成功。
|
||||
<img src="README_imgs/代码库-服务器配置界面.png" alt="代码库-服务器配置界面" width="600">
|
||||
|
||||
- 点击右上角"保存"按钮,保存Git仓库配置
|
||||
|
||||
1. 更新数据源配置
|
||||
- 点击左侧数据源列表中的数据源
|
||||
- 修改配置后点击保存,后台会自动删除原来同步的数据并重新同步
|
||||
|
||||
|
|
@ -292,7 +327,7 @@ docker-compose restart rag-api
|
|||
**解决**:
|
||||
- 确认 RAG API 服务已启动: `docker-compose ps`
|
||||
- 检查端口配置是否正确: `API_PORT=8001`
|
||||
- 尝试访问: http://localhost:8001/static/config/index.html
|
||||
- 尝试访问: http://localhost:8001/config 或 http://localhost:8001/static/config/index.html
|
||||
|
||||
### 6. 数据源配置保存失败
|
||||
|
||||
|
|
@ -313,6 +348,27 @@ docker-compose restart rag-api
|
|||
- 查看同步服务日志: `docker-compose logs rag-api | grep sync`
|
||||
- 手动触发同步: 在配置管理界面中点击"同步"按钮
|
||||
|
||||
### 8. Git连接失败
|
||||
|
||||
**错误**: `Git连接失败` 或 `Failed to connect to Git repository`
|
||||
|
||||
**解决**:
|
||||
- 确保 Git 仓库 URL 正确
|
||||
- 检查网络连接是否正常
|
||||
- 验证 Git 凭证(HTTPS Token 或 SSH 密钥)是否有效
|
||||
- 确保目标 Git 仓库存在且可访问
|
||||
- 查看详细错误信息: `docker-compose logs rag-api | grep git`
|
||||
|
||||
### 9. Git同步失败
|
||||
|
||||
**错误**: `Git同步失败` 或 `Failed to sync Git repository`
|
||||
|
||||
**解决**:
|
||||
- 检查 Git 仓库是否有访问权限
|
||||
- 验证本地磁盘空间是否充足
|
||||
- 查看同步服务日志获取详细错误信息: `docker-compose logs rag-api | grep sync`
|
||||
- 尝试手动触发同步: 在配置管理界面中点击"同步"按钮
|
||||
|
||||
## 性能优化建议
|
||||
|
||||
### 1. 调整配置参数
|
||||
|
|
@ -356,6 +412,9 @@ docker-compose restart rag-api
|
|||
**启动步骤**:
|
||||
|
||||
```bash
|
||||
# 0. 确保 .env 文件中的host配置准确
|
||||
cp .env.zkxlocal .env
|
||||
|
||||
# 1. 创建虚拟环境
|
||||
uv venv --python 3.13.9
|
||||
source .venv/bin/activate # Windows: venv\Scripts\activate
|
||||
|
|
@ -377,4 +436,6 @@ curl http://localhost:8003/health
|
|||
|
||||
# 7. 启动 RAG API 服务
|
||||
python main.py
|
||||
|
||||
# 8. 如要调试,使用.vscode/launch.json 启动调试会话
|
||||
```
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 174 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 196 KiB |
736
api/main.py
736
api/main.py
|
|
@ -77,6 +77,7 @@ async def lifespan(app: FastAPI):
|
|||
logger.error(f"Failed to initialize FileParser: {e}")
|
||||
raise
|
||||
|
||||
|
||||
|
||||
logger.info("✓ Core RAG services initialized")
|
||||
|
||||
|
|
@ -180,13 +181,47 @@ if STATIC_ROOT_DIR.exists():
|
|||
@app.get("/chat", include_in_schema=False)
|
||||
async def chat_interface():
|
||||
"""Serve the chat interface"""
|
||||
return FileResponse(STATIC_ROOT_DIR / "chat" / "index.html")
|
||||
chat_file = STATIC_ROOT_DIR / "chat" / "index.html"
|
||||
if not chat_file.exists():
|
||||
return Response(content="Chat file not found", status_code=404)
|
||||
|
||||
# Use aiofiles to read the file asynchronously
|
||||
try:
|
||||
import aiofiles
|
||||
async with aiofiles.open(chat_file, 'r', encoding='utf-8') as f:
|
||||
content = await f.read()
|
||||
return Response(content=content, media_type="text/html")
|
||||
except Exception as e:
|
||||
# Fallback to synchronous reading if aiofiles is not available
|
||||
try:
|
||||
with open(chat_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
return Response(content=content, media_type="text/html")
|
||||
except Exception as fallback_error:
|
||||
return Response(content=f"Error reading chat file: {str(fallback_error)}", status_code=500)
|
||||
|
||||
# Add route for configuration management interface
|
||||
@app.get("/config", include_in_schema=False)
|
||||
async def config_interface():
|
||||
"""Serve the configuration management interface"""
|
||||
return FileResponse(STATIC_ROOT_DIR / "config" / "index.html")
|
||||
config_file = STATIC_ROOT_DIR / "config" / "index.html"
|
||||
if not config_file.exists():
|
||||
return Response(content="Config file not found", status_code=404)
|
||||
|
||||
# Use aiofiles to read the file asynchronously
|
||||
try:
|
||||
import aiofiles
|
||||
async with aiofiles.open(config_file, 'r', encoding='utf-8') as f:
|
||||
content = await f.read()
|
||||
return Response(content=content, media_type="text/html")
|
||||
except Exception as e:
|
||||
# Fallback to synchronous reading if aiofiles is not available
|
||||
try:
|
||||
with open(config_file, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
return Response(content=content, media_type="text/html")
|
||||
except Exception as fallback_error:
|
||||
return Response(content=f"Error reading config file: {str(fallback_error)}", status_code=500)
|
||||
|
||||
|
||||
# Add favicon route to prevent 404 errors
|
||||
|
|
@ -281,6 +316,9 @@ class RetrieveResponse(BaseModel):
|
|||
count: int = Field(..., description="Number of documents retrieved")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
class SyncRequest(BaseModel):
|
||||
"""Manual sync request model"""
|
||||
full_sync: bool = Field(False, description="Whether to perform full sync")
|
||||
|
|
@ -919,12 +957,30 @@ async def retrieve(request: RetrieveRequest):
|
|||
try:
|
||||
|
||||
# Get retriever (works even if collection is empty, will return empty results)
|
||||
# Don't pass collection_key to search across all collections (code + non_code)
|
||||
retriever = vector_store_manager.get_retriever(top_k=request.top_k or settings.TOP_K)
|
||||
|
||||
# Retrieve documents (run in thread pool since retriever.retrieve() is synchronous)
|
||||
def retrieve_docs():
|
||||
nodes = retriever.retrieve(request.query)
|
||||
return nodes
|
||||
# Check if retriever is a list (multiple collections) or single retriever
|
||||
if isinstance(retriever, list):
|
||||
# Multiple retrievers: search across all collections
|
||||
all_nodes = []
|
||||
for collection_key, r in retriever:
|
||||
nodes = r.retrieve(request.query)
|
||||
# Add collection_key to node metadata for tracking
|
||||
for node in nodes:
|
||||
actual_node = node.node if hasattr(node, 'node') else node
|
||||
if hasattr(actual_node, 'metadata'):
|
||||
actual_node.metadata['_collection_key'] = collection_key
|
||||
all_nodes.extend(nodes)
|
||||
|
||||
# Sort by score and take top_k
|
||||
all_nodes.sort(key=lambda x: x.score if hasattr(x, 'score') else 0, reverse=True)
|
||||
return all_nodes[:request.top_k or settings.TOP_K]
|
||||
else:
|
||||
# Single retriever
|
||||
return retriever.retrieve(request.query)
|
||||
|
||||
nodes = await asyncio.to_thread(retrieve_docs)
|
||||
|
||||
|
|
@ -972,6 +1028,9 @@ async def retrieve(request: RetrieveRequest):
|
|||
raise HTTPException(status_code=500, detail=f"Error retrieving documents: {str(e)}")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@app.delete("/documents/source/{source_name}")
|
||||
async def delete_documents_by_source(source_name: str):
|
||||
"""
|
||||
|
|
@ -1170,6 +1229,17 @@ async def upload_document(
|
|||
|
||||
# Add to vector store (this may take time for large documents due to embedding generation)
|
||||
logger.info(f"Adding {len(chunked_documents)} chunks to vector store (this may take time for large files)...")
|
||||
|
||||
# Determine collection_key based on file extension
|
||||
# Code files → use 'code' collection
|
||||
# Other files → use 'non_code' collection
|
||||
code_extensions = {'.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.cpp', '.c', '.h', '.go', '.rs',
|
||||
'.rb', '.php', '.swift', '.kt', '.scala', '.cs', '.sh', '.bat', '.ps1',
|
||||
'.sql', '.r', '.m', '.lua', '.pl', '.vb', '.dart', '.groovy', '.clj'}
|
||||
file_ext = Path(file.filename).suffix.lower()
|
||||
collection_key = 'code' if file_ext in code_extensions else 'non_code'
|
||||
logger.info(f"File extension: {file_ext}, using collection: {collection_key}")
|
||||
|
||||
try:
|
||||
# Run in thread pool to avoid blocking event loop during embedding generation
|
||||
loop = asyncio.get_event_loop()
|
||||
|
|
@ -1177,7 +1247,8 @@ async def upload_document(
|
|||
None,
|
||||
vector_store_manager.add_documents,
|
||||
chunked_documents,
|
||||
False # skip_existing
|
||||
False, # skip_existing
|
||||
collection_key # collection_key based on file type
|
||||
)
|
||||
logger.info(f"Successfully added {len(chunked_documents)} chunks to vector store")
|
||||
except Exception as add_error:
|
||||
|
|
@ -1365,6 +1436,38 @@ async def create_config(config: Dict[str, Any]):
|
|||
config["host"].lower(),
|
||||
folder_path
|
||||
])
|
||||
elif config_type == "git":
|
||||
# 检查Git配置模式
|
||||
git_mode = config.get("git_mode", "single")
|
||||
|
||||
if git_mode == "single":
|
||||
# 单个Git仓库配置需要:仓库URL
|
||||
if not config.get("git_url"):
|
||||
raise HTTPException(status_code=400, detail="Git配置必须包含仓库URL")
|
||||
# 添加Git仓库URL到唯一标识符
|
||||
# 替换URL中的特殊字符为下划线
|
||||
git_url = config["git_url"].lower().replace("/", "_").replace(":", "_").replace(".", "_")
|
||||
# 截取URL的一部分作为唯一标识
|
||||
git_url_part = git_url[:100] # 限制长度
|
||||
unique_id_parts.extend([
|
||||
git_url_part
|
||||
])
|
||||
elif git_mode == "server":
|
||||
# Git服务器配置需要:服务器地址和仓库列表
|
||||
if not config.get("git_server_host"):
|
||||
raise HTTPException(status_code=400, detail="Git服务器配置必须包含服务器地址")
|
||||
if not config.get("git_repositories") or len(config.get("git_repositories")) == 0:
|
||||
raise HTTPException(status_code=400, detail="Git服务器配置必须包含仓库名称")
|
||||
# 处理所有仓库的分支,将"loading"替换为空字符串,并确保每个仓库都有path字段
|
||||
for repo in config["git_repositories"]:
|
||||
if repo.get("branch") == "loading":
|
||||
repo["branch"] = ""
|
||||
if "path" not in repo:
|
||||
repo["path"] = repo.get("repository", "")
|
||||
# 只使用服务器地址作为唯一标识,不包含仓库名
|
||||
server_name = config["git_server_host"].lower().replace("/", "_").replace(":", "_").replace(".", "_")
|
||||
server_part = server_name[:50] # 限制长度
|
||||
unique_id_parts.extend(["server", server_part])
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail=f"不支持的配置类型: {config_type}")
|
||||
|
||||
|
|
@ -1403,6 +1506,27 @@ async def create_config(config: Dict[str, Any]):
|
|||
status_code=409,
|
||||
detail=f"已存在相同服务器和路径的文件夹配置。如需调整,请点击配置列表中的配置并修改配置内容。"
|
||||
)
|
||||
elif config_type == 'git':
|
||||
# 检查Git配置模式
|
||||
existing_git_mode = existing_config_data.get('git_mode', 'single')
|
||||
current_git_mode = config.get('git_mode', 'single')
|
||||
|
||||
if existing_git_mode == current_git_mode:
|
||||
if current_git_mode == 'single':
|
||||
# 单个Git仓库配置,相同源意味着相同的git url
|
||||
if existing_config_data.get('git_url') == config.get('git_url'):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"已存在相同Git仓库的配置。如需调整,请点击配置列表中的配置并修改配置内容。"
|
||||
)
|
||||
elif current_git_mode == 'server':
|
||||
# Git服务器配置,相同源意味着相同的服务器地址和仓库名称
|
||||
if (existing_config_data.get('git_server_host') == config.get('git_server_host') and
|
||||
existing_config_data.get('git_repository') == config.get('git_repository')):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"已存在相同Git服务器和仓库的配置。如需调整,请点击配置列表中的配置并修改配置内容。"
|
||||
)
|
||||
except sqlite3.OperationalError as e:
|
||||
# 表不存在的情况,会在后面创建表
|
||||
logger.warning(f"SQLite table error: {e}. This is expected if the table doesn't exist yet.")
|
||||
|
|
@ -1433,7 +1557,7 @@ async def create_config(config: Dict[str, Any]):
|
|||
global sync_manager
|
||||
if sync_manager is not None:
|
||||
# Create appropriate data source config object
|
||||
from config import BaseDataSourceConfig, DatabaseDataSourceConfig, FolderDataSourceConfig
|
||||
from config import BaseDataSourceConfig, DatabaseDataSourceConfig, FolderDataSourceConfig, GitDataSourceConfig
|
||||
|
||||
if config_type == "database":
|
||||
source_config = DatabaseDataSourceConfig(
|
||||
|
|
@ -1470,6 +1594,37 @@ async def create_config(config: Dict[str, Any]):
|
|||
recursive=config.get("recursive", True),
|
||||
ignore_patterns=config.get("ignore_patterns")
|
||||
)
|
||||
elif config_type == "git":
|
||||
# 检查Git配置模式
|
||||
git_mode = config.get("git_mode", "single")
|
||||
|
||||
# 创建Git数据源配置
|
||||
source_config = GitDataSourceConfig(
|
||||
name=config_id,
|
||||
git_url=config.get("git_url"),
|
||||
branch=config.get("branch", config.get("git_branch", "")),
|
||||
protocol=config.get("protocol", "https"),
|
||||
ssh_key=config.get("ssh_key"),
|
||||
https_token=config.get("https_token"),
|
||||
local_repo_path=config.get("local_repo_path"),
|
||||
poll_interval=config.get("poll_interval", 300),
|
||||
support_lang=config.get("support_lang"),
|
||||
latest_commit_id=config.get("latest_commit_id"),
|
||||
last_sync_time=config.get("last_sync_time"),
|
||||
git_repositories=config.get("git_repositories", [])
|
||||
)
|
||||
|
||||
# 添加Git服务器模式的字段
|
||||
if git_mode == "server":
|
||||
source_config.git_mode = git_mode
|
||||
source_config.git_server_host = config.get("git_server_host")
|
||||
source_config.git_server_port = config.get("git_server_port", 22)
|
||||
source_config.git_server_username = config.get("git_server_username")
|
||||
source_config.git_server_password = config.get("git_server_password")
|
||||
source_config.git_server_root_path = config.get("git_server_root_path", "~")
|
||||
source_config.git_repository = config.get("git_repository")
|
||||
source_config.git_branch = config.get("git_branch", "")
|
||||
source_config.git_repositories = config.get("git_repositories", [])
|
||||
else:
|
||||
logger.warning(f"Unknown config type: {config_type}")
|
||||
# Create a base config as fallback
|
||||
|
|
@ -1540,6 +1695,466 @@ async def test_folder_connection(connection_data: Dict[str, Any]):
|
|||
raise HTTPException(status_code=500, detail=f"SSH连接失败: {str(e)}")
|
||||
|
||||
|
||||
@app.post("/git/test-connection")
|
||||
async def test_git_connection(connection_data: Dict[str, Any]):
|
||||
"""
|
||||
Test Git connection for Git repository configuration
|
||||
|
||||
Args:
|
||||
connection_data: Connection data including git_url, protocol, branch, https_token, ssh_key, password
|
||||
|
||||
Returns:
|
||||
Success message if connection is successful
|
||||
"""
|
||||
try:
|
||||
git_url = connection_data.get("git_url")
|
||||
protocol = connection_data.get("protocol", "https")
|
||||
branch = connection_data.get("branch", "main")
|
||||
https_token = connection_data.get("https_token")
|
||||
ssh_key = connection_data.get("ssh_key")
|
||||
password = connection_data.get("password")
|
||||
|
||||
if not git_url:
|
||||
raise HTTPException(status_code=400, detail="Git仓库URL是必填项")
|
||||
|
||||
# Import GitTool here to avoid circular imports
|
||||
from utils.git_tool import GitTool
|
||||
|
||||
# Create a temporary GitTool instance to test connection
|
||||
git_tool = GitTool(
|
||||
git_url=git_url,
|
||||
branch=branch,
|
||||
protocol=protocol,
|
||||
https_token=https_token,
|
||||
ssh_key=ssh_key,
|
||||
password=password,
|
||||
local_repo_path=None # 测试连接不需要本地路径
|
||||
)
|
||||
|
||||
# Try to test connection
|
||||
success = git_tool.test_connection()
|
||||
|
||||
if success:
|
||||
return {"message": "Git连接成功!"}
|
||||
else:
|
||||
raise HTTPException(status_code=500, detail="Git连接失败")
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error testing Git connection: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Git连接失败: {str(e)}")
|
||||
|
||||
|
||||
@app.post("/git/test-server-connection")
|
||||
async def test_git_server_connection(connection_data: Dict[str, Any]):
|
||||
"""
|
||||
Test Git server connection
|
||||
|
||||
Args:
|
||||
connection_data: Connection data including host, port, username, password
|
||||
|
||||
Returns:
|
||||
Success message if connection is successful
|
||||
"""
|
||||
try:
|
||||
host = connection_data.get("host")
|
||||
port = connection_data.get("port", 22)
|
||||
username = connection_data.get("username")
|
||||
password = connection_data.get("password")
|
||||
|
||||
if not host:
|
||||
raise HTTPException(status_code=400, detail="服务器地址是必填项")
|
||||
|
||||
# 对于Git daemon (端口9418),不需要用户名和密码
|
||||
if port != 9418 and not username:
|
||||
raise HTTPException(status_code=400, detail="用户名是必填项")
|
||||
|
||||
# 测试SSH连接到Git服务器
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
# 如果是Git daemon,使用git ls-remote命令测试连接
|
||||
if port == 9418:
|
||||
# 尝试连接到Git daemon
|
||||
git_daemon_url = f"git://{host}:{port}/"
|
||||
test_repos = ["repo1", "repo2", "project"]
|
||||
|
||||
for repo in test_repos:
|
||||
cmd = ["git", "ls-remote", f"git://{host}:{port}/{repo}"]
|
||||
try:
|
||||
res = subprocess.run(cmd, capture_output=True, text=True, timeout=5)
|
||||
if res.returncode == 0:
|
||||
return {"message": "Git服务器连接测试成功!"}
|
||||
except:
|
||||
continue
|
||||
|
||||
raise HTTPException(status_code=500, detail="Git服务器连接测试失败: 无法连接到Git daemon")
|
||||
else:
|
||||
# 使用paramiko库处理SSH连接
|
||||
import paramiko
|
||||
|
||||
try:
|
||||
# 创建SSH客户端
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
# 连接到服务器
|
||||
if password:
|
||||
# 使用密码认证
|
||||
client.connect(host, port=port, username=username, password=password, timeout=10)
|
||||
else:
|
||||
# 使用SSH密钥认证
|
||||
client.connect(host, port=port, username=username, timeout=10)
|
||||
|
||||
# 执行git --version命令
|
||||
stdin, stdout, stderr = client.exec_command("git --version")
|
||||
|
||||
# 获取输出
|
||||
output = stdout.read().decode('utf-8')
|
||||
error = stderr.read().decode('utf-8')
|
||||
|
||||
# 关闭连接
|
||||
client.close()
|
||||
|
||||
if output:
|
||||
return {"message": "Git服务器连接测试成功!"}
|
||||
else:
|
||||
raise HTTPException(status_code=500, detail=f"Git服务器连接测试失败: {error}")
|
||||
|
||||
except paramiko.AuthenticationException:
|
||||
raise HTTPException(status_code=401, detail="Git服务器连接测试失败: 认证失败,请检查用户名和密码")
|
||||
except paramiko.SSHException as e:
|
||||
raise HTTPException(status_code=500, detail=f"Git服务器连接测试失败: {str(e)}")
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Git服务器连接测试失败: {str(e)}")
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error testing Git server connection: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"Git服务器连接测试失败: {str(e)}")
|
||||
|
||||
|
||||
@app.post("/git/repositories")
|
||||
async def get_git_repositories(connection_data: Dict[str, Any]):
|
||||
"""
|
||||
Get Git repositories from server
|
||||
|
||||
Args:
|
||||
connection_data: Connection data including host, port, username, password
|
||||
|
||||
Returns:
|
||||
List of repositories
|
||||
"""
|
||||
try:
|
||||
host = connection_data.get("host")
|
||||
port = connection_data.get("port", 22)
|
||||
username = connection_data.get("username")
|
||||
password = connection_data.get("password")
|
||||
|
||||
if not host:
|
||||
raise HTTPException(status_code=400, detail="服务器地址是必填项")
|
||||
|
||||
# 对于Git daemon (端口9418),不需要用户名和密码
|
||||
if port != 9418 and not username:
|
||||
raise HTTPException(status_code=400, detail="用户名是必填项")
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
# 对于Git daemon,使用git ls-remote命令获取仓库列表
|
||||
# 问题 @xmy:这里是啥意思,固定了是9418端口、test-git-server目录?
|
||||
if port == 9418:
|
||||
# 尝试获取仓库列表
|
||||
# 直接返回已知的仓库列表,因为Git daemon可能不会直接返回仓库列表
|
||||
# 从test-git-server目录中获取仓库列表
|
||||
test_repos_dir = os.path.join(os.path.dirname(__file__), "..", "test-git-server")
|
||||
repositories = []
|
||||
|
||||
if os.path.exists(test_repos_dir):
|
||||
for item in os.listdir(test_repos_dir):
|
||||
item_path = os.path.join(test_repos_dir, item)
|
||||
if os.path.isdir(item_path):
|
||||
# 检查是否是Git仓库(包含.git目录或本身就是Git仓库)
|
||||
git_dir = os.path.join(item_path, '.git')
|
||||
if os.path.isdir(git_dir) or (os.path.exists(os.path.join(item_path, 'HEAD')) and os.path.exists(os.path.join(item_path, 'config'))):
|
||||
# 检查仓库是否已存在
|
||||
if item not in [r['name'] for r in repositories]:
|
||||
# 存储为字典,包含名称和路径
|
||||
repositories.append({
|
||||
"name": item,
|
||||
"path": item_path
|
||||
})
|
||||
|
||||
# 如果没有找到仓库,尝试使用git ls-remote命令
|
||||
if not repositories:
|
||||
git_daemon_url = f"git://{host}:{port}/"
|
||||
test_repos = ["project", "repo1", "repo2", "test-repo"]
|
||||
|
||||
for repo in test_repos:
|
||||
cmd = ["git", "ls-remote", f"git://{host}:{port}/{repo}"]
|
||||
try:
|
||||
res = subprocess.run(cmd, capture_output=True, text=True, timeout=5)
|
||||
if res.returncode == 0:
|
||||
# 存储为字典,包含名称和路径
|
||||
repositories.append({
|
||||
"name": repo,
|
||||
"path": f"/{repo}"
|
||||
})
|
||||
except:
|
||||
continue
|
||||
else:
|
||||
# 对于SSH连接,登录服务器并列出git仓库
|
||||
import paramiko
|
||||
import os
|
||||
|
||||
try:
|
||||
# 创建SSH客户端
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
# 连接到服务器
|
||||
if password:
|
||||
# 使用密码认证
|
||||
client.connect(host, port=port, username=username, password=password, timeout=10)
|
||||
else:
|
||||
# 使用SSH密钥认证
|
||||
client.connect(host, port=port, username=username, timeout=10)
|
||||
|
||||
# 优先使用用户指定的仓库根目录地址
|
||||
root_path = connection_data.get("git_server_root_path", "~")
|
||||
repositories = []
|
||||
|
||||
# 执行find命令在指定根目录下查找所有.git目录
|
||||
stdin, stdout, stderr = client.exec_command(f"find {root_path} -type d -name \"*.git\"")
|
||||
|
||||
# 获取输出
|
||||
output = stdout.read().decode('utf-8')
|
||||
error = stderr.read().decode('utf-8')
|
||||
|
||||
# 解析输出
|
||||
if not error:
|
||||
for line in output.splitlines():
|
||||
if line.endswith('.git'):
|
||||
# 验证是否为有效的git仓库
|
||||
repo_path = line
|
||||
stdin, stdout, stderr = client.exec_command(f"git -C {repo_path} rev-parse --is-bare-repository")
|
||||
git_output = stdout.read().decode('utf-8').strip()
|
||||
git_error = stderr.read().decode('utf-8')
|
||||
|
||||
# 如果命令执行成功,说明是有效的git仓库
|
||||
if not git_error:
|
||||
# 提取仓库名称
|
||||
repo_name = os.path.basename(line)[:-4] # 移除.git后缀
|
||||
if repo_name and repo_name not in [r['name'] for r in repositories]:
|
||||
# 存储为字典,包含名称和完整路径
|
||||
repositories.append({
|
||||
"name": repo_name,
|
||||
"path": line
|
||||
})
|
||||
logger.info(f"找到仓库: {repo_name} 在路径: {line}")
|
||||
|
||||
# 如果没有找到仓库,尝试在常见目录中搜索
|
||||
if not repositories:
|
||||
# 尝试多个可能的Git仓库目录
|
||||
git_dirs = ["/home/admin/", "~/git-repos/", "~/repos/", "~/git/", "~/repositories/", "~"]
|
||||
|
||||
for git_dir in git_dirs:
|
||||
# 执行ls命令列出git仓库
|
||||
stdin, stdout, stderr = client.exec_command(f"ls -la {git_dir}")
|
||||
|
||||
# 获取输出
|
||||
output = stdout.read().decode('utf-8')
|
||||
error = stderr.read().decode('utf-8')
|
||||
|
||||
if not error:
|
||||
for line in output.splitlines():
|
||||
if line.endswith('.git'):
|
||||
repo_path = f"{git_dir}/{line.split()[-1]}"
|
||||
# 验证是否为有效的git仓库
|
||||
stdin, stdout, stderr = client.exec_command(f"git -C {repo_path} rev-parse --is-bare-repository")
|
||||
git_output = stdout.read().decode('utf-8').strip()
|
||||
git_error = stderr.read().decode('utf-8')
|
||||
|
||||
if not git_error:
|
||||
repo_name = line.split()[-1][:-4] # 移除.git后缀
|
||||
if repo_name and repo_name not in [r['name'] for r in repositories]:
|
||||
repositories.append({
|
||||
"name": repo_name,
|
||||
"path": repo_path
|
||||
})
|
||||
logger.info(f"找到仓库: {repo_name} 在路径: {repo_path}")
|
||||
|
||||
# 关闭连接
|
||||
client.close()
|
||||
|
||||
# 如果仍然没有找到仓库,返回空列表
|
||||
if not repositories:
|
||||
repositories = []
|
||||
|
||||
except paramiko.AuthenticationException:
|
||||
raise HTTPException(status_code=401, detail="获取仓库列表失败: 认证失败,请检查用户名和密码")
|
||||
except paramiko.SSHException as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取仓库列表失败: {str(e)}")
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取仓库列表失败: {str(e)}")
|
||||
|
||||
return {"repositories": repositories}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting Git repositories: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"获取仓库列表失败: {str(e)}")
|
||||
|
||||
|
||||
@app.post("/git/branches")
|
||||
async def get_git_branches(connection_data: Dict[str, Any]):
|
||||
"""
|
||||
Get Git branches from repository
|
||||
|
||||
Args:
|
||||
connection_data: Connection data including host, port, username, password, repository
|
||||
|
||||
Returns:
|
||||
List of branches
|
||||
"""
|
||||
try:
|
||||
host = connection_data.get("host")
|
||||
port = connection_data.get("port", 22)
|
||||
username = connection_data.get("username")
|
||||
password = connection_data.get("password")
|
||||
repository = connection_data.get("repository")
|
||||
|
||||
if not host or not repository:
|
||||
raise HTTPException(status_code=400, detail="服务器地址和仓库名称是必填项")
|
||||
|
||||
# 对于Git daemon (端口9418),不需要用户名和密码
|
||||
if port != 9418 and not username:
|
||||
raise HTTPException(status_code=400, detail="用户名是必填项")
|
||||
|
||||
import subprocess
|
||||
|
||||
# 构建仓库URL
|
||||
if port == 9418:
|
||||
# 使用git ls-remote命令获取分支列表
|
||||
repo_url = f"git://{host}:{port}/{repository}"
|
||||
cmd = ["git", "ls-remote", "--heads", repo_url]
|
||||
logger.info(f"获取Git分支列表: {' '.join(cmd)}")
|
||||
|
||||
try:
|
||||
res = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||
if res.returncode != 0:
|
||||
raise HTTPException(status_code=500, detail=f"获取分支列表失败: {res.stderr}")
|
||||
|
||||
# 解析分支列表
|
||||
branches = []
|
||||
for line in res.stdout.splitlines():
|
||||
if line:
|
||||
parts = line.split()
|
||||
if len(parts) > 1:
|
||||
branch_path = parts[1]
|
||||
if branch_path.startswith('refs/heads/'):
|
||||
branch_name = branch_path[11:] # 移除refs/heads/前缀
|
||||
branches.append(branch_name)
|
||||
|
||||
# 如果没有分支,返回空列表
|
||||
if not branches:
|
||||
branches = []
|
||||
except Exception as e:
|
||||
logger.error(f"获取分支列表失败: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"获取分支列表失败: {str(e)}")
|
||||
else:
|
||||
# 使用paramiko库处理SSH连接
|
||||
import paramiko
|
||||
|
||||
try:
|
||||
# 创建SSH客户端
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
# 连接到服务器
|
||||
if password:
|
||||
# 使用密码认证
|
||||
client.connect(host, port=port, username=username, password=password, timeout=10)
|
||||
else:
|
||||
# 使用SSH密钥认证
|
||||
client.connect(host, port=port, username=username, timeout=10)
|
||||
|
||||
# 尝试多个可能的Git仓库路径
|
||||
repo_paths = []
|
||||
|
||||
# 如果repository已经是完整路径,直接使用
|
||||
if repository.startswith('/'):
|
||||
repo_paths.append(repository)
|
||||
else:
|
||||
# 否则尝试多个可能的Git仓库路径
|
||||
repo_paths = [
|
||||
f"~/git-repos/{repository}.git",
|
||||
f"~/repos/{repository}.git",
|
||||
f"~/git/{repository}.git",
|
||||
f"~/repositories/{repository}.git",
|
||||
f"~/{repository}.git",
|
||||
f"/home/admin/{repository}.git"
|
||||
]
|
||||
|
||||
# 尝试找到正确的仓库路径
|
||||
found = False
|
||||
output = ""
|
||||
error = ""
|
||||
|
||||
for repo_path in repo_paths:
|
||||
# 执行git ls-remote命令获取分支列表
|
||||
stdin, stdout, stderr = client.exec_command(f"git ls-remote --heads {repo_path}")
|
||||
|
||||
# 获取输出
|
||||
current_output = stdout.read().decode('utf-8')
|
||||
current_error = stderr.read().decode('utf-8')
|
||||
|
||||
# 如果成功,使用这个路径
|
||||
if not current_error and current_output:
|
||||
output = current_output
|
||||
error = current_error
|
||||
found = True
|
||||
break
|
||||
|
||||
# 关闭连接
|
||||
client.close()
|
||||
|
||||
if not found:
|
||||
raise HTTPException(status_code=500, detail="获取分支列表失败: 未找到仓库路径")
|
||||
|
||||
if error:
|
||||
raise HTTPException(status_code=500, detail=f"获取分支列表失败: {error}")
|
||||
|
||||
# 解析分支列表
|
||||
branches = []
|
||||
for line in output.splitlines():
|
||||
if line:
|
||||
parts = line.split()
|
||||
if len(parts) > 1:
|
||||
branch_path = parts[1]
|
||||
if branch_path.startswith('refs/heads/'):
|
||||
branch_name = branch_path[11:] # 移除refs/heads/前缀
|
||||
branches.append(branch_name)
|
||||
|
||||
except paramiko.AuthenticationException:
|
||||
raise HTTPException(status_code=401, detail="获取分支列表失败: 认证失败,请检查用户名和密码")
|
||||
except paramiko.SSHException as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取分支列表失败: {str(e)}")
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"获取分支列表失败: {str(e)}")
|
||||
|
||||
return {"branches": branches}
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting Git branches: {e}")
|
||||
raise HTTPException(status_code=500, detail=f"获取分支列表失败: {str(e)}")
|
||||
|
||||
|
||||
@app.post("/folder-configs/remote")
|
||||
async def create_remote_folder_config(config: Dict[str, Any]):
|
||||
"""
|
||||
|
|
@ -1579,8 +2194,10 @@ async def update_folder_config(config_id: str, config: Dict[str, Any]):
|
|||
cursor.execute('SELECT * FROM data_sources WHERE name = ?', (config_id,))
|
||||
existing_config = cursor.fetchone()
|
||||
|
||||
# If config doesn't exist, consider deletion successful (idempotent operation)
|
||||
if not existing_config:
|
||||
raise HTTPException(status_code=404, detail=f"Configuration with ID '{config_id}' not found")
|
||||
logger.info(f"Configuration '{config_id}' already deleted or does not exist")
|
||||
return {"status": "success", "message": f"Configuration '{config_id}' deleted successfully"}
|
||||
|
||||
# Ensure config has required fields
|
||||
if "type" not in config:
|
||||
|
|
@ -1615,6 +2232,43 @@ async def update_folder_config(config_id: str, config: Dict[str, Any]):
|
|||
else:
|
||||
# 本地文件夹:使用文件夹路径生成ID
|
||||
new_config_id = f"{config_type}_{folder_path}"
|
||||
elif config_type == "git":
|
||||
# Git配置:根据配置模式生成ID
|
||||
git_mode = config.get("git_mode", "single")
|
||||
if git_mode == "single":
|
||||
# 单个Git仓库配置
|
||||
if not config.get("git_url"):
|
||||
raise HTTPException(status_code=400, detail="Git URL is required for single Git configuration")
|
||||
# 替换URL中的特殊字符为下划线
|
||||
git_url = config["git_url"].lower().replace("/", "_").replace(":", "_").replace(".", "_")
|
||||
# 截取URL的一部分作为唯一标识
|
||||
git_url_part = git_url[:100] # 限制长度
|
||||
branch = config.get("branch", "main")
|
||||
# 如果分支为"loading",保存为空字符串
|
||||
if branch == "loading":
|
||||
branch = ""
|
||||
config["branch"] = ""
|
||||
new_config_id = f"{config_type}_{git_url_part}_{branch}"
|
||||
elif git_mode == "server":
|
||||
# Git服务器配置
|
||||
if not config.get("git_server_host"):
|
||||
raise HTTPException(status_code=400, detail="Git server host is required for server Git configuration")
|
||||
if not config.get("git_repositories") or len(config.get("git_repositories")) == 0:
|
||||
raise HTTPException(status_code=400, detail="Git repository is required for server Git configuration")
|
||||
|
||||
# 处理所有仓库的分支,将"loading"替换为空字符串
|
||||
for repo in config["git_repositories"]:
|
||||
if repo.get("branch") == "loading":
|
||||
repo["branch"] = ""
|
||||
# 确保每个仓库都有path字段
|
||||
if "path" not in repo:
|
||||
repo["path"] = repo.get("repository", "")
|
||||
|
||||
# 替换服务器地址中的特殊字符为下划线
|
||||
server_name = config["git_server_host"].lower().replace("/", "_").replace(":", "_").replace(".", "_")
|
||||
# 只使用服务器地址作为配置ID,不包含仓库名
|
||||
server_part = server_name[:50] # 限制长度
|
||||
new_config_id = f"{config_type}_server_{server_part}"
|
||||
|
||||
# 如果无法生成新的有意义的ID,保留原来的ID
|
||||
if not new_config_id:
|
||||
|
|
@ -1640,7 +2294,7 @@ async def update_folder_config(config_id: str, config: Dict[str, Any]):
|
|||
global sync_manager
|
||||
if sync_manager is not None:
|
||||
# Create appropriate data source config object
|
||||
from config import BaseDataSourceConfig, DatabaseDataSourceConfig, FolderDataSourceConfig
|
||||
from config import BaseDataSourceConfig, DatabaseDataSourceConfig, FolderDataSourceConfig, GitDataSourceConfig
|
||||
|
||||
if config_type == "database":
|
||||
source_config = DatabaseDataSourceConfig(
|
||||
|
|
@ -1677,6 +2331,38 @@ async def update_folder_config(config_id: str, config: Dict[str, Any]):
|
|||
recursive=config.get("recursive", True),
|
||||
ignore_patterns=config.get("ignore_patterns")
|
||||
)
|
||||
elif config_type == "git":
|
||||
# 检查Git配置模式
|
||||
git_mode = config.get("git_mode", "single")
|
||||
|
||||
# 创建Git数据源配置
|
||||
source_config = GitDataSourceConfig(
|
||||
name=new_config_id,
|
||||
git_url=config.get("git_url"),
|
||||
branch=config.get("branch", config.get("git_branch", "main")),
|
||||
protocol=config.get("protocol", "https"),
|
||||
ssh_key=config.get("ssh_key"),
|
||||
https_token=config.get("https_token"),
|
||||
local_repo_path=config.get("local_repo_path"),
|
||||
poll_interval=config.get("poll_interval", 300),
|
||||
support_lang=config.get("support_lang"),
|
||||
latest_commit_id=config.get("latest_commit_id"),
|
||||
last_sync_time=config.get("last_sync_time"),
|
||||
git_repositories=config.get("git_repositories", [])
|
||||
)
|
||||
|
||||
# 添加Git服务器模式的字段
|
||||
if git_mode == "server":
|
||||
source_config.git_mode = git_mode
|
||||
source_config.git_server_host = config.get("git_server_host")
|
||||
source_config.git_server_port = config.get("git_server_port", 22)
|
||||
source_config.git_server_username = config.get("git_server_username")
|
||||
source_config.git_server_password = config.get("git_server_password")
|
||||
# 使用第一个仓库作为默认仓库,保持向后兼容
|
||||
if config.get("git_repositories") and len(config.get("git_repositories")) > 0:
|
||||
first_repo = config["git_repositories"][0]
|
||||
source_config.git_repository = first_repo.get("repository")
|
||||
source_config.git_branch = first_repo.get("branch", "main")
|
||||
else:
|
||||
logger.warning(f"Unknown config type: {config_type}")
|
||||
# Create a base config as fallback
|
||||
|
|
@ -1724,6 +2410,35 @@ async def delete_folder_config(config_id: str):
|
|||
if not existing_config:
|
||||
raise HTTPException(status_code=404, detail=f"Configuration with ID '{config_id}' not found")
|
||||
|
||||
# BUG(但不影响): 不存在config_json。
|
||||
# NOTE:不影响删除逻辑,都是直接按config_id ~对应上~ db_source 进行删除
|
||||
# # Parse config to check git_mode
|
||||
# config_json = existing_config[2] if len(existing_config) > 2 else None
|
||||
# config_dict = None
|
||||
# if config_json:
|
||||
# try:
|
||||
# config_dict = json.loads(config_json)
|
||||
# logger.info(f"Config JSON: {config_json}")
|
||||
# logger.info(f"Parsed config_dict: {config_dict}")
|
||||
# except:
|
||||
# pass
|
||||
|
||||
# # Get git_repositories if it's a git server mode
|
||||
# repo_ids_to_delete = [config_id]
|
||||
# if config_dict and config_dict.get('type') == 'git':
|
||||
# git_mode = config_dict.get('git_mode')
|
||||
# git_repositories = config_dict.get('git_repositories', [])
|
||||
|
||||
# if git_mode == 'server' and git_repositories:
|
||||
# # For git server mode, generate repo_id for each repository
|
||||
# for repo in git_repositories:
|
||||
# if isinstance(repo, dict):
|
||||
# repo_name = repo.get('repository')
|
||||
# if repo_name:
|
||||
# repo_ids_to_delete.append(f"{config_id}_{repo_name}")
|
||||
|
||||
logger.info(f"Deleting config: {config_id}")
|
||||
|
||||
# Delete config from data_sources table
|
||||
cursor.execute('DELETE FROM data_sources WHERE name = ?', (config_id,))
|
||||
|
||||
|
|
@ -1734,6 +2449,11 @@ async def delete_folder_config(config_id: str):
|
|||
if sync_manager is not None:
|
||||
sync_manager.remove_sync_service(config_id)
|
||||
|
||||
# Delete documents from ChromaDB for this data source
|
||||
global vector_store_manager
|
||||
if vector_store_manager is not None:
|
||||
vector_store_manager.delete_documents_by_source(target_source=config_id, collection_key=None)
|
||||
|
||||
return {"status": "success", "message": f"Configuration '{config_id}' deleted successfully"}
|
||||
|
||||
except HTTPException:
|
||||
|
|
|
|||
163
config.py
163
config.py
|
|
@ -111,6 +111,39 @@ class FolderDataSourceConfig(BaseDataSourceConfig):
|
|||
self.ignore_patterns = ignore_patterns # 忽略的文件模式列表
|
||||
|
||||
|
||||
class GitDataSourceConfig(BaseDataSourceConfig):
|
||||
"""Git data source configuration"""
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
git_url: Optional[str] = None,
|
||||
branch: str = "main",
|
||||
protocol: str = "https", # https 或 ssh
|
||||
ssh_key: Optional[str] = None, # SSH私钥
|
||||
https_token: Optional[str] = None, # HTTPS令牌
|
||||
local_repo_path: Optional[str] = None, # 本地存储路径
|
||||
poll_interval: int = 300, # 轮询间隔(秒)
|
||||
support_lang: Optional[List[str]] = None, # 支持的编程语言
|
||||
latest_commit_id: Optional[str] = None, # 最新commit ID
|
||||
last_sync_time: Optional[str] = None, # 最后同步时间
|
||||
git_repositories: Optional[List[Dict[str, str]]] = None, # Git服务器模式下的仓库列表,每个仓库包含repository和branch字段
|
||||
git_server_root_path: Optional[str] = None, # Git服务器模式下的仓库根目录地址
|
||||
):
|
||||
super().__init__(name, "git")
|
||||
self.git_url = git_url # Git仓库地址
|
||||
self.branch = branch # 分支名称
|
||||
self.protocol = protocol # 协议类型
|
||||
self.ssh_key = ssh_key # SSH私钥(加密存储)
|
||||
self.https_token = https_token # HTTPS令牌(加密存储)
|
||||
self.local_repo_path = local_repo_path # 本地存储路径
|
||||
self.poll_interval = poll_interval # 轮询间隔
|
||||
self.support_lang = support_lang # 支持的编程语言
|
||||
self.latest_commit_id = latest_commit_id # 最新commit ID
|
||||
self.last_sync_time = last_sync_time # 最后同步时间
|
||||
self.git_repositories = git_repositories # Git服务器模式下的仓库列表
|
||||
self.git_server_root_path = git_server_root_path # Git服务器模式下的仓库根目录地址
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""
|
||||
Application settings
|
||||
|
|
@ -138,31 +171,25 @@ class Settings(BaseSettings):
|
|||
CHROMA_DB_PATH: str = "./chroma_db" # Only used for PersistentClient mode
|
||||
CHROMA_COLLECTION_NAME: str = "rag_collection"
|
||||
|
||||
# LLM Settings
|
||||
# LLM provider: "ollama", "vllm", "openai", "deepseek" etc.
|
||||
LLM_PROVIDER: str = "ollama"
|
||||
LLM_BASE_URL: str = "http://localhost:11434"
|
||||
LLM_MODEL: str = "qwen3:8b"
|
||||
LLM_API_KEY: Optional[str] = None
|
||||
|
||||
# Embedding Settings
|
||||
# Embedding provider: "ollama", "vllm", "openai" etc.
|
||||
EMBEDDING_PROVIDER: str = "ollama"
|
||||
EMBEDDING_BASE_URL: str = "http://localhost:11434"
|
||||
EMBEDDING_MODEL: str = "qwen3-embedding:0.6b"
|
||||
EMBEDDING_API_KEY: Optional[str] = None
|
||||
|
||||
# Legacy Ollama Settings (for backward compatibility)
|
||||
# Ollama Settings
|
||||
# Configure OLLAMA_BASE_URL in .env file based on your deployment
|
||||
# - Local: http://localhost:11434
|
||||
# - Remote: http://192.168.1.100:11434
|
||||
OLLAMA_BASE_URL: str = "http://localhost:11434"
|
||||
OLLAMA_MODEL: str = "qwen3:8b"
|
||||
OLLAMA_EMBEDDING_MODEL: str = "qwen3-embedding:0.6b"
|
||||
OLLAMA_MODEL: str = "qwen3:1.7b" # LLM model for text generation
|
||||
OLLAMA_EMBEDDING_MODEL: str = "qwen3-embedding:0.6b" # Embedding model for vectorization
|
||||
|
||||
# RAG Settings
|
||||
EMBEDDING_DIMENSION: int = 768
|
||||
CHUNK_SIZE: int = 1024
|
||||
CHUNK_SIZE: int = 4000
|
||||
CHUNK_OVERLAP: int = 200
|
||||
TOP_K: int = 5 # Number of documents to retrieve
|
||||
|
||||
# RAG Query Classification Settings
|
||||
CODE_RELATED_THRESHOLD_LOW: float = 0.3 # 低于此值认为是非代码问题
|
||||
CODE_RELATED_THRESHOLD_HIGH: float = 0.7 # 高于此值认为是代码相关问题
|
||||
FILTER_METADATA_FIELDS: str = "class_name,func_name,file_path" # 从query中提取的metadata字段
|
||||
|
||||
# Sync Settings
|
||||
SYNC_INTERVAL: int = 300 # Sync interval in seconds
|
||||
AUTO_SYNC: bool = True
|
||||
|
|
@ -187,6 +214,12 @@ class Settings(BaseSettings):
|
|||
SOFFICE_HOST: str = "127.0.0.1"
|
||||
SOFFICE_PORT: int = 8003
|
||||
|
||||
# Git 相关配置
|
||||
GIT_LOCAL_STORAGE_ROOT: str = "./git_repos" # Git仓库本地存储根目录
|
||||
GIT_DEFAULT_BRANCH: str = "main" # 默认分支
|
||||
GIT_POLL_INTERVAL: int = 300 # 默认轮询间隔(秒)
|
||||
GIT_MAX_REPO_SIZE_MB: int = 500 # 最大仓库大小(MB)
|
||||
|
||||
# Pydantic v2 configuration
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
|
|
@ -195,71 +228,6 @@ class Settings(BaseSettings):
|
|||
extra="ignore" # Ignore extra fields in .env file that are not defined in Settings
|
||||
)
|
||||
|
||||
def get_llm_config(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get LLM configuration with backward compatibility for legacy OLLAMA_* settings.
|
||||
|
||||
Returns:
|
||||
Dictionary with provider, base_url, model, and api_key
|
||||
"""
|
||||
if self.LLM_PROVIDER == "ollama":
|
||||
return {
|
||||
"provider": "ollama",
|
||||
"base_url": self.LLM_BASE_URL or self.OLLAMA_BASE_URL,
|
||||
"model": self.LLM_MODEL or self.OLLAMA_MODEL,
|
||||
"api_key": self.LLM_API_KEY
|
||||
}
|
||||
elif self.LLM_PROVIDER == "openai":
|
||||
return {
|
||||
"provider": "openai",
|
||||
"base_url": self.LLM_BASE_URL or "https://api.openai.com/v1",
|
||||
"model": self.LLM_MODEL,
|
||||
"api_key": self.LLM_API_KEY
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"provider": self.LLM_PROVIDER,
|
||||
"base_url": self.LLM_BASE_URL,
|
||||
"model": self.LLM_MODEL,
|
||||
"api_key": self.LLM_API_KEY
|
||||
}
|
||||
|
||||
def get_embedding_config(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get Embedding configuration with backward compatibility for legacy OLLAMA_* settings.
|
||||
|
||||
Returns:
|
||||
Dictionary with provider, base_url, model, and api_key
|
||||
"""
|
||||
if self.EMBEDDING_PROVIDER == "ollama":
|
||||
return {
|
||||
"provider": "ollama",
|
||||
"base_url": self.EMBEDDING_BASE_URL or self.OLLAMA_BASE_URL,
|
||||
"model": self.EMBEDDING_MODEL or self.OLLAMA_EMBEDDING_MODEL,
|
||||
"api_key": self.EMBEDDING_API_KEY
|
||||
}
|
||||
elif self.EMBEDDING_PROVIDER == "vllm":
|
||||
return {
|
||||
"provider": "vllm",
|
||||
"base_url": self.EMBEDDING_BASE_URL,
|
||||
"model": self.EMBEDDING_MODEL,
|
||||
"api_key": self.EMBEDDING_API_KEY
|
||||
}
|
||||
elif self.EMBEDDING_PROVIDER == "openai":
|
||||
return {
|
||||
"provider": "openai",
|
||||
"base_url": self.EMBEDDING_BASE_URL or "https://api.openai.com/v1",
|
||||
"model": self.EMBEDDING_MODEL,
|
||||
"api_key": self.EMBEDDING_API_KEY
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"provider": self.EMBEDDING_PROVIDER,
|
||||
"base_url": self.EMBEDDING_BASE_URL,
|
||||
"model": self.EMBEDDING_MODEL,
|
||||
"api_key": self.EMBEDDING_API_KEY
|
||||
}
|
||||
|
||||
def get_data_sources(self) -> List[BaseDataSourceConfig]:
|
||||
"""
|
||||
Get list of data source configurations
|
||||
|
|
@ -336,6 +304,35 @@ class Settings(BaseSettings):
|
|||
recursive=ds_config.get('recursive', True),
|
||||
ignore_patterns=ds_config.get('ignore_patterns', None)
|
||||
))
|
||||
elif source_type == 'git':
|
||||
# Create git data source
|
||||
git_config = GitDataSourceConfig(
|
||||
name=name, # 使用数据库表中的name列
|
||||
git_url=ds_config.get('git_url'),
|
||||
branch=ds_config.get('branch', ds_config.get('git_branch', 'main')),
|
||||
protocol=ds_config.get('protocol', 'https'),
|
||||
ssh_key=ds_config.get('ssh_key'),
|
||||
https_token=ds_config.get('https_token'),
|
||||
local_repo_path=ds_config.get('local_repo_path'),
|
||||
poll_interval=ds_config.get('poll_interval', 300),
|
||||
support_lang=ds_config.get('support_lang'),
|
||||
latest_commit_id=ds_config.get('latest_commit_id'),
|
||||
last_sync_time=ds_config.get('last_sync_time')
|
||||
)
|
||||
|
||||
# 添加Git服务器模式的字段
|
||||
git_mode = ds_config.get('git_mode', 'single')
|
||||
if git_mode == 'server':
|
||||
git_config.git_mode = git_mode
|
||||
git_config.git_server_host = ds_config.get('git_server_host')
|
||||
git_config.git_server_port = ds_config.get('git_server_port', 22)
|
||||
git_config.git_server_username = ds_config.get('git_server_username')
|
||||
git_config.git_server_password = ds_config.get('git_server_password')
|
||||
git_config.git_repository = ds_config.get('git_repository')
|
||||
git_config.git_branch = ds_config.get('git_branch', 'main')
|
||||
git_config.git_repositories = ds_config.get('git_repositories', [])
|
||||
|
||||
configs.append(git_config)
|
||||
else:
|
||||
from loguru import logger
|
||||
logger.warning(f"Unknown data source type: {source_type}, skipping")
|
||||
|
|
|
|||
70
db_utils.py
70
db_utils.py
|
|
@ -2,6 +2,7 @@
|
|||
Database utilities for RAG system
|
||||
"""
|
||||
import sqlite3
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Tuple, Optional
|
||||
|
|
@ -90,6 +91,75 @@ def update_data_source_update_at(source_name: str, update_at: datetime) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def add_git_datasource(user_id: str, repo_config: dict):
|
||||
"""
|
||||
新增Git仓库配置
|
||||
|
||||
Args:
|
||||
user_id: 用户ID
|
||||
repo_config: 仓库配置
|
||||
"""
|
||||
conn, cursor = get_db_connection()
|
||||
try:
|
||||
cursor.execute("""
|
||||
INSERT INTO datasource (user_id, name, type, git_config, create_time)
|
||||
VALUES (?, ?, 'git', ?, datetime('now'))
|
||||
""", (user_id, repo_config["name"], json.dumps(repo_config)))
|
||||
conn.commit()
|
||||
logger.info(f"新增Git数据源: {repo_config['name']}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def update_git_sync_status(user_id: str, repo_id: str, sync_status: dict):
|
||||
"""
|
||||
更新Git仓库同步状态
|
||||
|
||||
Args:
|
||||
user_id: 用户ID
|
||||
repo_id: 仓库ID
|
||||
sync_status: 同步状态
|
||||
"""
|
||||
conn, cursor = get_db_connection()
|
||||
try:
|
||||
cursor.execute("""
|
||||
UPDATE datasource SET git_config = json_set(git_config, '$.latest_commit_id', ?, '$.last_sync_time', ?)
|
||||
WHERE user_id = ? AND id = ?
|
||||
""", (sync_status["latest_commit_id"], sync_status["last_sync_time"], user_id, repo_id))
|
||||
conn.commit()
|
||||
logger.info(f"更新Git同步状态: {repo_id}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def update_git_repo_config(user_id: str, repo_id: str, config: dict):
|
||||
"""
|
||||
更新Git仓库配置
|
||||
|
||||
Args:
|
||||
user_id: 用户ID
|
||||
repo_id: 仓库ID
|
||||
config: 配置信息
|
||||
"""
|
||||
conn, cursor = get_db_connection()
|
||||
try:
|
||||
# 获取当前配置
|
||||
cursor.execute("SELECT git_config FROM datasource WHERE user_id = ? AND id = ?", (user_id, repo_id))
|
||||
result = cursor.fetchone()
|
||||
if result:
|
||||
current_config = json.loads(result[0])
|
||||
# 更新配置
|
||||
current_config.update(config)
|
||||
cursor.execute("""
|
||||
UPDATE datasource SET git_config = ?
|
||||
WHERE user_id = ? AND id = ?
|
||||
""", (json.dumps(current_config), user_id, repo_id))
|
||||
conn.commit()
|
||||
logger.info(f"更新Git仓库配置: {repo_id}")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def init_session_db():
|
||||
"""
|
||||
Initialize session database with users and sessions tables
|
||||
|
|
|
|||
|
|
@ -52,7 +52,10 @@ services:
|
|||
container_name: rag-api
|
||||
# 使用宿主机网络模式,可以直接访问宿主机上的服务(Ollama、MySQL 等)
|
||||
# 注意:使用 host 网络模式时,不能使用 ports 映射,容器直接使用宿主机的网络
|
||||
network_mode: host
|
||||
# network_mode: host # 注释/删除host网络模式(Windows下无效)
|
||||
# 添加端口映射(Windows下开发)
|
||||
ports:
|
||||
- "${API_PORT:-8001}:8001"
|
||||
# 自动读取 .env 文件(如果存在)
|
||||
env_file:
|
||||
- .env
|
||||
|
|
@ -96,7 +99,7 @@ services:
|
|||
|
||||
# RAG 配置
|
||||
- EMBEDDING_DIMENSION=${EMBEDDING_DIMENSION:-768}
|
||||
- CHUNK_SIZE=${CHUNK_SIZE:-1024}
|
||||
- CHUNK_SIZE=${CHUNK_SIZE:-4000}
|
||||
- CHUNK_OVERLAP=${CHUNK_OVERLAP:-200}
|
||||
- TOP_K=${TOP_K:-5}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ RUN apt-get update \
|
|||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
RUN pip install --no-cache-dir -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
|
||||
COPY docker/soffice/app.py /app/app.py
|
||||
COPY docker/soffice/README.md /app/README.md
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -52,6 +52,7 @@ dependencies = [
|
|||
"pandas>=2.0.0", # CSV parsing (required by LlamaIndex for CSV files)
|
||||
"markdown>=3.5.0", # Markdown parsing
|
||||
# Note: Text (.txt) and JSON (.json) are handled by Python standard library, no extra dependencies needed
|
||||
# "rank-bm25", # Note: for hybrid search (deprecated)
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
|
@ -73,6 +74,11 @@ Issues = "https://github.com/your-org/rag-api/issues"
|
|||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "tuna"
|
||||
url = "https://pypi.tuna.tsinghua.edu.cn/simple/"
|
||||
default = true
|
||||
|
||||
# 配置 hatchling 不尝试打包项目(这是一个应用,不是库)
|
||||
# 使用 --no-install-project 时,这个配置不会被使用,但保留以避免构建错误
|
||||
[tool.hatch.build.targets.wheel]
|
||||
|
|
@ -88,10 +94,6 @@ dev = [
|
|||
"mypy>=1.0.0",
|
||||
]
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "tuna"
|
||||
url = "https://pypi.tuna.tsinghua.edu.cn/simple/"
|
||||
|
||||
[tool.black]
|
||||
line-length = 100
|
||||
target-version = ['py38', 'py39', 'py310', 'py311', 'py312', 'py313']
|
||||
|
|
|
|||
|
|
@ -5,15 +5,16 @@ from llama_index.core.query_engine import RetrieverQueryEngine
|
|||
from llama_index.core.response_synthesizers import ResponseMode
|
||||
from llama_index.core.base.response.schema import StreamingResponse
|
||||
from llama_index.llms.ollama import Ollama
|
||||
from llama_index.llms.openai import OpenAI
|
||||
from llama_index.core import PromptTemplate
|
||||
from typing import AsyncIterator, Optional, Tuple
|
||||
from typing import AsyncIterator, Optional, Tuple, Dict, Any
|
||||
import asyncio
|
||||
import httpx
|
||||
from loguru import logger
|
||||
from config import settings
|
||||
from .vector_store import VectorStoreManager
|
||||
from .chunk_handler import OptimizedDeltaThinkFilter
|
||||
from utils.query_processor import QueryProcessor
|
||||
from utils.code_prompt_manager import generate_dynamic_code_prompt
|
||||
|
||||
|
||||
# Single module-level prompt string for easy editing in one place
|
||||
|
|
@ -83,71 +84,42 @@ class RAGEngine:
|
|||
"""Main RAG engine for query processing"""
|
||||
|
||||
@staticmethod
|
||||
def check_llm_connection(provider: str, base_url: str, model: str, api_key: Optional[str] = None) -> Tuple[bool, str]:
|
||||
def check_ollama_connection() -> Tuple[bool, str]:
|
||||
"""
|
||||
Check if LLM server is accessible and connection can be established
|
||||
Check if Ollama server is accessible and connection can be established
|
||||
|
||||
Args:
|
||||
provider: LLM provider (ollama, vllm, openai)
|
||||
base_url: Base URL for the LLM service
|
||||
model: Model name
|
||||
api_key: Optional API key for authentication
|
||||
|
||||
Returns:
|
||||
Tuple of (is_connected: bool, error_message: str)
|
||||
If connected, error_message will be empty string
|
||||
"""
|
||||
try:
|
||||
headers = {}
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
logger.info(f"Checking Ollama connection to {settings.OLLAMA_BASE_URL}...")
|
||||
|
||||
if provider == "ollama":
|
||||
logger.info(f"Checking Ollama connection to {base_url}...")
|
||||
with httpx.Client(timeout=10.0) as client:
|
||||
response = client.get(f"{base_url}/api/tags")
|
||||
if response.status_code == 200:
|
||||
models = response.json().get("models", [])
|
||||
model_names = [m.get("name", "unknown") for m in models]
|
||||
logger.info(f"✓ Ollama connection successful (found {len(models)} model(s): {', '.join(model_names[:3])}{'...' if len(model_names) > 3 else ''})")
|
||||
return True, ""
|
||||
else:
|
||||
error_message = f"Ollama API returned status {response.status_code}: {response.text}"
|
||||
logger.error(f"✗ Ollama connection failed: {error_message}")
|
||||
return False, error_message
|
||||
elif provider == "openai":
|
||||
logger.info(f"Checking OpenAI-compatible API connection to {base_url}...")
|
||||
if not api_key:
|
||||
logger.warning("OpenAI provider requires API key, skipping connection check")
|
||||
# Test connection by calling Ollama API
|
||||
with httpx.Client(timeout=10.0) as client:
|
||||
response = client.get(f"{settings.OLLAMA_BASE_URL}/api/tags")
|
||||
if response.status_code == 200:
|
||||
models = response.json().get("models", [])
|
||||
model_names = [m.get("name", "unknown") for m in models]
|
||||
logger.info(f"✓ Ollama connection successful (found {len(models)} model(s): {', '.join(model_names[:3])}{'...' if len(model_names) > 3 else ''})")
|
||||
return True, ""
|
||||
with httpx.Client(timeout=10.0, headers=headers) as client:
|
||||
response = client.get(f"{base_url}/models")
|
||||
if response.status_code == 200:
|
||||
models = response.json().get("data", [])
|
||||
model_names = [m.get("id", "unknown") for m in models]
|
||||
logger.info(f"✓ OpenAI-compatible API connection successful (found {len(models)} model(s): {', '.join(model_names[:3])}{'...' if len(model_names) > 3 else ''})")
|
||||
return True, ""
|
||||
else:
|
||||
error_message = f"OpenAI API returned status {response.status_code}: {response.text}"
|
||||
logger.error(f"✗ OpenAI connection failed: {error_message}")
|
||||
return False, error_message
|
||||
else:
|
||||
logger.warning(f"Unknown LLM provider: {provider}, skipping connection check")
|
||||
return True, ""
|
||||
|
||||
else:
|
||||
error_message = f"Ollama API returned status {response.status_code}: {response.text}"
|
||||
logger.error(f"✗ Ollama connection failed: {error_message}")
|
||||
return False, error_message
|
||||
except httpx.ConnectError as e:
|
||||
error_message = f"Cannot connect to {provider} server at {base_url}. " \
|
||||
f"Please check if server is running and accessible."
|
||||
logger.error(f"✗ {provider} connection failed: {error_message}")
|
||||
error_message = f"Cannot connect to Ollama server at {settings.OLLAMA_BASE_URL}. " \
|
||||
f"Please check if Ollama server is running and accessible."
|
||||
logger.error(f"✗ Ollama connection failed: {error_message}")
|
||||
return False, error_message
|
||||
except httpx.TimeoutException:
|
||||
error_message = f"Connection to {provider} server at {base_url} timed out. " \
|
||||
f"Please check if server is running and accessible."
|
||||
logger.error(f"✗ {provider} connection failed: {error_message}")
|
||||
error_message = f"Connection to Ollama server at {settings.OLLAMA_BASE_URL} timed out. " \
|
||||
f"Please check if Ollama server is running and accessible."
|
||||
logger.error(f"✗ Ollama connection failed: {error_message}")
|
||||
return False, error_message
|
||||
except Exception as e:
|
||||
error_message = f"Unexpected error while checking {provider} connection: {str(e)}"
|
||||
logger.error(f"✗ {provider} connection check failed: {error_message}")
|
||||
error_message = f"Unexpected error while checking Ollama connection: {str(e)}"
|
||||
logger.error(f"✗ Ollama connection check failed: {error_message}")
|
||||
return False, error_message
|
||||
|
||||
def __init__(
|
||||
|
|
@ -156,59 +128,48 @@ class RAGEngine:
|
|||
prompt_template: Optional[PromptTemplate] = None,
|
||||
system_prompt: Optional[str] = None,
|
||||
temperature: float = 0.7,
|
||||
request_timeout: float = 120.0,
|
||||
request_timeout: float = 1200.0,
|
||||
):
|
||||
self.vector_store_manager = vector_store_manager
|
||||
# Configurable LLM / prompt parameters
|
||||
self._user_prompt_template = prompt_template
|
||||
self._system_prompt = system_prompt
|
||||
self._temperature = temperature
|
||||
self._request_timeout = request_timeout
|
||||
|
||||
llm_config = settings.get_llm_config()
|
||||
provider = llm_config["provider"]
|
||||
base_url = llm_config["base_url"]
|
||||
model = llm_config["model"]
|
||||
api_key = llm_config["api_key"]
|
||||
|
||||
is_connected, error_message = self.check_llm_connection(provider, base_url, model, api_key)
|
||||
# Check Ollama connection before initializing
|
||||
is_connected, error_message = self.check_ollama_connection()
|
||||
if not is_connected:
|
||||
error_msg = (
|
||||
f"Error: Cannot connect to {provider} server.\n"
|
||||
f"Error: Cannot connect to Ollama server.\n"
|
||||
f" {error_message}\n"
|
||||
f"Connection info: {base_url}\n"
|
||||
f"Connection info: {settings.OLLAMA_BASE_URL}\n"
|
||||
f"Please check:\n"
|
||||
f" 1. {provider} server is running\n"
|
||||
f" 2. {provider} server is accessible from this host\n"
|
||||
f" 3. LLM_BASE_URL is correctly configured\n"
|
||||
f" 4. Firewall rules allow connection"
|
||||
f" 1. Ollama server is running\n"
|
||||
f" 2. Ollama server is accessible from this host\n"
|
||||
f" 3. OLLAMA_BASE_URL is correctly configured\n"
|
||||
f" 4. Firewall rules allow connection to Ollama port"
|
||||
)
|
||||
logger.error(error_msg)
|
||||
raise RuntimeError(error_msg)
|
||||
|
||||
if provider == "ollama":
|
||||
self.llm = Ollama(
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
temperature=self._temperature,
|
||||
request_timeout=self._request_timeout,
|
||||
)
|
||||
elif provider == "openai":
|
||||
self.llm = OpenAI(
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
temperature=self._temperature,
|
||||
timeout=self._request_timeout,
|
||||
)
|
||||
else:
|
||||
self.llm = Ollama(
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
temperature=self._temperature,
|
||||
request_timeout=self._request_timeout,
|
||||
)
|
||||
self.llm = Ollama(
|
||||
model=settings.OLLAMA_MODEL,
|
||||
base_url=settings.OLLAMA_BASE_URL,
|
||||
temperature=self._temperature,
|
||||
request_timeout=self._request_timeout,
|
||||
)
|
||||
|
||||
self._llm_provider = provider
|
||||
# 初始化小型模型用于代码问题判断
|
||||
self.small_llm = Ollama(
|
||||
model="qwen3:0.6b",
|
||||
base_url=settings.OLLAMA_BASE_URL,
|
||||
temperature=0.3,
|
||||
request_timeout=30.0,
|
||||
)
|
||||
|
||||
# 初始化查询处理器
|
||||
self.query_processor = QueryProcessor(llm=self.llm)
|
||||
|
||||
def extract_text_from_chunk(self, chunk) -> Optional[str]:
|
||||
if hasattr(chunk, 'delta'):
|
||||
|
|
@ -225,6 +186,69 @@ class RAGEngine:
|
|||
str(chunk)
|
||||
except:
|
||||
pass
|
||||
|
||||
async def is_code_related(self, query: str, history: str) -> Dict[str, Any]:
|
||||
"""
|
||||
使用小型模型判断是否是代码相关问题,并提取可能的filter
|
||||
|
||||
Args:
|
||||
query: 用户查询字符串
|
||||
history: 对话历史字符串
|
||||
|
||||
Returns:
|
||||
dict: 包含 confidence (float) 和 filters (dict)
|
||||
"""
|
||||
try:
|
||||
# 获取需要提取的metadata字段
|
||||
filter_fields = settings.FILTER_METADATA_FIELDS.split(',')
|
||||
filter_fields_str = ', '.join(filter_fields)
|
||||
|
||||
prompt = f"""你是一个分类器,需要分析用户的问题。
|
||||
|
||||
用户问题:{query}
|
||||
|
||||
对话历史:{history}
|
||||
|
||||
请分析这个问题并返回JSON格式的分析结果:
|
||||
{{
|
||||
"confidence": 0.0-1.0之间的置信度,1表示完全确定是代码相关问题,0表示完全确定不是代码相关问题,
|
||||
"filters": {{}} 或 {{"字段名": "从问题中提取的值"}},只有当问题中明确提到"xxx文件/xxx函数/xxx类"时才提取
|
||||
}}
|
||||
|
||||
提取规则:
|
||||
- 只有当用户明确说明了"xxx文件"、"xxx函数"、"xxx类"时才提取对应的metadata
|
||||
- class_name: 用户提到具体类名时提取,如"User类"、"ArrayList"
|
||||
- func_name: 用户提到具体函数名时提取,如"main函数"、"delete方法"
|
||||
- file_path: 用户提到具体文件时提取,如"utils.py"、"config.json"
|
||||
|
||||
请仅返回JSON,不要添加任何其他内容。"""
|
||||
|
||||
response = await self.small_llm.acomplete(prompt=prompt)
|
||||
response_text = response.text.strip()
|
||||
|
||||
logger.info(f"小型模型分析结果: {response_text}")
|
||||
|
||||
# 尝试解析JSON
|
||||
import json
|
||||
json_start = response_text.find('{')
|
||||
json_end = response_text.rfind('}')
|
||||
if json_start != -1 and json_end != -1:
|
||||
json_str = response_text[json_start:json_end + 1]
|
||||
result = json.loads(json_str)
|
||||
confidence = float(result.get('confidence', 0.5))
|
||||
filters = result.get('filters', {})
|
||||
# 过滤掉空的filter
|
||||
filters = {k: v for k, v in filters.items() if v}
|
||||
logger.info(f"解析成功: confidence={confidence}, filters={filters}")
|
||||
return {'confidence': confidence, 'filters': filters}
|
||||
else:
|
||||
logger.warning(f"无法解析JSON,使用默认结果")
|
||||
return {'confidence': 0.5, 'filters': {}}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"判断代码问题时出错: {e}")
|
||||
# 出错时返回中间值,不过滤
|
||||
return {'confidence': 0.5, 'filters': {}}
|
||||
|
||||
async def query_stream(self, query: str, history: str, top_k: Optional[int] = None) -> AsyncIterator[str]:
|
||||
"""
|
||||
|
|
@ -239,49 +263,107 @@ class RAGEngine:
|
|||
Response text chunks
|
||||
"""
|
||||
try:
|
||||
# Create query engine with streaming mode
|
||||
retriever = self.vector_store_manager.get_retriever(top_k=top_k or settings.TOP_K)
|
||||
# query index
|
||||
retrieved_nodes = await retriever.aretrieve(query)
|
||||
# 1. 不区分是否代码相关问题,直接在所有collection中检索
|
||||
logger.info("在所有collection中进行向量检索")
|
||||
|
||||
# 2. 构建上下文
|
||||
context_parts = []
|
||||
for i, node in enumerate(retrieved_nodes[:top_k or settings.TOP_K], 1): # 限制数量
|
||||
text = node.text if hasattr(node, 'text') else str(node)
|
||||
# 清理和截断
|
||||
# 2. 使用纯向量检索
|
||||
logger.info("使用纯向量检索")
|
||||
k = top_k or settings.TOP_K
|
||||
|
||||
retriever = self.vector_store_manager.get_retriever(
|
||||
top_k=k * 2,
|
||||
filters=None,
|
||||
collection_key=None
|
||||
)
|
||||
|
||||
if isinstance(retriever, list):
|
||||
all_nodes = []
|
||||
for key, r in retriever:
|
||||
nodes = r.retrieve(query)
|
||||
for node in nodes:
|
||||
actual_node = node.node if hasattr(node, 'node') else node
|
||||
if hasattr(actual_node, 'metadata'):
|
||||
actual_node.metadata['_collection_key'] = key
|
||||
all_nodes.extend(nodes)
|
||||
vector_nodes = all_nodes
|
||||
else:
|
||||
vector_nodes = retriever.retrieve(query)
|
||||
|
||||
# 去重并按得分排序
|
||||
seen_doc_ids = set()
|
||||
unique_results = []
|
||||
for node in vector_nodes:
|
||||
doc_id = getattr(node, 'id_', None) or getattr(node, 'node_id', None)
|
||||
if doc_id and doc_id not in seen_doc_ids:
|
||||
seen_doc_ids.add(doc_id)
|
||||
unique_results.append(node)
|
||||
|
||||
unique_results.sort(key=lambda x: getattr(x, 'score', 0), reverse=True)
|
||||
vector_nodes = unique_results[:k]
|
||||
|
||||
logger.info(f"检索到 {len(vector_nodes)} 个结果")
|
||||
|
||||
# 4. 构建检索结果
|
||||
retrieved_results = []
|
||||
for i, node in enumerate(vector_nodes, 1):
|
||||
metadata = getattr(node, 'metadata', {})
|
||||
if 'func_body' in metadata:
|
||||
text = metadata['func_body']
|
||||
else:
|
||||
text = node.text if hasattr(node, 'text') else str(node)
|
||||
text = text.strip()
|
||||
if len(text) > 400:
|
||||
text = text[:400] + "..."
|
||||
context_parts.append(f"【参考信息{i}】{text}")
|
||||
|
||||
retrieved_results.append({
|
||||
'id': i,
|
||||
'text': text,
|
||||
'metadata': metadata
|
||||
})
|
||||
|
||||
# 5. 构建上下文
|
||||
context_parts = []
|
||||
for result in retrieved_results:
|
||||
metadata = result['metadata']
|
||||
file_path = metadata.get('file_path', '')
|
||||
func_name = metadata.get('func_name', '')
|
||||
|
||||
context_part = f"【参考信息{result['id']}】"
|
||||
if file_path:
|
||||
context_part += f"(来源:{file_path})"
|
||||
if func_name:
|
||||
context_part += f"\n函数:{func_name}"
|
||||
context_part += f"\n{result['text']}"
|
||||
context_parts.append(context_part)
|
||||
|
||||
context_str = "\n\n".join(context_parts) if context_parts else "未找到相关参考信息"
|
||||
logger.debug(f"上下文字符串: {context_str}")
|
||||
|
||||
if history is not None:
|
||||
# 6. 生成Prompt - 根据是否有历史对话选择不同模板
|
||||
logger.info("生成Prompt")
|
||||
if history:
|
||||
qa_prompt = QA_PROMPT_HISTORY
|
||||
filled_prompt = qa_prompt.format(history=history, context_str=context_str, query_str=query)
|
||||
else:
|
||||
qa_prompt = QA_PROMPT_NO_HISTORY
|
||||
filled_prompt = qa_prompt.format(context_str=context_str, query_str=query)
|
||||
|
||||
stream_response = await self.llm.astream_complete(
|
||||
prompt=filled_prompt
|
||||
)
|
||||
logger.info("根据历史对话选择prompt模板")
|
||||
|
||||
# 7. 流式生成回答
|
||||
stream_response = await self.llm.astream_complete(prompt=filled_prompt)
|
||||
|
||||
full_response = ""
|
||||
think_filter = OptimizedDeltaThinkFilter()
|
||||
|
||||
async for chunk in stream_response:
|
||||
# 提取文本内容
|
||||
# text_chunk = self.extract_text_from_chunk(chunk)
|
||||
delta, full_text, has_output = think_filter.process_delta_robust(chunk)
|
||||
|
||||
if delta is not None:
|
||||
full_response += delta
|
||||
yield delta.encode('utf-8')
|
||||
await asyncio.sleep(0.001) # slight delay to yield control
|
||||
await asyncio.sleep(0.001)
|
||||
|
||||
logger.info(f"响应完成,长度: {len(full_response)}字符")
|
||||
|
||||
print(full_response)
|
||||
except Exception as e:
|
||||
logger.error(f"Error in RAG query: {e}")
|
||||
|
||||
|
|
@ -298,33 +380,97 @@ class RAGEngine:
|
|||
Complete response string
|
||||
"""
|
||||
try:
|
||||
# Create query engine with streaming mode
|
||||
retriever = self.vector_store_manager.get_retriever(top_k=top_k or settings.TOP_K)
|
||||
# query index
|
||||
retrieved_nodes = await retriever.aretrieve(query)
|
||||
# 1. 不区分是否代码相关问题,直接在所有collection中检索
|
||||
logger.info("在所有collection中进行向量检索")
|
||||
|
||||
# 2. 构建上下文
|
||||
context_parts = []
|
||||
for i, node in enumerate(retrieved_nodes[:top_k or settings.TOP_K], 1): # 限制数量
|
||||
text = node.text if hasattr(node, 'text') else str(node)
|
||||
# 清理和截断
|
||||
# 2. 使用纯向量检索
|
||||
logger.info("使用纯向量检索")
|
||||
k = top_k or settings.TOP_K
|
||||
|
||||
# 获取retriever
|
||||
retriever = self.vector_store_manager.get_retriever(
|
||||
top_k=k * 2, # 获取更多结果用于去重
|
||||
filters=None,
|
||||
collection_key=None
|
||||
)
|
||||
|
||||
# 执行检索
|
||||
if isinstance(retriever, list):
|
||||
# 多个collection
|
||||
all_nodes = []
|
||||
for key, r in retriever:
|
||||
nodes = r.retrieve(query)
|
||||
for node in nodes:
|
||||
actual_node = node.node if hasattr(node, 'node') else node
|
||||
if hasattr(actual_node, 'metadata'):
|
||||
actual_node.metadata['_collection_key'] = key
|
||||
all_nodes.extend(nodes)
|
||||
vector_nodes = all_nodes
|
||||
else:
|
||||
vector_nodes = retriever.retrieve(query)
|
||||
|
||||
# 去重并按得分排序
|
||||
seen_doc_ids = set()
|
||||
unique_results = []
|
||||
for node in vector_nodes:
|
||||
doc_id = getattr(node, 'id_', None) or getattr(node, 'node_id', None)
|
||||
if doc_id and doc_id not in seen_doc_ids:
|
||||
seen_doc_ids.add(doc_id)
|
||||
unique_results.append(node)
|
||||
|
||||
# 按得分排序,取top_k
|
||||
unique_results.sort(key=lambda x: getattr(x, 'score', 0), reverse=True)
|
||||
vector_nodes = unique_results[:k]
|
||||
|
||||
logger.info(f"检索到 {len(vector_nodes)} 个结果")
|
||||
|
||||
# 4. 获取文档内容
|
||||
retrieved_results = []
|
||||
for i, node in enumerate(vector_nodes, 1):
|
||||
# 优先使用metadata中的func_body字段
|
||||
metadata = getattr(node, 'metadata', {})
|
||||
if 'func_body' in metadata:
|
||||
text = metadata['func_body']
|
||||
else:
|
||||
text = node.text if hasattr(node, 'text') else str(node)
|
||||
text = text.strip()
|
||||
if len(text) > 400:
|
||||
text = text[:400] + "..."
|
||||
context_parts.append(f"【参考信息{i}】{text}")
|
||||
|
||||
retrieved_results.append({
|
||||
'id': i,
|
||||
'text': text,
|
||||
'metadata': metadata
|
||||
})
|
||||
|
||||
# 5. 构建上下文
|
||||
context_parts = []
|
||||
for result in retrieved_results:
|
||||
metadata = result['metadata']
|
||||
file_path = metadata.get('file_path', '')
|
||||
func_name = metadata.get('func_name', '')
|
||||
|
||||
context_part = f"【参考信息{result['id']}】"
|
||||
if file_path:
|
||||
context_part += f"(来源:{file_path})"
|
||||
if func_name:
|
||||
context_part += f"\n函数:{func_name}"
|
||||
context_part += f"\n{result['text']}"
|
||||
context_parts.append(context_part)
|
||||
|
||||
context_str = "\n\n".join(context_parts) if context_parts else "未找到相关参考信息"
|
||||
|
||||
if history is not None:
|
||||
# 6. 生成Prompt - 根据是否有历史对话选择不同模板
|
||||
logger.info("生成Prompt")
|
||||
if history:
|
||||
qa_prompt = QA_PROMPT_HISTORY
|
||||
filled_prompt = qa_prompt.format(history=history, context_str=context_str, query_str=query)
|
||||
else:
|
||||
qa_prompt = QA_PROMPT_NO_HISTORY
|
||||
filled_prompt = qa_prompt.format(context_str=context_str, query_str=query)
|
||||
|
||||
response = await self.llm.acomplete(
|
||||
prompt=filled_prompt
|
||||
)
|
||||
logger.info("根据历史对话选择prompt模板")
|
||||
|
||||
# 7. 调用LLM生成回答
|
||||
response = await self.llm.acomplete(prompt=filled_prompt)
|
||||
|
||||
return response.text
|
||||
except Exception as e:
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,253 @@
|
|||
# 云端服务器Git服务器搭建指南
|
||||
|
||||
## 1. 选择云服务提供商
|
||||
|
||||
主流的云服务提供商包括:
|
||||
- **AWS (Amazon Web Services)**
|
||||
- **阿里云**
|
||||
- **腾讯云**
|
||||
- **华为云**
|
||||
- **Google Cloud Platform (GCP)**
|
||||
- **Microsoft Azure**
|
||||
|
||||
本指南以AWS为例,其他云服务提供商的步骤类似。
|
||||
|
||||
## 2. 创建云服务器实例
|
||||
|
||||
### 2.1 AWS EC2实例创建
|
||||
1. **登录AWS控制台**:https://console.aws.amazon.com/
|
||||
2. **导航到EC2服务**:点击 "服务" → "EC2"
|
||||
3. **启动实例**:
|
||||
- 点击 "启动实例"
|
||||
- 选择AMI(Amazon Machine Image):推荐选择Ubuntu Server 20.04 LTS或Amazon Linux 2
|
||||
- 选择实例类型:根据需要选择,推荐t2.micro(免费套餐)
|
||||
- 配置实例:保持默认设置
|
||||
- 添加存储:保持默认设置
|
||||
- 添加标签:可选,添加名称标签
|
||||
- 配置安全组:
|
||||
- 允许SSH访问(端口22)
|
||||
- 可选:允许HTTP/HTTPS访问(端口80/443)
|
||||
- 审核并启动:点击 "启动"
|
||||
- 创建密钥对:创建新密钥对,下载并保存私钥文件(.pem)
|
||||
|
||||
### 2.2 其他云服务提供商
|
||||
对于阿里云、腾讯云等其他提供商,步骤类似:
|
||||
1. 登录控制台
|
||||
2. 创建云服务器实例
|
||||
3. 选择操作系统(推荐Ubuntu或CentOS)
|
||||
4. 配置网络安全组,允许SSH访问
|
||||
5. 设置登录凭据(密钥对或密码)
|
||||
|
||||
## 3. 连接到云服务器
|
||||
|
||||
### 3.1 使用SSH连接
|
||||
|
||||
#### AWS EC2实例:
|
||||
```bash
|
||||
# 在本地终端执行
|
||||
ssh -i /path/to/your-key.pem ubuntu@your-instance-public-ip
|
||||
```
|
||||
|
||||
#### 其他云服务器:
|
||||
```bash
|
||||
# 使用密码认证
|
||||
ssh username@your-server-public-ip
|
||||
|
||||
# 或使用密钥认证
|
||||
ssh -i /path/to/your-key.pem username@your-server-public-ip
|
||||
```
|
||||
|
||||
## 4. 安装和配置Git服务器
|
||||
|
||||
### 4.1 安装Git
|
||||
|
||||
#### Ubuntu/Debian:
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt install git
|
||||
```
|
||||
|
||||
#### Amazon Linux/CentOS:
|
||||
```bash
|
||||
sudo yum update
|
||||
sudo yum install git
|
||||
```
|
||||
|
||||
### 4.2 创建Git用户
|
||||
```bash
|
||||
# 创建git用户
|
||||
sudo adduser git
|
||||
|
||||
# 设置密码
|
||||
sudo passwd git
|
||||
```
|
||||
|
||||
### 4.3 配置SSH访问
|
||||
|
||||
1. **在本地生成SSH密钥**:
|
||||
```bash
|
||||
ssh-keygen -t rsa -b 4096 -C "your_email@example.com"
|
||||
```
|
||||
|
||||
2. **将公钥复制到服务器**:
|
||||
```bash
|
||||
# AWS EC2实例
|
||||
ssh-copy-id -i ~/.ssh/id_rsa.pub ubuntu@your-instance-public-ip
|
||||
|
||||
# 然后切换到git用户并配置
|
||||
ssh -i /path/to/your-key.pem ubuntu@your-instance-public-ip
|
||||
sudo su - git
|
||||
mkdir -p ~/.ssh
|
||||
chmod 700 ~/.ssh
|
||||
touch ~/.ssh/authorized_keys
|
||||
chmod 600 ~/.ssh/authorized_keys
|
||||
exit
|
||||
|
||||
# 将本地公钥添加到git用户的authorized_keys
|
||||
cat ~/.ssh/id_rsa.pub | ssh -i /path/to/your-key.pem ubuntu@your-instance-public-ip "sudo su - git -c 'cat >> ~/.ssh/authorized_keys'"
|
||||
```
|
||||
|
||||
### 4.4 创建Git仓库
|
||||
|
||||
```bash
|
||||
# 登录服务器
|
||||
ssh -i /path/to/your-key.pem ubuntu@your-instance-public-ip
|
||||
|
||||
# 切换到git用户
|
||||
sudo su - git
|
||||
|
||||
# 创建仓库存储目录
|
||||
mkdir -p ~/git-repos
|
||||
|
||||
# 创建仓库
|
||||
cd ~/git-repos
|
||||
git init --bare project1.git
|
||||
git init --bare project2.git
|
||||
git init --bare project3.git
|
||||
|
||||
# 设置权限
|
||||
chmod -R 755 ~/git-repos
|
||||
```
|
||||
|
||||
## 5. 配置防火墙
|
||||
|
||||
### 5.1 Ubuntu/Debian:
|
||||
```bash
|
||||
sudo ufw allow ssh
|
||||
sudo ufw enable
|
||||
```
|
||||
|
||||
### 5.2 Amazon Linux/CentOS:
|
||||
```bash
|
||||
sudo firewall-cmd --permanent --add-service=ssh
|
||||
sudo firewall-cmd --reload
|
||||
```
|
||||
|
||||
## 6. 测试连接
|
||||
|
||||
### 6.1 从本地克隆仓库
|
||||
```bash
|
||||
# 克隆仓库
|
||||
git clone git@your-server-public-ip:~/git-repos/project1.git
|
||||
|
||||
# 测试推送
|
||||
cd project1
|
||||
echo "Hello, Cloud Git Server!" > README.md
|
||||
git add README.md
|
||||
git commit -m "Initial commit"
|
||||
git push origin main
|
||||
```
|
||||
|
||||
### 6.2 在配置管理界面测试连接
|
||||
|
||||
1. **打开配置管理界面**
|
||||
2. **选择Git配置类型**
|
||||
3. **选择"服务器同步"模式**
|
||||
4. **填写服务器信息**:
|
||||
- 服务器地址:您的云服务器公网IP
|
||||
- 端口:22
|
||||
- 用户名:git
|
||||
- 密码:您设置的git用户密码
|
||||
5. **点击"测试服务器连接"按钮**
|
||||
6. **点击"获取仓库列表"按钮**
|
||||
7. **选择一个仓库并点击"获取分支列表"按钮**
|
||||
|
||||
## 7. 高级配置
|
||||
|
||||
### 7.1 使用弹性IP
|
||||
为了避免服务器重启后IP地址变化,建议使用弹性IP:
|
||||
|
||||
1. **AWS EC2**:在EC2控制台分配弹性IP并关联到实例
|
||||
2. **阿里云**:在ECS控制台申请弹性公网IP并绑定到实例
|
||||
3. **腾讯云**:在CVM控制台申请弹性公网IP并绑定到实例
|
||||
|
||||
### 7.2 使用域名
|
||||
为了方便访问,可以将域名解析到服务器IP:
|
||||
|
||||
1. 在域名注册商处添加A记录,指向服务器公网IP
|
||||
2. 等待DNS解析生效
|
||||
3. 使用域名代替IP地址访问Git服务器
|
||||
|
||||
### 7.3 配置HTTPS
|
||||
如果需要通过HTTPS访问,可以配置Web服务器(如Apache或Nginx):
|
||||
|
||||
```bash
|
||||
# 安装Apache
|
||||
sudo apt install apache2
|
||||
|
||||
# 安装GitWeb(可选)
|
||||
sudo apt install gitweb
|
||||
|
||||
# 配置Apache虚拟主机
|
||||
# 参考Apache文档配置HTTPS
|
||||
```
|
||||
|
||||
## 8. 安全最佳实践
|
||||
|
||||
1. **使用SSH密钥认证**:禁用密码认证,只允许SSH密钥登录
|
||||
2. **限制SSH访问**:在安全组中只允许特定IP访问SSH端口
|
||||
3. **定期更新系统**:定期执行系统更新
|
||||
4. **使用防火墙**:配置防火墙只允许必要的端口
|
||||
5. **备份仓库**:定期备份Git仓库数据
|
||||
6. **使用IAM角色**:在AWS等云服务中使用IAM角色进行权限管理
|
||||
7. **监控服务器**:启用云服务的监控功能,及时发现异常
|
||||
|
||||
## 9. 常见问题及解决方案
|
||||
|
||||
| 问题 | 可能原因 | 解决方案 |
|
||||
|------|---------|---------|
|
||||
| 无法连接到服务器 | 安全组未开放SSH端口 | 检查云服务安全组配置,确保端口22开放 |
|
||||
| 认证失败 | SSH密钥未正确配置 | 检查SSH密钥配置和权限设置 |
|
||||
| 无法推送代码 | 权限不足 | 确保git用户拥有仓库的写权限 |
|
||||
| 仓库列表为空 | 路径错误或权限不足 | 检查仓库路径和用户权限 |
|
||||
| 服务器IP变化 | 未使用弹性IP | 配置弹性IP或更新DNS记录 |
|
||||
|
||||
## 10. 云服务特定配置
|
||||
|
||||
### 10.1 AWS
|
||||
- **使用EC2 Instance Connect**:无需密钥对即可连接实例
|
||||
- **使用S3备份**:将仓库备份到S3存储
|
||||
- **使用CloudWatch监控**:监控服务器状态和性能
|
||||
|
||||
### 10.2 阿里云
|
||||
- **使用安全组**:配置精细化的网络访问控制
|
||||
- **使用快照**:定期创建服务器快照
|
||||
- **使用OSS备份**:将仓库备份到OSS存储
|
||||
|
||||
### 10.3 腾讯云
|
||||
- **使用安全组**:配置网络访问控制
|
||||
- **使用快照**:创建服务器快照进行备份
|
||||
- **使用COS备份**:将仓库备份到COS存储
|
||||
|
||||
## 11. 替代方案
|
||||
|
||||
如果不想自己搭建Git服务器,可以使用以下托管服务:
|
||||
|
||||
1. **GitHub**:公共和私有仓库
|
||||
2. **GitLab**:自托管或托管版本
|
||||
3. **Bitbucket**:适合团队协作
|
||||
4. **Gitee**:国内Git托管服务
|
||||
|
||||
这些服务提供了更完善的功能,如Web界面、CI/CD集成、问题跟踪等。
|
||||
|
||||
通过以上步骤,您可以在云端服务器上搭建自己的Git服务器,创建多个Git仓库,并在配置管理界面中测试连接和同步功能。
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
# Windows环境下Git服务器搭建指南
|
||||
|
||||
## 1. 准备工作
|
||||
|
||||
### 1.1 系统要求
|
||||
- Windows 7或更高版本
|
||||
- .NET Framework 4.6.1或更高版本
|
||||
- IIS (Internet Information Services)
|
||||
- Git for Windows(已安装)
|
||||
|
||||
### 1.2 安装IIS
|
||||
1. **打开控制面板** → **程序** → **程序和功能**
|
||||
2. 点击 **打开或关闭Windows功能**
|
||||
3. 勾选 **Internet Information Services**,确保包含以下组件:
|
||||
- Web管理工具
|
||||
- 万维网服务 → 应用程序开发功能 → ASP.NET 4.8
|
||||
- 万维网服务 → 常见HTTP功能
|
||||
- 万维网服务 → 安全性
|
||||
4. 点击 **确定** 安装
|
||||
|
||||
## 2. 安装Bonobo Git Server
|
||||
|
||||
### 2.1 下载Bonobo Git Server
|
||||
1. 访问Bonobo Git Server官方网站:https://bonobogitserver.com/
|
||||
2. 下载最新版本的Bonobo Git Server
|
||||
|
||||
### 2.2 安装Bonobo Git Server
|
||||
1. 解压下载的压缩包
|
||||
2. 将解压后的文件夹复制到IIS网站目录,默认路径为:`C:\inetpub\wwwroot`
|
||||
3. 重命名文件夹为 `Bonobo.Git.Server`
|
||||
|
||||
### 2.3 配置IIS
|
||||
1. **打开IIS管理器**:搜索 "IIS管理器" 并打开
|
||||
2. **添加网站**:
|
||||
- 站点名称:`BonoboGitServer`
|
||||
- 物理路径:`C:\inetpub\wwwroot\Bonobo.Git.Server`
|
||||
- 端口:8080(或其他未使用的端口)
|
||||
- 主机名:留空
|
||||
3. **配置应用程序池**:
|
||||
- 选择新创建的网站
|
||||
- 点击 "基本设置"
|
||||
- 应用程序池:选择 ".NET v4.5 Classic" 或更高版本
|
||||
4. **设置权限**:
|
||||
- 右键点击 `Bonobo.Git.Server` 文件夹
|
||||
- 选择 "属性" → "安全" → "编辑"
|
||||
- 添加 `IIS_IUSRS` 用户,并授予 "修改" 权限
|
||||
|
||||
## 3. 配置Bonobo Git Server
|
||||
|
||||
### 3.1 访问Bonobo Git Server
|
||||
1. 打开浏览器,访问:`http://localhost:8080`
|
||||
2. 默认登录凭据:
|
||||
- 用户名:admin
|
||||
- 密码:admin
|
||||
|
||||
### 3.2 修改默认密码
|
||||
1. 登录后,点击右上角的 "admin" → "Profile"
|
||||
2. 输入新密码并保存
|
||||
|
||||
### 3.3 创建用户
|
||||
1. 点击左侧菜单 "Users"
|
||||
2. 点击 "Add New User"
|
||||
3. 填写用户名、密码和邮箱
|
||||
4. 点击 "Save"
|
||||
|
||||
## 4. 创建Git仓库
|
||||
|
||||
### 4.1 创建仓库
|
||||
1. 点击左侧菜单 "Repositories"
|
||||
2. 点击 "Create New Repository"
|
||||
3. 填写仓库名称(例如:project1)
|
||||
4. 选择访问权限(Public或Private)
|
||||
5. 点击 "Create"
|
||||
|
||||
### 4.2 创建多个仓库
|
||||
重复上述步骤,创建多个仓库,例如:
|
||||
- project1
|
||||
- project2
|
||||
- project3
|
||||
|
||||
## 5. 测试连接
|
||||
|
||||
### 5.1 从客户端克隆仓库
|
||||
1. 打开Git Bash
|
||||
2. 克隆仓库:
|
||||
```bash
|
||||
git clone http://localhost:8080/Bonobo.Git.Server/project1.git
|
||||
```
|
||||
3. 输入Bonobo Git Server的用户名和密码
|
||||
4. 进入仓库目录并测试:
|
||||
```bash
|
||||
cd project1
|
||||
echo "Hello, Git Server!" > README.md
|
||||
git add README.md
|
||||
git commit -m "Initial commit"
|
||||
git push origin main
|
||||
```
|
||||
|
||||
### 5.2 在配置管理界面测试连接
|
||||
|
||||
1. **打开配置管理界面**
|
||||
2. **选择Git配置类型**
|
||||
3. **选择"服务器同步"模式**
|
||||
4. **填写服务器信息**:
|
||||
- 服务器地址:localhost
|
||||
- 端口:8080
|
||||
- 用户名:您在Bonobo中创建的用户名
|
||||
- 密码:对应的密码
|
||||
5. **点击"测试服务器连接"按钮**
|
||||
6. **点击"获取仓库列表"按钮**,应该能看到您创建的仓库
|
||||
7. **选择一个仓库并点击"获取分支列表"按钮**
|
||||
|
||||
## 6. 高级配置
|
||||
|
||||
### 6.1 配置HTTPS
|
||||
1. **打开IIS管理器**
|
||||
2. 选择Bonobo Git Server网站
|
||||
3. 点击 "绑定..."
|
||||
4. 添加HTTPS绑定,选择或创建SSL证书
|
||||
5. 点击 "确定"
|
||||
|
||||
### 6.2 配置邮件通知
|
||||
1. 编辑 `C:\inetpub\wwwroot\Bonobo.Git.Server\Web.config` 文件
|
||||
2. 找到 `<appSettings>` 部分,配置SMTP设置
|
||||
|
||||
## 7. 常见问题及解决方案
|
||||
|
||||
| 问题 | 可能原因 | 解决方案 |
|
||||
|------|---------|---------|
|
||||
| 无法访问Bonobo Git Server | IIS未启动或端口被占用 | 检查IIS服务状态和端口设置 |
|
||||
| 认证失败 | 用户名或密码错误 | 检查Bonobo Git Server的用户凭据 |
|
||||
| 无法推送代码 | 权限不足 | 确保用户有仓库的写权限 |
|
||||
| 仓库列表为空 | 权限不足或仓库未创建 | 检查用户权限和仓库创建状态 |
|
||||
|
||||
## 8. 安全建议
|
||||
|
||||
1. **修改默认密码**:登录后立即修改admin用户的默认密码
|
||||
2. **创建专用用户**:为每个开发者创建专用用户账号
|
||||
3. **使用HTTPS**:配置HTTPS以加密传输
|
||||
4. **定期备份**:定期备份Bonobo Git Server的仓库数据
|
||||
5. **限制访问**:通过防火墙限制只有必要的IP可以访问Git服务器
|
||||
|
||||
## 9. 替代方案
|
||||
|
||||
如果Bonobo Git Server不满足您的需求,还可以考虑以下Windows环境的Git服务器解决方案:
|
||||
|
||||
1. **GitLab CE**:功能强大的Git服务器,支持Windows部署
|
||||
2. **GitHub Enterprise**:企业级Git解决方案
|
||||
3. **Gitea**:轻量级Git服务器,支持Windows
|
||||
|
||||
通过以上步骤,您可以在Windows环境下搭建自己的Git服务器,创建多个Git仓库,并在配置管理界面中测试连接和同步功能。
|
||||
|
|
@ -12,13 +12,17 @@
|
|||
<!-- 侧边栏 -->
|
||||
<aside class="sidebar">
|
||||
<div class="sidebar-header">
|
||||
<h1>⚙️ 配置管理</h1>
|
||||
<div style="display: flex; justify-content: space-between; align-items: center; width: 100%;">
|
||||
<h1>⚙️ 配置管理</h1>
|
||||
<button id="themeToggle" class="theme-toggle-btn">🌙</button>
|
||||
</div>
|
||||
<div class="add-config-container">
|
||||
<label for="configTypeSelect" class="config-type-label">新增数据源:</label>
|
||||
<select id="configTypeSelect">
|
||||
<option value="">选择类型</option>
|
||||
<option value="database">数据库</option>
|
||||
<option value="folder">文件夹</option>
|
||||
<option value="git">Git代码库</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -75,6 +79,7 @@
|
|||
<select id="configType" name="type" required>
|
||||
<option value="database">数据库 (database)</option>
|
||||
<option value="folder">文件夹 (folder)</option>
|
||||
<option value="git">Git代码库 (git)</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="form-group" id="dbTypeGroup" style="display: none;">
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,235 @@
|
|||
"""
|
||||
代码AST解析工具类
|
||||
实现跨语言函数级切片
|
||||
"""
|
||||
import ast
|
||||
import os
|
||||
from typing import List, Dict, Optional
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class ASTParser:
|
||||
def __init__(self, file_path: str, lang: str):
|
||||
"""
|
||||
初始化AST解析器
|
||||
|
||||
Args:
|
||||
file_path: 文件路径
|
||||
lang: 编程语言
|
||||
"""
|
||||
self.file_path = file_path
|
||||
self.lang = lang
|
||||
self.func_list: List[Dict] = [] # 提取的函数列表
|
||||
|
||||
def parse_functions(self) -> List[Dict]:
|
||||
"""
|
||||
统一入口:根据语言调用对应解析方法
|
||||
|
||||
Returns:
|
||||
List[Dict]: 函数信息列表
|
||||
"""
|
||||
if not os.path.exists(self.file_path):
|
||||
raise Exception(f"文件不存在: {self.file_path}")
|
||||
|
||||
# 读取文件内容
|
||||
try:
|
||||
with open(self.file_path, "r", encoding="utf-8") as f:
|
||||
self.code = f.read()
|
||||
except Exception as e:
|
||||
logger.error(f"读取文件失败: {e}")
|
||||
raise
|
||||
|
||||
# 按语言解析
|
||||
if self.lang == "python":
|
||||
self._parse_python()
|
||||
elif self.lang == "java":
|
||||
self._parse_java()
|
||||
elif self.lang == "go":
|
||||
self._parse_go()
|
||||
elif self.lang == "javascript" or self.lang == "typescript":
|
||||
self._parse_javascript()
|
||||
else:
|
||||
logger.warning(f"暂不支持的编程语言: {self.lang}")
|
||||
|
||||
logger.info(f"解析文件 {self.file_path},提取到 {len(self.func_list)} 个函数")
|
||||
return self.func_list
|
||||
|
||||
def _parse_python(self):
|
||||
"""
|
||||
解析Python代码
|
||||
"""
|
||||
try:
|
||||
tree = ast.parse(self.code)
|
||||
for node in ast.walk(tree):
|
||||
# 提取函数定义(普通函数/类方法/异步函数)
|
||||
if isinstance(node, ast.FunctionDef) or isinstance(node, ast.AsyncFunctionDef):
|
||||
func_info = self._extract_python_func_info(node)
|
||||
self.func_list.append(func_info)
|
||||
except SyntaxError as e:
|
||||
logger.error(f"Python代码语法错误: {e}")
|
||||
raise Exception(f"Python代码语法错误: {e}")
|
||||
|
||||
def _extract_python_func_info(self, node) -> Dict:
|
||||
"""
|
||||
提取Python函数的标准化信息
|
||||
|
||||
Args:
|
||||
node: AST节点
|
||||
|
||||
Returns:
|
||||
Dict: 函数信息
|
||||
"""
|
||||
# 提取函数名
|
||||
func_name = node.name
|
||||
|
||||
# 提取参数
|
||||
params = []
|
||||
for arg in node.args.args:
|
||||
param_info = {
|
||||
"name": arg.arg,
|
||||
"type": None
|
||||
}
|
||||
# 提取类型注解
|
||||
if arg.annotation:
|
||||
try:
|
||||
param_info["type"] = ast.unparse(arg.annotation)
|
||||
except Exception:
|
||||
pass
|
||||
params.append(param_info)
|
||||
|
||||
# 提取返回值类型
|
||||
return_type = None
|
||||
if node.returns:
|
||||
try:
|
||||
return_type = ast.unparse(node.returns)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 提取函数体代码
|
||||
func_body = self._get_func_body(node)
|
||||
|
||||
# 提取所属类名
|
||||
class_name = None
|
||||
parent = node
|
||||
while hasattr(parent, "parent"):
|
||||
parent = parent.parent
|
||||
if isinstance(parent, ast.ClassDef):
|
||||
class_name = parent.name
|
||||
break
|
||||
|
||||
# 提取函数文档字符串
|
||||
docstring = ast.get_docstring(node)
|
||||
|
||||
return {
|
||||
"file_path": self.file_path,
|
||||
"lang": "python",
|
||||
"func_name": func_name,
|
||||
"class_name": class_name,
|
||||
"params": params,
|
||||
"return_type": return_type,
|
||||
"func_body": func_body,
|
||||
"docstring": docstring,
|
||||
"start_line": node.lineno,
|
||||
"end_line": node.end_lineno
|
||||
}
|
||||
|
||||
def _parse_java(self):
|
||||
"""
|
||||
解析Java代码
|
||||
注意:这里使用简单的正则解析,实际项目中建议使用专业的Java解析库
|
||||
"""
|
||||
logger.warning("Java解析功能暂未完全实现,使用简单的正则解析")
|
||||
# TODO: 实现Java代码的AST解析
|
||||
|
||||
def _parse_go(self):
|
||||
"""
|
||||
解析Go代码
|
||||
注意:这里使用简单的正则解析,实际项目中建议使用专业的Go解析库
|
||||
"""
|
||||
logger.warning("Go解析功能暂未完全实现,使用简单的正则解析")
|
||||
# TODO: 实现Go代码的AST解析
|
||||
|
||||
def _parse_javascript(self):
|
||||
"""
|
||||
解析JavaScript/TypeScript代码
|
||||
注意:这里使用简单的正则解析,实际项目中建议使用专业的JS解析库
|
||||
"""
|
||||
logger.warning("JavaScript解析功能暂未完全实现,使用简单的正则解析")
|
||||
# TODO: 实现JavaScript代码的AST解析
|
||||
|
||||
def _get_func_body(self, node) -> str:
|
||||
"""
|
||||
获取函数体代码
|
||||
|
||||
Args:
|
||||
node: AST节点
|
||||
|
||||
Returns:
|
||||
str: 函数体代码
|
||||
"""
|
||||
try:
|
||||
# 使用ast.unparse获取函数体代码
|
||||
return ast.unparse(node)
|
||||
except Exception:
|
||||
# 降级方案:根据行号提取代码
|
||||
lines = self.code.splitlines()
|
||||
start_line = node.lineno - 1 # 转换为0-based索引
|
||||
end_line = node.end_lineno # 转换为0-based索引
|
||||
if start_line >= 0 and end_line <= len(lines):
|
||||
return "\n".join(lines[start_line:end_line])
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def detect_language(file_path: str) -> Optional[str]:
|
||||
"""
|
||||
根据文件扩展名检测编程语言
|
||||
|
||||
Args:
|
||||
file_path: 文件路径
|
||||
|
||||
Returns:
|
||||
Optional[str]: 编程语言
|
||||
"""
|
||||
ext = os.path.splitext(file_path)[1].lower()
|
||||
|
||||
lang_map = {
|
||||
".py": "python",
|
||||
".java": "java",
|
||||
".go": "go",
|
||||
".js": "javascript",
|
||||
".ts": "typescript",
|
||||
".jsx": "javascript",
|
||||
".tsx": "typescript",
|
||||
".c": "c",
|
||||
".cpp": "cpp",
|
||||
".h": "c",
|
||||
".hpp": "cpp",
|
||||
".cs": "csharp",
|
||||
".rs": "rust",
|
||||
".php": "php",
|
||||
".rb": "ruby",
|
||||
".swift": "swift",
|
||||
".kt": "kotlin",
|
||||
".scala": "scala"
|
||||
}
|
||||
|
||||
return lang_map.get(ext)
|
||||
|
||||
@staticmethod
|
||||
def parse_file(file_path: str) -> List[Dict]:
|
||||
"""
|
||||
静态方法:解析文件
|
||||
|
||||
Args:
|
||||
file_path: 文件路径
|
||||
|
||||
Returns:
|
||||
List[Dict]: 函数信息列表
|
||||
"""
|
||||
lang = ASTParser.detect_language(file_path)
|
||||
if not lang:
|
||||
logger.warning(f"无法检测文件类型: {file_path}")
|
||||
return []
|
||||
|
||||
parser = ASTParser(file_path, lang)
|
||||
return parser.parse_functions()
|
||||
|
|
@ -105,19 +105,20 @@ class BaseSync(ABC):
|
|||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def doc_to_llamaindex_doc(self, doc: Dict) -> 'Document':
|
||||
"""
|
||||
Convert data source document to LlamaIndex Document
|
||||
|
||||
Args:
|
||||
doc: Document from the data source
|
||||
|
||||
Returns:
|
||||
LlamaIndex Document object
|
||||
"""
|
||||
pass
|
||||
|
||||
# @abstractmethod
|
||||
# def doc_to_llamaindex_doc(self, doc: Dict) -> 'Document':
|
||||
# """
|
||||
# Convert data source document to LlamaIndex Document
|
||||
#
|
||||
# Args:
|
||||
# doc: Document from the data source
|
||||
#
|
||||
# Returns:
|
||||
# LlamaIndex Document object
|
||||
# """
|
||||
# pass
|
||||
|
||||
|
||||
def process_documents(self, docs: List[Dict]) -> List['Document']:
|
||||
"""
|
||||
Process multiple documents into LlamaIndex Documents
|
||||
|
|
@ -164,7 +165,7 @@ class BaseSync(ABC):
|
|||
from config import settings
|
||||
|
||||
node_parser = SentenceSplitter(
|
||||
chunk_size=settings.CHUNK_SIZE,
|
||||
chunk_size=settings.CHUNK_SIZE, #NOTE: 从settings中获取,默认1024
|
||||
chunk_overlap=settings.CHUNK_OVERLAP
|
||||
)
|
||||
|
||||
|
|
@ -285,7 +286,7 @@ def get_sync_class(source_type: str, db_type: str = 'mysql') -> type[BaseSync]:
|
|||
Get the appropriate sync class based on data source type
|
||||
|
||||
Args:
|
||||
source_type: Type of data source (database, folder)
|
||||
source_type: Type of data source (database, folder, git)
|
||||
db_type: Type of database (mysql, dameng, etc.) - only used when source_type is 'database'
|
||||
|
||||
Returns:
|
||||
|
|
@ -297,6 +298,7 @@ def get_sync_class(source_type: str, db_type: str = 'mysql') -> type[BaseSync]:
|
|||
from sync.mysql_sync import MySQLSync
|
||||
from sync.folder_sync import FolderSync
|
||||
from sync.dameng_sync import DaMengSync
|
||||
from sync.git_sync import GitSync
|
||||
|
||||
if source_type == 'database':
|
||||
# 根据数据库类型选择相应的同步类
|
||||
|
|
@ -307,5 +309,7 @@ def get_sync_class(source_type: str, db_type: str = 'mysql') -> type[BaseSync]:
|
|||
return MySQLSync
|
||||
elif source_type == 'folder':
|
||||
return FolderSync
|
||||
elif source_type == 'git':
|
||||
return GitSync
|
||||
else:
|
||||
raise ValueError(f"Unsupported data source type: {source_type}")
|
||||
|
|
|
|||
|
|
@ -222,7 +222,8 @@ class DaMengSync(BaseSync):
|
|||
"source": "dameng",
|
||||
# Dameng中没有database的概念
|
||||
"schema": self.db_config.user,
|
||||
"table": self.db_config.table_name
|
||||
"table": self.db_config.table_name,
|
||||
"db_source": self.db_config.name
|
||||
}
|
||||
|
||||
if title:
|
||||
|
|
|
|||
|
|
@ -216,6 +216,7 @@ class FolderSync(BaseSync):
|
|||
# Ensure metadata has source information
|
||||
metadata['source'] = 'folder'
|
||||
metadata['host'] = self.config.host
|
||||
metadata['db_source'] = self.config.name
|
||||
|
||||
# Create Document
|
||||
return Document(
|
||||
|
|
@ -260,6 +261,9 @@ class FolderSync(BaseSync):
|
|||
|
||||
# Create SFTP client
|
||||
self._sftp_client = self._ssh_client.open_sftp()
|
||||
|
||||
# 检测服务器操作系统类型,确定路径分隔符
|
||||
self._detect_server_os()
|
||||
except paramiko.AuthenticationException:
|
||||
raise Exception(f"SSH connection failed: Authentication failed for user {username} on {self.config.host}")
|
||||
except paramiko.SSHException as ssh_error:
|
||||
|
|
@ -267,6 +271,32 @@ class FolderSync(BaseSync):
|
|||
except Exception as e:
|
||||
raise Exception(f"Connection failed: {str(e)}")
|
||||
|
||||
def _detect_server_os(self):
|
||||
"""
|
||||
检测服务器操作系统类型,确定路径分隔符
|
||||
"""
|
||||
try:
|
||||
# 尝试执行uname命令(Linux/Unix系统)
|
||||
stdin, stdout, stderr = self._ssh_client.exec_command('uname')
|
||||
output = stdout.read().decode('utf-8').strip().lower()
|
||||
|
||||
if output:
|
||||
# Linux/Unix系统
|
||||
self._path_sep = '/'
|
||||
else:
|
||||
# 尝试执行ver命令(Windows系统)
|
||||
stdin, stdout, stderr = self._ssh_client.exec_command('ver')
|
||||
output = stdout.read().decode('utf-8').strip().lower()
|
||||
if 'windows' in output:
|
||||
# Windows系统
|
||||
self._path_sep = '\\'
|
||||
else:
|
||||
# 默认使用正斜杠(大多数SFTP服务器支持)
|
||||
self._path_sep = '/'
|
||||
except Exception:
|
||||
# 如果命令执行失败,默认使用正斜杠
|
||||
self._path_sep = '/'
|
||||
|
||||
def _disconnect(self):
|
||||
"""
|
||||
Disconnect from the server
|
||||
|
|
@ -309,7 +339,8 @@ class FolderSync(BaseSync):
|
|||
items = self._sftp_client.listdir_attr(folder_path)
|
||||
|
||||
for item in items:
|
||||
item_path = os.path.join(folder_path, item.filename)
|
||||
# 使用检测到的路径分隔符连接路径
|
||||
item_path = f"{folder_path}{getattr(self, '_path_sep', '/')}{item.filename}"
|
||||
|
||||
if item.filename not in ('.', '..'):
|
||||
if item.st_mode & 0o040000: # Check if it's a directory
|
||||
|
|
@ -335,8 +366,8 @@ class FolderSync(BaseSync):
|
|||
if not hasattr(self.config, 'ignore_patterns') or not self.config.ignore_patterns:
|
||||
return False
|
||||
|
||||
# Get relative path from folder root
|
||||
relative_path = os.path.relpath(file_path, self.config.folder_path)
|
||||
# Get relative path from folder root and ensure forward slashes for pattern matching
|
||||
relative_path = os.path.relpath(file_path, self.config.folder_path).replace('\\', '/')
|
||||
|
||||
for pattern in self.config.ignore_patterns:
|
||||
if self._match_pattern(relative_path, pattern):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,652 @@
|
|||
"""
|
||||
Git代码库同步子类
|
||||
继承BaseSync,实现代码拉取/增量同步/函数解析
|
||||
"""
|
||||
import os
|
||||
from typing import List, Dict, Any, Set, Optional
|
||||
from datetime import datetime
|
||||
from loguru import logger
|
||||
from config import BaseDataSourceConfig, GitDataSourceConfig, settings
|
||||
from sync.base_sync import BaseSync
|
||||
from sync.ast_parser import ASTParser
|
||||
from utils.git_tool import GitTool
|
||||
from utils.func_id_generator import generate_func_unique_id
|
||||
|
||||
|
||||
class GitSync(BaseSync):
|
||||
def __init__(self, config: GitDataSourceConfig, vector_store_manager=None):
|
||||
"""
|
||||
初始化Git同步器
|
||||
|
||||
Args:
|
||||
config: Git数据源配置
|
||||
vector_store_manager: 向量存储管理器
|
||||
"""
|
||||
super().__init__(config, vector_store_manager)
|
||||
self.config = config
|
||||
self.git_tools = [] # 存储多个GitTool实例
|
||||
|
||||
# 处理多仓库配置
|
||||
if hasattr(config, 'git_mode') and config.git_mode == 'server' and hasattr(config, 'git_repositories') and config.git_repositories:
|
||||
# Git服务器模式,多仓库
|
||||
host = getattr(config, 'git_server_host', 'localhost')
|
||||
port = getattr(config, 'git_server_port', 9418)
|
||||
username = getattr(config, 'git_server_username', '')
|
||||
|
||||
for repo_config in config.git_repositories:
|
||||
# 获取仓库名称、路径和分支
|
||||
if isinstance(repo_config, dict):
|
||||
repo_name = repo_config.get('repository')
|
||||
repo_path = repo_config.get('path', repo_name) # 优先使用path字段,如果没有则使用repository
|
||||
branch = repo_config.get('branch', '') # 如果没有指定分支,使用空字符串
|
||||
else:
|
||||
# 向后兼容:如果是字符串格式,使用默认分支
|
||||
repo_name = repo_config
|
||||
repo_path = repo_config
|
||||
branch = getattr(config, 'git_branch', '') # 如果没有指定分支,使用空字符串
|
||||
|
||||
if not repo_name:
|
||||
continue
|
||||
|
||||
# 构建git_url
|
||||
if port == 9418:
|
||||
# Git daemon协议
|
||||
# 尝试两种格式:带.git后缀和不带.git后缀
|
||||
git_url = f"git://{host}:{port}/{repo_path}"
|
||||
# 同时支持带.git后缀的格式
|
||||
if not repo_path.endswith('.git'):
|
||||
git_url_with_suffix = f"git://{host}:{port}/{repo_path}.git"
|
||||
else:
|
||||
git_url_with_suffix = git_url
|
||||
else:
|
||||
# SSH协议
|
||||
# 检查repo_path是否已经是完整路径或已经包含.git后缀
|
||||
if repo_path.startswith('/') or repo_path.endswith('.git'):
|
||||
# 如果是完整路径或已经包含.git后缀,直接使用
|
||||
if username:
|
||||
git_url = f"ssh://{username}@{host}:{port}{repo_path}"
|
||||
else:
|
||||
git_url = f"ssh://{host}:{port}{repo_path}"
|
||||
else:
|
||||
# 否则,添加.git后缀
|
||||
if username:
|
||||
git_url = f"ssh://{username}@{host}:{port}/{repo_path}.git"
|
||||
else:
|
||||
git_url = f"ssh://{host}:{port}/{repo_path}.git"
|
||||
|
||||
# 初始化Git工具
|
||||
git_tool = GitTool(
|
||||
user_id="default", # 暂时使用默认用户ID
|
||||
repo_id=f"{config.name}_{repo_name}",
|
||||
git_config={
|
||||
"git_url": git_url,
|
||||
"branch": branch,
|
||||
"ssh_key": config.ssh_key,
|
||||
"https_token": config.https_token,
|
||||
"password": getattr(config, 'git_server_password', None),
|
||||
"local_repo_path": config.local_repo_path
|
||||
}
|
||||
)
|
||||
# 存储仓库的分支信息,以便后续使用
|
||||
git_tool.branch = branch
|
||||
self.git_tools.append(git_tool)
|
||||
else:
|
||||
# 单个Git仓库
|
||||
# 构建git_url
|
||||
git_url = config.git_url
|
||||
if hasattr(config, 'git_mode') and config.git_mode == 'server':
|
||||
# 对于Git服务器模式,构建git_url
|
||||
host = getattr(config, 'git_server_host', 'localhost')
|
||||
port = getattr(config, 'git_server_port', 9418)
|
||||
repository = getattr(config, 'git_repository', '')
|
||||
repo_path = repository
|
||||
|
||||
# 尝试从git_repositories中获取对应仓库的路径
|
||||
if hasattr(config, 'git_repositories') and config.git_repositories:
|
||||
for repo in config.git_repositories:
|
||||
if isinstance(repo, dict) and repo.get('repository') == repository:
|
||||
repo_path = repo.get('path', repository) # 优先使用path字段
|
||||
break
|
||||
|
||||
if port == 9418:
|
||||
# Git daemon协议
|
||||
git_url = f"git://{host}:{port}/{repo_path}"
|
||||
else:
|
||||
# SSH协议
|
||||
username = getattr(config, 'git_server_username', '')
|
||||
# 检查repo_path是否已经是完整路径或已经包含.git后缀
|
||||
if repo_path.startswith('/') or repo_path.endswith('.git'):
|
||||
# 如果是完整路径或已经包含.git后缀,直接使用
|
||||
if username:
|
||||
git_url = f"ssh://{username}@{host}:{port}{repo_path}"
|
||||
else:
|
||||
git_url = f"ssh://{host}:{port}{repo_path}"
|
||||
else:
|
||||
# 否则,添加.git后缀
|
||||
if username:
|
||||
git_url = f"ssh://{username}@{host}:{port}/{repo_path}.git"
|
||||
else:
|
||||
git_url = f"ssh://{host}:{port}/{repo_path}.git"
|
||||
|
||||
# 初始化Git工具
|
||||
git_tool = GitTool(
|
||||
user_id="default", # 暂时使用默认用户ID
|
||||
repo_id=config.name,
|
||||
git_config={
|
||||
"git_url": git_url,
|
||||
"branch": getattr(config, 'branch', ''), # 如果没有指定分支,使用空字符串
|
||||
"ssh_key": config.ssh_key,
|
||||
"https_token": config.https_token,
|
||||
"password": getattr(config, 'git_server_password', None),
|
||||
"local_repo_path": config.local_repo_path
|
||||
}
|
||||
)
|
||||
self.git_tools.append(git_tool)
|
||||
|
||||
def fetch_all_documents(self) -> List[Dict[str, Any]]:
|
||||
"""获取所有文档,基于Git blob SHA进行文件级别去重"""
|
||||
func_list = []
|
||||
|
||||
# 遍历所有Git工具实例(支持多仓库)
|
||||
for git_tool in self.git_tools:
|
||||
# 克隆/更新仓库
|
||||
git_tool.clone_repo()
|
||||
|
||||
# 获取当前仓库所有文件的blob SHA
|
||||
current_file_shas = git_tool.get_all_file_shas()
|
||||
|
||||
# 从ChromaDB获取已处理的文件SHA(使用git_tool的repo_id和branch)
|
||||
processed_file_shas = self._get_processed_file_shas_from_chroma(
|
||||
repo_id=git_tool.repo_id,
|
||||
branch=git_tool.branch
|
||||
)
|
||||
|
||||
# 识别需要处理的新文件/修改文件
|
||||
files_to_process = []
|
||||
for file_path, current_sha in current_file_shas.items():
|
||||
if file_path not in processed_file_shas or processed_file_shas[file_path] != current_sha:
|
||||
files_to_process.append(file_path)
|
||||
|
||||
logger.info(f"仓库 {git_tool.repo_id} 文件去重结果: 总数{len(current_file_shas)}, 已处理{len(processed_file_shas)}, 待处理{len(files_to_process)}")
|
||||
|
||||
# 解析需要处理的文件
|
||||
for file_path in files_to_process:
|
||||
lang = ASTParser.detect_language(file_path)
|
||||
if lang:
|
||||
parser = ASTParser(file_path, lang)
|
||||
try:
|
||||
functions = parser.parse_functions()
|
||||
for func in functions:
|
||||
if func is None:
|
||||
continue
|
||||
|
||||
func_id = generate_func_unique_id(
|
||||
user_id="default",
|
||||
repo_id=git_tool.repo_id,
|
||||
branch=git_tool.branch,
|
||||
file_path=func["file_path"],
|
||||
class_name=func.get("class_name"),
|
||||
func_name=func["func_name"]
|
||||
)
|
||||
func['id'] = func_id
|
||||
func['repo_id'] = git_tool.repo_id # 存储正确的 repo_id
|
||||
func['branch'] = git_tool.branch # 存储正确的 branch
|
||||
func['file_blob_sha'] = current_file_shas[file_path]
|
||||
func_list.append(func)
|
||||
except Exception as e:
|
||||
logger.error(f"解析文件失败 {file_path}: {e}")
|
||||
|
||||
logger.info(f"获取到 {len(func_list)} 个函数")
|
||||
return func_list
|
||||
|
||||
def _get_processed_file_shas_from_chroma(self, repo_id: str = None, branch: str = None, collection_key: str = 'code') -> Dict[str, str]:
|
||||
"""
|
||||
从ChromaDB获取已处理的文件SHA映射
|
||||
|
||||
Args:
|
||||
repo_id: 仓库ID(可选,不传则使用config.name)
|
||||
branch: 分支名(可选,不传则使用config.branch)
|
||||
collection_key: Collection key
|
||||
"""
|
||||
try:
|
||||
if not self.vector_store_manager:
|
||||
return {}
|
||||
|
||||
collection = self.vector_store_manager.collections.get(collection_key)
|
||||
if not collection:
|
||||
logger.warning(f"Collection {collection_key} not initialized")
|
||||
return {}
|
||||
|
||||
# 使用传入的参数或默认值
|
||||
query_repo_id = repo_id if repo_id else self.config.name
|
||||
query_branch = branch if branch else getattr(self.config, 'branch', None)
|
||||
|
||||
# 构建查询条件
|
||||
if query_branch:
|
||||
where_clause = {
|
||||
"$and": [
|
||||
{"repo_id": {"$eq": query_repo_id}},
|
||||
{"branch": {"$eq": query_branch}}
|
||||
]
|
||||
}
|
||||
else:
|
||||
where_clause = {"repo_id": {"$eq": query_repo_id}}
|
||||
|
||||
results = collection.get(where=where_clause)
|
||||
|
||||
processed_shas = {}
|
||||
for metadata in results.get('metadatas', []):
|
||||
if metadata and 'file_path' in metadata and 'file_blob_sha' in metadata:
|
||||
file_path = metadata['file_path']
|
||||
blob_sha = metadata['file_blob_sha']
|
||||
processed_shas[file_path] = blob_sha
|
||||
|
||||
logger.debug(f"从ChromaDB获取到 {len(processed_shas)} 个已处理文件SHA (repo_id: {query_repo_id}, branch: {query_branch})")
|
||||
return processed_shas
|
||||
except Exception as e:
|
||||
logger.warning(f"从ChromaDB获取已处理文件SHA失败: {e}")
|
||||
return {}
|
||||
|
||||
def doc_to_llamaindex_doc(self, doc: Dict) -> 'Document':
|
||||
"""
|
||||
转换函数信息为LlamaIndex Document
|
||||
|
||||
Args:
|
||||
doc: 函数信息
|
||||
|
||||
Returns:
|
||||
Document: LlamaIndex Document对象
|
||||
"""
|
||||
from llama_index.core import Document
|
||||
|
||||
# 生成函数唯一ID
|
||||
func_id = doc.get('id')
|
||||
|
||||
# 生成函数描述
|
||||
func_desc = self.generate_func_desc(doc)
|
||||
|
||||
# 获取函数体
|
||||
func_body = doc.get("func_body", "")
|
||||
|
||||
|
||||
# 创建Document对象
|
||||
document = Document(
|
||||
text=func_desc, # 使用描述作为文本
|
||||
id_=func_id,
|
||||
metadata={
|
||||
"func_id": func_id,
|
||||
"func_name": doc["func_name"],
|
||||
"class_name": doc.get("class_name") if doc.get("class_name")!=None else "None",
|
||||
"file_path": doc["file_path"],
|
||||
"lang": doc["lang"],
|
||||
"params": len(doc.get("params", [])), # 只存储参数数量,不存储完整参数列表
|
||||
"return_type": doc.get("return_type") if doc.get("return_type")!=None else "None",
|
||||
"docstring": (doc.get("docstring") or "")[:200], # 进一步限制文档字符串长度
|
||||
"start_line": doc.get("start_line"),
|
||||
"end_line": doc.get("end_line"),
|
||||
"repo_id": doc.get("repo_id", self.config.name), # 使用 doc 中的 repo_id
|
||||
"db_source": self.config.name, # 保存 git server 配置名称,用于删除时匹配
|
||||
"branch": doc.get("branch", self.config.branch), # 使用 doc 中的 branch
|
||||
"func_body": doc["func_body"][:1000],
|
||||
"file_blob_sha": doc.get("file_blob_sha", "")
|
||||
}
|
||||
)
|
||||
|
||||
return document
|
||||
|
||||
def generate_func_desc(self, func_info: Dict) -> str:
|
||||
"""
|
||||
生成函数描述
|
||||
|
||||
Args:
|
||||
func_info: 函数信息
|
||||
|
||||
Returns:
|
||||
str: 函数描述
|
||||
"""
|
||||
# 构建函数描述
|
||||
parts = []
|
||||
|
||||
# 函数类型
|
||||
if func_info.get("class_name"):
|
||||
parts.append(f"{func_info['class_name']}类的{func_info['func_name']}方法")
|
||||
else:
|
||||
parts.append(f"{func_info['func_name']}函数")
|
||||
|
||||
# 参数信息
|
||||
params = func_info.get("params", [])
|
||||
if params:
|
||||
param_str = []
|
||||
for param in params:
|
||||
if param.get("type"):
|
||||
param_str.append(f"{param['name']}: {param['type']}")
|
||||
else:
|
||||
param_str.append(param['name'])
|
||||
parts.append(f"接收参数: {', '.join(param_str)}")
|
||||
|
||||
# 返回值信息
|
||||
return_type = func_info.get("return_type")
|
||||
if return_type:
|
||||
parts.append(f"返回类型: {return_type}")
|
||||
|
||||
# 文档字符串
|
||||
docstring = func_info.get("docstring")
|
||||
if docstring:
|
||||
parts.append(f"功能描述: {docstring.strip()}")
|
||||
else:
|
||||
# NOTE:如果没有文档字符串,使用本地大模型根据函数体生成描述
|
||||
# BUG: 同步服务终止后(删除对应配置 or 整个服务终止),仍在继续生成文档字符串
|
||||
parts.append(f"功能描述: {self._generate_docstring_from_body(func_info.get('func_body', ''))}")
|
||||
|
||||
return ". ".join(parts)
|
||||
|
||||
def _generate_docstring_from_body(self, func_body: str) -> str:
|
||||
"""
|
||||
使用本地大模型根据函数体生成文档字符串
|
||||
|
||||
Args:
|
||||
func_body: 函数体代码
|
||||
|
||||
Returns:
|
||||
str: 生成的文档字符串
|
||||
"""
|
||||
if not func_body:
|
||||
return "无文档字符串"
|
||||
|
||||
try:
|
||||
from config import settings
|
||||
from llama_index.llms.ollama import Ollama
|
||||
|
||||
# 初始化Ollama LLM
|
||||
llm = Ollama(
|
||||
model=settings.OLLAMA_MODEL,
|
||||
base_url=settings.OLLAMA_BASE_URL,
|
||||
temperature=0.3, # 降低温度,生成更确定的结果
|
||||
request_timeout=300.0
|
||||
)
|
||||
|
||||
# 构建提示词
|
||||
prompt = f"""
|
||||
请为以下函数生成简洁的文档字符串,描述其功能、参数和返回值:
|
||||
|
||||
{func_body}
|
||||
|
||||
要求:
|
||||
1. 语言简洁明了,不超过100字
|
||||
2. 只返回文档字符串内容,不要包含其他内容
|
||||
3. 重点描述函数的核心功能
|
||||
"""
|
||||
|
||||
# 生成文档字符串
|
||||
response = llm.complete(prompt)
|
||||
generated_docstring = response.text.strip()
|
||||
|
||||
# 限制长度
|
||||
if len(generated_docstring) > 200:
|
||||
generated_docstring = generated_docstring[:200] + "..."
|
||||
|
||||
logger.debug(f"生成的文档字符串: {generated_docstring}")
|
||||
return generated_docstring
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"生成文档字符串失败: {e}")
|
||||
# 降级方案:返回基于函数名的简单描述
|
||||
return "执行相关操作的函数"
|
||||
|
||||
def fetch_new_documents(self, last_sync_time=None) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
获取新文档(增量同步)
|
||||
|
||||
Args:
|
||||
last_sync_time: 上次同步时间
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 新函数信息列表
|
||||
"""
|
||||
func_list = []
|
||||
|
||||
# 遍历所有Git工具实例
|
||||
for git_tool in self.git_tools:
|
||||
# 检测远程更新
|
||||
try:
|
||||
has_update, local_commit, remote_commit = git_tool.detect_remote_update()
|
||||
|
||||
if not has_update:
|
||||
logger.info(f"Git仓库 {git_tool.repo_id} 无更新")
|
||||
continue
|
||||
|
||||
# 增量拉取
|
||||
delta_files = git_tool.incremental_pull(local_commit, remote_commit)
|
||||
|
||||
#==== 处理重命名文件
|
||||
rename_file_pairs = delta_files.get("RENAME", [])
|
||||
if rename_file_pairs and self.vector_store_manager:
|
||||
logger.info(f"处理重命名的文件: {len(rename_file_pairs)} 个")
|
||||
for old_path, new_path in rename_file_pairs:
|
||||
logger.info(f"重命名文件: {old_path} -> {new_path}")
|
||||
# 从向量存储中更新元数据
|
||||
try:
|
||||
self.vector_store_manager.update_document_metadata(old_path, new_path)
|
||||
except Exception as e:
|
||||
logger.error(f"更新重命名文件元数据失败: {e}")
|
||||
|
||||
#==== 处理删除的文件
|
||||
delete_files = delta_files.get("DELETE", [])
|
||||
if delete_files and self.vector_store_manager:
|
||||
logger.info(f"处理删除的文件: {len(delete_files)} 个")
|
||||
for file_path in delete_files:
|
||||
# 从向量存储中删除相关文档
|
||||
try:
|
||||
collection = self.vector_store_manager.collections.get('default')
|
||||
if not collection:
|
||||
logger.warning("Collection not initialized")
|
||||
continue
|
||||
# 获取所有文档的元数据
|
||||
results = collection.get(include=['metadatas'])
|
||||
metadatas = results.get('metadatas', [])
|
||||
ids = results.get('ids', [])
|
||||
|
||||
# 找出需要删除的文档ID
|
||||
to_delete_ids = []
|
||||
for doc_id, metadata in zip(ids, metadatas):
|
||||
if metadata and metadata.get('file_path') == file_path:
|
||||
to_delete_ids.append(doc_id)
|
||||
|
||||
if to_delete_ids:
|
||||
logger.info(f"删除文件 {file_path} 相关的 {len(to_delete_ids)} 个文档")
|
||||
self.vector_store_manager.delete_documents(to_delete_ids)
|
||||
except Exception as e:
|
||||
logger.error(f"删除文件 {file_path} 相关文档失败: {e}")
|
||||
|
||||
#==== 解析新增/修改的文件
|
||||
processed_files = set()
|
||||
logger.info(f"增量更新处理文件: {len(delta_files.get('ADD', []))} 个新增, {len(delta_files.get('MODIFY', []))} 个修改")
|
||||
for file_path in delta_files.get("ADD", []) + delta_files.get("MODIFY", []):
|
||||
logger.debug(f"处理文件: {file_path}")
|
||||
if file_path in processed_files:
|
||||
logger.warning(f"文件 {file_path} 已被处理,跳过")
|
||||
continue
|
||||
processed_files.add(file_path)
|
||||
|
||||
lang = ASTParser.detect_language(file_path)
|
||||
if lang:
|
||||
parser = ASTParser(file_path, lang)
|
||||
try:
|
||||
functions = parser.parse_functions()
|
||||
logger.debug(f"文件 {file_path} 解析出 {len(functions)} 个函数")
|
||||
# 为每个函数生成doc_id并设置到字典中
|
||||
for func in functions:
|
||||
if func is None:
|
||||
logger.warning(f"解析出空函数,跳过: {file_path}")
|
||||
continue
|
||||
# 生成唯一的文档ID
|
||||
func_id = generate_func_unique_id(
|
||||
user_id="default",
|
||||
repo_id=git_tool.repo_id,
|
||||
branch=git_tool.branch,
|
||||
file_path=func["file_path"],
|
||||
class_name=func.get("class_name"),
|
||||
func_name=func["func_name"]
|
||||
)
|
||||
logger.debug(f"生成函数ID: {func_id}")
|
||||
func['id'] = func_id
|
||||
func['repo_id'] = git_tool.repo_id # 存储正确的 repo_id
|
||||
func['branch'] = git_tool.branch # 存储正确的 branch
|
||||
func_list.extend(functions)
|
||||
except Exception as e:
|
||||
logger.error(f"解析文件失败 {file_path}: {e}")
|
||||
except Exception as e:
|
||||
logger.warning(f"处理Git仓库 {git_tool.repo_id} 增量同步失败: {e}")
|
||||
|
||||
return func_list
|
||||
|
||||
def get_synced_document_ids(self) -> Set[str]:
|
||||
"""
|
||||
获取已同步的文档ID
|
||||
|
||||
Returns:
|
||||
Set[str]: 文档ID集合
|
||||
"""
|
||||
# 从向量存储中获取已同步的函数ID
|
||||
if not self.vector_store_manager:
|
||||
return set()
|
||||
|
||||
try:
|
||||
# 获取所有已存在的文档ID
|
||||
all_doc_ids = self.vector_store_manager.get_existing_doc_ids()
|
||||
|
||||
# 过滤出与当前Git仓库相关的文档ID
|
||||
synced_ids = set()
|
||||
|
||||
# 获取所有文档的元数据,用于过滤
|
||||
collection = self.vector_store_manager.collections.get('default')
|
||||
if not collection:
|
||||
logger.warning("Collection not initialized")
|
||||
return set()
|
||||
results = collection.get(include=['metadatas'])
|
||||
metadatas = results.get('metadatas', [])
|
||||
ids = results.get('ids', [])
|
||||
|
||||
for doc_id, metadata in zip(ids, metadatas):
|
||||
if metadata and metadata.get('repo_id') == self.config.name:
|
||||
synced_ids.add(doc_id)
|
||||
|
||||
logger.info(f"获取到 {len(synced_ids)} 个已同步的Git函数ID")
|
||||
return synced_ids
|
||||
except Exception as e:
|
||||
logger.error(f"获取已同步文档ID失败: {e}")
|
||||
return set()
|
||||
|
||||
def generate_doc_id(self, identifier: str) -> str:
|
||||
"""
|
||||
生成唯一的文档ID
|
||||
|
||||
Args:
|
||||
identifier: 文档的唯一标识符(文件路径等)
|
||||
|
||||
Returns:
|
||||
str: 唯一的文档ID
|
||||
"""
|
||||
from utils.func_id_generator import generate_func_unique_id
|
||||
|
||||
# 对于Git数据源,使用函数唯一ID生成器
|
||||
# 假设identifier是文件路径
|
||||
return generate_func_unique_id(
|
||||
user_id="default",
|
||||
repo_id=self.config.name,
|
||||
branch=self.config.branch,
|
||||
file_path=identifier,
|
||||
class_name="",
|
||||
func_name=identifier.split('/')[-1].split('.')[0]
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def check_data_source_exists(config: BaseDataSourceConfig) -> bool:
|
||||
"""
|
||||
检查数据源是否存在
|
||||
|
||||
Args:
|
||||
config: 数据源配置
|
||||
|
||||
Returns:
|
||||
bool: 是否存在
|
||||
"""
|
||||
try:
|
||||
# 构建git_url
|
||||
git_url = config.git_url
|
||||
if hasattr(config, 'git_mode') and config.git_mode == 'server':
|
||||
# 对于Git服务器模式,构建git_url
|
||||
host = getattr(config, 'git_server_host', 'localhost')
|
||||
port = getattr(config, 'git_server_port', 9418)
|
||||
username = getattr(config, 'git_server_username', '')
|
||||
repository = getattr(config, 'git_repository', '')
|
||||
repo_path = repository
|
||||
|
||||
# 检查仓库名称是否存在
|
||||
if not repository:
|
||||
# 尝试使用git_repositories中的第一个仓库
|
||||
if hasattr(config, 'git_repositories') and config.git_repositories:
|
||||
first_repo = config.git_repositories[0]
|
||||
if isinstance(first_repo, dict):
|
||||
repository = first_repo.get('repository', '')
|
||||
repo_path = first_repo.get('path', repository) # 优先使用path字段
|
||||
else:
|
||||
repository = first_repo
|
||||
repo_path = repository
|
||||
else:
|
||||
# 尝试从git_repositories中获取对应仓库的路径
|
||||
if hasattr(config, 'git_repositories') and config.git_repositories:
|
||||
for repo in config.git_repositories:
|
||||
if isinstance(repo, dict) and repo.get('repository') == repository:
|
||||
repo_path = repo.get('path', repository) # 优先使用path字段
|
||||
break
|
||||
|
||||
if port == 9418:
|
||||
# Git daemon协议
|
||||
git_url = f"git://{host}:{port}/{repo_path}"
|
||||
else:
|
||||
# SSH协议
|
||||
# 检查repo_path是否已经是完整路径或已经包含.git后缀
|
||||
if repo_path.startswith('/') or repo_path.endswith('.git'):
|
||||
# 如果是完整路径或已经包含.git后缀,直接使用
|
||||
if username:
|
||||
git_url = f"ssh://{username}@{host}{repo_path}"
|
||||
else:
|
||||
git_url = f"ssh://{host}{repo_path}"
|
||||
else:
|
||||
# 否则,添加.git后缀
|
||||
if username:
|
||||
git_url = f"ssh://{username}@{host}/{repo_path}.git"
|
||||
else:
|
||||
git_url = f"ssh://{host}/{repo_path}.git"
|
||||
|
||||
# 尝试克隆仓库
|
||||
# 获取分支信息
|
||||
branch = getattr(config, 'branch', '')
|
||||
if not branch:
|
||||
branch = getattr(config, 'git_branch', '')
|
||||
# 尝试从git_repositories中获取第一个仓库的分支
|
||||
if not branch and hasattr(config, 'git_repositories') and config.git_repositories:
|
||||
first_repo = config.git_repositories[0]
|
||||
if isinstance(first_repo, dict):
|
||||
branch = first_repo.get('branch', '')
|
||||
else:
|
||||
branch = ''
|
||||
|
||||
git_tool = GitTool(
|
||||
user_id="default",
|
||||
repo_id=config.name,
|
||||
git_config={
|
||||
"git_url": git_url,
|
||||
"branch": branch,
|
||||
"ssh_key": config.ssh_key,
|
||||
"https_token": config.https_token,
|
||||
"password": getattr(config, 'git_server_password', None)
|
||||
}
|
||||
)
|
||||
git_tool.clone_repo()
|
||||
logger.info(f"Git数据源检查成功: {config.name}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Git数据源检查失败: {e}")
|
||||
return False
|
||||
|
|
@ -212,7 +212,8 @@ class MySQLSync(BaseSync):
|
|||
"doc_id": doc_id,
|
||||
"source": "mysql",
|
||||
"database": self.db_config.database,
|
||||
"table": self.db_config.table_name
|
||||
"table": self.db_config.table_name,
|
||||
"db_source": self.db_config.name
|
||||
}
|
||||
|
||||
if title:
|
||||
|
|
|
|||
|
|
@ -197,6 +197,35 @@ class SyncService:
|
|||
chunked_docs = self.syncer.chunk_documents(processed_docs)
|
||||
all_chunked_docs.extend(chunked_docs)
|
||||
|
||||
# Update last sync time
|
||||
self.last_sync_time = datetime.now()
|
||||
elif self.source_config.type == "git":
|
||||
if not force:
|
||||
new_documents = []
|
||||
for doc in documents:
|
||||
# 检查服务运行状态:仅在非手动同步时检查
|
||||
if not is_manual and not self._running:
|
||||
logger.info(f"Sync interrupted during document filtering: {self.source_name}")
|
||||
return all_chunked_docs, total_docs, skipped_docs_count # 返回结果,不退出程序
|
||||
|
||||
doc_id = doc.get('id')
|
||||
if not self.vector_store_manager.document_exists(doc_id):
|
||||
new_documents.append(doc)
|
||||
else:
|
||||
skipped_docs_count += 1
|
||||
|
||||
if not new_documents:
|
||||
self.last_sync_time = datetime.now()
|
||||
return all_chunked_docs, total_docs, skipped_docs_count
|
||||
documents = new_documents
|
||||
|
||||
# Process and chunk documents
|
||||
processed_docs = self.syncer.process_documents(documents)
|
||||
chunked_docs = self.syncer.chunk_documents(processed_docs)
|
||||
all_chunked_docs.extend(chunked_docs)
|
||||
|
||||
|
||||
|
||||
# Update last sync time
|
||||
self.last_sync_time = datetime.now()
|
||||
else:
|
||||
|
|
@ -223,11 +252,18 @@ class SyncService:
|
|||
|
||||
# Add all documents to vector store (run in thread pool to avoid blocking event loop)
|
||||
if all_chunked_docs:
|
||||
# Determine collection_key based on data source type
|
||||
# Git repositories contain code → use 'code' collection
|
||||
# Other sources (database, folder) → use 'non_code' collection
|
||||
collection_key = 'code' if self.source_config.type == 'git' else 'non_code'
|
||||
logger.info(f"Adding documents to collection: {collection_key} (data source type: {self.source_config.type})")
|
||||
|
||||
await loop.run_in_executor(
|
||||
None,
|
||||
self.vector_store_manager.add_documents,
|
||||
all_chunked_docs,
|
||||
not force # skip_existing true
|
||||
not force, # skip_existing true
|
||||
collection_key # collection_key based on data source type
|
||||
)
|
||||
|
||||
sync_duration = (datetime.now() - sync_start_time).total_seconds()
|
||||
|
|
@ -317,12 +353,19 @@ class SyncService:
|
|||
|
||||
# Add all new documents to vector store
|
||||
if all_chunked_docs:
|
||||
# Determine collection_key based on data source type
|
||||
# Git repositories contain code → use 'code' collection
|
||||
# Other sources (database, folder) → use 'non_code' collection
|
||||
collection_key = 'code' if self.source_config.type == 'git' else 'non_code'
|
||||
logger.info(f"Adding documents to collection: {collection_key} (data source type: {self.source_config.type})")
|
||||
|
||||
# Run in thread pool to avoid blocking event loop
|
||||
await loop.run_in_executor(
|
||||
None,
|
||||
self.vector_store_manager.add_documents,
|
||||
all_chunked_docs,
|
||||
False # skip_existing,不跳过已存在的文档,默认是更新了内容
|
||||
False, # skip_existing,不跳过已存在的文档,默认是更新了内容
|
||||
collection_key # collection_key based on data source type
|
||||
)
|
||||
logger.info(f"Incremental sync completed: {len(all_chunked_docs)} chunks from {total_docs} documents in {self.source_name}")
|
||||
else:
|
||||
|
|
@ -358,7 +401,7 @@ class SyncService:
|
|||
return
|
||||
|
||||
max_restart_attempts = 10 # Maximum number of restart attempts
|
||||
restart_delay = 60 # Wait 60 seconds before restarting after an error
|
||||
restart_delay = settings.SYNC_INTERVAL # Wait 60 seconds before restarting after an error
|
||||
restart_count = 0
|
||||
|
||||
self._running = True
|
||||
|
|
@ -432,7 +475,7 @@ class SyncService:
|
|||
|
||||
# Check if another sync is in progress (e.g., initial sync or previous incremental sync still running)
|
||||
# Wait for it to complete before starting incremental sync (no timeout - wait indefinitely)
|
||||
wait_interval = 10 # Check every 10 seconds
|
||||
wait_interval = settings.SYNC_INTERVAL # Check every 10 seconds
|
||||
waited_time = 0
|
||||
while self._sync_in_progress:
|
||||
logger.info(f"Another sync is in progress for {self.source_name}, waiting... (waited {waited_time}s, will wait until completion)")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
用于检查chromadb数据库中所有文档的metadata字段
|
||||
运行方式:项目根目录下执行
|
||||
python utils/check_metadata.py
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
# 将上一级目录添加到sys.path
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from rag.vector_store import VectorStoreManager
|
||||
|
||||
|
||||
def check_metadata():
|
||||
"""检查chromadb数据库中所有文档的metadata字段"""
|
||||
try:
|
||||
# 初始化VectorStoreManager
|
||||
print("正在初始化VectorStoreManager...")
|
||||
vector_store_manager = VectorStoreManager()
|
||||
|
||||
# 获取所有文档的metadata
|
||||
print("正在获取所有文档的metadata...")
|
||||
results = vector_store_manager.collections['non_code'].get(include=['metadatas', 'documents'])
|
||||
print(results)
|
||||
input()
|
||||
# print(vector_store_manager)
|
||||
results = vector_store_manager.collections['code'].get(include=['metadatas', 'documents'])
|
||||
|
||||
processed_shas = {}
|
||||
for metadata in results.get('metadatas', []):
|
||||
if metadata and 'file_path' in metadata and 'file_blob_sha' in metadata:
|
||||
file_path = metadata['file_path']
|
||||
blob_sha = metadata['file_blob_sha']
|
||||
processed_shas[file_path] = blob_sha
|
||||
print(f"已处理文件SHA: {file_path} -> {blob_sha}")
|
||||
# input("按任意键继续...")
|
||||
collection = vector_store_manager.collections['code']
|
||||
target_source = 'git_server_172_26_120_125'
|
||||
result = collection.get(where={"repo_id": {"$contains": target_source}})
|
||||
print(f"{target_source}查询结果: {str(result)[:50]}")
|
||||
target_source = 'git_server_172_26_120_125_testrepo'
|
||||
result = collection.get(where={"repo_id": {"$contains": target_source}})
|
||||
print(f"{target_source}查询结果: {str(result)[:50]}")
|
||||
result = collection.get(where={"repo_id": target_source})
|
||||
print(f"{target_source}查询结果: {str(result)[:50]}")
|
||||
exit(0)
|
||||
|
||||
# 提取数据
|
||||
ids = results.get('ids', [])
|
||||
metadatas = results.get('metadatas', [])
|
||||
documents = results.get('documents', [])
|
||||
|
||||
print(f"共找到 {len(ids)} 个文档")
|
||||
print("\n检查metadata中的file_path字段:")
|
||||
print("-" * 80)
|
||||
|
||||
# 统计信息
|
||||
total_docs = len(ids)
|
||||
hello_docs = 0
|
||||
hello_copy_docs = 0
|
||||
hello_copy_copy_docs = 0
|
||||
other_docs = 0
|
||||
|
||||
# 检查每个文档的metadata
|
||||
for i, (doc_id, metadata, document) in enumerate(zip(ids, metadatas, documents)):
|
||||
if metadata:
|
||||
file_path = metadata.get('file_path', 'N/A')
|
||||
func_name = metadata.get('func_name', 'N/A')
|
||||
doc_id_meta = metadata.get('doc_id', 'N/A')
|
||||
chunk_id = metadata.get('chunk_id', 'N/A')
|
||||
|
||||
if "check_metadata" in file_path:
|
||||
print(f"文档ID: {doc_id}")
|
||||
print(f" file_path: {file_path}")
|
||||
print(f" func_name: {func_name}")
|
||||
print(f" doc_id: {doc_id_meta}")
|
||||
print(f" chunk_id: {chunk_id}")
|
||||
for key, value in metadata.items():
|
||||
if "mysql_sync" in str(value):
|
||||
print(f" {key}: {value}")
|
||||
print("+++++")
|
||||
hello_copy_copy_docs += 1
|
||||
|
||||
# 统计file_path中的hello和hello_copy
|
||||
# if 'hello' in file_path and 'hello_copy' not in file_path:
|
||||
# hello_docs += 1
|
||||
# print(" 状态: 仍为hello,未更新")
|
||||
# elif 'hello_copy' in file_path:
|
||||
# hello_copy_docs += 1
|
||||
# print(" 状态: 已更新为hello_copy")
|
||||
# else:
|
||||
# other_docs += 1
|
||||
# print(" 状态: 不包含hello或hello_copy")
|
||||
else:
|
||||
print(f"文档ID: {doc_id}")
|
||||
print(" 无metadata")
|
||||
other_docs += 1
|
||||
|
||||
# print("-" * 80)
|
||||
|
||||
# 打印统计结果
|
||||
print("\n统计结果:")
|
||||
print(f"总文档数: {total_docs}")
|
||||
print(f"包含'hello_copy_copy'的文档数: {hello_copy_copy_docs}")
|
||||
|
||||
if hello_docs > 0:
|
||||
print("\n警告: 仍有文档的file_path包含'hello',未更新为'hello_copy'")
|
||||
else:
|
||||
print("\n所有文档的file_path已更新为'hello_copy'")
|
||||
|
||||
except Exception as e:
|
||||
print(f"错误: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_metadata()
|
||||
|
|
@ -0,0 +1,724 @@
|
|||
"""
|
||||
代码Prompt管理模块
|
||||
用于生成和管理代码相关问答的专属Prompt模板
|
||||
支持基于代码意图分类的动态Prompt选择和生成
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
from typing import Dict, Any, Optional, List, Tuple
|
||||
from enum import Enum
|
||||
from loguru import logger
|
||||
|
||||
# 添加项目根目录到 Python 模块搜索路径
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
project_root = os.path.dirname(current_dir)
|
||||
if project_root not in sys.path:
|
||||
sys.path.insert(0, project_root)
|
||||
|
||||
from utils.query_processor import CodeIntentCategory, PromptTemplateType
|
||||
from utils.prompt import (
|
||||
CODE_EXPLANATION_TEMPLATE,
|
||||
CODE_DEBUGGING_TEMPLATE,
|
||||
CODE_GENERATION_TEMPLATE,
|
||||
ALGORITHM_EXPLANATION_TEMPLATE,
|
||||
CODE_OPTIMIZATION_TEMPLATE,
|
||||
GENERAL_QA_TEMPLATE,
|
||||
)
|
||||
|
||||
|
||||
class CodePromptManager:
|
||||
"""代码Prompt管理类"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化代码Prompt管理器
|
||||
"""
|
||||
self._templates = self._load_templates()
|
||||
self._context_cache: Dict[str, List[Dict[str, str]]] = {}
|
||||
logger.info("代码Prompt管理器初始化完成")
|
||||
|
||||
def _load_templates(self) -> Dict[PromptTemplateType, str]:
|
||||
"""
|
||||
加载Prompt模板
|
||||
|
||||
Returns:
|
||||
Dict[PromptTemplateType, str]: Prompt模板字典
|
||||
"""
|
||||
return {
|
||||
PromptTemplateType.CODE_EXPLANATION: CODE_EXPLANATION_TEMPLATE,
|
||||
PromptTemplateType.CODE_DEBUGGING: CODE_DEBUGGING_TEMPLATE,
|
||||
PromptTemplateType.CODE_GENERATION: CODE_GENERATION_TEMPLATE,
|
||||
PromptTemplateType.ALGORITHM_EXPLANATION: ALGORITHM_EXPLANATION_TEMPLATE,
|
||||
PromptTemplateType.CODE_OPTIMIZATION: CODE_OPTIMIZATION_TEMPLATE,
|
||||
PromptTemplateType.GENERAL_QA: GENERAL_QA_TEMPLATE
|
||||
}
|
||||
|
||||
def _map_intent_to_prompt_type(self, intent_category: str) -> PromptTemplateType:
|
||||
"""
|
||||
将代码意图分类映射到Prompt类型
|
||||
|
||||
Args:
|
||||
intent_category: 代码意图分类(字符串)
|
||||
|
||||
Returns:
|
||||
PromptTemplateType: 对应的Prompt类型
|
||||
"""
|
||||
mapping = {
|
||||
# 代码解释与逻辑类 -> CODE_EXPLANATION
|
||||
"logic_explanation": PromptTemplateType.CODE_EXPLANATION,
|
||||
"entity_introduction": PromptTemplateType.CODE_EXPLANATION,
|
||||
"code_structure": PromptTemplateType.CODE_EXPLANATION,
|
||||
|
||||
# 代码生成与实现类 -> CODE_GENERATION
|
||||
"code_generation": PromptTemplateType.CODE_GENERATION,
|
||||
"boilerplate_implementation": PromptTemplateType.CODE_GENERATION,
|
||||
|
||||
# 调试、优化与理论类
|
||||
"error_debugging": PromptTemplateType.CODE_DEBUGGING,
|
||||
"code_optimization": PromptTemplateType.CODE_OPTIMIZATION,
|
||||
"algorithm_theory": PromptTemplateType.ALGORITHM_EXPLANATION,
|
||||
|
||||
# 非代码问题 -> GENERAL_QA
|
||||
"general_technical": PromptTemplateType.GENERAL_QA,
|
||||
"non_technical": PromptTemplateType.GENERAL_QA,
|
||||
"unknown": PromptTemplateType.GENERAL_QA
|
||||
}
|
||||
|
||||
return mapping.get(intent_category, PromptTemplateType.GENERAL_QA)
|
||||
|
||||
def _build_conversation_history(self, history: Optional[Any]) -> str:
|
||||
"""
|
||||
构建对话历史字符串
|
||||
|
||||
Args:
|
||||
history: 对话历史,可以是字符串或字典列表
|
||||
|
||||
Returns:
|
||||
str: 格式化的对话历史
|
||||
"""
|
||||
if not history:
|
||||
return "无"
|
||||
|
||||
# 如果是字符串,直接返回
|
||||
if isinstance(history, str):
|
||||
return history
|
||||
|
||||
# 如果是字典列表,格式化为字符串
|
||||
if isinstance(history, list):
|
||||
history_str = []
|
||||
for item in history:
|
||||
if isinstance(item, dict):
|
||||
role = item.get('role', 'user')
|
||||
content = item.get('content', '')
|
||||
if role == 'user':
|
||||
history_str.append(f"用户: {content}")
|
||||
else:
|
||||
history_str.append(f"助手: {content}")
|
||||
return "\n".join(history_str)
|
||||
|
||||
# 其他类型,转换为字符串
|
||||
return str(history)
|
||||
|
||||
def _extract_code_from_context(self, code_context: str) -> str:
|
||||
"""
|
||||
从上下文中提取代码
|
||||
|
||||
Args:
|
||||
code_context: 代码上下文
|
||||
|
||||
Returns:
|
||||
str: 提取的代码
|
||||
"""
|
||||
if not code_context:
|
||||
return "无"
|
||||
|
||||
# 尝试提取代码块
|
||||
code_blocks = re.findall(r'```[\w]*\n[\s\S]*?```', code_context)
|
||||
if code_blocks:
|
||||
# 提取所有代码块并合并
|
||||
extracted_code = []
|
||||
for block in code_blocks:
|
||||
# 提取语言标记
|
||||
lang_match = re.match(r'```([\w]*)\n', block)
|
||||
language = lang_match.group(1) if lang_match else ""
|
||||
|
||||
# 去除代码块标记
|
||||
code = re.sub(r'```[\w]*\n|```', '', block)
|
||||
code = code.strip()
|
||||
|
||||
if code:
|
||||
if language:
|
||||
extracted_code.append(f"语言: {language}\n{code}")
|
||||
else:
|
||||
extracted_code.append(code)
|
||||
|
||||
return "\n\n".join(extracted_code)
|
||||
|
||||
# 如果没有代码块标记,尝试提取看起来像代码的部分
|
||||
# 查找连续的多行代码(以缩进或常见代码关键字开头)
|
||||
lines = code_context.split('\n')
|
||||
code_lines = []
|
||||
in_code = False
|
||||
|
||||
for line in lines:
|
||||
# 检查是否是代码行
|
||||
line_stripped = line.strip()
|
||||
if (line_stripped and
|
||||
(line.startswith(' ') or line.startswith('\t') or # 缩进
|
||||
line_stripped.startswith('def ') or line_stripped.startswith('class ') or # Python关键字
|
||||
line_stripped.startswith('import ') or line_stripped.startswith('from ') or # 导入
|
||||
line_stripped.startswith('if ') or line_stripped.startswith('for ') or # 控制流
|
||||
line_stripped.startswith('while ') or line_stripped.startswith('try ') or
|
||||
line_stripped.startswith('except ') or line_stripped.startswith('finally ') or
|
||||
line_stripped.startswith('return ') or line_stripped.startswith('print(') or
|
||||
line_stripped.startswith('// ') or line_stripped.startswith('# ') or # 注释
|
||||
line_stripped.endswith(';') or # 分号结尾(如Java、C++等)
|
||||
line_stripped.startswith('{') or line_stripped.startswith('}') or # 大括号
|
||||
re.match(r'^[\w_]+\s*=\s*', line_stripped) or # 变量赋值
|
||||
re.match(r'^[\w_]+\s*\(.*\)\s*\{{?', line_stripped))): # 函数定义
|
||||
code_lines.append(line)
|
||||
in_code = True
|
||||
elif in_code and line.strip() == '':
|
||||
# 保留代码中的空行
|
||||
code_lines.append(line)
|
||||
elif in_code and len(code_lines) > 3:
|
||||
# 如果已经收集了多行代码,并且遇到非代码行,停止收集
|
||||
break
|
||||
else:
|
||||
# 非代码行,重置
|
||||
code_lines = []
|
||||
in_code = False
|
||||
|
||||
if len(code_lines) > 3:
|
||||
return "\n".join(code_lines)
|
||||
|
||||
return code_context
|
||||
|
||||
def _format_code_block(self, code: str, language: str = "") -> str:
|
||||
"""
|
||||
格式化代码块,提高显示质量
|
||||
|
||||
Args:
|
||||
code: 代码内容
|
||||
language: 代码语言
|
||||
|
||||
Returns:
|
||||
str: 格式化的代码块
|
||||
"""
|
||||
if not code:
|
||||
return ""
|
||||
|
||||
# 添加语言标记
|
||||
lang_tag = language if language else ""
|
||||
|
||||
# 确保代码块格式正确
|
||||
formatted_code = f"```{lang_tag}\n{code}\n```"
|
||||
|
||||
return formatted_code
|
||||
|
||||
def _build_enhanced_context(self, retrieved_results: List[Dict[str, Any]], intent_result: Optional[Dict[str, Any]]) -> str:
|
||||
"""
|
||||
根据意图和检索结果构建增强的上下文
|
||||
|
||||
Args:
|
||||
retrieved_results: 检索结果列表,每个元素包含 id、text 和 metadata
|
||||
intent_result: 意图识别结果
|
||||
|
||||
Returns:
|
||||
str: 增强的上下文
|
||||
"""
|
||||
if not retrieved_results:
|
||||
return "未找到相关参考信息"
|
||||
|
||||
context_parts = []
|
||||
intent = intent_result.get('intent', '') if intent_result else ''
|
||||
|
||||
for result in retrieved_results:
|
||||
metadata = result.get('metadata', {})
|
||||
text = result.get('text', '')
|
||||
result_id = result.get('id', 1)
|
||||
|
||||
# 提取所有 metadata 字段
|
||||
func_id = metadata.get('func_id', '')
|
||||
func_name = metadata.get('func_name', '')
|
||||
class_name = metadata.get('class_name', 'None')
|
||||
file_path = metadata.get('file_path', '')
|
||||
lang = metadata.get('lang', '')
|
||||
params = metadata.get('params', 0)
|
||||
return_type = metadata.get('return_type', 'None')
|
||||
docstring = metadata.get('docstring', '')
|
||||
start_line = metadata.get('start_line', '')
|
||||
end_line = metadata.get('end_line', '')
|
||||
repo_id = metadata.get('repo_id', '')
|
||||
branch = metadata.get('branch', '')
|
||||
func_body = metadata.get('func_body', '')
|
||||
|
||||
# 根据意图构建不同的上下文
|
||||
if intent == "code_understanding":
|
||||
# 代码理解意图,强调语言、函数名、类名、参数、返回类型和函数体
|
||||
context_part = f"【参考信息{result_id}】这是由{lang}实现的函数{func_name}"
|
||||
if class_name and class_name != "None":
|
||||
context_part += f",属于{class_name}类"
|
||||
context_part += f",它接收{params}个参数,返回类型为{return_type}"
|
||||
if docstring:
|
||||
context_part += f"。函数说明:{docstring}"
|
||||
context_part += f"\n文件路径:{file_path},位置:{start_line}-{end_line}\n"
|
||||
context_part += f"具体实现:\n{func_body}\n"
|
||||
context_part += f"仓库:{repo_id},分支:{branch}\n"
|
||||
context_part += f"原始文本:\n{text}"
|
||||
elif intent == "code_modification":
|
||||
# 代码修改意图,强调文件路径、位置和函数体
|
||||
context_part = f"【参考信息{result_id}】需要修改的代码位于文件:{file_path},位置:{start_line}-{end_line}"
|
||||
context_part += f"\n函数名:{func_name}"
|
||||
if class_name and class_name != "None":
|
||||
context_part += f"({class_name}类)"
|
||||
context_part += f",由{lang}实现\n"
|
||||
context_part += f"函数签名:接收{params}个参数,返回类型为{return_type}\n"
|
||||
if docstring:
|
||||
context_part += f"函数说明:{docstring}\n"
|
||||
context_part += f"具体实现:\n{func_body}\n"
|
||||
context_part += f"仓库:{repo_id},分支:{branch}\n"
|
||||
context_part += f"原始文本:\n{text}"
|
||||
elif intent == "functionality_question":
|
||||
# 功能询问意图,强调函数名、文档、参数和返回类型
|
||||
context_part = f"【参考信息{result_id}】函数{func_name}"
|
||||
if class_name and class_name != "None":
|
||||
context_part += f"({class_name}类)"
|
||||
context_part += f"的功能说明:\n{docstring}\n"
|
||||
context_part += f"由{lang}实现,接收{params}个参数,返回类型为{return_type}\n"
|
||||
context_part += f"文件路径:{file_path},位置:{start_line}-{end_line}\n"
|
||||
context_part += f"具体实现:\n{func_body}\n"
|
||||
context_part += f"仓库:{repo_id},分支:{branch}\n"
|
||||
context_part += f"原始文本:\n{text}"
|
||||
else:
|
||||
# 其他意图,综合所有信息
|
||||
context_part = f"【参考信息{result_id}】(来源:{file_path})"
|
||||
context_part += f"\n函数:{func_name}"
|
||||
if class_name and class_name != "None":
|
||||
context_part += f"({class_name}类)"
|
||||
context_part += f",语言:{lang}\n"
|
||||
context_part += f"参数:{params}个,返回类型:{return_type}\n"
|
||||
if docstring:
|
||||
context_part += f"说明:{docstring}\n"
|
||||
context_part += f"位置:{start_line}-{end_line}\n"
|
||||
context_part += f"仓库:{repo_id},分支:{branch}\n"
|
||||
context_part += f"实现:\n{func_body}\n"
|
||||
context_part += f"原始文本:\n{text}"
|
||||
|
||||
context_parts.append(context_part)
|
||||
|
||||
return "\n\n".join(context_parts)
|
||||
|
||||
def generate_prompt(
|
||||
self,
|
||||
user_query: str,
|
||||
intent_category: CodeIntentCategory,
|
||||
code_context: Optional[str] = None,
|
||||
conversation_history: Optional[List[Dict[str, str]]] = None,
|
||||
error_message: Optional[str] = None,
|
||||
target_language: Optional[str] = None,
|
||||
user_requirement: Optional[str] = None
|
||||
) -> str:
|
||||
"""
|
||||
生成代码专用Prompt
|
||||
|
||||
Args:
|
||||
user_query: 用户问题
|
||||
intent_category: 代码意图分类
|
||||
code_context: 代码上下文
|
||||
conversation_history: 对话历史
|
||||
error_message: 错误信息(仅Bug修复场景)
|
||||
target_language: 目标编程语言(仅代码生成场景)
|
||||
user_requirement: 用户需求(仅代码生成场景)
|
||||
|
||||
Returns:
|
||||
str: 生成的Prompt
|
||||
"""
|
||||
try:
|
||||
logger.info(f"生成代码Prompt,意图分类: {intent_category}")
|
||||
|
||||
# 映射意图到Prompt类型
|
||||
prompt_type = self._map_intent_to_prompt_type(intent_category)
|
||||
logger.info(f"选择Prompt类型: {prompt_type}")
|
||||
|
||||
# 获取对应模板
|
||||
template = self._templates.get(prompt_type)
|
||||
if not template:
|
||||
logger.warning(f"未找到对应Prompt模板: {prompt_type}")
|
||||
template = self._templates[PromptTemplateType.GENERAL_QA]
|
||||
|
||||
# 准备参数
|
||||
params = {
|
||||
"user_query": user_query,
|
||||
"code_context": code_context or "无",
|
||||
"conversation_history": self._build_conversation_history(conversation_history),
|
||||
"error_message": error_message or "无",
|
||||
"target_language": target_language or "根据上下文判断",
|
||||
"user_requirement": user_requirement or user_query,
|
||||
"algorithm_code": self._extract_code_from_context(code_context) if code_context else "无"
|
||||
}
|
||||
|
||||
# 填充模板
|
||||
prompt = template
|
||||
for key, value in params.items():
|
||||
placeholder = f"{{{key}}}"
|
||||
prompt = prompt.replace(placeholder, value)
|
||||
|
||||
logger.info(f"Prompt生成完成,长度: {len(prompt)}字符")
|
||||
return prompt
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"生成Prompt失败: {e}")
|
||||
# 返回通用模板
|
||||
return self._templates[PromptTemplateType.GENERAL_QA].format(
|
||||
user_query=user_query,
|
||||
code_context=code_context or "无",
|
||||
conversation_history=self._build_conversation_history(conversation_history),
|
||||
error_message="无",
|
||||
target_language="根据上下文判断",
|
||||
user_requirement=user_query,
|
||||
algorithm_code="无"
|
||||
)
|
||||
|
||||
def generate_dynamic_prompt(
|
||||
self,
|
||||
user_query: str,
|
||||
intent_result: Optional[Dict[str, Any]] = None,
|
||||
code_context: Optional[str] = None,
|
||||
conversation_history: Optional[List[Dict[str, str]]] = None,
|
||||
**kwargs
|
||||
) -> str:
|
||||
"""
|
||||
生成动态Prompt(基于意图识别结果)
|
||||
|
||||
Args:
|
||||
user_query: 用户问题
|
||||
intent_result: 意图识别结果
|
||||
code_context: 代码上下文
|
||||
conversation_history: 对话历史
|
||||
**kwargs: 其他参数
|
||||
|
||||
Returns:
|
||||
str: 生成的动态Prompt
|
||||
"""
|
||||
try:
|
||||
if intent_result:
|
||||
# 从意图结果中提取分类
|
||||
category_str = intent_result.get('category', 'unknown')
|
||||
# 直接使用category_str,因为CodeIntentCategory是一个普通的类,不是枚举类型
|
||||
intent_category = category_str
|
||||
else:
|
||||
# 默认使用通用分类
|
||||
intent_category = CodeIntentCategory.UNKNOWN
|
||||
|
||||
# 提取其他参数
|
||||
error_message = kwargs.get('error_message')
|
||||
target_language = kwargs.get('target_language')
|
||||
user_requirement = kwargs.get('user_requirement')
|
||||
retrieved_results = kwargs.get('retrieved_results', [])
|
||||
|
||||
# 根据意图和 retrieved_results 构建增强的上下文
|
||||
enhanced_context = code_context
|
||||
if retrieved_results:
|
||||
enhanced_context = self._build_enhanced_context(retrieved_results, intent_result)
|
||||
|
||||
# 生成Prompt
|
||||
return self.generate_prompt(
|
||||
user_query=user_query,
|
||||
intent_category=intent_category,
|
||||
code_context=enhanced_context,
|
||||
conversation_history=conversation_history,
|
||||
error_message=error_message,
|
||||
target_language=target_language,
|
||||
user_requirement=user_requirement
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"生成动态Prompt失败: {e}")
|
||||
# 返回通用Prompt
|
||||
return self._templates[PromptTemplateType.GENERAL_QA].format(
|
||||
user_query=user_query,
|
||||
code_context=code_context or "无",
|
||||
conversation_history=self._build_conversation_history(conversation_history),
|
||||
error_message="无",
|
||||
target_language="根据上下文判断",
|
||||
user_requirement=user_query,
|
||||
algorithm_code="无"
|
||||
)
|
||||
|
||||
def optimize_prompt(
|
||||
self,
|
||||
prompt: str,
|
||||
max_length: int = 4000,
|
||||
preserve_structure: bool = True
|
||||
) -> str:
|
||||
"""
|
||||
优化Prompt长度
|
||||
|
||||
Args:
|
||||
prompt: 原始Prompt
|
||||
max_length: 最大长度
|
||||
preserve_structure: 是否保留结构
|
||||
|
||||
Returns:
|
||||
str: 优化后的Prompt
|
||||
"""
|
||||
if len(prompt) <= max_length:
|
||||
return prompt
|
||||
|
||||
logger.warning(f"Prompt过长 ({len(prompt)} > {max_length}),需要优化")
|
||||
|
||||
if preserve_structure:
|
||||
# 保留结构,只优化内容部分
|
||||
# 1. 保留角色设定和核心指令
|
||||
# 2. 精简分析要求
|
||||
# 3. 缩短代码上下文
|
||||
|
||||
# 提取角色设定和核心指令
|
||||
role_match = re.search(r'# 角色设定[\s\S]*?# 核心指令[\s\S]*?\n', prompt)
|
||||
if role_match:
|
||||
role_section = role_match.group(0)
|
||||
else:
|
||||
role_section = ""
|
||||
|
||||
# 提取分析要求
|
||||
req_match = re.search(r'# 分析要求[\s\S]*?(?=# |$)', prompt)
|
||||
if req_match:
|
||||
req_section = req_match.group(0)
|
||||
# 精简分析要求
|
||||
req_lines = req_section.split('\n')
|
||||
# 只保留前3条要求
|
||||
req_section = '\n'.join(req_lines[:4]) # 保留标题和前3条
|
||||
else:
|
||||
req_section = ""
|
||||
|
||||
# 提取其他部分
|
||||
rest_match = re.search(r'# (代码上下文|错误信息|对话历史|用户问题|输出格式)[\s\S]*$', prompt)
|
||||
if rest_match:
|
||||
rest_section = rest_match.group(0)
|
||||
# 缩短代码上下文
|
||||
if '# 代码上下文' in rest_section:
|
||||
code_match = re.search(r'# 代码上下文[\s\S]*?(?=# |$)', rest_section)
|
||||
if code_match:
|
||||
code_section = code_match.group(0)
|
||||
# 只保留前500个字符
|
||||
if len(code_section) > 600:
|
||||
code_lines = code_section.split('\n')
|
||||
if len(code_lines) > 3:
|
||||
# 保留标题和前几行
|
||||
code_section = '\n'.join(code_lines[:2]) + '\n...\n(代码已截断)'
|
||||
rest_section = rest_section.replace(code_match.group(0), code_section)
|
||||
else:
|
||||
rest_section = ""
|
||||
|
||||
optimized = role_section + '\n' + req_section + '\n' + rest_section
|
||||
|
||||
if len(optimized) > max_length:
|
||||
# 进一步缩短
|
||||
optimized = optimized[:max_length - 3] + '...'
|
||||
|
||||
else:
|
||||
# 直接截断
|
||||
optimized = prompt[:max_length - 3] + '...'
|
||||
|
||||
logger.info(f"Prompt优化完成,长度: {len(optimized)}字符")
|
||||
return optimized
|
||||
|
||||
def save_prompt_template(
|
||||
self,
|
||||
template_type: PromptTemplateType,
|
||||
template_content: str,
|
||||
description: Optional[str] = None
|
||||
) -> bool:
|
||||
"""
|
||||
保存自定义Prompt模板
|
||||
|
||||
Args:
|
||||
template_type: Prompt类型
|
||||
template_content: 模板内容
|
||||
description: 模板描述
|
||||
|
||||
Returns:
|
||||
bool: 保存是否成功
|
||||
"""
|
||||
try:
|
||||
# 这里可以扩展为持久化存储
|
||||
# 目前只是在内存中更新
|
||||
self._templates[template_type] = template_content
|
||||
logger.info(f"保存Prompt模板成功: {template_type.value}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"保存Prompt模板失败: {e}")
|
||||
return False
|
||||
|
||||
def get_prompt_template(self, template_type: PromptTemplateType) -> Optional[str]:
|
||||
"""
|
||||
获取Prompt模板
|
||||
|
||||
Args:
|
||||
template_type: Prompt类型
|
||||
|
||||
Returns:
|
||||
Optional[str]: 模板内容
|
||||
"""
|
||||
return self._templates.get(template_type)
|
||||
|
||||
def list_available_templates(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
列出可用的Prompt模板
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 模板列表
|
||||
"""
|
||||
templates = []
|
||||
for template_type, content in self._templates.items():
|
||||
templates.append({
|
||||
"type": template_type.value,
|
||||
"name": template_type.name,
|
||||
"length": len(content),
|
||||
"sample": content[:100] + "..." if len(content) > 100 else content
|
||||
})
|
||||
return templates
|
||||
|
||||
|
||||
# 全局Prompt管理器实例
|
||||
_prompt_manager = None
|
||||
|
||||
def get_prompt_manager() -> CodePromptManager:
|
||||
"""
|
||||
获取全局Prompt管理器实例
|
||||
|
||||
Returns:
|
||||
CodePromptManager: Prompt管理器实例
|
||||
"""
|
||||
global _prompt_manager
|
||||
if _prompt_manager is None:
|
||||
_prompt_manager = CodePromptManager()
|
||||
return _prompt_manager
|
||||
|
||||
|
||||
def generate_code_prompt(
|
||||
user_query: str,
|
||||
intent_category: CodeIntentCategory,
|
||||
code_context: Optional[str] = None,
|
||||
conversation_history: Optional[List[Dict[str, str]]] = None,
|
||||
**kwargs
|
||||
) -> str:
|
||||
"""
|
||||
生成代码专用Prompt
|
||||
|
||||
Args:
|
||||
user_query: 用户问题
|
||||
intent_category: 代码意图分类
|
||||
code_context: 代码上下文
|
||||
conversation_history: 对话历史
|
||||
**kwargs: 其他参数
|
||||
|
||||
Returns:
|
||||
str: 生成的Prompt
|
||||
"""
|
||||
manager = get_prompt_manager()
|
||||
return manager.generate_prompt(
|
||||
user_query=user_query,
|
||||
intent_category=intent_category,
|
||||
code_context=code_context,
|
||||
conversation_history=conversation_history,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
def generate_dynamic_code_prompt(
|
||||
user_query: str,
|
||||
intent_result: Optional[Dict[str, Any]] = None,
|
||||
code_context: Optional[str] = None,
|
||||
conversation_history: Optional[List[Dict[str, str]]] = None,
|
||||
**kwargs
|
||||
) -> str:
|
||||
"""
|
||||
生成动态代码Prompt
|
||||
|
||||
Args:
|
||||
user_query: 用户问题
|
||||
intent_result: 意图识别结果
|
||||
code_context: 代码上下文
|
||||
conversation_history: 对话历史
|
||||
**kwargs: 其他参数
|
||||
|
||||
Returns:
|
||||
str: 生成的动态Prompt
|
||||
"""
|
||||
manager = get_prompt_manager()
|
||||
return manager.generate_dynamic_prompt(
|
||||
user_query=user_query,
|
||||
intent_result=intent_result,
|
||||
code_context=code_context,
|
||||
conversation_history=conversation_history,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""测试代码"""
|
||||
import asyncio
|
||||
from utils.code_intent import CodeIntentDetector
|
||||
|
||||
async def test_prompt_generation():
|
||||
"""测试Prompt生成"""
|
||||
print("=" * 80)
|
||||
print("测试代码Prompt生成")
|
||||
print("=" * 80)
|
||||
|
||||
# 初始化管理器
|
||||
manager = CodePromptManager()
|
||||
detector = CodeIntentDetector()
|
||||
|
||||
# 测试用例
|
||||
test_cases = [
|
||||
{
|
||||
"query": "这个函数是做什么的?如何使用它?",
|
||||
"code": "def calculate_factorial(n):\n if n <= 1:\n return 1\n return n * calculate_factorial(n-1)",
|
||||
"category": CodeIntentCategory.ENTITY_INTRODUCTION
|
||||
},
|
||||
{
|
||||
"query": "为什么会报语法错误?",
|
||||
"code": "for i in range(10)\n print(i)",
|
||||
"error": "SyntaxError: invalid syntax",
|
||||
"category": CodeIntentCategory.ERROR_DEBUGGING
|
||||
},
|
||||
{
|
||||
"query": "如何实现快速排序算法?",
|
||||
"category": CodeIntentCategory.CODE_GENERATION
|
||||
},
|
||||
{
|
||||
"query": "如何优化这段代码的性能?",
|
||||
"code": "def slow_function():\n result = []\n for i in range(100000):\n result.append(i * 2)\n return result",
|
||||
"category": CodeIntentCategory.CODE_OPTIMIZATION
|
||||
}
|
||||
]
|
||||
|
||||
for i, test_case in enumerate(test_cases):
|
||||
print(f"\n测试用例 {i+1}: {test_case['query']}")
|
||||
print("-" * 60)
|
||||
|
||||
# 生成Prompt
|
||||
prompt = manager.generate_prompt(
|
||||
user_query=test_case['query'],
|
||||
intent_category=test_case['category'],
|
||||
code_context=test_case.get('code'),
|
||||
error_message=test_case.get('error')
|
||||
)
|
||||
|
||||
# 打印结果
|
||||
print(f"Prompt类型: {manager._map_intent_to_prompt_type(test_case['category']).value}")
|
||||
print(f"Prompt长度: {len(prompt)}字符")
|
||||
print("\nPrompt内容:")
|
||||
print(prompt[:300] + "..." if len(prompt) > 300 else prompt)
|
||||
print("-" * 60)
|
||||
|
||||
print("\n" + "=" * 80)
|
||||
print("测试完成")
|
||||
print("=" * 80)
|
||||
|
||||
asyncio.run(test_prompt_generation())
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
"""
|
||||
函数全局唯一ID生成工具类
|
||||
按用户/仓库/分支/文件/函数生成唯一ID
|
||||
"""
|
||||
import os
|
||||
from typing import Optional, Dict
|
||||
from config import settings
|
||||
|
||||
|
||||
def generate_func_unique_id(
|
||||
user_id: str,
|
||||
repo_id: str,
|
||||
branch: str,
|
||||
file_path: str,
|
||||
class_name: Optional[str],
|
||||
func_name: str
|
||||
) -> str:
|
||||
"""
|
||||
生成函数全局唯一ID
|
||||
|
||||
Args:
|
||||
user_id: 用户ID
|
||||
repo_id: 仓库ID
|
||||
branch: 分支名
|
||||
file_path: 文件路径
|
||||
class_name: 类名
|
||||
func_name: 函数名
|
||||
|
||||
Returns:
|
||||
str: 函数唯一ID
|
||||
"""
|
||||
# 类名为None则使用空字符串
|
||||
class_name = class_name if class_name else "None"
|
||||
|
||||
# 直接使用文件路径的绝对路径部分,确保唯一性
|
||||
# 替换路径分隔符为下划线
|
||||
file_path = file_path.split(os.sep)[3:]
|
||||
file_path = "_".join(file_path)
|
||||
normalized_file_path = file_path.replace(os.sep, "_")
|
||||
|
||||
# 生成唯一ID
|
||||
unique_id = f"{user_id}_{repo_id}_{branch}_{normalized_file_path}_{class_name}_{func_name}"
|
||||
|
||||
# 替换特殊字符,避免ChromaDB主键冲突
|
||||
unique_id = unique_id.replace("/", "_").replace("\\", "_").replace(":", "_").replace(" ", "_")
|
||||
|
||||
return unique_id
|
||||
|
||||
|
||||
def parse_func_unique_id(func_id: str) -> Dict[str, str]:
|
||||
"""
|
||||
解析函数唯一ID
|
||||
|
||||
Args:
|
||||
func_id: 函数唯一ID
|
||||
|
||||
Returns:
|
||||
Dict[str, str]: 解析后的信息
|
||||
"""
|
||||
parts = func_id.split("_")
|
||||
if len(parts) < 6:
|
||||
raise Exception(f"无效的函数ID格式: {func_id}")
|
||||
|
||||
# 解析各部分
|
||||
user_id = parts[0]
|
||||
repo_id = parts[1]
|
||||
branch = parts[2]
|
||||
|
||||
# 解析文件路径(可能包含下划线)
|
||||
# 从第3个部分开始,到倒数第2个部分结束
|
||||
file_path_parts = parts[3:-2]
|
||||
file_path = "_".join(file_path_parts).replace("_", os.sep)
|
||||
|
||||
class_name = parts[-2]
|
||||
if class_name == "None":
|
||||
class_name = None
|
||||
|
||||
func_name = parts[-1]
|
||||
|
||||
return {
|
||||
"user_id": user_id,
|
||||
"repo_id": repo_id,
|
||||
"branch": branch,
|
||||
"file_path": file_path,
|
||||
"class_name": class_name,
|
||||
"func_name": func_name
|
||||
}
|
||||
|
||||
|
||||
def get_repo_path_from_func_id(func_id: str) -> str:
|
||||
"""
|
||||
从函数ID获取仓库路径
|
||||
|
||||
Args:
|
||||
func_id: 函数唯一ID
|
||||
|
||||
Returns:
|
||||
str: 仓库路径
|
||||
"""
|
||||
info = parse_func_unique_id(func_id)
|
||||
return os.path.join(settings.GIT_LOCAL_STORAGE_ROOT, info["user_id"], info["repo_id"])
|
||||
|
||||
|
||||
def get_file_path_from_func_id(func_id: str) -> str:
|
||||
"""
|
||||
从函数ID获取文件路径
|
||||
|
||||
Args:
|
||||
func_id: 函数唯一ID
|
||||
|
||||
Returns:
|
||||
str: 文件路径
|
||||
"""
|
||||
info = parse_func_unique_id(func_id)
|
||||
repo_path = get_repo_path_from_func_id(func_id)
|
||||
return os.path.join(repo_path, info["file_path"])
|
||||
|
|
@ -0,0 +1,549 @@
|
|||
"""
|
||||
Git命令封装工具类
|
||||
实现Git仓库的克隆、更新检测、增量拉取等功能
|
||||
"""
|
||||
import subprocess
|
||||
import os
|
||||
from typing import Tuple, Dict, List
|
||||
from loguru import logger
|
||||
from config import settings
|
||||
|
||||
|
||||
class GitTool:
|
||||
def __init__(self, user_id: str = "test", repo_id: str = "test", git_config: dict = None,
|
||||
git_url: str = None, branch: str = None, protocol: str = None,
|
||||
https_token: str = None, ssh_key: str = None, password: str = None, local_repo_path: str = None):
|
||||
"""
|
||||
初始化Git工具类
|
||||
|
||||
Args:
|
||||
user_id: 用户ID
|
||||
repo_id: 仓库ID
|
||||
git_config: Git配置信息
|
||||
git_url: Git仓库URL(直接参数,优先级高于git_config)
|
||||
branch: Git分支(直接参数,优先级高于git_config)
|
||||
protocol: Git协议(直接参数,优先级高于git_config)
|
||||
https_token: HTTPS令牌(直接参数,优先级高于git_config)
|
||||
ssh_key: SSH密钥(直接参数,优先级高于git_config)
|
||||
password: SSH密码(直接参数,优先级高于git_config)
|
||||
local_repo_path: 本地仓库路径(直接参数,优先级高于git_config)
|
||||
"""
|
||||
self.user_id = user_id
|
||||
self.repo_id = repo_id
|
||||
|
||||
# 优先使用直接参数,如果没有则使用git_config
|
||||
if git_config:
|
||||
self.git_url = git_url or git_config.get("git_url")
|
||||
# 如果branch为空字符串,保持为空字符串,不使用默认值
|
||||
self.branch = branch if branch is not None else git_config.get("branch", "")
|
||||
self.protocol = protocol or git_config.get("protocol", "https")
|
||||
self.ssh_key = ssh_key or git_config.get("ssh_key")
|
||||
self.https_token = https_token or git_config.get("https_token")
|
||||
self.password = password or git_config.get("password")
|
||||
# 本地结构化存储路径
|
||||
self.local_repo_path = local_repo_path or git_config.get("local_repo_path")
|
||||
else:
|
||||
self.git_url = git_url
|
||||
# 如果branch为空字符串,保持为空字符串,不使用默认值
|
||||
self.branch = branch if branch is not None else ""
|
||||
self.protocol = protocol or "https"
|
||||
self.ssh_key = ssh_key
|
||||
self.https_token = https_token
|
||||
self.password = password
|
||||
self.local_repo_path = local_repo_path
|
||||
|
||||
if not self.local_repo_path:
|
||||
self.local_repo_path = os.path.join(settings.GIT_LOCAL_STORAGE_ROOT, user_id, repo_id)
|
||||
# 初始化Git环境
|
||||
try:
|
||||
self._init_git_env()
|
||||
except Exception as e:
|
||||
import traceback
|
||||
logger.error(f"初始化Git环境失败: {e}")
|
||||
logger.error(f"错误类型: {type(e).__name__}")
|
||||
logger.error(f"错误堆栈: {traceback.format_exc()}")
|
||||
raise
|
||||
|
||||
def _init_git_env(self):
|
||||
"""
|
||||
初始化Git环境(SSH密钥配置)
|
||||
"""
|
||||
logger.info(f"初始化Git环境,密码:{self.password is not None}, SSH密钥:{self.ssh_key is not None}")
|
||||
|
||||
# 设置 SSH variant 为 ssh,避免 "simple" variant 不支持端口问题
|
||||
os.environ["GIT_SSH_VARIANT"] = "ssh"
|
||||
|
||||
# 提取SSH端口(如果在URL中指定)
|
||||
ssh_port = 22
|
||||
if self.git_url and self.git_url.startswith('ssh://'):
|
||||
# 解析SSH URL以获取端口
|
||||
import re
|
||||
port_match = re.search(r'ssh://[^:]+:([0-9]+)/', self.git_url)
|
||||
if port_match:
|
||||
ssh_port = port_match.group(1)
|
||||
|
||||
if self.ssh_key:
|
||||
# 解密SSH私钥,写入临时文件,配置Git SSH
|
||||
ssh_key_path = f"/tmp/ssh_key_{self.user_id}_{self.repo_id}"
|
||||
with open(ssh_key_path, "w") as f:
|
||||
f.write(self.ssh_key)
|
||||
if not self.ssh_key.endswith('\n'):
|
||||
f.write('\n')
|
||||
os.chmod(ssh_key_path, 0o600)
|
||||
os.environ["GIT_SSH_COMMAND"] = f"ssh -i {ssh_key_path} -p {ssh_port} -o StrictHostKeyChecking=no"
|
||||
logger.info(f"使用SSH密钥认证,命令:{os.environ['GIT_SSH_COMMAND']}")
|
||||
elif self.password:
|
||||
# 检查是否在Windows系统上
|
||||
import platform
|
||||
system = platform.system()
|
||||
logger.info(f"本地系统类型:{system}")
|
||||
|
||||
# 使用 GIT_ASKPASS 机制,这是 Git 官方推荐的密码输入方式
|
||||
# 创建 askpass 脚本
|
||||
askpass_script = r'''#!/usr/bin/env python3
|
||||
import sys
|
||||
# 密码文件路径通过环境变量传递
|
||||
import os
|
||||
password_file = os.environ.get('GIT_ASKPASS_PASSWORD_FILE', '')
|
||||
if password_file and os.path.exists(password_file):
|
||||
with open(password_file, 'r', encoding='utf-8') as f:
|
||||
print(f.read().strip())
|
||||
else:
|
||||
print('111111') # 默认密码
|
||||
'''
|
||||
|
||||
import tempfile
|
||||
import sys as sys_module
|
||||
|
||||
# 写入 askpass 脚本
|
||||
askpass_file = tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False, encoding='utf-8')
|
||||
askpass_file.write(askpass_script)
|
||||
askpass_file.close()
|
||||
askpass_path = askpass_file.name
|
||||
|
||||
# 写入密码文件
|
||||
password_file = tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False, encoding='utf-8')
|
||||
password_file.write(self.password)
|
||||
password_file.close()
|
||||
password_file_path = password_file.name
|
||||
|
||||
# 设置环境变量
|
||||
os.environ['GIT_ASKPASS'] = sys_module.executable.replace('\\', '/') + ' "' + askpass_path.replace('\\', '/') + '"'
|
||||
os.environ['GIT_ASKPASS_PASSWORD_FILE'] = password_file_path.replace('\\', '/')
|
||||
os.environ['GIT_TERMINAL_PROMPT'] = '0' # 禁用 Git 的终端提示
|
||||
|
||||
# 设置 SSH 使用交互式模式
|
||||
os.environ['GIT_SSH_COMMAND'] = f'ssh -o StrictHostKeyChecking=no -o BatchMode=no'
|
||||
|
||||
logger.info(f"使用 GIT_ASKPASS 机制进行密码认证")
|
||||
logger.info(f"GIT_ASKPASS: {os.environ['GIT_ASKPASS']}")
|
||||
logger.info(f"GIT_SSH_COMMAND: {os.environ['GIT_SSH_COMMAND']}")
|
||||
logger.info("使用密码认证,自动使用配置中的密码")
|
||||
|
||||
def clone_repo(self) -> bool:
|
||||
"""
|
||||
克隆Git仓库
|
||||
|
||||
Returns:
|
||||
bool: 是否成功克隆
|
||||
"""
|
||||
if not os.path.exists(self.local_repo_path):
|
||||
os.makedirs(os.path.dirname(self.local_repo_path), exist_ok=True)
|
||||
|
||||
# 尝试两种URL格式:原始URL和带/不带.git后缀的URL
|
||||
test_urls = [self.git_url]
|
||||
|
||||
# 如果是Git daemon协议,尝试两种格式
|
||||
if self.git_url.startswith("git://"):
|
||||
if ".git" in self.git_url:
|
||||
# 尝试不带.git后缀的格式
|
||||
test_urls.append(self.git_url.replace(".git", ""))
|
||||
else:
|
||||
# 尝试带.git后缀的格式
|
||||
test_urls.append(f"{self.git_url}.git")
|
||||
# 如果是SSH协议且路径不包含.git后缀,尝试带.git后缀的格式
|
||||
elif self.git_url.startswith("ssh://") and ".git" not in self.git_url:
|
||||
test_urls.append(f"{self.git_url}.git")
|
||||
|
||||
for url in test_urls:
|
||||
# 根据分支是否为空决定克隆方式
|
||||
if self.branch:
|
||||
# 有指定分支,使用带分支克隆
|
||||
cmd = [
|
||||
"git", "clone", "--single-branch",
|
||||
"--branch", self.branch, url, self.local_repo_path
|
||||
]
|
||||
logger.info(f"执行Git克隆命令: {' '.join(cmd)}")
|
||||
# 捕获输出,避免密码提示 #BUG: 对于云服务器方式不能使用git pass密码验证,显示permission denied
|
||||
res = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8', errors='replace')
|
||||
|
||||
if res.returncode == 0:
|
||||
# 克隆成功,更新git_url
|
||||
self.git_url = url
|
||||
# 克隆后校验
|
||||
self._check_repo_integrity()
|
||||
logger.info(f"Git仓库克隆成功: {self.local_repo_path}")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Git克隆失败: {res.stderr}")
|
||||
# 清理失败的克隆尝试
|
||||
if os.path.exists(self.local_repo_path):
|
||||
import shutil
|
||||
shutil.rmtree(self.local_repo_path)
|
||||
|
||||
# 尝试不带分支克隆(适用于空分支或分支不存在的情况)
|
||||
cmd_no_branch = [
|
||||
"git", "clone", url, self.local_repo_path
|
||||
]
|
||||
logger.info(f"尝试不带分支克隆: {' '.join(cmd_no_branch)}")
|
||||
res_no_branch = subprocess.run(cmd_no_branch, capture_output=True, text=True, encoding='utf-8', errors='replace')
|
||||
if res_no_branch.returncode == 0:
|
||||
# 克隆成功,更新git_url
|
||||
self.git_url = url
|
||||
# 克隆后校验
|
||||
self._check_repo_integrity()
|
||||
logger.info(f"Git仓库克隆成功(不带分支): {self.local_repo_path}")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"不带分支克隆失败: {res_no_branch.stderr}")
|
||||
|
||||
# 继续尝试其他URL格式
|
||||
continue
|
||||
|
||||
# 所有格式都尝试失败
|
||||
raise Exception("所有Git克隆尝试都失败")
|
||||
else:
|
||||
# 仓库已存在,执行完整性检查
|
||||
logger.info(f"Git仓库已存在: {self.local_repo_path}")
|
||||
# 检查仓库完整性
|
||||
self._check_repo_integrity()
|
||||
# 尝试获取当前分支
|
||||
try:
|
||||
current_branch = subprocess.run(
|
||||
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
cwd=self.local_repo_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding='utf-8',
|
||||
errors='replace'
|
||||
).stdout.strip()
|
||||
if current_branch:
|
||||
self.branch = current_branch
|
||||
logger.info(f"检测到本地仓库分支: {current_branch}")
|
||||
# 尝试获取远程URL
|
||||
remote_url = subprocess.run(
|
||||
["git", "config", "--get", "remote.origin.url"],
|
||||
cwd=self.local_repo_path,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding='utf-8',
|
||||
errors='replace'
|
||||
).stdout.strip()
|
||||
if remote_url:
|
||||
self.git_url = remote_url
|
||||
logger.info(f"检测到远程URL: {remote_url}")
|
||||
except Exception as e:
|
||||
logger.warning(f"获取本地仓库信息失败: {e}")
|
||||
return False
|
||||
|
||||
def _check_repo_integrity(self):
|
||||
"""
|
||||
仓库完整性校验(git fsck)+ 支持的编程语言检测
|
||||
"""
|
||||
# 执行git fsck
|
||||
try:
|
||||
subprocess.run(["git", "fsck"], cwd=self.local_repo_path, check=True, capture_output=True, text=True, encoding='utf-8', errors='replace')
|
||||
logger.info(f"Git仓库完整性校验成功: {self.local_repo_path}")
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning(f"Git仓库完整性校验失败: {e.stderr}")
|
||||
|
||||
# 扫描文件类型,记录支持的编程语言
|
||||
support_lang = self._detect_support_lang()
|
||||
logger.info(f"检测到支持的编程语言: {support_lang}")
|
||||
return support_lang
|
||||
|
||||
def _detect_support_lang(self) -> List[str]:
|
||||
"""
|
||||
检测仓库支持的编程语言
|
||||
|
||||
Returns:
|
||||
List[str]: 支持的编程语言列表
|
||||
"""
|
||||
lang_extensions = {
|
||||
"python": [".py"],
|
||||
"java": [".java"],
|
||||
"go": [".go"],
|
||||
"javascript": [".js", ".jsx"],
|
||||
"typescript": [".ts", ".tsx"],
|
||||
"c": [".c", ".h"],
|
||||
"cpp": [".cpp", ".hpp", ".cc"],
|
||||
"csharp": [".cs"],
|
||||
"rust": [".rs"],
|
||||
"php": [".php"],
|
||||
"ruby": [".rb"],
|
||||
"swift": [".swift"],
|
||||
"kotlin": [".kt"],
|
||||
"scala": [".scala"]
|
||||
}
|
||||
|
||||
support_lang = []
|
||||
for root, dirs, files in os.walk(self.local_repo_path):
|
||||
# 跳过.git目录
|
||||
if ".git" in dirs:
|
||||
dirs.remove(".git")
|
||||
# 跳过其他常见的非代码目录
|
||||
dirs_to_skip = ["node_modules", "venv", "dist", "build", "__pycache__"]
|
||||
dirs[:] = [d for d in dirs if d not in dirs_to_skip]
|
||||
|
||||
for file in files:
|
||||
for lang, extensions in lang_extensions.items():
|
||||
if any(file.endswith(ext) for ext in extensions):
|
||||
if lang not in support_lang:
|
||||
support_lang.append(lang)
|
||||
break
|
||||
|
||||
return support_lang
|
||||
|
||||
def detect_remote_update(self) -> Tuple[bool, str, str]:
|
||||
"""
|
||||
远程更新检测
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str, str]: (是否有更新, 本地commit ID, 远程commit ID)
|
||||
"""
|
||||
# 确保仓库存在
|
||||
if not os.path.exists(self.local_repo_path):
|
||||
raise Exception(f"Git仓库不存在: {self.local_repo_path}")
|
||||
|
||||
# 拉取远程commit记录
|
||||
try:
|
||||
# 修改fetch命令,只获取远程更新而不直接更新本地分支
|
||||
subprocess.run(["git", "fetch", "origin", self.branch],
|
||||
cwd=self.local_repo_path, check=True, capture_output=True, text=True, encoding='utf-8',errors="replace")
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"Git fetch失败: {e.stderr}")
|
||||
raise
|
||||
|
||||
# 获取本地/远程commit ID
|
||||
local_commit = subprocess.run(["git", "rev-parse", "HEAD"],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True, encoding='utf-8', errors='replace').stdout.strip()
|
||||
remote_commit = subprocess.run(["git", "rev-parse", f"origin/{self.branch}"],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True, encoding='utf-8', errors='replace').stdout.strip()
|
||||
|
||||
has_update = local_commit != remote_commit
|
||||
logger.info(f"Git更新检测: 本地={local_commit[:7]}, 远程={remote_commit[:7]}, 有更新={has_update}")
|
||||
return has_update, local_commit, remote_commit
|
||||
|
||||
def incremental_pull(self, local_commit: str, remote_commit: str) -> Dict[str, List[str]]:
|
||||
"""
|
||||
增量拉取代码+解析文件变更
|
||||
|
||||
Args:
|
||||
local_commit: 本地commit ID
|
||||
remote_commit: 远程commit ID
|
||||
|
||||
Returns:
|
||||
Dict[str, List[str]]: 文件变更集
|
||||
"""
|
||||
# 快进合并到远程最新版本
|
||||
try:
|
||||
subprocess.run(["git", "merge", "--ff-only", f"origin/{self.branch}"],
|
||||
cwd=self.local_repo_path, check=True, capture_output=True, text=True, encoding='utf-8', errors="replace")
|
||||
logger.info(f"Git快进合并成功: {self.branch}")
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f"Git合并失败: {e.stderr}")
|
||||
raise
|
||||
|
||||
# 提取增量commit的文件变更
|
||||
delta_commits = subprocess.run(["git", "log", "--pretty=format:%H", f"{local_commit}..{remote_commit}"],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True, encoding='utf-8', errors='replace').stdout.split()
|
||||
|
||||
# 解析文件变更为ADD/MODIFY/DELETE
|
||||
delta_files = self._parse_delta_files(delta_commits)
|
||||
logger.info(f"Git增量变更: ADD={len(delta_files['ADD'])}, MODIFY={len(delta_files['MODIFY'])}, DELETE={len(delta_files['DELETE'])}, RENAME={len(delta_files['RENAME'])}")
|
||||
return delta_files
|
||||
|
||||
def _parse_delta_files(self, delta_commits: List[str]) -> Dict[str, List[str]]:
|
||||
"""
|
||||
解析文件变更集
|
||||
|
||||
Args:
|
||||
delta_commits: 增量commit列表
|
||||
|
||||
Returns:
|
||||
Dict[str, List[str]]: 文件变更集
|
||||
"""
|
||||
add_files, modify_files, delete_files, rename_files = [], [], [], []
|
||||
|
||||
for commit in delta_commits:
|
||||
# git show --name-status 获取文件变更
|
||||
res = subprocess.run(["git", "show", "--name-status", commit],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True, encoding='utf-8', errors='replace').stdout
|
||||
|
||||
for line in res.splitlines():
|
||||
if not line:
|
||||
continue
|
||||
# 解析状态和文件路径
|
||||
if "\t" in line:
|
||||
status, file_path = line.split("\t", 1)
|
||||
|
||||
if status.startswith("R"):
|
||||
# 处理重命名:分割旧路径和新路径
|
||||
old_path, new_path = file_path.split("\t", 1)
|
||||
old_full_path = os.path.join(self.local_repo_path, old_path)
|
||||
new_full_path = os.path.join(self.local_repo_path, new_path)
|
||||
if status == "R100":
|
||||
# 对于R100(仅重命名无内容变更),处理为仅修改对应document的元数据(file_path、chunk_id、func_id)
|
||||
rename_files.append((old_full_path, new_full_path))
|
||||
else:
|
||||
# 旧路径视为删除,新路径视为添加
|
||||
delete_files.append(old_full_path)
|
||||
add_files.append(new_full_path)
|
||||
elif status == "A":
|
||||
full_path = os.path.join(self.local_repo_path, file_path)
|
||||
add_files.append(full_path)
|
||||
elif status == "M":
|
||||
full_path = os.path.join(self.local_repo_path, file_path)
|
||||
modify_files.append(full_path)
|
||||
elif status == "D":
|
||||
full_path = os.path.join(self.local_repo_path, file_path)
|
||||
delete_files.append(full_path)
|
||||
|
||||
# 去重并返回
|
||||
return {
|
||||
"ADD": list(set(add_files)),
|
||||
"MODIFY": list(set(modify_files)),
|
||||
"DELETE": list(set(delete_files)),
|
||||
"RENAME": list(set(rename_files)),
|
||||
}
|
||||
|
||||
def get_current_commit(self) -> str:
|
||||
"""
|
||||
获取当前commit ID
|
||||
|
||||
Returns:
|
||||
str: 当前commit ID
|
||||
"""
|
||||
if not os.path.exists(self.local_repo_path):
|
||||
raise Exception(f"Git仓库不存在: {self.local_repo_path}")
|
||||
|
||||
commit_id = subprocess.run(["git", "rev-parse", "HEAD"],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True, encoding='utf-8', errors='replace').stdout.strip()
|
||||
return commit_id
|
||||
|
||||
def get_repo_info(self) -> Dict[str, str]:
|
||||
"""
|
||||
获取仓库信息
|
||||
|
||||
Returns:
|
||||
Dict[str, str]: 仓库信息
|
||||
"""
|
||||
if not os.path.exists(self.local_repo_path):
|
||||
raise Exception(f"Git仓库不存在: {self.local_repo_path}")
|
||||
|
||||
# 获取仓库URL
|
||||
remote_url = subprocess.run(["git", "config", "--get", "remote.origin.url"],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True, encoding='utf-8', errors='replace').stdout.strip()
|
||||
|
||||
# 获取当前分支
|
||||
current_branch = subprocess.run(["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
cwd=self.local_repo_path, capture_output=True, text=True, encoding='utf-8', errors='replace').stdout.strip()
|
||||
|
||||
# 获取当前commit
|
||||
current_commit = self.get_current_commit()
|
||||
|
||||
return {
|
||||
"remote_url": remote_url,
|
||||
"current_branch": current_branch,
|
||||
"current_commit": current_commit,
|
||||
"local_path": self.local_repo_path
|
||||
}
|
||||
|
||||
def test_connection(self) -> bool:
|
||||
"""
|
||||
测试Git连接
|
||||
|
||||
Returns:
|
||||
bool: 是否连接成功
|
||||
"""
|
||||
if not self.git_url:
|
||||
raise Exception("Git仓库URL未设置")
|
||||
|
||||
# 尝试两种URL格式:原始URL和带/不带.git后缀的URL
|
||||
test_urls = [self.git_url]
|
||||
|
||||
# 如果是Git daemon协议,尝试两种格式
|
||||
if self.git_url.startswith("git://"):
|
||||
if ".git" in self.git_url:
|
||||
# 尝试不带.git后缀的格式
|
||||
test_urls.append(self.git_url.replace(".git", ""))
|
||||
else:
|
||||
# 尝试带.git后缀的格式
|
||||
test_urls.append(f"{self.git_url}.git")
|
||||
|
||||
for url in test_urls:
|
||||
logger.info(f"测试Git连接: {url}")
|
||||
|
||||
# 尝试执行git ls-remote命令来测试连接
|
||||
try:
|
||||
cmd = ["git", "ls-remote", "--heads", url, f"refs/heads/{self.branch}"]
|
||||
logger.info(f"执行Git连接测试命令: {' '.join(cmd)}")
|
||||
|
||||
res = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8', errors='replace')
|
||||
|
||||
if res.returncode == 0:
|
||||
if self.branch in res.stdout or not res.stdout:
|
||||
logger.info("Git连接测试成功!")
|
||||
self.git_url = url
|
||||
return True
|
||||
else:
|
||||
logger.warning(f"Git连接测试失败:分支 {self.branch} 不存在")
|
||||
continue
|
||||
else:
|
||||
logger.error(f"Git连接测试失败: {res.stderr}")
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Git连接测试异常: {e}")
|
||||
continue
|
||||
|
||||
logger.error("所有Git连接测试格式都失败")
|
||||
return False
|
||||
|
||||
def get_file_blob_sha(self, file_path: str) -> str:
|
||||
"""获取文件的Git blob SHA值"""
|
||||
try:
|
||||
rel_path = os.path.relpath(file_path, self.local_repo_path)
|
||||
|
||||
cmd = ["git", "ls-files", "-s", rel_path]
|
||||
result = subprocess.run(cmd, cwd=self.local_repo_path, capture_output=True, text=True)
|
||||
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
parts = result.stdout.strip().split()
|
||||
if len(parts) >= 3:
|
||||
return parts[1]
|
||||
except Exception as e:
|
||||
logger.warning(f"获取文件blob SHA失败 {file_path}: {e}")
|
||||
|
||||
return ""
|
||||
|
||||
def get_all_file_shas(self) -> Dict[str, str]:
|
||||
"""获取仓库中所有支持语言文件的blob SHA"""
|
||||
from sync.ast_parser import ASTParser
|
||||
|
||||
file_shas = {}
|
||||
support_lang = self._detect_support_lang()
|
||||
|
||||
for root, dirs, files in os.walk(self.local_repo_path):
|
||||
if ".git" in dirs:
|
||||
dirs.remove(".git")
|
||||
|
||||
for file in files:
|
||||
file_path = os.path.join(root, file)
|
||||
lang = ASTParser.detect_language(file_path)
|
||||
if lang and lang in support_lang:
|
||||
blob_sha = self.get_file_blob_sha(file_path)
|
||||
if blob_sha:
|
||||
file_shas[file_path] = blob_sha
|
||||
|
||||
return file_shas
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
"""Prompt模板包"""
|
||||
|
||||
from .answer_generator import (
|
||||
CODE_EXPLANATION_TEMPLATE,
|
||||
CODE_DEBUGGING_TEMPLATE,
|
||||
CODE_GENERATION_TEMPLATE,
|
||||
ALGORITHM_EXPLANATION_TEMPLATE,
|
||||
CODE_OPTIMIZATION_TEMPLATE,
|
||||
GENERAL_QA_TEMPLATE
|
||||
)
|
||||
from .integrated_query_processing import INTEGRATED_QUERY_PROCESSING_TEMPLATE
|
||||
|
||||
__all__ = [
|
||||
"CODE_EXPLANATION_TEMPLATE",
|
||||
"CODE_DEBUGGING_TEMPLATE",
|
||||
"CODE_GENERATION_TEMPLATE",
|
||||
"ALGORITHM_EXPLANATION_TEMPLATE",
|
||||
"CODE_OPTIMIZATION_TEMPLATE",
|
||||
"GENERAL_QA_TEMPLATE",
|
||||
"INTEGRATED_QUERY_PROCESSING_TEMPLATE"
|
||||
]
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
"""答案生成相关的Prompt模板"""
|
||||
|
||||
from .code_explanation import CODE_EXPLANATION_TEMPLATE
|
||||
from .code_generation import CODE_GENERATION_TEMPLATE
|
||||
from .algorithm_explanation import ALGORITHM_EXPLANATION_TEMPLATE
|
||||
from .code_debugging import CODE_DEBUGGING_TEMPLATE
|
||||
from .code_optimization import CODE_OPTIMIZATION_TEMPLATE
|
||||
from .general_qa import GENERAL_QA_TEMPLATE
|
||||
|
||||
__all__ = [
|
||||
"CODE_EXPLANATION_TEMPLATE",
|
||||
"CODE_DEBUGGING_TEMPLATE",
|
||||
"CODE_GENERATION_TEMPLATE",
|
||||
"ALGORITHM_EXPLANATION_TEMPLATE",
|
||||
"CODE_OPTIMIZATION_TEMPLATE",
|
||||
"GENERAL_QA_TEMPLATE"
|
||||
]
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
"""算法解释相关的Prompt模板"""
|
||||
|
||||
ALGORITHM_EXPLANATION_TEMPLATE = """# 角色设定
|
||||
你是一位算法专家,擅长深入解析算法原理和实现,能够根据不同类型的问题提供精准的专业解答。
|
||||
|
||||
## 核心指令
|
||||
请基于提供的算法代码、上下文和问题类型,为用户提供详细、专业的算法分析。
|
||||
|
||||
## 分析要求(根据问题类型调整重点)
|
||||
|
||||
### 算法解释(algorithm_explanation)
|
||||
- **算法原理**:详细解释算法的基本原理和设计思想
|
||||
- **算法步骤**:逐步说明算法的执行流程
|
||||
- **时间复杂度**:分析时间复杂度(最好、平均、最坏情况)
|
||||
- **空间复杂度**:分析空间复杂度,说明内存使用情况
|
||||
- **优缺点**:分析算法的优势和局限性
|
||||
- **适用场景**:说明算法的适用场景和典型应用
|
||||
- **比较分析**:与其他同类算法进行对比
|
||||
|
||||
### 算法实现(algorithm_implementation)
|
||||
- **算法选择**:选择最适合该问题的算法
|
||||
- **实现细节**:提供完整的算法实现代码
|
||||
- **复杂度分析**:分析实现的时间和空间复杂度
|
||||
- **边界处理**:考虑边界情况和特殊输入
|
||||
- **优化建议**:提供可能的优化方向
|
||||
- **测试用例**:提供测试算法的示例用例
|
||||
|
||||
### 数据结构(data_structure)
|
||||
- **结构定义**:详细说明数据结构的定义和特点
|
||||
- **操作方法**:说明数据结构支持的操作及其复杂度
|
||||
- **实现方式**:提供数据结构的实现代码
|
||||
- **适用场景**:说明数据结构的适用场景和典型应用
|
||||
- **性能对比**:与其他数据结构进行性能对比
|
||||
- **使用示例**:提供数据结构的使用示例
|
||||
|
||||
## 算法代码
|
||||
{algorithm_code}
|
||||
|
||||
## 对话历史
|
||||
{conversation_history}
|
||||
|
||||
## 用户问题
|
||||
{user_query}
|
||||
|
||||
请开始你的分析:"""
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
"""代码调试相关的Prompt模板"""
|
||||
|
||||
CODE_DEBUGGING_TEMPLATE = """# 角色设定
|
||||
你是一位专业的代码调试专家,擅长快速定位和解决代码中的错误,能够根据不同类型的错误提供精准的诊断和解决方案。
|
||||
|
||||
## 核心指令
|
||||
请基于提供的代码、错误信息、上下文和错误类型,为用户提供详细的错误分析和解决方案。
|
||||
|
||||
## 分析要求(根据错误类型调整重点)
|
||||
|
||||
### 语法错误(syntax_error)
|
||||
- **错误定位**:准确指出语法错误的具体位置(行号、列号)
|
||||
- **错误原因**:解释违反了哪条语法规则
|
||||
- **修正方案**:提供修正后的完整代码
|
||||
- **预防建议**:说明如何避免类似的语法错误
|
||||
- **常见模式**:列出该语法错误的常见触发场景
|
||||
|
||||
### 运行时错误(runtime_error)
|
||||
- **错误分析**:详细分析异常类型和错误信息
|
||||
- **堆栈跟踪**:解释错误堆栈中的关键信息
|
||||
- **根本原因**:深入分析导致错误的根本原因
|
||||
- **修复方案**:提供具体的修复代码和实施步骤
|
||||
- **异常处理**:建议如何添加异常处理来预防此类错误
|
||||
- **测试建议**:说明如何测试修复是否有效
|
||||
|
||||
### 调试问题(debugging)
|
||||
- **调试方法**:提供适合该问题的调试策略
|
||||
- **断点设置**:建议在哪些位置设置断点
|
||||
- **日志分析**:说明如何通过日志分析问题
|
||||
- **变量检查**:建议检查哪些关键变量的值
|
||||
- **逐步排查**:提供逐步排查问题的流程
|
||||
- **工具推荐**:推荐适合的调试工具和技巧
|
||||
|
||||
## 代码上下文
|
||||
{code_context}
|
||||
|
||||
## 错误信息
|
||||
{error_message}
|
||||
|
||||
## 对话历史
|
||||
{conversation_history}
|
||||
|
||||
## 用户问题
|
||||
{user_query}
|
||||
|
||||
请开始你的分析:"""
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
"""代码解释相关的Prompt模板"""
|
||||
|
||||
CODE_EXPLANATION_TEMPLATE = """# 角色设定
|
||||
你是一位资深的代码分析专家,擅长深入解析代码结构和功能,能够根据不同类型的问题提供精准的专业解答。
|
||||
|
||||
## 核心指令
|
||||
请基于提供的代码、上下文和用户问题类型,为用户提供详细、专业的代码分析。
|
||||
|
||||
## 分析要求(根据问题类型调整重点)
|
||||
|
||||
### 逻辑解释问题(logic_explanation)
|
||||
- **功能说明**:详细解释代码的功能和用途
|
||||
- **逻辑分析**:说明代码的执行流程和核心逻辑
|
||||
- **实现原理**:解释代码的实现原理和技术细节
|
||||
- **使用示例**:提供实际可运行的代码示例
|
||||
- **注意事项**:指出使用时需要注意的要点和常见错误
|
||||
|
||||
### 实体介绍问题(entity_introduction)
|
||||
- **实体结构**:说明函数、类、API等实体的结构和组成
|
||||
- **参数分析**:说明每个参数的类型、含义和默认值
|
||||
- **返回值说明**:解释返回值的类型、含义和可能的取值
|
||||
- **使用示例**:提供实际可运行的代码示例
|
||||
- **注意事项**:指出使用时需要注意的要点和常见错误
|
||||
|
||||
### 代码结构问题(code_structure)
|
||||
- **项目结构**:详细说明项目的目录和文件组织
|
||||
- **模块划分**:说明各个模块的功能和职责
|
||||
- **依赖关系**:解释模块之间的依赖和调用关系
|
||||
- **架构设计**:说明项目的整体架构和设计思路
|
||||
- **文件说明**:解释关键文件的作用和内容
|
||||
|
||||
## 代码上下文
|
||||
{code_context}
|
||||
|
||||
## 对话历史
|
||||
{conversation_history}
|
||||
|
||||
## 用户问题
|
||||
{user_query}
|
||||
|
||||
请开始你的分析:"""
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
"""代码生成相关的Prompt模板"""
|
||||
|
||||
CODE_GENERATION_TEMPLATE = '''# 角色设定
|
||||
你是一位经验丰富的代码生成专家,擅长根据不同类型的需求编写高质量、可维护的代码。
|
||||
|
||||
## 核心指令
|
||||
请基于用户的需求、上下文和生成类型,为用户提供完整、专业、可直接使用的代码。
|
||||
|
||||
## 代码生成要求(根据生成类型调整重点)
|
||||
|
||||
### 完整代码生成(code_generation)
|
||||
- **需求分析**:深入理解用户的功能需求
|
||||
- **架构设计**:设计合理的代码结构和模块划分
|
||||
- **完整实现**:提供完整可运行的代码,包括所有必要的导入
|
||||
- **最佳实践**:遵循目标语言的编码规范和最佳实践
|
||||
- **错误处理**:添加适当的异常处理和边界检查
|
||||
- **代码注释**:添加清晰的注释,解释关键逻辑
|
||||
- **使用示例**:提供如何使用该代码的示例
|
||||
|
||||
### 函数实现(function_implementation)
|
||||
- **函数签名**:设计清晰的函数名、参数和返回值
|
||||
- **参数验证**:添加参数类型检查和验证逻辑
|
||||
- **边界处理**:考虑边界情况和特殊输入
|
||||
- **错误处理**:使用适当的异常处理机制
|
||||
- **文档字符串**:添加详细的docstring说明函数用途
|
||||
- **类型提示**:使用类型注解提高代码可读性
|
||||
- **单元测试**:提供简单的测试用例
|
||||
|
||||
### 类实现(class_implementation)
|
||||
- **类设计**:设计合理的类结构和方法划分
|
||||
- **构造函数**:实现__init__方法,正确初始化属性
|
||||
- **封装性**:合理使用私有属性和公共方法
|
||||
- **方法实现**:实现所有必要的方法,确保功能完整
|
||||
- **特殊方法**:根据需要实现__str__、__repr__等特殊方法
|
||||
- **文档字符串**:为类和主要方法添加docstring
|
||||
- **使用示例**:提供类的使用示例
|
||||
|
||||
## 通用代码质量要求
|
||||
1. **语法正确性**:确保代码语法完全正确,可直接运行
|
||||
2. **代码风格**:遵循PEP 8(Python)或其他语言的编码规范
|
||||
3. **可读性**:使用有意义的变量名和函数名,添加必要的注释
|
||||
4. **可维护性**:代码结构清晰,易于理解和修改
|
||||
5. **性能考虑**:在保证正确性的前提下,考虑性能优化
|
||||
6. **安全性**:注意常见的安全问题(如SQL注入、XSS等)
|
||||
|
||||
## 目标语言
|
||||
{target_language}
|
||||
|
||||
## 代码上下文
|
||||
{code_context}
|
||||
|
||||
## 对话历史
|
||||
{conversation_history}
|
||||
|
||||
## 用户需求
|
||||
{user_requirement}
|
||||
|
||||
请开始生成代码:'''
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
"""代码优化相关的Prompt模板"""
|
||||
|
||||
CODE_OPTIMIZATION_TEMPLATE = """你是一个专业的代码优化专家。请根据用户的问题和检索到的代码上下文,提供专业的优化建议。
|
||||
|
||||
## 意图上下文
|
||||
当前意图:{{intent_type}}
|
||||
检索策略:{{retrieval_strategy}}
|
||||
|
||||
## 对话历史
|
||||
{{conversation_history}}
|
||||
|
||||
## 检索上下文
|
||||
{{code_context}}
|
||||
|
||||
## 响应指南
|
||||
1. **必须使用标准 Markdown 格式**
|
||||
2. **确保流式输出体验**:首句直接入题,段落间使用 \n\n 分隔
|
||||
3. **中英文之间自动添加空格**
|
||||
4. **代码块必须闭合且标注语言**
|
||||
5. **针对 {{intent_type}} 采用对应的回答模板**
|
||||
|
||||
## 回答模板
|
||||
|
||||
### 代码优化问题
|
||||
当用户询问如何优化代码时,请按以下结构回答:
|
||||
|
||||
#### 1. 代码分析
|
||||
- 指出当前代码存在的问题
|
||||
- 分析性能瓶颈
|
||||
- 说明可优化的地方
|
||||
|
||||
#### 2. 优化建议
|
||||
- 提供具体的优化方案
|
||||
- 说明优化原理
|
||||
- 给出优化后的代码示例
|
||||
|
||||
#### 3. 性能对比
|
||||
- 对比优化前后的性能
|
||||
- 说明优化的效果
|
||||
- 给出具体的性能指标
|
||||
|
||||
#### 4. 最佳实践
|
||||
- 提供相关的编程最佳实践
|
||||
- 说明代码规范
|
||||
- 给出可维护性建议
|
||||
|
||||
### 性能调优问题
|
||||
当用户询问性能调优时,请按以下结构回答:
|
||||
|
||||
#### 1. 性能分析
|
||||
- 分析当前性能问题
|
||||
- 定位性能瓶颈
|
||||
- 说明影响性能的因素
|
||||
|
||||
#### 2. 调优策略
|
||||
- 提供具体的调优方案
|
||||
- 说明调优的原理
|
||||
- 给出调优的步骤
|
||||
|
||||
#### 3. 优化效果
|
||||
- 说明调优后的效果
|
||||
- 给出性能提升的数据
|
||||
- 对比调优前后的差异
|
||||
|
||||
#### 4. 注意事项
|
||||
- 说明调优时的注意事项
|
||||
- 提供避免问题的建议
|
||||
- 给出监控和评估方法
|
||||
|
||||
## 代码示例要求
|
||||
- 代码块必须使用 ```python 或 ```javascript 等标注语言
|
||||
- 代码必须完整可运行
|
||||
- 添加必要的注释说明
|
||||
- 保持代码风格一致
|
||||
|
||||
现在请根据用户问题和检索上下文,提供专业的代码优化建议:"""
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
"""并发编程相关的Prompt模板"""
|
||||
|
||||
CONCURRENCY_EXPLANATION_TEMPLATE = '''你是一个专业的并发编程专家。请根据用户的问题和检索到的代码上下文,提供专业的并发编程指导。
|
||||
|
||||
## 意图上下文
|
||||
当前意图:{{intent_type}}
|
||||
检索策略:{{retrieval_strategy}}
|
||||
|
||||
## 对话历史
|
||||
{{conversation_history}}
|
||||
|
||||
## 检索上下文
|
||||
{{code_context}}
|
||||
|
||||
## 响应指南
|
||||
1. **必须使用标准 Markdown 格式**
|
||||
2. **确保流式输出体验**:首句直接入题,段落间使用 \n\n 分隔
|
||||
3. **中英文之间自动添加空格**
|
||||
4. **代码块必须闭合且标注语言**
|
||||
5. **针对 {{intent_type}} 采用对应的回答模板**
|
||||
|
||||
## 回答模板
|
||||
|
||||
### 并发问题
|
||||
当用户询问并发问题时,请按以下结构回答:
|
||||
|
||||
#### 1. 并发模型
|
||||
- 说明并发的基本概念
|
||||
- 解释并发模型
|
||||
- 对比不同并发模型
|
||||
|
||||
#### 2. 实现方式
|
||||
- 提供具体的实现代码
|
||||
- 说明实现原理
|
||||
- 给出使用示例
|
||||
|
||||
#### 3. 同步机制
|
||||
- 说明同步的必要性
|
||||
- 提供同步方法
|
||||
- 解释同步原理
|
||||
|
||||
#### 4. 竞争条件处理
|
||||
- 说明竞争条件的概念
|
||||
- 提供避免竞争条件的方法
|
||||
- 给出并发设计模式
|
||||
|
||||
### 线程问题
|
||||
当用户询问线程问题时,请按以下结构回答:
|
||||
|
||||
#### 1. 线程基础
|
||||
- 说明线程的概念
|
||||
- 解释线程的生命周期
|
||||
- 说明线程的创建和管理
|
||||
|
||||
#### 2. 线程同步
|
||||
- 说明线程同步的必要性
|
||||
- 提供同步方法(锁、信号量等)
|
||||
- 给出同步示例代码
|
||||
|
||||
#### 3. 死锁预防
|
||||
- 说明死锁的概念
|
||||
- 提供死锁预防方法
|
||||
- 给出避免死锁的最佳实践
|
||||
|
||||
#### 4. 线程安全
|
||||
- 说明线程安全的概念
|
||||
- 提供线程安全的实现方法
|
||||
- 给出线程安全的编程建议
|
||||
|
||||
### 异步编程问题
|
||||
当用户询问异步编程时,请按以下结构回答:
|
||||
|
||||
#### 1. 异步编程模型
|
||||
- 说明异步编程的概念
|
||||
- 解释异步与同步的区别
|
||||
- 说明异步的优势
|
||||
|
||||
#### 2. 异步实现
|
||||
- 提供异步编程的代码示例
|
||||
- 说明 async/await 的使用
|
||||
- 解释事件循环的原理
|
||||
|
||||
#### 3. 异步 I/O
|
||||
- 说明异步 I/O 的概念
|
||||
- 提供异步 I/O 的实现方法
|
||||
- 给出异步 I/O 的使用示例
|
||||
|
||||
#### 4. 错误处理
|
||||
- 说明异步编程中的错误处理
|
||||
- 提供异常处理的方法
|
||||
- 给出调试和测试建议
|
||||
|
||||
## 代码示例要求
|
||||
- 代码块必须使用 ```python 或 ```javascript 等标注语言
|
||||
- 代码必须完整可运行
|
||||
- 添加必要的注释说明
|
||||
- 展示并发/异步的完整流程
|
||||
|
||||
现在请根据用户问题和检索上下文,提供专业的并发编程指导:'''
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
"""测试部署相关的Prompt模板"""
|
||||
|
||||
DEPLOYMENT_EXPLANATION_TEMPLATE = """你是一个专业的测试和部署专家。请根据用户的问题和检索到的代码上下文,提供专业的测试和部署指导。
|
||||
|
||||
## 意图上下文
|
||||
当前意图:{{intent_type}}
|
||||
检索策略:{{retrieval_strategy}}
|
||||
|
||||
## 对话历史
|
||||
{{conversation_history}}
|
||||
|
||||
## 检索上下文
|
||||
{{code_context}}
|
||||
|
||||
## 响应指南
|
||||
1. **必须使用标准 Markdown 格式**
|
||||
2. **确保流式输出体验**:首句直接入题,段落间使用 \n\n 分隔
|
||||
3. **中英文之间自动添加空格**
|
||||
4. **代码块必须闭合且标注语言**
|
||||
5. **针对 {{intent_type}} 采用对应的回答模板**
|
||||
|
||||
## 回答模板
|
||||
|
||||
### 测试问题
|
||||
当用户询问测试问题时,请按以下结构回答:
|
||||
|
||||
#### 1. 测试类型
|
||||
- 说明不同类型的测试(单元测试、集成测试、端到端测试)
|
||||
- 解释各种测试的适用场景
|
||||
- 提供测试策略建议
|
||||
|
||||
#### 2. 测试框架
|
||||
- 推荐适合的测试框架
|
||||
- 说明框架的特点和优势
|
||||
- 提供框架的使用示例
|
||||
|
||||
#### 3. 测试实现
|
||||
- 提供具体的测试代码
|
||||
- 说明测试的编写方法
|
||||
- 给出测试的最佳实践
|
||||
|
||||
#### 4. Mock 和测试数据
|
||||
- 说明 Mock 的使用场景
|
||||
- 提供 Mock 的实现方法
|
||||
- 给出测试数据的准备策略
|
||||
|
||||
### 部署问题
|
||||
当用户询问部署问题时,请按以下结构回答:
|
||||
|
||||
#### 1. 部署策略
|
||||
- 说明不同的部署方式(手动部署、自动化部署)
|
||||
- 解释部署的流程
|
||||
- 提供部署策略建议
|
||||
|
||||
#### 2. 环境配置
|
||||
- 说明开发、测试、生产环境的配置
|
||||
- 提供环境变量的管理方法
|
||||
- 给出配置文件的组织方式
|
||||
|
||||
#### 3. CI/CD 流程
|
||||
- 说明 CI/CD 的概念
|
||||
- 提供主流 CI/CD 工具的使用方法
|
||||
- 给出 CI/CD 流程的配置示例
|
||||
|
||||
#### 4. 监控和告警
|
||||
- 说明监控的重要性
|
||||
- 提供监控工具的推荐
|
||||
- 给出告警策略的配置方法
|
||||
|
||||
### 配置问题
|
||||
当用户询问配置问题时,请按以下结构回答:
|
||||
|
||||
#### 1. 配置文件格式
|
||||
- 说明不同配置文件的格式(JSON、YAML、INI)
|
||||
- 解释各种格式的优缺点
|
||||
- 提供格式选择的建议
|
||||
|
||||
#### 2. 环境变量管理
|
||||
- 说明环境变量的使用场景
|
||||
- 提供环境变量的管理方法
|
||||
- 给出环境变量的最佳实践
|
||||
|
||||
#### 3. 参数配置
|
||||
- 说明参数配置的原则
|
||||
- 提供参数验证的方法
|
||||
- 给出参数管理的建议
|
||||
|
||||
#### 4. 配置验证
|
||||
- 说明配置验证的重要性
|
||||
- 提供配置验证的方法
|
||||
- 给出配置错误的处理建议
|
||||
|
||||
## 代码示例要求
|
||||
- 代码块必须使用 ```bash、```yaml、```python 等标注语言
|
||||
- 配置文件必须完整且格式正确
|
||||
- 添加必要的注释说明
|
||||
- 提供可执行的命令或脚本
|
||||
|
||||
现在请根据用户问题和检索上下文,提供专业的测试和部署指导:"""
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
"""通用代码相关的Prompt模板"""
|
||||
|
||||
GENERAL_QA_TEMPLATE = """# 角色设定
|
||||
你是一位专业的代码顾问,能够回答各种代码相关问题,擅长根据不同类型的问题提供精准的专业解答。
|
||||
|
||||
## 核心指令
|
||||
请基于提供的代码、上下文和问题类型,为用户提供全面、准确、专业的回答。
|
||||
|
||||
## 回答要求(根据问题类型调整重点)
|
||||
|
||||
### 测试问题(testing)
|
||||
- **测试方法**:说明适合的测试方法(单元测试、集成测试等)
|
||||
- **测试框架**:推荐适合的测试框架(如pytest、unittest等)
|
||||
- **测试用例**:提供具体的测试用例示例
|
||||
- **Mock技术**:说明如何mock外部依赖
|
||||
- **覆盖率**:解释测试覆盖率的概念和如何提高覆盖率
|
||||
- **最佳实践**:提供测试的最佳实践和常见陷阱
|
||||
|
||||
### 部署问题(deployment)
|
||||
- **部署策略**:说明适合的部署方式(容器化、云部署等)
|
||||
- **环境配置**:详细说明环境变量的配置方法
|
||||
- **CI/CD流程**:解释持续集成和持续部署的流程
|
||||
- **依赖管理**:说明如何管理生产环境的依赖
|
||||
- **监控告警**:建议部署后的监控和告警方案
|
||||
- **回滚策略**:说明如何处理部署失败的情况
|
||||
|
||||
### 配置问题(configuration)
|
||||
- **配置文件**:说明配置文件的格式和位置
|
||||
- **环境变量**:解释如何设置和使用环境变量
|
||||
- **参数配置**:详细说明各个配置参数的含义和取值
|
||||
- **配置验证**:提供验证配置是否正确的方法
|
||||
- **常见问题**:列出配置相关的常见错误和解决方案
|
||||
- **最佳实践**:提供配置管理的最佳实践
|
||||
|
||||
### 通用知识问题(general_knowledge)
|
||||
- **概念解释**:详细解释相关概念和术语
|
||||
- **原理说明**:深入说明技术原理和机制
|
||||
- **应用场景**:说明技术的适用场景和典型应用
|
||||
- **发展趋势**:介绍技术的发展趋势和未来方向
|
||||
- **学习资源**:推荐相关的学习资源和文档
|
||||
- **实践建议**:提供实际应用的建议和注意事项
|
||||
|
||||
### 非技术问题(non_technical)
|
||||
- **友好回应**:保持友好、自然的对话风格
|
||||
- **相关信息**:提供与问题相关的有用信息
|
||||
- **引导澄清**:如果问题模糊,引导用户明确需求
|
||||
- **保持自然**:避免过度技术化,保持对话的自然流畅
|
||||
|
||||
## 代码上下文
|
||||
{code_context}
|
||||
|
||||
## 对话历史
|
||||
{conversation_history}
|
||||
|
||||
## 用户问题
|
||||
{user_query}
|
||||
|
||||
请开始你的回答:"""
|
||||
|
|
@ -0,0 +1,148 @@
|
|||
"""集成查询处理Prompt模板
|
||||
整合意图识别、Metadata过滤条件提取和查询转换功能
|
||||
"""
|
||||
|
||||
INTEGRATED_QUERY_PROCESSING_TEMPLATE = """### 角色定义
|
||||
你是一个全面的查询处理助手,需要完成以下三个任务:
|
||||
1. 代码意图识别:分析用户问题的意图类型
|
||||
2. 元数据过滤条件提取:从问题中提取显示限制的过滤条件
|
||||
3. 查询转换:重写查询、生成更广泛的查询、分解复杂查询
|
||||
|
||||
### 对话历史
|
||||
{history_str}
|
||||
|
||||
### 当前用户问题
|
||||
{query}
|
||||
|
||||
---
|
||||
|
||||
### 任务1:代码意图识别
|
||||
请分析用户问题的意图,判断其属于以下分类之一:
|
||||
- logic_explanation:解释既有代码的底层逻辑
|
||||
- entity_introduction:介绍具体的代码实体
|
||||
- code_structure:询问项目组织
|
||||
- code_generation:请求从零编写完整代码或功能块
|
||||
- boilerplate_implementation:请求提供标准算法/模板
|
||||
- error_debugging:排查 Bug 或异常
|
||||
- code_optimization:改进既有代码的性能或质量
|
||||
- algorithm_theory:算法原理或复杂度分析
|
||||
- general_technical:通用技术咨询
|
||||
- non_technical:非技术问题
|
||||
- unknown:未知类型
|
||||
|
||||
#### 分类决策树 (判定逻辑)
|
||||
在判定分类前,请严格执行以下优先级逻辑:
|
||||
1. **上下文回溯**:如果 query 中提到的实体(函数、变量、类名)在对话历史或上下文代码中出现过,优先判定为【代码解释/架构类】。
|
||||
2. **句式辨析**:
|
||||
- **[实体/功能] 是怎么实现的/怎么做的?** -> 倾向于【代码解释】,语态为"对既有状态的追溯"。
|
||||
- **怎么实现 [功能]?/ 帮我写一个...** -> 倾向于【代码生成】,语态为"对未知实现的请求"。
|
||||
3. **理论深度**:若问题涉及性能瓶颈、数学原理或复杂度,优先归类为【算法与优化类】。
|
||||
|
||||
#### 语义微调示例 (Few-Shot)
|
||||
- **输入**: "find_median 是如何实现的?"
|
||||
**判定**: logic_explanation | **原因**: 指向特定函数名且询问其现状。
|
||||
- **输入**: "如何实现查找中位数的算法?"
|
||||
**判定**: code_generation | **原因**: 泛指功能实现,表现为编程请求。
|
||||
- **输入**: "这段代码能跑快一点吗?"
|
||||
**判定**: code_optimization | **原因**: 基于现有代码的性能改进请求。
|
||||
- **输入**: "什么是深度优先搜索?"
|
||||
**判定**: algorithm_theory | **原因**: 概念性理论询问。
|
||||
|
||||
---
|
||||
|
||||
### 任务2:严格元数据过滤条件提取
|
||||
请从用户问题中提取显示限制的metadata过滤条件,只提取与以下key一致的条件:
|
||||
- func_id: 函数ID
|
||||
- func_name: 函数名
|
||||
- class_name: 类名
|
||||
- file_path: 文件路径
|
||||
- lang: 编程语言
|
||||
- params: 参数数量
|
||||
- return_type: 返回类型
|
||||
- docstring: 文档字符串
|
||||
- start_line: 开始行号
|
||||
- end_line: 结束行号
|
||||
- repo_id: 仓库ID
|
||||
- branch: 分支名
|
||||
- func_body: 函数体
|
||||
|
||||
**重要规则**:
|
||||
- 只提取查询中**明确提到**的条件,不要进行任何推测
|
||||
- 只有当查询中明确使用了与某个key相关的词汇时,才提取该key的value
|
||||
- **value必须为小写**
|
||||
- **一个key只对应一个value**
|
||||
- **value的字符串长度尽可能短**
|
||||
- 例如:对于查询"在 algorithms 目录下,用java实现的排序算法",
|
||||
只提取 {{"lang": "java"}},不要提取其他任何key
|
||||
|
||||
**强制性约束:**
|
||||
1. **零推测原则**:仅提取用户明确指定的属性限定。若用户说“计算斐波那契的函数”,由于未指定函数名、文件名或语言,提取结果应为空 `{{}}`。
|
||||
2. **关键词触发**:
|
||||
- 提取 `file_path`:原文必须包含路径特征(如 .py, /path, 文件夹等)。
|
||||
- 提取 `func_name` / `class_name`:原文必须包含“名为”、“叫作”或明显的标识符引用。
|
||||
- 提取 `return_type` / `params`:原文必须明确提到“返回类型为...”或“参数个数为...”。
|
||||
3. **格式规范**:value 一律小写,保持极简,严禁包含任何描述性文字。
|
||||
|
||||
---
|
||||
|
||||
### 任务3:查询转换
|
||||
请完成以下三个转换:
|
||||
|
||||
#### 3.1 重写查询
|
||||
将查询重写为更具体、详细且对RAG系统中的信息检索更有效的形式。
|
||||
- 更具体和详细
|
||||
- 如果适用,包含来自对话历史的相关上下文
|
||||
- 保持原始意图
|
||||
- 适合向量搜索
|
||||
|
||||
#### 3.2 生成更广泛的查询
|
||||
生成给定用户查询的更广泛版本,以帮助在RAG系统中检索更全面的上下文信息。
|
||||
- 涵盖与原始查询相关的更一般方面
|
||||
- 能够帮助检索相关的背景信息
|
||||
- 保持原始查询的核心主题
|
||||
- 适合向量搜索
|
||||
|
||||
#### 3.3 分解查询
|
||||
将复杂用户查询分解为更简单、更集中的子查询,这些子查询可用于RAG系统中的全面信息检索。
|
||||
- 2-5个更简单的子查询
|
||||
- 每个子查询应关注原始查询的特定方面
|
||||
- 所有子查询一起应涵盖整个原始查询
|
||||
- 每个子查询应适合向量搜索
|
||||
|
||||
---
|
||||
|
||||
### 输出格式要求
|
||||
请以JSON格式返回所有结果,包含以下字段:
|
||||
{{
|
||||
"intent": {{
|
||||
"is_code_related": true/false,
|
||||
"category": "分类名称",
|
||||
"confidence": 0.0-1.0,
|
||||
"keywords": ["关键词列表"],
|
||||
"reasoning": "分类理由",
|
||||
"requires_code_context": true/false,
|
||||
"suggested_search_terms": ["搜索词列表"]
|
||||
}},
|
||||
"filters": {{
|
||||
"file_path": "value",
|
||||
"lang": "value",
|
||||
...
|
||||
}},
|
||||
"transformed": {{
|
||||
"rewritten": "重写后的查询",
|
||||
"backward": "更广泛的查询",
|
||||
"sub_queries": ["子查询1", "子查询2", ...]
|
||||
}}
|
||||
}}
|
||||
|
||||
### 输出规则
|
||||
1. 必须输出有效的JSON格式,不要包含其他内容
|
||||
2. confidence表示分类的置信度,范围0.0-1.0
|
||||
3. keywords从问题和对话历史中提取的关键词,最多8个
|
||||
4. reasoning简要说明为什么这样分类,要考虑对话历史的内容
|
||||
5. requires_code_context表示是否需要代码上下文来回答
|
||||
6. suggested_search_terms建议的检索词,最多5个,要考虑对话历史中提到的技术或库
|
||||
7. 只提取明确提到的信息,不要进行推测
|
||||
8. 确保所有字段都有合理的值
|
||||
|
||||
现在请分析用户问题和对话历史并输出JSON结果:"""
|
||||
|
|
@ -0,0 +1,418 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
查询处理器
|
||||
整合意图识别、filter生成和查询转换功能,使用一次LLM调用生成所有结果
|
||||
"""
|
||||
import json
|
||||
from typing import Dict, Any, Optional, List
|
||||
from loguru import logger
|
||||
from config import settings
|
||||
from llama_index.llms.ollama import Ollama
|
||||
|
||||
|
||||
class CodeIntentCategory:
|
||||
"""代码意图分类枚举 - 细粒度分类体系"""
|
||||
|
||||
# 代码解释与逻辑类 (Existing Code Focus)
|
||||
LOGIC_EXPLANATION = "logic_explanation" # 解释既有代码的底层逻辑
|
||||
ENTITY_INTRODUCTION = "entity_introduction" # 介绍具体的代码实体(函数定义、类属性、API参数)
|
||||
CODE_STRUCTURE = "code_structure" # 询问项目组织
|
||||
|
||||
# 代码生成与实现类 (New Code Focus)
|
||||
CODE_GENERATION = "code_generation" # 请求从零编写完整代码或功能块
|
||||
BOILERPLATE_IMPLEMENTATION = "boilerplate_implementation" # 请求提供标准算法/模板
|
||||
|
||||
# 调试、优化与理论类
|
||||
ERROR_DEBUGGING = "error_debugging" # 排查 Bug 或异常
|
||||
CODE_OPTIMIZATION = "code_optimization" # 改进既有代码的性能或质量
|
||||
ALGORITHM_THEORY = "algorithm_theory" # 算法原理或复杂度分析
|
||||
|
||||
# 非代码类
|
||||
GENERAL_TECHNICAL = "general_technical" # 通用技术咨询
|
||||
NON_TECHNICAL = "non_technical" # 非技术问题
|
||||
UNKNOWN = "unknown" # 未知类型
|
||||
|
||||
|
||||
class PromptTemplateType:
|
||||
"""Prompt模板类型枚举"""
|
||||
CODE_EXPLANATION = "code_explanation" # 代码解释模板(逻辑解释、实体介绍、代码结构)
|
||||
CODE_GENERATION = "code_generation" # 代码生成模板(代码生成、模板实现)
|
||||
CODE_DEBUGGING = "code_debugging" # 代码调试模板(错误调试)
|
||||
CODE_OPTIMIZATION = "code_optimization" # 代码优化模板(代码优化)
|
||||
ALGORITHM_EXPLANATION = "algorithm_explanation" # 算法解释模板(算法理论)
|
||||
GENERAL_QA = "general_qa" # 通用问答模板(通用技术咨询、非技术问题)
|
||||
|
||||
|
||||
class CodeIntentResult:
|
||||
"""代码意图识别结果"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
is_code_related: bool,
|
||||
category: str,
|
||||
confidence: float,
|
||||
prompt_template_type: str,
|
||||
keywords: List[str],
|
||||
reasoning: str,
|
||||
requires_code_context: bool,
|
||||
suggested_search_terms: List[str],
|
||||
):
|
||||
self.is_code_related = is_code_related # 是否与代码相关(True/False)
|
||||
self.category = category # 代码意图分类
|
||||
self.confidence = confidence # 置信度分数,范围0-1之间
|
||||
self.prompt_template_type = prompt_template_type # Prompt模板类型
|
||||
self.keywords = keywords # 相关关键词列表
|
||||
self.reasoning = reasoning # 解释或理由
|
||||
self.requires_code_context = requires_code_context # 是否需要代码上下文(True/False)
|
||||
self.suggested_search_terms = suggested_search_terms # 建议搜索条款列表
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""转换为字典格式"""
|
||||
return {
|
||||
"is_code_related": self.is_code_related,
|
||||
"category": self.category,
|
||||
"confidence": self.confidence,
|
||||
"prompt_template_type": self.prompt_template_type,
|
||||
"keywords": self.keywords,
|
||||
"reasoning": self.reasoning,
|
||||
"requires_code_context": self.requires_code_context,
|
||||
"suggested_search_terms": self.suggested_search_terms,
|
||||
}
|
||||
|
||||
def to_json(self) -> str:
|
||||
"""转换为JSON格式"""
|
||||
return json.dumps(self.to_dict(), ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
class QueryProcessor:
|
||||
"""
|
||||
查询处理器
|
||||
整合意图识别、filter生成和查询转换功能
|
||||
"""
|
||||
|
||||
def __init__(self, llm: Optional[Ollama] = None):
|
||||
"""
|
||||
初始化查询处理器
|
||||
|
||||
Args:
|
||||
llm: LLM实例,如果为None则使用默认配置
|
||||
"""
|
||||
if llm is None:
|
||||
self.llm = Ollama(
|
||||
model=settings.OLLAMA_MODEL,
|
||||
base_url=settings.OLLAMA_BASE_URL,
|
||||
temperature=0.1, # 低温度确保输出稳定
|
||||
request_timeout=1200.0
|
||||
)
|
||||
else:
|
||||
self.llm = llm
|
||||
|
||||
logger.info("查询处理器初始化完成")
|
||||
|
||||
def _build_integrated_prompt(self, query: str, history: Optional[str] = None) -> str:
|
||||
"""
|
||||
构建集成Prompt,一次调用生成所有结果
|
||||
|
||||
Args:
|
||||
query: 用户问题
|
||||
history: 对话历史
|
||||
|
||||
Returns:
|
||||
str: 集成Prompt
|
||||
"""
|
||||
from utils.prompt.integrated_query_processing import INTEGRATED_QUERY_PROCESSING_TEMPLATE
|
||||
|
||||
history_str = "无" if not history else history
|
||||
prompt = INTEGRATED_QUERY_PROCESSING_TEMPLATE.format(
|
||||
history_str=history_str,
|
||||
query=query
|
||||
)
|
||||
|
||||
return prompt
|
||||
|
||||
def _map_to_prompt_template_type(self, category: str) -> str:
|
||||
"""
|
||||
根据分类确定Prompt模板类型
|
||||
|
||||
Args:
|
||||
category: 意图分类
|
||||
|
||||
Returns:
|
||||
str: Prompt模板类型
|
||||
"""
|
||||
# 代码解释与逻辑类 -> CODE_EXPLANATION
|
||||
if category in [CodeIntentCategory.LOGIC_EXPLANATION, CodeIntentCategory.ENTITY_INTRODUCTION, CodeIntentCategory.CODE_STRUCTURE]:
|
||||
return PromptTemplateType.CODE_EXPLANATION
|
||||
|
||||
# 代码生成与实现类 -> CODE_GENERATION
|
||||
elif category in [CodeIntentCategory.CODE_GENERATION, CodeIntentCategory.BOILERPLATE_IMPLEMENTATION]:
|
||||
return PromptTemplateType.CODE_GENERATION
|
||||
|
||||
# 调试、优化与理论类
|
||||
elif category == CodeIntentCategory.ERROR_DEBUGGING:
|
||||
return PromptTemplateType.CODE_DEBUGGING
|
||||
elif category == CodeIntentCategory.CODE_OPTIMIZATION:
|
||||
return PromptTemplateType.CODE_OPTIMIZATION
|
||||
elif category == CodeIntentCategory.ALGORITHM_THEORY:
|
||||
return PromptTemplateType.ALGORITHM_EXPLANATION
|
||||
|
||||
# 非代码问题 -> GENERAL_QA
|
||||
elif category in [CodeIntentCategory.GENERAL_TECHNICAL, CodeIntentCategory.NON_TECHNICAL, CodeIntentCategory.UNKNOWN]:
|
||||
return PromptTemplateType.GENERAL_QA
|
||||
|
||||
else:
|
||||
return PromptTemplateType.GENERAL_QA
|
||||
|
||||
def _parse_llm_response(self, response_text: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
解析LLM响应
|
||||
|
||||
Args:
|
||||
response_text: LLM响应文本
|
||||
|
||||
Returns:
|
||||
解析后的字典,如果解析失败返回None
|
||||
"""
|
||||
try:
|
||||
response_text = response_text.strip()
|
||||
|
||||
# 尝试提取JSON部分
|
||||
json_start = response_text.find('{')
|
||||
json_end = response_text.rfind('}')
|
||||
|
||||
if json_start == -1 or json_end == -1:
|
||||
logger.warning(f"未找到JSON格式响应: {response_text}")
|
||||
return None
|
||||
|
||||
json_str = response_text[json_start:json_end + 1]
|
||||
logger.debug(f"提取的JSON字符串: {json_str}")
|
||||
result = json.loads(json_str)
|
||||
|
||||
return result
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"JSON解析失败: {e}, 响应: {response_text}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"解析响应失败: {e}")
|
||||
return None
|
||||
|
||||
def process_query(self, query: str, history: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
处理查询,一次调用生成所有结果
|
||||
|
||||
Args:
|
||||
query: 用户问题
|
||||
history: 对话历史
|
||||
|
||||
Returns:
|
||||
包含意图识别、filter生成和查询转换结果的字典
|
||||
"""
|
||||
try:
|
||||
# 构建集成Prompt
|
||||
prompt = self._build_integrated_prompt(query, history)
|
||||
|
||||
# 调用LLM
|
||||
response = self.llm.complete(prompt)
|
||||
response_text = response.text
|
||||
logger.debug(f"LLM响应: {response_text}")
|
||||
|
||||
# 解析响应
|
||||
parsed_result = self._parse_llm_response(response_text)
|
||||
|
||||
if parsed_result is None:
|
||||
logger.error("解析LLM响应失败,使用默认结果")
|
||||
return self._get_default_result(query)
|
||||
|
||||
# 验证并处理结果
|
||||
result = {
|
||||
"intent": None,
|
||||
"filters": {},
|
||||
"transformed": {
|
||||
"rewritten": query,
|
||||
"backward": query,
|
||||
"sub_queries": [query]
|
||||
}
|
||||
}
|
||||
|
||||
# 处理意图识别结果
|
||||
try:
|
||||
if "intent" in parsed_result:
|
||||
intent_data = parsed_result["intent"]
|
||||
# 确保所有必需字段都存在
|
||||
intent_data.setdefault("is_code_related", False)
|
||||
intent_data.setdefault("category", CodeIntentCategory.UNKNOWN)
|
||||
intent_data.setdefault("confidence", 0.5)
|
||||
intent_data.setdefault("keywords", [])
|
||||
intent_data.setdefault("reasoning", "")
|
||||
intent_data.setdefault("requires_code_context", False)
|
||||
intent_data.setdefault("suggested_search_terms", [])
|
||||
|
||||
# 确定Prompt模板类型
|
||||
prompt_template_type = self._map_to_prompt_template_type(intent_data["category"])
|
||||
|
||||
# 构建CodeIntentResult对象
|
||||
intent_result = CodeIntentResult(
|
||||
is_code_related=intent_data["is_code_related"],
|
||||
category=intent_data["category"],
|
||||
confidence=intent_data["confidence"],
|
||||
prompt_template_type=prompt_template_type,
|
||||
keywords=intent_data["keywords"],
|
||||
reasoning=intent_data["reasoning"],
|
||||
requires_code_context=intent_data["requires_code_context"],
|
||||
suggested_search_terms=intent_data["suggested_search_terms"]
|
||||
)
|
||||
result["intent"] = intent_result
|
||||
except Exception as e:
|
||||
logger.error(f"处理意图识别结果失败: {e}")
|
||||
|
||||
# 处理过滤条件
|
||||
try:
|
||||
if "filters" in parsed_result:
|
||||
filters = parsed_result["filters"]
|
||||
if isinstance(filters, dict):
|
||||
# 验证并过滤结果,确保只包含有效的metadata key
|
||||
valid_keys = ['func_id', 'func_name', 'class_name', 'file_path', 'lang', 'params', 'return_type', 'docstring', 'start_line', 'end_line', 'repo_id', 'branch', 'func_body']
|
||||
filtered_filters = {}
|
||||
for key, value in filters.items():
|
||||
if key in valid_keys and value:
|
||||
# 确保value是字符串类型
|
||||
if isinstance(value, str):
|
||||
filtered_filters[key] = value
|
||||
result["filters"] = filtered_filters
|
||||
except Exception as e:
|
||||
logger.error(f"处理过滤条件失败: {e}")
|
||||
|
||||
# 处理查询转换结果
|
||||
try:
|
||||
if "transformed" in parsed_result:
|
||||
transformed = parsed_result["transformed"]
|
||||
if isinstance(transformed, dict):
|
||||
result["transformed"].update({
|
||||
"rewritten": transformed.get("rewritten", query),
|
||||
"backward": transformed.get("backward", query),
|
||||
"sub_queries": transformed.get("sub_queries", [query])
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"处理查询转换结果失败: {e}")
|
||||
|
||||
logger.info("查询处理完成")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"查询处理失败: {e}")
|
||||
return self._get_default_result(query)
|
||||
|
||||
def _get_default_result(self, query: str) -> Dict[str, Any]:
|
||||
"""
|
||||
获取默认结果(当处理失败时使用)
|
||||
|
||||
Args:
|
||||
query: 用户问题
|
||||
|
||||
Returns:
|
||||
默认结果
|
||||
"""
|
||||
logger.warning(f"使用默认查询处理结果: {query}")
|
||||
|
||||
# 构建默认的意图识别结果
|
||||
default_intent = CodeIntentResult(
|
||||
is_code_related=False,
|
||||
category=CodeIntentCategory.UNKNOWN,
|
||||
confidence=0.0,
|
||||
prompt_template_type=PromptTemplateType.GENERAL_QA,
|
||||
keywords=[],
|
||||
reasoning="查询处理失败,使用默认结果",
|
||||
requires_code_context=False,
|
||||
suggested_search_terms=[query]
|
||||
)
|
||||
|
||||
return {
|
||||
"intent": default_intent,
|
||||
"filters": {},
|
||||
"transformed": {
|
||||
"rewritten": query,
|
||||
"backward": query,
|
||||
"sub_queries": [query]
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# 辅助函数
|
||||
def create_query_processor(llm: Optional[Ollama] = None) -> QueryProcessor:
|
||||
"""
|
||||
创建查询处理器实例
|
||||
|
||||
Args:
|
||||
llm: LLM实例,如果为None则使用默认配置
|
||||
|
||||
Returns:
|
||||
QueryProcessor: 处理器实例
|
||||
"""
|
||||
return QueryProcessor(llm)
|
||||
|
||||
|
||||
class MetadataFilter:
|
||||
"""
|
||||
Metadata过滤工具类
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def apply_filter(metadata: Dict[str, Any], filters: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
应用过滤条件到metadata
|
||||
|
||||
Args:
|
||||
metadata: 文档的metadata
|
||||
filters: 过滤条件
|
||||
|
||||
Returns:
|
||||
bool: 如果metadata符合过滤条件返回True,否则返回False
|
||||
"""
|
||||
if not filters:
|
||||
return True
|
||||
|
||||
for key, value in filters.items():
|
||||
if key not in metadata:
|
||||
return False
|
||||
|
||||
metadata_value = metadata[key]
|
||||
if isinstance(metadata_value, str) and isinstance(value, str):
|
||||
# 对于字符串类型,使用大小写不敏感的模糊匹配
|
||||
if value.lower() not in metadata_value.lower():
|
||||
return False
|
||||
else:
|
||||
if metadata_value != value:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""测试查询处理器"""
|
||||
processor = QueryProcessor()
|
||||
|
||||
test_queries = [
|
||||
"在 data_structures 目录下,二叉搜索树(Binary Search Tree)的删除操作依赖于哪些辅助方法来寻找后继节点?",
|
||||
"如何实现快速排序算法?",
|
||||
"kth_number的时间复杂度是多少?",
|
||||
"今天天气怎么样?"
|
||||
]
|
||||
|
||||
for query in test_queries:
|
||||
print(f"\n{'='*60}")
|
||||
print(f"问题: {query}")
|
||||
print('='*60)
|
||||
|
||||
result = processor.process_query(query)
|
||||
|
||||
print("意图识别结果:")
|
||||
if result["intent"]:
|
||||
print(result["intent"].to_json())
|
||||
|
||||
print("\n过滤条件:")
|
||||
import json
|
||||
print(json.dumps(result["filters"], ensure_ascii=False, indent=2))
|
||||
|
||||
print("\n查询转换结果:")
|
||||
print(json.dumps(result["transformed"], ensure_ascii=False, indent=2))
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
"""
|
||||
Query transformer for RAG system
|
||||
Implements three query transformation techniques:
|
||||
1. Query rewriting: Make queries more specific and detailed
|
||||
2. Backward prompt generation: Generate broader queries for context retrieval
|
||||
3. Sub-query decomposition: Break down complex queries into simpler components
|
||||
"""
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
from loguru import logger
|
||||
from config import settings
|
||||
from llama_index.llms.ollama import Ollama
|
||||
from utils.prompt.query_trans import REWRITE_PROMPT, BACKWARD_PROMPT, DECOMPOSE_PROMPT
|
||||
|
||||
|
||||
class QueryTransformer:
|
||||
"""
|
||||
Query transformer for RAG system
|
||||
"""
|
||||
|
||||
def __init__(self, llm: Ollama):
|
||||
"""
|
||||
Initialize query transformer
|
||||
|
||||
Args:
|
||||
llm: Ollama LLM instance for generating transformations
|
||||
"""
|
||||
self.llm = llm
|
||||
|
||||
async def rewrite_query(self, query: str, history: str = "") -> str:
|
||||
"""
|
||||
Rewrite query to be more specific and detailed
|
||||
|
||||
Args:
|
||||
query: Original user query
|
||||
history: Chat history (optional)
|
||||
|
||||
Returns:
|
||||
Rewritten query
|
||||
"""
|
||||
try:
|
||||
prompt = REWRITE_PROMPT.format(query=query, history=history)
|
||||
|
||||
response = await self.llm.acomplete(prompt=prompt)
|
||||
rewritten_query = response.text.strip()
|
||||
logger.info(f"Rewritten query: {rewritten_query}")
|
||||
return rewritten_query
|
||||
except Exception as e:
|
||||
logger.error(f"Error in query rewriting: {e}")
|
||||
return query
|
||||
|
||||
async def backward_query(self, query: str, history: str = "") -> str:
|
||||
"""
|
||||
Generate broader query for context retrieval
|
||||
|
||||
Args:
|
||||
query: Original user query
|
||||
history: Chat history (optional)
|
||||
|
||||
Returns:
|
||||
Broader query for context retrieval
|
||||
"""
|
||||
try:
|
||||
prompt = BACKWARD_PROMPT.format(query=query, history=history)
|
||||
|
||||
response = await self.llm.acomplete(prompt=prompt)
|
||||
backward_query = response.text.strip()
|
||||
logger.info(f"Backward query: {backward_query}")
|
||||
return backward_query
|
||||
except Exception as e:
|
||||
logger.error(f"Error in backward query generation: {e}")
|
||||
return query
|
||||
|
||||
async def decompose_query(self, query: str, history: str = "") -> List[str]:
|
||||
"""
|
||||
Decompose complex query into simpler sub-queries
|
||||
|
||||
Args:
|
||||
query: Original user query
|
||||
history: Chat history (optional)
|
||||
|
||||
Returns:
|
||||
List of sub-queries
|
||||
"""
|
||||
try:
|
||||
prompt = DECOMPOSE_PROMPT.format(query=query, history=history)
|
||||
|
||||
response = await self.llm.acomplete(prompt=prompt)
|
||||
sub_queries_text = response.text.strip()
|
||||
|
||||
# Parse sub-queries from response
|
||||
sub_queries = []
|
||||
for line in sub_queries_text.split('\n'):
|
||||
line = line.strip()
|
||||
if line and (line.startswith('1.') or line.startswith('2.') or line.startswith('3.') or line.startswith('4.') or line.startswith('5.')):
|
||||
sub_query = line.split('.', 1)[1].strip()
|
||||
if sub_query:
|
||||
sub_queries.append(sub_query)
|
||||
|
||||
# If no valid sub-queries found, return original query as single sub-query
|
||||
if not sub_queries:
|
||||
sub_queries = [query]
|
||||
|
||||
logger.info(f"Original query: {query}")
|
||||
logger.info(f"Decomposed sub-queries: {sub_queries}")
|
||||
return sub_queries
|
||||
except Exception as e:
|
||||
logger.error(f"Error in query decomposition: {e}")
|
||||
return [query]
|
||||
|
||||
async def transform_query(self, query: str, history: str = "") -> Dict[str, any]:
|
||||
"""
|
||||
Transform query using all three techniques
|
||||
|
||||
Args:
|
||||
query: Original user query
|
||||
history: Chat history (optional)
|
||||
|
||||
Returns:
|
||||
Dict with transformed queries
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Transforming query: {query}")
|
||||
|
||||
# Run all transformations in parallel
|
||||
from asyncio import gather
|
||||
rewritten, backward, sub_queries = await gather(
|
||||
self.rewrite_query(query, history),
|
||||
self.backward_query(query, history),
|
||||
self.decompose_query(query, history)
|
||||
)
|
||||
|
||||
# Combine all transformed queries for comprehensive retrieval
|
||||
all_transformed_queries = [rewritten, backward] + sub_queries
|
||||
|
||||
return {
|
||||
"original": query,
|
||||
"rewritten": rewritten,
|
||||
"backward": backward,
|
||||
"sub_queries": sub_queries,
|
||||
"all_transformed": all_transformed_queries
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error in query transformation: {e}")
|
||||
return {
|
||||
"original": query,
|
||||
"rewritten": query,
|
||||
"backward": query,
|
||||
"sub_queries": [query],
|
||||
"all_transformed": [query]
|
||||
}
|
||||
Loading…
Reference in New Issue