fix: 修复Git服务器模式下文件去重失败的问题
- _get_processed_file_shas_from_chroma 支持传入 repo_id 和 branch 参数 - fetch_all_documents 为每个仓库分别查询已处理的文件SHA - 使用 git_tool.repo_id 和 git_tool.branch 而不是 config.name
This commit is contained in:
parent
6dfc217ba6
commit
e008cd76ee
|
|
@ -155,8 +155,11 @@ class GitSync(BaseSync):
|
|||
# 获取当前仓库所有文件的blob SHA
|
||||
current_file_shas = git_tool.get_all_file_shas()
|
||||
|
||||
# 从ChromaDB获取已处理的文件SHA
|
||||
processed_file_shas = self._get_processed_file_shas_from_chroma()
|
||||
# 从ChromaDB获取已处理的文件SHA(使用git_tool的repo_id和branch)
|
||||
processed_file_shas = self._get_processed_file_shas_from_chroma(
|
||||
repo_id=git_tool.repo_id,
|
||||
branch=git_tool.branch
|
||||
)
|
||||
|
||||
# 识别需要处理的新文件/修改文件
|
||||
files_to_process = []
|
||||
|
|
@ -194,8 +197,15 @@ class GitSync(BaseSync):
|
|||
logger.info(f"获取到 {len(func_list)} 个函数")
|
||||
return func_list
|
||||
|
||||
def _get_processed_file_shas_from_chroma(self, collection_key: str = 'default') -> Dict[str, str]:
|
||||
"""从ChromaDB获取已处理的文件SHA映射"""
|
||||
def _get_processed_file_shas_from_chroma(self, repo_id: str = None, branch: str = None, collection_key: str = 'default') -> Dict[str, str]:
|
||||
"""
|
||||
从ChromaDB获取已处理的文件SHA映射
|
||||
|
||||
Args:
|
||||
repo_id: 仓库ID(可选,不传则使用config.name)
|
||||
branch: 分支名(可选,不传则使用config.branch)
|
||||
collection_key: Collection key
|
||||
"""
|
||||
try:
|
||||
if not self.vector_store_manager:
|
||||
return {}
|
||||
|
|
@ -205,14 +215,22 @@ class GitSync(BaseSync):
|
|||
logger.warning(f"Collection {collection_key} not initialized")
|
||||
return {}
|
||||
|
||||
results = collection.get(
|
||||
where={
|
||||
# 使用传入的参数或默认值
|
||||
query_repo_id = repo_id if repo_id else self.config.name
|
||||
query_branch = branch if branch else getattr(self.config, 'branch', None)
|
||||
|
||||
# 构建查询条件
|
||||
if query_branch:
|
||||
where_clause = {
|
||||
"$and": [
|
||||
{"repo_id": {"$eq": self.config.name}},
|
||||
{"branch": {"$eq": self.config.branch}}
|
||||
{"repo_id": {"$eq": query_repo_id}},
|
||||
{"branch": {"$eq": query_branch}}
|
||||
]
|
||||
}
|
||||
)
|
||||
else:
|
||||
where_clause = {"repo_id": {"$eq": query_repo_id}}
|
||||
|
||||
results = collection.get(where=where_clause)
|
||||
|
||||
processed_shas = {}
|
||||
for metadata in results.get('metadatas', []):
|
||||
|
|
@ -221,6 +239,7 @@ class GitSync(BaseSync):
|
|||
blob_sha = metadata['file_blob_sha']
|
||||
processed_shas[file_path] = blob_sha
|
||||
|
||||
logger.debug(f"从ChromaDB获取到 {len(processed_shas)} 个已处理文件SHA (repo_id: {query_repo_id}, branch: {query_branch})")
|
||||
return processed_shas
|
||||
except Exception as e:
|
||||
logger.warning(f"从ChromaDB获取已处理文件SHA失败: {e}")
|
||||
|
|
|
|||
Loading…
Reference in New Issue