From 6a8a29d68fabb35bdfe80792e2d71b2a486bf7de Mon Sep 17 00:00:00 2001 From: zhangxunhui Date: Mon, 5 Jan 2026 00:06:13 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=86soffice=E7=9A=84fast?= =?UTF-8?q?api=E6=9C=8D=E5=8A=A1=EF=BC=8C=E7=94=A8=E4=BA=8E=E5=B0=86doc?= =?UTF-8?q?=E8=BD=AC=E6=8D=A2=E4=B8=BAdocx?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 3 + README.md | 2 - docker-compose.yml | 19 ++++++ docker/soffice/Dockerfile | 37 +++++++++++ docker/soffice/README.md | 15 +++++ docker/soffice/app.py | 131 ++++++++++++++++++++++++++++++++++++ rag/__init__.py | 3 +- rag/doc_converter.py | 135 ++++++++++++++++++++++++++++++++++++++ rag/file_parser.py | 33 +++++----- 9 files changed, 357 insertions(+), 21 deletions(-) create mode 100644 docker/soffice/Dockerfile create mode 100644 docker/soffice/README.md create mode 100644 docker/soffice/app.py create mode 100644 rag/doc_converter.py diff --git a/.env.example b/.env.example index 9c36683..4e3005f 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,9 @@ API_VERSION=1.0.0 # 文件上传大小限制（单位：MB，默认：5MB） MAX_UPLOAD_SIZE_MB=5 +# LibreOffice soffice service port (used by docker/soffice service) +SOFFICE_PORT=8003 + # ============================================ # ChromaDB 配置 # ============================================ diff --git a/README.md b/README.md index ce33a91..b93f6c5 100644 --- a/README.md +++ b/README.md @@ -600,8 +600,6 @@ source .venv/bin/activate # Windows: venv\Scripts\activate # 2. 安装依赖 uv pip install -r requirements.txt -uv pip install python-office - # 3. 启动 ChromaDB 服务（必须） docker-compose up -d chromadb diff --git a/docker-compose.yml b/docker-compose.yml index 8e91c63..dc0c430 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -115,6 +115,8 @@ services: depends_on: chromadb: condition: service_healthy + soffice-service: + condition: service_healthy restart: unless-stopped healthcheck: # 健康检查端点使用变量替换，参考 .env 中的 API_PORT（默认 8001） @@ -124,3 +126,20 @@ services: retries: 3 start_period: 60s + soffice-service: + build: + context: . + dockerfile: docker/soffice/Dockerfile + container_name: rag-soffice + ports: + - "${SOFFICE_PORT:-8003}:8003" + environment: + - SOFFICE_PORT=${SOFFICE_PORT:-8003} + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:${SOFFICE_PORT:-8003}/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + diff --git a/docker/soffice/Dockerfile b/docker/soffice/Dockerfile new file mode 100644 index 0000000..3f7dea7 --- /dev/null +++ b/docker/soffice/Dockerfile @@ -0,0 +1,37 @@ +FROM python:3.11-slim + +ENV DEBIAN_FRONTEND=noninteractive + +# Ensure LibreOffice program libraries are on PYTHONPATH and LD_LIBRARY_PATH +ENV PYTHONPATH=/usr/lib/libreoffice/program:$PYTHONPATH +ENV LD_LIBRARY_PATH=/usr/lib/libreoffice/program:$LD_LIBRARY_PATH + +# 配置国内 apt 镜像源（加速包下载） +# 使用阿里云 Debian 镜像源（适用于 Debian/Ubuntu 系统） +RUN sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \ + sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || \ + (echo "deb https://mirrors.aliyun.com/debian/ bookworm main" > /etc/apt/sources.list && \ + echo "deb https://mirrors.aliyun.com/debian/ bookworm-updates main" >> /etc/apt/sources.list && \ + echo "deb https://mirrors.aliyun.com/debian-security/ bookworm-security main" >> /etc/apt/sources.list) + + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libreoffice-core libreoffice-writer libreoffice-common \ + libreoffice-base libreoffice-math libreoffice-impress \ + ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +COPY docker/soffice/app.py /app/app.py +COPY docker/soffice/README.md /app/README.md + +EXPOSE 8003 + +# Start uvicorn with python3 to ensure sys.executable points to python3 +# Use port 8003 to match the compose mapping and .env example +CMD ["python3", "-m", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8003"] diff --git a/docker/soffice/README.md b/docker/soffice/README.md new file mode 100644 index 0000000..4f3324a --- /dev/null +++ b/docker/soffice/README.md @@ -0,0 +1,15 @@ +soffice-service +================ + +Minimal service that provides a REST endpoint to convert .doc files (server-side paths) to .docx using LibreOffice (`soffice`). + +Usage: +- Build: `docker-compose build soffice-service` +- Start: `docker-compose up -d soffice-service` +- Health: `GET /health` +- Convert (server-path): `POST /convert` with JSON body `{ "path": "/path/to/file.doc", "delete_original": true }` + +Notes: +- Ensure `soffice` (LibreOffice) is available in the container (Dockerfile installs it). +- The endpoint performs atomic replace: the resulting `.docx` replaces or is created next to the original file, and the original is removed when `delete_original` is true. +- This service does no auth; in production, restrict access to trusted networks or add auth/proxy. diff --git a/docker/soffice/app.py b/docker/soffice/app.py new file mode 100644 index 0000000..399089b --- /dev/null +++ b/docker/soffice/app.py @@ -0,0 +1,131 @@ +import os +import tempfile +import subprocess +import shutil +import glob +from fastapi import FastAPI, HTTPException, UploadFile, File +from fastapi.responses import FileResponse +from pydantic import BaseModel +from starlette.concurrency import run_in_threadpool +from starlette.background import BackgroundTask + +app = FastAPI() + + +@app.get('/health') +async def health(): + return {'status': 'ok'} + + +@app.get('/diagnose') +async def diagnose(): + info = {} + try: + info['soffice_path'] = shutil.which('soffice') + except Exception as e: + info['soffice_path_error'] = str(e) + + try: + proc = subprocess.run(['soffice', '--version'], capture_output=True, text=True, timeout=5) + info['soffice_version'] = proc.stdout.strip() or proc.stderr.strip() + except Exception as e: + info['soffice_version_error'] = str(e) + + try: + info['libreoffice_program_list'] = os.listdir('/usr/lib/libreoffice/program')[:50] + except Exception as e: + info['libreoffice_program_list_error'] = str(e) + + return info + + +@app.post('/convert') +async def convert(file: UploadFile = File(...)): + """Accept a file upload, convert it to .docx via soffice, and return the resulting file.""" + # Save upload to a temp file + tmpdir = tempfile.mkdtemp(prefix='soffice_upload_') + # print("Created temp dir", tmpdir) + try: + original_name = file.filename or 'uploaded' + # print(original_name) + _, ext = os.path.splitext(original_name) + in_path = os.path.join(tmpdir, 'input' + (ext or '')) + with open(in_path, 'wb') as f: + content = await file.read() + f.write(content) + # print("Saved uploaded file to", in_path) + + # ========== 修改点 1: 添加文件存在检查 ========== + if not os.path.exists(in_path): + raise HTTPException(status_code=500, detail="Uploaded file was not saved properly") + + # If already .docx, return it directly + if in_path.lower().endswith('.docx'): + # ========== 修改点 2: 为 .docx 文件添加背景清理任务 ========== + return FileResponse( + in_path, + media_type='application/vnd.openxmlformats-officedocument.wordprocessingml.document', + filename=os.path.basename(in_path), + background=BackgroundTask(lambda: cleanup_temp_dir(tmpdir)) # 使用 BackgroundTask + ) + + def run_conv(): + return subprocess.run( + ['soffice', '--headless', '--convert-to', 'docx', '--outdir', tmpdir, in_path], + check=True, capture_output=True, text=True, timeout=120 + ) + + try: + await run_in_threadpool(run_conv) + except subprocess.CalledProcessError as e: + stderr = (e.stderr or '').strip() + raise HTTPException(status_code=500, detail=f'soffice conversion failed: {stderr}') + except subprocess.TimeoutExpired: + raise HTTPException(status_code=504, detail='conversion timeout') + + candidates = glob.glob(os.path.join(tmpdir, '*.docx')) + # print("Conversion produced candidates:", candidates) + if not candidates: + raise HTTPException(status_code=500, detail='soffice produced no .docx') + out_file = candidates[0] + + # ========== 修改点 3: 添加输出文件存在检查 ========== + if not os.path.exists(out_file): + raise HTTPException(status_code=500, detail="Converted file was not created properly") + + out_fname = os.path.splitext(original_name)[0] + '.docx' + # print("Returning converted file", out_file, "as", out_fname) + + # ========== 修改点 4: 主要修改 - 使用 BackgroundTask 延迟清理 ========== + return FileResponse( + out_file, + media_type='application/vnd.openxmlformats-officedocument.wordprocessingml.document', + filename=out_fname, + background=BackgroundTask(lambda: cleanup_temp_dir(tmpdir)) # 使用 BackgroundTask + ) + + # ========== 修改点 5: 移除 finally 块中的立即清理，改为只在异常时清理 ========== + except Exception as e: + # 只有在发生异常时才立即清理 + cleanup_temp_dir(tmpdir) + # 重新抛出异常 + if isinstance(e, HTTPException): + raise e + else: + raise HTTPException(status_code=500, detail=f"Conversion error: {str(e)}") + +# ========== 修改点 6: 添加清理函数 ========== +def cleanup_temp_dir(tmpdir: str): + """清理临时目录，忽略错误""" + try: + if tmpdir and os.path.exists(tmpdir): + shutil.rmtree(tmpdir, ignore_errors=True) + print(f"Cleaned up temp dir: {tmpdir}") + except Exception as e: + print(f"Warning: Failed to cleanup temp dir {tmpdir}: {e}") + + +if __name__ == '__main__': + import uvicorn + port = int(os.environ.get('SOFFICE_PORT') or 8003) + uvicorn.run('app:app', host='0.0.0.0', port=port) diff --git a/rag/__init__.py b/rag/__init__.py index 4d6c46e..543b388 100644 --- a/rag/__init__.py +++ b/rag/__init__.py @@ -6,6 +6,7 @@ from .document_processor import DocumentProcessor from .rag_engine import RAGEngine from .file_parser import FileParser from .chunk_handler import OptimizedDeltaThinkFilter +from .anti_word_doc_reader import AntiwordDocReader -__all__ = ["VectorStoreManager", "DocumentProcessor", "RAGEngine", "FileParser", "OptimizedDeltaThinkFilter"] +__all__ = ["VectorStoreManager", "DocumentProcessor", "RAGEngine", "FileParser", "OptimizedDeltaThinkFilter", "AntiwordDocReader"] diff --git a/rag/doc_converter.py b/rag/doc_converter.py new file mode 100644 index 0000000..311b731 --- /dev/null +++ b/rag/doc_converter.py @@ -0,0 +1,135 @@ +from llama_index.core.readers.base import BaseReader +from llama_index.core import Document +import subprocess +import os +import tempfile +import shutil +import glob +from typing import List, Optional + +class DocConverter(BaseReader): + """ + DOC 文件读取器 — 使用 LibreOffice (`soffice`) 将 `.doc` 转为 `.docx` + """ + + def __init__(self, soffice_path: str = "soffice"): + """ + 初始化读取器。 + + 参数: + soffice_path: LibreOffice 可执行路径（默认 'soffice'） + """ + self.soffice_path = soffice_path + + def load_data(self, file_path: str, extra_info: Optional[dict] = None) -> List[Document]: + """ + 加载.doc文件并返回LlamaIndex Document对象列表。 + + 参数: + file_path: .doc文件的路径 + extra_info: 可选的额外元数据 + + 返回: + Document对象列表 + """ + # 检查文件是否存在 + if not os.path.exists(file_path): + raise FileNotFoundError(f"文件 '{file_path}' 不存在") + + # If the file is already a .docx, extract directly; otherwise convert + # using LibreOffice (soffice) into a temporary directory and extract + # from the produced .docx. + text_content = "" + tmpdir = None + try: + if file_path.lower().endswith('.docx'): + docx_path = file_path + else: + # create temporary dir for conversion output + tmpdir = tempfile.mkdtemp(prefix='docconv_') + try: + conv = subprocess.run( + [self.soffice_path, '--headless', '--convert-to', 'docx', '--outdir', tmpdir, file_path], + capture_output=True, + text=True, + timeout=120 + ) + except FileNotFoundError: + raise RuntimeError( + f"soffice not found: '{self.soffice_path}'. Please install LibreOffice (soffice) in the environment." + ) + + if conv.returncode != 0: + stderr = (conv.stderr or conv.stdout or '').strip() + raise RuntimeError(f"soffice conversion failed: {stderr}") + + # find resulting docx + docx_candidates = glob.glob(os.path.join(tmpdir, '*.docx')) + if not docx_candidates: + raise RuntimeError('soffice produced no .docx output') + docx_path = docx_candidates[0] + + # extract text from docx: prefer docx2txt, fallback to python-docx + try: + import docx2txt + text_content = docx2txt.process(docx_path) or "" + except Exception: + try: + from docx import Document as DocxDocument + doc = DocxDocument(docx_path) + text_content = "\n".join(p.text for p in doc.paragraphs) + except Exception as e: + raise RuntimeError(f"failed to extract text from docx: {e}") from e + + # 准备文档元数据 + metadata = { + "file_path": file_path, + "file_name": os.path.basename(file_path), + "file_size": os.path.getsize(file_path), + "parser": "soffice" + } + + # 合并额外信息（如果有） + if extra_info: + metadata.update(extra_info) + + # 创建并返回Document对象 + document = Document(text=text_content, metadata=metadata) + return [document] + + + + + + finally: + # ensure temporary conversion dir is removed + try: + if tmpdir and os.path.exists(tmpdir): + shutil.rmtree(tmpdir, ignore_errors=True) + except Exception: + pass + + def batch_load_data(self, file_paths: List[str]) -> List[Document]: + """批量加载多个.doc文件""" + all_documents = [] + for file_path in file_paths: + documents = self.load_data(file_path) + all_documents.extend(documents) + return all_documents + +# 使用示例：集成到LlamaIndex流程中 +if __name__ == "__main__": + # 1. 初始化读取器 + reader = DocConverter() + + # 2. 加载.doc文件 + try: + documents = reader.load_data("/home/zxh/programs/RAG/test_files/测试.doc", extra_info={"category": "报告"}) + + # 3. 现在你可以将documents用于LlamaIndex的后续处理 + print(f"成功加载 {len(documents)} 个文档") + print(f"文档文本长度：{len(documents[0].text)} 字符") + print(f"文档元数据：{documents[0].metadata}") + + except Exception as e: + print(f"加载文档失败：{e}") \ No newline at end of file diff --git a/rag/file_parser.py b/rag/file_parser.py index af81263..e70719b 100644 --- a/rag/file_parser.py +++ b/rag/file_parser.py @@ -2,12 +2,15 @@ File parsing module for various document formats """ import os -import office import mimetypes from typing import List, Dict, Optional from pathlib import Path from loguru import logger from llama_index.core import Document +from llama_index.readers.alibabacloud_aisearch import ( + AlibabaCloudAISearchDocumentReader, + AlibabaCloudAISearchImageReader, +) from llama_index.core.readers import SimpleDirectoryReader @@ -66,30 +69,24 @@ class FileParser: if not doc_id: doc_id = Path(filename).stem - # 特殊处理.doc文件：转换为.docx格式 - if ext == '.doc': - logger.info(f"检测到.doc文件，开始转换为.docx格式: {filename}") - # 保存.doc临时文件 - try: - tmp_doc_path = file_path - # 准备.docx临时文件路径 - tmp_docx_path = tmp_doc_path.replace('.doc', '.docx') - tmp_dir_path = os.path.dirname(tmp_doc_path) - - office.word.doc2docx(tmp_doc_path, tmp_dir_path) - logger.info(f".doc文件转换为.docx成功: {filename}") - file_path = tmp_docx_path - except Exception as e: - logger.error(f"转换.doc文件为.docx格式时出错: {e}", exc_info=True) - raise - try: logger.info(f"Starting to parse file: {filename} (type: {ext})") + # 阿里巴巴的llama index reader插件可以解析doc文档 + document_reader = AlibabaCloudAISearchDocumentReader() + image_reader = AlibabaCloudAISearchImageReader() + + file_extractor = {} + for suffix in (".pdf", ".docx", ".doc", ".ppt", ".pptx"): + file_extractor[suffix] = document_reader + for suffix in (".jpg", ".jpeg", ".png", ".bmp", ".tiff"): + file_extractor[suffix] = image_reader + # Use LlamaIndex's SimpleDirectoryReader for parsing # It supports many formats out of the box reader = SimpleDirectoryReader( input_files=[file_path], + file_extractor=file_extractor, filename_as_id=False # We'll set custom IDs )