添加了soffice的fastapi服务，用于将doc转换为docx
This commit is contained in:
parent
c584e45666
commit
6a8a29d68f
|
|
@ -15,6 +15,9 @@ API_VERSION=1.0.0
|
|||
# 文件上传大小限制（单位：MB，默认：5MB）
|
||||
MAX_UPLOAD_SIZE_MB=5
|
||||
|
||||
# LibreOffice soffice service port (used by docker/soffice service)
|
||||
SOFFICE_PORT=8003
|
||||
|
||||
# ============================================
|
||||
# ChromaDB 配置
|
||||
# ============================================
|
||||
|
|
|
|||
|
|
@ -600,8 +600,6 @@ source .venv/bin/activate # Windows: venv\Scripts\activate
|
|||
# 2. 安装依赖
|
||||
uv pip install -r requirements.txt
|
||||
|
||||
uv pip install python-office
|
||||
|
||||
# 3. 启动 ChromaDB 服务（必须）
|
||||
docker-compose up -d chromadb
|
||||
|
||||
|
|
|
|||
|
|
@ -115,6 +115,8 @@ services:
|
|||
depends_on:
|
||||
chromadb:
|
||||
condition: service_healthy
|
||||
soffice-service:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
# 健康检查端点使用变量替换，参考 .env 中的 API_PORT（默认 8001）
|
||||
|
|
@ -124,3 +126,20 @@ services:
|
|||
retries: 3
|
||||
start_period: 60s
|
||||
|
||||
soffice-service:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/soffice/Dockerfile
|
||||
container_name: rag-soffice
|
||||
ports:
|
||||
- "${SOFFICE_PORT:-8003}:8003"
|
||||
environment:
|
||||
- SOFFICE_PORT=${SOFFICE_PORT:-8003}
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:${SOFFICE_PORT:-8003}/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
FROM python:3.11-slim
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Ensure LibreOffice program libraries are on PYTHONPATH and LD_LIBRARY_PATH
|
||||
ENV PYTHONPATH=/usr/lib/libreoffice/program:$PYTHONPATH
|
||||
ENV LD_LIBRARY_PATH=/usr/lib/libreoffice/program:$LD_LIBRARY_PATH
|
||||
|
||||
# 配置国内 apt 镜像源（加速包下载）
|
||||
# 使用阿里云 Debian 镜像源（适用于 Debian/Ubuntu 系统）
|
||||
RUN sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
|
||||
sed -i 's|deb.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list 2>/dev/null || \
|
||||
(echo "deb https://mirrors.aliyun.com/debian/ bookworm main" > /etc/apt/sources.list && \
|
||||
echo "deb https://mirrors.aliyun.com/debian/ bookworm-updates main" >> /etc/apt/sources.list && \
|
||||
echo "deb https://mirrors.aliyun.com/debian-security/ bookworm-security main" >> /etc/apt/sources.list)
|
||||
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
libreoffice-core libreoffice-writer libreoffice-common \
|
||||
libreoffice-base libreoffice-math libreoffice-impress \
|
||||
ca-certificates curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY docker/soffice/app.py /app/app.py
|
||||
COPY docker/soffice/README.md /app/README.md
|
||||
|
||||
EXPOSE 8003
|
||||
|
||||
# Start uvicorn with python3 to ensure sys.executable points to python3
|
||||
# Use port 8003 to match the compose mapping and .env example
|
||||
CMD ["python3", "-m", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8003"]
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
soffice-service
|
||||
================
|
||||
|
||||
Minimal service that provides a REST endpoint to convert .doc files (server-side paths) to .docx using LibreOffice (`soffice`).
|
||||
|
||||
Usage:
|
||||
- Build: `docker-compose build soffice-service`
|
||||
- Start: `docker-compose up -d soffice-service`
|
||||
- Health: `GET /health`
|
||||
- Convert (server-path): `POST /convert` with JSON body `{ "path": "/path/to/file.doc", "delete_original": true }`
|
||||
|
||||
Notes:
|
||||
- Ensure `soffice` (LibreOffice) is available in the container (Dockerfile installs it).
|
||||
- The endpoint performs atomic replace: the resulting `.docx` replaces or is created next to the original file, and the original is removed when `delete_original` is true.
|
||||
- This service does no auth; in production, restrict access to trusted networks or add auth/proxy.
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
import os
|
||||
import tempfile
|
||||
import subprocess
|
||||
import shutil
|
||||
import glob
|
||||
from fastapi import FastAPI, HTTPException, UploadFile, File
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.background import BackgroundTask
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
|
||||
@app.get('/health')
|
||||
async def health():
|
||||
return {'status': 'ok'}
|
||||
|
||||
|
||||
@app.get('/diagnose')
|
||||
async def diagnose():
|
||||
info = {}
|
||||
try:
|
||||
info['soffice_path'] = shutil.which('soffice')
|
||||
except Exception as e:
|
||||
info['soffice_path_error'] = str(e)
|
||||
|
||||
try:
|
||||
proc = subprocess.run(['soffice', '--version'], capture_output=True, text=True, timeout=5)
|
||||
info['soffice_version'] = proc.stdout.strip() or proc.stderr.strip()
|
||||
except Exception as e:
|
||||
info['soffice_version_error'] = str(e)
|
||||
|
||||
try:
|
||||
info['libreoffice_program_list'] = os.listdir('/usr/lib/libreoffice/program')[:50]
|
||||
except Exception as e:
|
||||
info['libreoffice_program_list_error'] = str(e)
|
||||
|
||||
return info
|
||||
|
||||
|
||||
@app.post('/convert')
|
||||
async def convert(file: UploadFile = File(...)):
|
||||
"""Accept a file upload, convert it to .docx via soffice, and return the resulting file."""
|
||||
# Save upload to a temp file
|
||||
tmpdir = tempfile.mkdtemp(prefix='soffice_upload_')
|
||||
# print("Created temp dir", tmpdir)
|
||||
try:
|
||||
original_name = file.filename or 'uploaded'
|
||||
# print(original_name)
|
||||
_, ext = os.path.splitext(original_name)
|
||||
in_path = os.path.join(tmpdir, 'input' + (ext or ''))
|
||||
with open(in_path, 'wb') as f:
|
||||
content = await file.read()
|
||||
f.write(content)
|
||||
# print("Saved uploaded file to", in_path)
|
||||
|
||||
# ========== 修改点 1: 添加文件存在检查 ==========
|
||||
if not os.path.exists(in_path):
|
||||
raise HTTPException(status_code=500, detail="Uploaded file was not saved properly")
|
||||
|
||||
# If already .docx, return it directly
|
||||
if in_path.lower().endswith('.docx'):
|
||||
# ========== 修改点 2: 为 .docx 文件添加背景清理任务 ==========
|
||||
return FileResponse(
|
||||
in_path,
|
||||
media_type='application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
filename=os.path.basename(in_path),
|
||||
background=BackgroundTask(lambda: cleanup_temp_dir(tmpdir)) # 使用 BackgroundTask
|
||||
)
|
||||
|
||||
def run_conv():
|
||||
return subprocess.run(
|
||||
['soffice', '--headless', '--convert-to', 'docx', '--outdir', tmpdir, in_path],
|
||||
check=True, capture_output=True, text=True, timeout=120
|
||||
)
|
||||
|
||||
try:
|
||||
await run_in_threadpool(run_conv)
|
||||
except subprocess.CalledProcessError as e:
|
||||
stderr = (e.stderr or '').strip()
|
||||
raise HTTPException(status_code=500, detail=f'soffice conversion failed: {stderr}')
|
||||
except subprocess.TimeoutExpired:
|
||||
raise HTTPException(status_code=504, detail='conversion timeout')
|
||||
|
||||
candidates = glob.glob(os.path.join(tmpdir, '*.docx'))
|
||||
# print("Conversion produced candidates:", candidates)
|
||||
if not candidates:
|
||||
raise HTTPException(status_code=500, detail='soffice produced no .docx')
|
||||
out_file = candidates[0]
|
||||
|
||||
# ========== 修改点 3: 添加输出文件存在检查 ==========
|
||||
if not os.path.exists(out_file):
|
||||
raise HTTPException(status_code=500, detail="Converted file was not created properly")
|
||||
|
||||
out_fname = os.path.splitext(original_name)[0] + '.docx'
|
||||
# print("Returning converted file", out_file, "as", out_fname)
|
||||
|
||||
# ========== 修改点 4: 主要修改 - 使用 BackgroundTask 延迟清理 ==========
|
||||
return FileResponse(
|
||||
out_file,
|
||||
media_type='application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
filename=out_fname,
|
||||
background=BackgroundTask(lambda: cleanup_temp_dir(tmpdir)) # 使用 BackgroundTask
|
||||
)
|
||||
|
||||
# ========== 修改点 5: 移除 finally 块中的立即清理，改为只在异常时清理 ==========
|
||||
except Exception as e:
|
||||
# 只有在发生异常时才立即清理
|
||||
cleanup_temp_dir(tmpdir)
|
||||
# 重新抛出异常
|
||||
if isinstance(e, HTTPException):
|
||||
raise e
|
||||
else:
|
||||
raise HTTPException(status_code=500, detail=f"Conversion error: {str(e)}")
|
||||
|
||||
# ========== 修改点 6: 添加清理函数 ==========
|
||||
def cleanup_temp_dir(tmpdir: str):
|
||||
"""清理临时目录，忽略错误"""
|
||||
try:
|
||||
if tmpdir and os.path.exists(tmpdir):
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
print(f"Cleaned up temp dir: {tmpdir}")
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to cleanup temp dir {tmpdir}: {e}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import uvicorn
|
||||
port = int(os.environ.get('SOFFICE_PORT') or 8003)
|
||||
uvicorn.run('app:app', host='0.0.0.0', port=port)
|
||||
|
|
@ -6,6 +6,7 @@ from .document_processor import DocumentProcessor
|
|||
from .rag_engine import RAGEngine
|
||||
from .file_parser import FileParser
|
||||
from .chunk_handler import OptimizedDeltaThinkFilter
|
||||
from .anti_word_doc_reader import AntiwordDocReader
|
||||
|
||||
__all__ = ["VectorStoreManager", "DocumentProcessor", "RAGEngine", "FileParser", "OptimizedDeltaThinkFilter"]
|
||||
__all__ = ["VectorStoreManager", "DocumentProcessor", "RAGEngine", "FileParser", "OptimizedDeltaThinkFilter", "AntiwordDocReader"]
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,135 @@
|
|||
from llama_index.core.readers.base import BaseReader
|
||||
from llama_index.core import Document
|
||||
import subprocess
|
||||
import os
|
||||
import tempfile
|
||||
import shutil
|
||||
import glob
|
||||
from typing import List, Optional
|
||||
|
||||
class DocConverter(BaseReader):
|
||||
"""
|
||||
DOC 文件读取器 — 使用 LibreOffice (`soffice`) 将 `.doc` 转为 `.docx`
|
||||
"""
|
||||
|
||||
def __init__(self, soffice_path: str = "soffice"):
|
||||
"""
|
||||
初始化读取器。
|
||||
|
||||
参数:
|
||||
soffice_path: LibreOffice 可执行路径（默认 'soffice'）
|
||||
"""
|
||||
self.soffice_path = soffice_path
|
||||
|
||||
def load_data(self, file_path: str, extra_info: Optional[dict] = None) -> List[Document]:
|
||||
"""
|
||||
加载.doc文件并返回LlamaIndex Document对象列表。
|
||||
|
||||
参数:
|
||||
file_path: .doc文件的路径
|
||||
extra_info: 可选的额外元数据
|
||||
|
||||
返回:
|
||||
Document对象列表
|
||||
"""
|
||||
# 检查文件是否存在
|
||||
if not os.path.exists(file_path):
|
||||
raise FileNotFoundError(f"文件 '{file_path}' 不存在")
|
||||
|
||||
# If the file is already a .docx, extract directly; otherwise convert
|
||||
# using LibreOffice (soffice) into a temporary directory and extract
|
||||
# from the produced .docx.
|
||||
text_content = ""
|
||||
tmpdir = None
|
||||
try:
|
||||
if file_path.lower().endswith('.docx'):
|
||||
docx_path = file_path
|
||||
else:
|
||||
# create temporary dir for conversion output
|
||||
tmpdir = tempfile.mkdtemp(prefix='docconv_')
|
||||
try:
|
||||
conv = subprocess.run(
|
||||
[self.soffice_path, '--headless', '--convert-to', 'docx', '--outdir', tmpdir, file_path],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise RuntimeError(
|
||||
f"soffice not found: '{self.soffice_path}'. Please install LibreOffice (soffice) in the environment."
|
||||
)
|
||||
|
||||
if conv.returncode != 0:
|
||||
stderr = (conv.stderr or conv.stdout or '').strip()
|
||||
raise RuntimeError(f"soffice conversion failed: {stderr}")
|
||||
|
||||
# find resulting docx
|
||||
docx_candidates = glob.glob(os.path.join(tmpdir, '*.docx'))
|
||||
if not docx_candidates:
|
||||
raise RuntimeError('soffice produced no .docx output')
|
||||
docx_path = docx_candidates[0]
|
||||
|
||||
# extract text from docx: prefer docx2txt, fallback to python-docx
|
||||
try:
|
||||
import docx2txt
|
||||
text_content = docx2txt.process(docx_path) or ""
|
||||
except Exception:
|
||||
try:
|
||||
from docx import Document as DocxDocument
|
||||
doc = DocxDocument(docx_path)
|
||||
text_content = "\n".join(p.text for p in doc.paragraphs)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"failed to extract text from docx: {e}") from e
|
||||
|
||||
# 准备文档元数据
|
||||
metadata = {
|
||||
"file_path": file_path,
|
||||
"file_name": os.path.basename(file_path),
|
||||
"file_size": os.path.getsize(file_path),
|
||||
"parser": "soffice"
|
||||
}
|
||||
|
||||
# 合并额外信息（如果有）
|
||||
if extra_info:
|
||||
metadata.update(extra_info)
|
||||
|
||||
# 创建并返回Document对象
|
||||
document = Document(text=text_content, metadata=metadata)
|
||||
return [document]
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
finally:
|
||||
# ensure temporary conversion dir is removed
|
||||
try:
|
||||
if tmpdir and os.path.exists(tmpdir):
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def batch_load_data(self, file_paths: List[str]) -> List[Document]:
|
||||
"""批量加载多个.doc文件"""
|
||||
all_documents = []
|
||||
for file_path in file_paths:
|
||||
documents = self.load_data(file_path)
|
||||
all_documents.extend(documents)
|
||||
return all_documents
|
||||
|
||||
# 使用示例：集成到LlamaIndex流程中
|
||||
if __name__ == "__main__":
|
||||
# 1. 初始化读取器
|
||||
reader = DocConverter()
|
||||
|
||||
# 2. 加载.doc文件
|
||||
try:
|
||||
documents = reader.load_data("/home/zxh/programs/RAG/test_files/测试.doc", extra_info={"category": "报告"})
|
||||
|
||||
# 3. 现在你可以将documents用于LlamaIndex的后续处理
|
||||
print(f"成功加载 {len(documents)} 个文档")
|
||||
print(f"文档文本长度：{len(documents[0].text)} 字符")
|
||||
print(f"文档元数据：{documents[0].metadata}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"加载文档失败：{e}")
|
||||
|
|
@ -2,12 +2,15 @@
|
|||
File parsing module for various document formats
|
||||
"""
|
||||
import os
|
||||
import office
|
||||
import mimetypes
|
||||
from typing import List, Dict, Optional
|
||||
from pathlib import Path
|
||||
from loguru import logger
|
||||
from llama_index.core import Document
|
||||
from llama_index.readers.alibabacloud_aisearch import (
|
||||
AlibabaCloudAISearchDocumentReader,
|
||||
AlibabaCloudAISearchImageReader,
|
||||
)
|
||||
from llama_index.core.readers import SimpleDirectoryReader
|
||||
|
||||
|
||||
|
|
@ -66,30 +69,24 @@ class FileParser:
|
|||
if not doc_id:
|
||||
doc_id = Path(filename).stem
|
||||
|
||||
# 特殊处理.doc文件：转换为.docx格式
|
||||
if ext == '.doc':
|
||||
logger.info(f"检测到.doc文件，开始转换为.docx格式: {filename}")
|
||||
# 保存.doc临时文件
|
||||
try:
|
||||
tmp_doc_path = file_path
|
||||
# 准备.docx临时文件路径
|
||||
tmp_docx_path = tmp_doc_path.replace('.doc', '.docx')
|
||||
tmp_dir_path = os.path.dirname(tmp_doc_path)
|
||||
|
||||
office.word.doc2docx(tmp_doc_path, tmp_dir_path)
|
||||
logger.info(f".doc文件转换为.docx成功: {filename}")
|
||||
file_path = tmp_docx_path
|
||||
except Exception as e:
|
||||
logger.error(f"转换.doc文件为.docx格式时出错: {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
try:
|
||||
logger.info(f"Starting to parse file: {filename} (type: {ext})")
|
||||
|
||||
# 阿里巴巴的llama index reader插件可以解析doc文档
|
||||
document_reader = AlibabaCloudAISearchDocumentReader()
|
||||
image_reader = AlibabaCloudAISearchImageReader()
|
||||
|
||||
file_extractor = {}
|
||||
for suffix in (".pdf", ".docx", ".doc", ".ppt", ".pptx"):
|
||||
file_extractor[suffix] = document_reader
|
||||
for suffix in (".jpg", ".jpeg", ".png", ".bmp", ".tiff"):
|
||||
file_extractor[suffix] = image_reader
|
||||
|
||||
# Use LlamaIndex's SimpleDirectoryReader for parsing
|
||||
# It supports many formats out of the box
|
||||
reader = SimpleDirectoryReader(
|
||||
input_files=[file_path],
|
||||
file_extractor=file_extractor,
|
||||
filename_as_id=False # We'll set custom IDs
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue