Add Builder and configurable text extraction to DoclingDocumentParser (#5818)
## Summary - Introduce a fluent `Builder` for `DoclingDocumentParser` with an optional `Function<InBodyConvertDocumentResponse, String>` (`documentTextExtractor`) to customize how document text is extracted from the Docling conversion response - Default behavior (markdown extraction) is preserved — the extractor defaults to `response.getDocument().getMarkdownContent()` - Deprecate the existing public constructors in favor of the builder, delegating internally to the builder-based constructor - Bump `docling-serve-api`/`docling-serve-client` version from 0.5.2 to 0.6.0 - Update documentation at `docs/docs/integrations/document-parsers/docling.md` with builder usage and custom extraction examples ## Test plan - [x] All 17 unit tests pass (7 existing + 10 new) covering builder construction, custom extractors (HTML, text, doctags), null/blank extractor results, full response metadata access, and builder with options - [x] Integration test with custom extractor against real Docling container (`DoclingDocumentParserIT`) - [x] Verify backward compatibility — existing constructor-based tests pass without modification --------- Signed-off-by: Eric Deandrea <eric.deandrea@ibm.com>
This commit is contained in:
Eric Deandrea
GitHub
parent
4b5b6675c1
commit
aea344c244
|
|
@ -43,19 +43,49 @@ DoclingServeApi api = DoclingServeApi.builder()
|
|||
.baseUrl("http://localhost:5001")
|
||||
.build();
|
||||
|
||||
DoclingDocumentParser parser = new DoclingDocumentParser(api);
|
||||
DoclingDocumentParser parser = DoclingDocumentParser.builder()
|
||||
.doclingClient(api)
|
||||
.build();
|
||||
|
||||
Document document = parser.parse(inputStream);
|
||||
String text = document.text();
|
||||
```
|
||||
|
||||
To customize Docling processing, use the constructor that also accepts a [`ConvertDocumentOptions`](https://docling-project.github.io/docling-java/dev/docling-serve/serve-api/#requests-convertdocumentrequest):
|
||||
### Conversion Options
|
||||
|
||||
To customize Docling processing, pass [`ConvertDocumentOptions`](https://docling-project.github.io/docling-java/dev/docling-serve/serve-api/#requests-convertdocumentrequest) to the builder:
|
||||
|
||||
```java
|
||||
ConvertDocumentOptions options = ConvertDocumentOptions.builder()
|
||||
// configure options here
|
||||
.build();
|
||||
|
||||
DoclingDocumentParser parser = new DoclingDocumentParser(api, options);
|
||||
DoclingDocumentParser parser = DoclingDocumentParser.builder()
|
||||
.doclingClient(api)
|
||||
.options(options)
|
||||
.build();
|
||||
```
|
||||
|
||||
### Custom Text Extraction
|
||||
|
||||
By default, the parser extracts markdown content from the Docling response. You can customize how text is extracted by providing a `Function<InBodyConvertDocumentResponse, String>` via the `documentTextExtractor` builder method. The function receives the full `InBodyConvertDocumentResponse`, giving access to the converted document in various formats (markdown, HTML, text, doctags, JSON), conversion errors, processing time, and status information.
|
||||
|
||||
For example, to extract HTML content instead of markdown:
|
||||
|
||||
```java
|
||||
DoclingDocumentParser parser = DoclingDocumentParser.builder()
|
||||
.doclingClient(api)
|
||||
.documentTextExtractor(response -> response.getDocument().getHtmlContent())
|
||||
.build();
|
||||
```
|
||||
|
||||
Or to extract plain text:
|
||||
|
||||
```java
|
||||
DoclingDocumentParser parser = DoclingDocumentParser.builder()
|
||||
.doclingClient(api)
|
||||
.documentTextExtractor(response -> response.getDocument().getTextContent())
|
||||
.build();
|
||||
```
|
||||
|
||||
## APIs
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
<name>LangChain4j :: Document Parser :: Docling</name>
|
||||
|
||||
<properties>
|
||||
<docling.version>0.5.2</docling.version>
|
||||
<docling.version>0.6.0</docling.version>
|
||||
</properties>
|
||||
|
||||
<dependencyManagement>
|
||||
|
|
|
|||
|
|
@ -14,6 +14,11 @@
|
|||
"new": "missing-class ai.docling.serve.api.convert.request.options.ConvertDocumentOptions",
|
||||
"justification": "Docling Serve API type intentionally exposed by the Docling document parser integration for document conversion options"
|
||||
},
|
||||
{
|
||||
"code": "java.class.externalClassExposedInAPI",
|
||||
"new": "missing-class ai.docling.serve.api.convert.response.InBodyConvertDocumentResponse",
|
||||
"justification": "Docling Serve API type intentionally exposed to allow custom document text extraction"
|
||||
},
|
||||
{
|
||||
"code": "java.class.externalClassExposedInAPI",
|
||||
"new": "missing-class dev.langchain4j.data.document.Document",
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
package dev.langchain4j.data.document.parser.docling;
|
||||
|
||||
import static dev.langchain4j.internal.Utils.getOrDefault;
|
||||
|
||||
import ai.docling.serve.api.DoclingServeApi;
|
||||
import ai.docling.serve.api.convert.request.ConvertDocumentRequest;
|
||||
import ai.docling.serve.api.convert.request.options.ConvertDocumentOptions;
|
||||
import ai.docling.serve.api.convert.request.source.FileSource;
|
||||
import ai.docling.serve.api.convert.response.ConvertDocumentResponse;
|
||||
import ai.docling.serve.api.convert.response.ErrorItem;
|
||||
import ai.docling.serve.api.convert.response.InBodyConvertDocumentResponse;
|
||||
import ai.docling.serve.api.convert.response.ResponseType;
|
||||
import dev.langchain4j.data.document.BlankDocumentException;
|
||||
|
|
@ -16,22 +17,38 @@ import dev.langchain4j.internal.ValidationUtils;
|
|||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Base64;
|
||||
import java.util.function.Function;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class DoclingDocumentParser implements DocumentParser {
|
||||
private static final Logger log = LoggerFactory.getLogger(DoclingDocumentParser.class);
|
||||
|
||||
private static final Function<InBodyConvertDocumentResponse, String> DEFAULT_DOCUMENT_TEXT_EXTRACTOR =
|
||||
response -> response.getDocument().getMarkdownContent();
|
||||
|
||||
private final DoclingServeApi doclingClient;
|
||||
private final ConvertDocumentOptions options;
|
||||
private final Function<InBodyConvertDocumentResponse, String> documentTextExtractor;
|
||||
|
||||
@Deprecated(forRemoval = true)
|
||||
public DoclingDocumentParser(DoclingServeApi doclingClient) {
|
||||
this(doclingClient, null);
|
||||
}
|
||||
|
||||
@Deprecated(forRemoval = true)
|
||||
public DoclingDocumentParser(DoclingServeApi doclingClient, ConvertDocumentOptions options) {
|
||||
this.doclingClient = ValidationUtils.ensureNotNull(doclingClient, "doclingClient");
|
||||
this.options = options;
|
||||
this(builder().doclingClient(doclingClient).options(options));
|
||||
}
|
||||
|
||||
private DoclingDocumentParser(Builder builder) {
|
||||
this.doclingClient = ValidationUtils.ensureNotNull(builder.doclingClient, "doclingClient");
|
||||
this.options = builder.options;
|
||||
this.documentTextExtractor = getOrDefault(builder.documentTextExtractor, DEFAULT_DOCUMENT_TEXT_EXTRACTOR);
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -41,16 +58,16 @@ public class DoclingDocumentParser implements DocumentParser {
|
|||
try {
|
||||
byte[] documentBytes = inputStream.readAllBytes();
|
||||
|
||||
Metadata metadata = new Metadata();
|
||||
var metadata = new Metadata();
|
||||
metadata.put("document_size_bytes", String.valueOf(documentBytes.length));
|
||||
|
||||
if (documentBytes.length == 0) {
|
||||
throw new BlankDocumentException();
|
||||
}
|
||||
|
||||
String base64Content = Base64.getEncoder().encodeToString(documentBytes);
|
||||
var base64Content = Base64.getEncoder().encodeToString(documentBytes);
|
||||
|
||||
ConvertDocumentRequest.Builder requestBuilder = ConvertDocumentRequest.builder()
|
||||
var requestBuilder = ConvertDocumentRequest.builder()
|
||||
.source(FileSource.builder()
|
||||
.base64String(base64Content)
|
||||
.filename("document")
|
||||
|
|
@ -68,10 +85,10 @@ public class DoclingDocumentParser implements DocumentParser {
|
|||
.formatted(ResponseType.IN_BODY, response.getResponseType()));
|
||||
}
|
||||
|
||||
InBodyConvertDocumentResponse inBodyResponse = (InBodyConvertDocumentResponse) response;
|
||||
var inBodyResponse = (InBodyConvertDocumentResponse) response;
|
||||
|
||||
if (!inBodyResponse.getErrors().isEmpty()) {
|
||||
ErrorItem first = inBodyResponse.getErrors().get(0);
|
||||
var first = inBodyResponse.getErrors().get(0);
|
||||
log.warn(
|
||||
"Docling reported {} error(s). First: [{}] {}",
|
||||
inBodyResponse.getErrors().size(),
|
||||
|
|
@ -79,8 +96,8 @@ public class DoclingDocumentParser implements DocumentParser {
|
|||
first.getErrorMessage());
|
||||
}
|
||||
|
||||
String parsedText = inBodyResponse.getDocument().getMarkdownContent();
|
||||
if ((parsedText == null) || parsedText.strip().isEmpty()) {
|
||||
var parsedText = documentTextExtractor.apply(inBodyResponse);
|
||||
if ((parsedText == null) || parsedText.isBlank()) {
|
||||
throw new BlankDocumentException();
|
||||
}
|
||||
|
||||
|
|
@ -93,4 +110,44 @@ public class DoclingDocumentParser implements DocumentParser {
|
|||
throw new RuntimeException("Docling failed to parse document: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
public static final class Builder {
|
||||
private DoclingServeApi doclingClient;
|
||||
private ConvertDocumentOptions options;
|
||||
private Function<InBodyConvertDocumentResponse, String> documentTextExtractor;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
public Builder doclingClient(DoclingServeApi doclingClient) {
|
||||
this.doclingClient = doclingClient;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder options(ConvertDocumentOptions options) {
|
||||
this.options = options;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets a custom function to extract text content from the Docling conversion response.
|
||||
* <p>
|
||||
* The function receives the full {@link InBodyConvertDocumentResponse}, giving access to
|
||||
* the converted document (in various formats: markdown, HTML, text, doctags, JSON),
|
||||
* conversion errors, processing time, and status information.
|
||||
* <p>
|
||||
* If not set, defaults to extracting markdown content:
|
||||
* {@code response -> response.getDocument().getMarkdownContent()}.
|
||||
*
|
||||
* @param documentTextExtractor the function to extract document text from the response
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder documentTextExtractor(Function<InBodyConvertDocumentResponse, String> documentTextExtractor) {
|
||||
this.documentTextExtractor = documentTextExtractor;
|
||||
return this;
|
||||
}
|
||||
|
||||
public DoclingDocumentParser build() {
|
||||
return new DoclingDocumentParser(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,4 +52,20 @@ class DoclingDocumentParserIT {
|
|||
assertThat(document.text()).isNotEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldParsePdfDocumentWithCustomExtractor() throws Exception {
|
||||
var parser = DoclingDocumentParser.builder()
|
||||
.doclingClient(client)
|
||||
.documentTextExtractor(
|
||||
response -> response.getDocument().getMarkdownContent().toUpperCase())
|
||||
.build();
|
||||
|
||||
try (InputStream inputStream = getClass().getResourceAsStream("/test-file.pdf")) {
|
||||
Document document = parser.parse(inputStream);
|
||||
|
||||
assertThat(document).isNotNull();
|
||||
assertThat(document.text()).isNotEmpty().isUpperCase();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import static org.mockito.Mockito.when;
|
|||
|
||||
import ai.docling.serve.api.DoclingServeApi;
|
||||
import ai.docling.serve.api.convert.request.ConvertDocumentRequest;
|
||||
import ai.docling.serve.api.convert.request.options.ConvertDocumentOptions;
|
||||
import ai.docling.serve.api.convert.response.DocumentResponse;
|
||||
import ai.docling.serve.api.convert.response.InBodyConvertDocumentResponse;
|
||||
import dev.langchain4j.data.document.BlankDocumentException;
|
||||
|
|
@ -48,12 +49,7 @@ class DoclingDocumentParserTest {
|
|||
|
||||
@Test
|
||||
void shouldReturnDocumentWithParsedText() {
|
||||
when(mockApi.convertSource(any(ConvertDocumentRequest.class)))
|
||||
.thenReturn(InBodyConvertDocumentResponse.builder()
|
||||
.document(DocumentResponse.builder()
|
||||
.markdownContent("# Parsed Content")
|
||||
.build())
|
||||
.build());
|
||||
mockResponseWithMarkdown("# Parsed Content");
|
||||
|
||||
DoclingDocumentParser parser = new DoclingDocumentParser(mockApi);
|
||||
Document document = parser.parse(new ByteArrayInputStream("some bytes".getBytes()));
|
||||
|
|
@ -64,12 +60,7 @@ class DoclingDocumentParserTest {
|
|||
@Test
|
||||
void shouldIncludeDocumentSizeBytesInMetadata() {
|
||||
byte[] content = "document content".getBytes();
|
||||
when(mockApi.convertSource(any(ConvertDocumentRequest.class)))
|
||||
.thenReturn(InBodyConvertDocumentResponse.builder()
|
||||
.document(DocumentResponse.builder()
|
||||
.markdownContent("Parsed text")
|
||||
.build())
|
||||
.build());
|
||||
mockResponseWithMarkdown("Parsed text");
|
||||
|
||||
DoclingDocumentParser parser = new DoclingDocumentParser(mockApi);
|
||||
Document document = parser.parse(new ByteArrayInputStream(content));
|
||||
|
|
@ -79,10 +70,7 @@ class DoclingDocumentParserTest {
|
|||
|
||||
@Test
|
||||
void shouldHandleEmptyDocumentWhenApiReturnsEmptyContent() {
|
||||
when(mockApi.convertSource(any(ConvertDocumentRequest.class)))
|
||||
.thenReturn(InBodyConvertDocumentResponse.builder()
|
||||
.document(DocumentResponse.builder().markdownContent("").build())
|
||||
.build());
|
||||
mockResponseWithMarkdown("");
|
||||
|
||||
assertThatThrownBy(() -> new DoclingDocumentParser(mockApi).parse(new ByteArrayInputStream("data".getBytes())))
|
||||
.isInstanceOf(BlankDocumentException.class);
|
||||
|
|
@ -92,4 +80,148 @@ class DoclingDocumentParserTest {
|
|||
void shouldImplementDocumentParserInterface() {
|
||||
assertThat(new DoclingDocumentParser(mockApi)).isInstanceOf(DocumentParser.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldReturnDocumentWithParsedText_usingBuilder() {
|
||||
mockResponseWithMarkdown("# Parsed Content");
|
||||
|
||||
var parser = DoclingDocumentParser.builder().doclingClient(mockApi).build();
|
||||
var document = parser.parse(new ByteArrayInputStream("some bytes".getBytes()));
|
||||
|
||||
assertThat(document.text()).isEqualTo("# Parsed Content");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldThrowWhenDoclingClientIsNull_usingBuilder() {
|
||||
assertThatThrownBy(() -> DoclingDocumentParser.builder().build()).isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldUseCustomDocumentTextExtractor() {
|
||||
mockResponseWith("# Markdown", "<h1>HTML</h1>", null, null);
|
||||
|
||||
var parser = DoclingDocumentParser.builder()
|
||||
.doclingClient(mockApi)
|
||||
.documentTextExtractor(response -> response.getDocument().getHtmlContent())
|
||||
.build();
|
||||
var document = parser.parse(new ByteArrayInputStream("some bytes".getBytes()));
|
||||
|
||||
assertThat(document.text()).isEqualTo("<h1>HTML</h1>");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldUseDefaultMarkdownExtractorWhenNoneSpecified() {
|
||||
mockResponseWith("# Markdown", "<h1>HTML</h1>", "Plain text", null);
|
||||
|
||||
var parser = DoclingDocumentParser.builder().doclingClient(mockApi).build();
|
||||
var document = parser.parse(new ByteArrayInputStream("some bytes".getBytes()));
|
||||
|
||||
assertThat(document.text()).isEqualTo("# Markdown");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldThrowBlankDocumentExceptionWhenExtractorReturnsNull() {
|
||||
mockResponseWithMarkdown("# Markdown");
|
||||
|
||||
var parser = DoclingDocumentParser.builder()
|
||||
.doclingClient(mockApi)
|
||||
.documentTextExtractor(response -> null)
|
||||
.build();
|
||||
|
||||
assertThatThrownBy(() -> parser.parse(new ByteArrayInputStream("data".getBytes())))
|
||||
.isInstanceOf(BlankDocumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldThrowBlankDocumentExceptionWhenExtractorReturnsBlank() {
|
||||
mockResponseWithMarkdown("# Markdown");
|
||||
|
||||
var parser = DoclingDocumentParser.builder()
|
||||
.doclingClient(mockApi)
|
||||
.documentTextExtractor(response -> " ")
|
||||
.build();
|
||||
|
||||
assertThatThrownBy(() -> parser.parse(new ByteArrayInputStream("data".getBytes())))
|
||||
.isInstanceOf(BlankDocumentException.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldAllowAccessToFullResponse() {
|
||||
when(mockApi.convertSource(any(ConvertDocumentRequest.class)))
|
||||
.thenReturn(InBodyConvertDocumentResponse.builder()
|
||||
.document(DocumentResponse.builder()
|
||||
.markdownContent("content")
|
||||
.build())
|
||||
.status("SUCCESS")
|
||||
.processingTime(1.5)
|
||||
.build());
|
||||
|
||||
var parser = DoclingDocumentParser.builder()
|
||||
.doclingClient(mockApi)
|
||||
.documentTextExtractor(response -> "%s (status=%s, time=%.1f)"
|
||||
.formatted(
|
||||
response.getDocument().getMarkdownContent(),
|
||||
response.getStatus(),
|
||||
response.getProcessingTime()))
|
||||
.build();
|
||||
var document = parser.parse(new ByteArrayInputStream("data".getBytes()));
|
||||
|
||||
assertThat(document.text()).isEqualTo("content (status=SUCCESS, time=1.5)");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldBuildWithOptions() {
|
||||
mockResponseWithMarkdown("# Content");
|
||||
|
||||
var options = ConvertDocumentOptions.builder().build();
|
||||
var parser = DoclingDocumentParser.builder()
|
||||
.doclingClient(mockApi)
|
||||
.options(options)
|
||||
.build();
|
||||
var document = parser.parse(new ByteArrayInputStream("data".getBytes()));
|
||||
|
||||
assertThat(document.text()).isEqualTo("# Content");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldExtractTextContent() {
|
||||
mockResponseWith(null, null, "Plain text content", null);
|
||||
|
||||
var parser = DoclingDocumentParser.builder()
|
||||
.doclingClient(mockApi)
|
||||
.documentTextExtractor(response -> response.getDocument().getTextContent())
|
||||
.build();
|
||||
var document = parser.parse(new ByteArrayInputStream("data".getBytes()));
|
||||
|
||||
assertThat(document.text()).isEqualTo("Plain text content");
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldExtractDoctagsContent() {
|
||||
mockResponseWith(null, null, null, "<doctag>content</doctag>");
|
||||
|
||||
var parser = DoclingDocumentParser.builder()
|
||||
.doclingClient(mockApi)
|
||||
.documentTextExtractor(response -> response.getDocument().getDoctagsContent())
|
||||
.build();
|
||||
var document = parser.parse(new ByteArrayInputStream("data".getBytes()));
|
||||
|
||||
assertThat(document.text()).isEqualTo("<doctag>content</doctag>");
|
||||
}
|
||||
|
||||
private void mockResponseWithMarkdown(String markdown) {
|
||||
mockResponseWith(markdown, null, null, null);
|
||||
}
|
||||
|
||||
private void mockResponseWith(String markdown, String html, String text, String doctags) {
|
||||
when(mockApi.convertSource(any(ConvertDocumentRequest.class)))
|
||||
.thenReturn(InBodyConvertDocumentResponse.builder()
|
||||
.document(DocumentResponse.builder()
|
||||
.markdownContent(markdown)
|
||||
.htmlContent(html)
|
||||
.textContent(text)
|
||||
.doctagsContent(doctags)
|
||||
.build())
|
||||
.build());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue