wanjia9506
/
langchain4j
mirror of https://github.com/langchain4j/langchain4j.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: warn only about metadata values that actually differ 

A shared metadata key is no longer reported as a conflict when the
document and the source hold equal values, since nothing observable is
discarded in that case. Conflicts are now collected and logged as a
single warning per document instead of one per key, which matters for
parsers that copy whole metadata dictionaries out of a file.

When two conflicting values render identically but differ in type, the
type is appended so the warning does not name two values that look the
same.
This commit is contained in:
Dmytro Liubarskyi 2026-07-31 11:49:58 +02:00
parent f11c7a78d2
commit 44db14bde7
2 changed files with 119 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
langchain4j-core/src/main/java/dev/langchain4j/data/document/DocumentLoader.java
View File

@ -1,7 +1,9 @@
package dev.langchain4j.data.document;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -17,10 +19,10 @@ public class DocumentLoader {
/**
* Loads a document from the given source using the given parser.
*
* <p>Forwards the source Metadata to the parsed Document. If both define the same metadata key,
* the source value is kept, the document value is discarded, and a warning is logged.
* To control which value is kept, remove the key in your {@link DocumentParser}
* before returning the {@link Document}.
* <p>Forwards the source Metadata to the parsed Document. If both define the same metadata key
* with a different value, the source value is kept, the document value is discarded,
* and a warning is logged. To control which value is kept, remove the key in your
* {@link DocumentParser} before returning the {@link Document}.
*
* @param source The source from which the document will be loaded.
* @param parser The parser that will be used to parse the document.
@ -42,17 +44,40 @@ public class DocumentLoader {
}
private static void warnAboutDiscardedValues(Metadata documentMetadata, Map<String, Object> sourceMetadata) {
sourceMetadata.forEach((key, sourceValue) -> {
if (documentMetadata.containsKey(key)) {
log.warn(
"Metadata key \"{}\" is set both by the document (\"{}\") and by the source (\"{}\"). "
+ "Keeping the source value and discarding the document value. "
+ "To control this, remove the key in your DocumentParser "
+ "before returning the Document.",
key,
documentMetadata.toMap().get(key),
sourceValue);
}
});
List<String> conflicts = describeConflicts(documentMetadata, sourceMetadata);
if (conflicts.isEmpty()) {
return;
}
log.warn(
"Metadata keys set by both the document and the source, with different values: {}. "
+ "Keeping the source values and discarding the document values. "
+ "To control this, remove these keys in your DocumentParser "
+ "before returning the Document.",
"{" + String.join(", ", conflicts) + "}");
}
static List<String> describeConflicts(Metadata documentMetadata, Map<String, Object> sourceMetadata) {
Map<String, Object> documentValues = documentMetadata.toMap();
return sourceMetadata.entrySet().stream()
.filter(sourceEntry -> documentValues.containsKey(sourceEntry.getKey())
&& !Objects.equals(documentValues.get(sourceEntry.getKey()), sourceEntry.getValue()))
.sorted(Map.Entry.comparingByKey())
.map(sourceEntry -> describeConflict(
sourceEntry.getKey(), documentValues.get(sourceEntry.getKey()), sourceEntry.getValue()))
.toList();
}
private static String describeConflict(String key, Object documentValue, Object sourceValue) {
if (documentValue.toString().equals(sourceValue.toString())) {
// the values differ only in type, so without it the two would be indistinguishable
return "%s=(document=\"%s\" (%s), source=\"%s\" (%s))"
.formatted(
key,
documentValue,
documentValue.getClass().getSimpleName(),
sourceValue,
sourceValue.getClass().getSimpleName());
}
return "%s=(document=\"%s\", source=\"%s\")".formatted(key, documentValue, sourceValue);
}
}

78
langchain4j-core/src/test/java/dev/langchain4j/data/document/DocumentLoaderTest.java
View File

@ -5,6 +5,7 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import org.assertj.core.api.WithAssertions;
import org.junit.jupiter.api.Test;
@ -123,4 +124,81 @@ class DocumentLoaderTest implements WithAssertions {
.withMessageContaining("Failed to load document")
.withCauseInstanceOf(IllegalArgumentException.class);
}
@Test
void describeConflicts_reportsSharedKeysWithDifferentValues() {
Metadata documentMetadata = new Metadata().put("file_name", "report.pdf");
Map<String, Object> sourceMetadata =
new Metadata().put("file_name", "2024-report.pdf").toMap();
assertThat(DocumentLoader.describeConflicts(documentMetadata, sourceMetadata))
.containsExactly("file_name=(document=\"report.pdf\", source=\"2024-report.pdf\")");
}
@Test
void describeConflicts_ignoresSharedKeysWithEqualValues() {
Metadata documentMetadata = new Metadata().put("file_name", "report.pdf");
Map<String, Object> sourceMetadata =
new Metadata().put("file_name", "report.pdf").toMap();
assertThat(DocumentLoader.describeConflicts(documentMetadata, sourceMetadata))
.isEmpty();
}
@Test
void describeConflicts_ignoresKeysSetByOnlyOneSide() {
Metadata documentMetadata = new Metadata().put("title", "Bar");
Map<String, Object> sourceMetadata =
new Metadata().put("absolute_directory_path", "/docs").toMap();
assertThat(DocumentLoader.describeConflicts(documentMetadata, sourceMetadata))
.isEmpty();
}
@Test
void describeConflicts_reportsAllConflictsSortedByKey() {
Metadata documentMetadata = new Metadata()
.put("title", "Bar")
.put("author", "Alice")
.put("file_name", "report.pdf")
.put("language", "en");
Map<String, Object> sourceMetadata = new Metadata()
.put("title", "Baz")
.put("author", "Alice")
.put("file_name", "2024-report.pdf")
.put("absolute_directory_path", "/docs")
.toMap();
assertThat(DocumentLoader.describeConflicts(documentMetadata, sourceMetadata))
.containsExactly(
"file_name=(document=\"report.pdf\", source=\"2024-report.pdf\")",
"title=(document=\"Bar\", source=\"Baz\")");
}
@Test
void describeConflicts_addsTypesWhenValuesOfDifferentTypesRenderTheSame() {
Metadata documentMetadata = new Metadata().put("page", 5);
Map<String, Object> sourceMetadata = new Metadata().put("page", "5").toMap();
assertThat(DocumentLoader.describeConflicts(documentMetadata, sourceMetadata))
.containsExactly("page=(document=\"5\" (Integer), source=\"5\" (String))");
}
@Test
void describeConflicts_omitsTypesWhenValuesRenderDifferently() {
Metadata documentMetadata = new Metadata().put("page", 5);
Map<String, Object> sourceMetadata = new Metadata().put("page", "6").toMap();
assertThat(DocumentLoader.describeConflicts(documentMetadata, sourceMetadata))
.containsExactly("page=(document=\"5\", source=\"6\")");
}
@Test
void load_keepsSourceValueWhenSharedKeysHaveEqualValues() {
StringSource source = new StringSource("Hello, world!", new Metadata().put("foo", "baz"));
Document document = DocumentLoader.load(source, COLLIDING_PARSER);
assertThat(document.metadata().toMap()).containsOnly(entry("foo", "baz"), entry("title", "Bar"));
}
}