diff --git a/langchain4j-azure-open-ai/src/main/java/dev/langchain4j/model/azure/AzureOpenAiTokenizer.java b/langchain4j-azure-open-ai/src/main/java/dev/langchain4j/model/azure/AzureOpenAiTokenizer.java index 17cd9fd4b..a4f0e5a54 100644 --- a/langchain4j-azure-open-ai/src/main/java/dev/langchain4j/model/azure/AzureOpenAiTokenizer.java +++ b/langchain4j-azure-open-ai/src/main/java/dev/langchain4j/model/azure/AzureOpenAiTokenizer.java @@ -1,5 +1,7 @@ package dev.langchain4j.model.azure; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.knuddels.jtokkit.Encodings; import com.knuddels.jtokkit.api.Encoding; import com.knuddels.jtokkit.api.IntArrayList; @@ -17,7 +19,6 @@ import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.model.Tokenizer; import static dev.langchain4j.internal.Exceptions.illegalArgument; -import static dev.langchain4j.internal.Json.fromJson; import static dev.langchain4j.internal.Utils.isNullOrBlank; import static dev.langchain4j.internal.ValidationUtils.ensureNotBlank; import static dev.langchain4j.model.azure.AzureOpenAiChatModelName.GPT_3_5_TURBO; @@ -43,6 +44,7 @@ public class AzureOpenAiTokenizer implements Tokenizer { private final String modelName; private final Optional encoding; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); /** * Creates an instance of the {@code AzureOpenAiTokenizer} for the "gpt-3.5-turbo" model. @@ -159,7 +161,12 @@ public class AzureOpenAiTokenizer implements Tokenizer { tokenCount += 7; tokenCount += estimateTokenCountInText(toolExecutionRequest.name()); - Map arguments = fromJson(toolExecutionRequest.arguments(), Map.class); + Map arguments; + try { + arguments = OBJECT_MAPPER.readValue(toolExecutionRequest.arguments(), Map.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } for (Map.Entry argument : arguments.entrySet()) { tokenCount += 2; tokenCount += estimateTokenCountInText(argument.getKey().toString()); @@ -383,7 +390,12 @@ public class AzureOpenAiTokenizer implements Tokenizer { if (isNullOrBlank(arguments)) { return 0; } - Map argumentsMap = fromJson(arguments, Map.class); + Map argumentsMap; + try { + argumentsMap = OBJECT_MAPPER.readValue(arguments, Map.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } return argumentsMap.size(); }