From c0b58733ac9c8c8c184af2224a493801c8d720c5 Mon Sep 17 00:00:00 2001 From: mzegla Date: Wed, 8 Jul 2026 13:46:26 +0200 Subject: [PATCH 1/2] init --- src/llm/BUILD | 2 + src/llm/apis/openai_completions.cpp | 7 +- src/llm/io_processing/input_processor.cpp | 28 +++--- .../empty_content_normalization_processor.cpp | 38 ++++++++ .../empty_content_normalization_processor.hpp | 31 +++++++ .../text_content_normalization_processor.cpp | 15 +++- .../text_content_normalization_processor.hpp | 7 +- src/test/http_openai_handler_test.cpp | 11 ++- ...y_content_normalization_processor_test.cpp | 89 +++++++++++++++++++ .../input_processing_integration_test.cpp | 21 +++-- ...t_content_normalization_processor_test.cpp | 8 +- 11 files changed, 221 insertions(+), 36 deletions(-) create mode 100644 src/llm/io_processing/input_processors/empty_content_normalization_processor.cpp create mode 100644 src/llm/io_processing/input_processors/empty_content_normalization_processor.hpp create mode 100644 src/test/llm/input_processing/empty_content_normalization_processor_test.cpp diff --git a/src/llm/BUILD b/src/llm/BUILD index 0ac3de44b8..120f813846 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -155,12 +155,14 @@ ovms_cc_library( "io_processing/input_processors/chat_template_processor.hpp", "io_processing/input_processors/chat_template_adapter.hpp", "io_processing/chat_template/caps.hpp", + "io_processing/input_processors/empty_content_normalization_processor.hpp", "io_processing/input_processors/raw_prompt_extractor.hpp", "io_processing/input_processors/text_content_normalization_processor.hpp", "io_processing/input_processors/tokenization_processor.hpp"], srcs = ["io_processing/input_processors/image_decoding_processor.cpp", "io_processing/input_processors/chat_template_processor.cpp", "io_processing/input_processors/chat_template_adapter.cpp", + "io_processing/input_processors/empty_content_normalization_processor.cpp", "io_processing/input_processors/text_content_normalization_processor.cpp", "io_processing/input_processors/tokenization_processor.cpp"], deps = [ diff --git a/src/llm/apis/openai_completions.cpp b/src/llm/apis/openai_completions.cpp index 2ac285e61e..d978bcbf13 100644 --- a/src/llm/apis/openai_completions.cpp +++ b/src/llm/apis/openai_completions.cpp @@ -195,10 +195,9 @@ absl::Status OpenAIChatCompletionsHandler::parseMessages(std::optionalvalue.IsArray()) { - // Validate the content array and check whether it contains images. - if (member->value.GetArray().Size() == 0) { - return absl::InvalidArgumentError("Invalid message structure - content array is empty"); - } + // Empty content arrays are accepted and preserved as-is. The + // EmptyContentNormalizationProcessor converts them to null before + // downstream processing. for (const auto& v : member->value.GetArray()) { if (!v.IsObject()) { return absl::InvalidArgumentError("Invalid message structure - content array should contain objects"); diff --git a/src/llm/io_processing/input_processor.cpp b/src/llm/io_processing/input_processor.cpp index 97cfd26d35..bb88e6e003 100644 --- a/src/llm/io_processing/input_processor.cpp +++ b/src/llm/io_processing/input_processor.cpp @@ -23,6 +23,7 @@ #include "../../config.hpp" #include "../../logging.hpp" #include "input_processors/chat_template_processor.hpp" +#include "input_processors/empty_content_normalization_processor.hpp" #include "input_processors/image_decoding_processor.hpp" #include "input_processors/chat_template_adapter.hpp" #include "input_processors/raw_prompt_extractor.hpp" @@ -37,22 +38,25 @@ InputProcessor::InputProcessor(InputProcessorContext& context, // Chat template already adds special tokens; completions path needs them added by the tokenizer. const bool addSpecialTokens = !isChatPath; - if (context.config.isVLM && isChatPath) { - const auto& settings = Config::instance().getServerSettings(); - processors.emplace_back(std::make_unique( - settings.allowedLocalMediaPath, - settings.allowedMediaDomains)); - } + if (isChatPath) { + // Normalize empty content arrays to null before any content-aware processor runs. + processors.emplace_back(std::make_unique()); - if (!context.config.isVLM && isChatPath) { + // Flatten text-only content arrays for both LM and VLM. Arrays that contain + // images (or other modalities) are left untouched for ImageDecodingProcessor. processors.emplace_back(std::make_unique()); - } - if (isChatPath && context.chatTemplateCaps.needsWorkarounds()) { - processors.emplace_back(std::make_unique(context.chatTemplateCaps)); - } + if (context.config.isVLM) { + const auto& settings = Config::instance().getServerSettings(); + processors.emplace_back(std::make_unique( + settings.allowedLocalMediaPath, + settings.allowedMediaDomains)); + } + + if (context.chatTemplateCaps.needsWorkarounds()) { + processors.emplace_back(std::make_unique(context.chatTemplateCaps)); + } - if (isChatPath) { #if (PYTHON_DISABLE == 0) // Select the path at construction time. If !useMinja but templateProcessor is null // (shouldn't happen on a properly initialized servable), fall back to the native path. diff --git a/src/llm/io_processing/input_processors/empty_content_normalization_processor.cpp b/src/llm/io_processing/input_processors/empty_content_normalization_processor.cpp new file mode 100644 index 0000000000..0fb2b9e713 --- /dev/null +++ b/src/llm/io_processing/input_processors/empty_content_normalization_processor.cpp @@ -0,0 +1,38 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "empty_content_normalization_processor.hpp" + +#include + +namespace ovms { + +absl::Status EmptyContentNormalizationProcessor::process(InputRequest& req) { + if (!std::holds_alternative(req.input)) { + return absl::Status(absl::StatusCode::kInternal, + "EmptyContentNormalizationProcessor received input that is not a ChatHistory"); + } + ov::genai::ChatHistory& chatHistory = std::get(req.input); + for (size_t i = 0; i < chatHistory.size(); i++) { + const auto content = chatHistory[i]["content"]; + if (content.is_array() && content.size() == 0) { + chatHistory[i]["content"] = ov::genai::JsonContainer(nullptr); + } + } + return absl::OkStatus(); +} + +} // namespace ovms diff --git a/src/llm/io_processing/input_processors/empty_content_normalization_processor.hpp b/src/llm/io_processing/input_processors/empty_content_normalization_processor.hpp new file mode 100644 index 0000000000..467e0e47b1 --- /dev/null +++ b/src/llm/io_processing/input_processors/empty_content_normalization_processor.hpp @@ -0,0 +1,31 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include "../base_input_processor.hpp" + +namespace ovms { + +// Replaces empty content arrays ("content": []) in ChatHistory messages with null. +// Runs for all chat paths (LM and VLM) and must execute before ImageDecodingProcessor +// and TextContentNormalizationProcessor so downstream processors and chat templates +// see a null content instead of an empty array. +class EmptyContentNormalizationProcessor : public BaseInputProcessor { +public: + absl::Status process(InputRequest& req) override; +}; + +} // namespace ovms diff --git a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp index 2818b0b8ba..b12e32e494 100644 --- a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp +++ b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp @@ -32,12 +32,21 @@ absl::Status TextContentNormalizationProcessor::process(InputRequest& req) { if (!content.is_array()) { continue; } + // Only flatten arrays that contain exclusively text parts. Arrays with + // images (or other modalities) are left untouched for ImageDecodingProcessor. + bool allText = true; + for (size_t j = 0; j < content.size(); j++) { + if (content[j]["type"].as_string().value_or("") != "text") { + allText = false; + break; + } + } + if (!allText) { + continue; + } std::string combined; for (size_t j = 0; j < content.size(); j++) { const auto part = content[j]; - if (part["type"].as_string().value_or("") != "text") { - continue; - } if (!combined.empty()) { combined += "\n"; } diff --git a/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp b/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp index 0899a13660..7280214920 100644 --- a/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp +++ b/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp @@ -19,9 +19,10 @@ namespace ovms { -// Normalizes text-only content arrays in ChatHistory messages to plain strings. -// Parts are joined with "\n" for backward compatibility with LM chat templates. -// Active when: !config.isVLM && input is ChatHistory variant. +// Flattens text-only content arrays in ChatHistory messages to plain strings. +// Parts are joined with "\n" for backward compatibility with chat templates. +// Runs for both LM and VLM chat paths: arrays that contain images (or other +// non-text modalities) are left untouched for ImageDecodingProcessor. // Must run before ChatTemplateProcessor. class TextContentNormalizationProcessor : public BaseInputProcessor { public: diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index a600872f62..99d94a6534 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -2325,7 +2325,7 @@ TEST_F(HttpOpenAIHandlerParsingTest, ParsingMessagesWithInvalidContentTypeFails) EXPECT_EQ(apiHandler->parseMessages(), absl::InvalidArgumentError("Unsupported content type")); } -TEST_F(HttpOpenAIHandlerParsingTest, ParsingMessagesEmptyContentArrayFails) { +TEST_F(HttpOpenAIHandlerParsingTest, ParsingMessagesEmptyContentArrayPreservesArray) { std::string json = R"({ "model": "llama", "messages": [ @@ -2338,7 +2338,14 @@ TEST_F(HttpOpenAIHandlerParsingTest, ParsingMessagesEmptyContentArrayFails) { doc.Parse(json.c_str()); ASSERT_FALSE(doc.HasParseError()); std::shared_ptr apiHandler = std::make_shared(doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer); - EXPECT_EQ(apiHandler->parseMessages(), absl::InvalidArgumentError("Invalid message structure - content array is empty")); + // Empty content arrays are accepted and preserved as-is. The + // EmptyContentNormalizationProcessor converts them to null downstream. + ASSERT_EQ(apiHandler->parseMessages(), absl::OkStatus()); + auto& chatHistory = apiHandler->getChatHistory(); + ASSERT_EQ(chatHistory.size(), 1u); + auto content = chatHistory[0]["content"]; + EXPECT_TRUE(content.is_array()); + EXPECT_EQ(content.size(), 0u); } TEST_F(HttpOpenAIHandlerParsingTest, ParsingMessagesWithNoContentFieldAddsEmptyStringToChatHistory) { diff --git a/src/test/llm/input_processing/empty_content_normalization_processor_test.cpp b/src/test/llm/input_processing/empty_content_normalization_processor_test.cpp new file mode 100644 index 0000000000..45da44b516 --- /dev/null +++ b/src/test/llm/input_processing/empty_content_normalization_processor_test.cpp @@ -0,0 +1,89 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include + +#include +#include + +#include "../../../llm/io_processing/input_processors/empty_content_normalization_processor.hpp" +#include "../../../llm/io_processing/input_request.hpp" + +using namespace ovms; + +// Helpers ---------------------------------------------------------------- + +static InputRequest makeChatRequest(ov::genai::ChatHistory chatHistory) { + InputRequest req; + req.input = std::move(chatHistory); + return req; +} + +// Tests ------------------------------------------------------------------ + +TEST(EmptyContentNormalizationProcessorTest, EmptyArrayConvertedToNull) { + ov::genai::ChatHistory history; + ov::AnyMap msg = {{"role", std::string("user")}}; + msg["content"] = ov::genai::JsonContainer::from_json_string("[]"); + history.push_back(msg); + + InputRequest req = makeChatRequest(history); + EmptyContentNormalizationProcessor processor; + const auto status = processor.process(req); + + EXPECT_TRUE(status.ok()); + const auto& result = std::get(req.input); + EXPECT_FALSE(result[0]["content"].is_array()); + EXPECT_TRUE(result[0]["content"].is_null()); +} + +TEST(EmptyContentNormalizationProcessorTest, NonEmptyArrayPreserved) { + ov::genai::ChatHistory history; + ov::AnyMap msg = {{"role", std::string("user")}}; + msg["content"] = ov::genai::JsonContainer::from_json_string( + R"([{"type":"text","text":"hello"}])"); + history.push_back(msg); + + InputRequest req = makeChatRequest(history); + EmptyContentNormalizationProcessor processor; + const auto status = processor.process(req); + + EXPECT_TRUE(status.ok()); + const auto& result = std::get(req.input); + ASSERT_TRUE(result[0]["content"].is_array()); + EXPECT_EQ(result[0]["content"].size(), 1u); +} + +TEST(EmptyContentNormalizationProcessorTest, StringContentPassedThrough) { + ov::genai::ChatHistory history; + history.push_back({{"role", "user"}, {"content", "Hello, world!"}}); + + InputRequest req = makeChatRequest(history); + EmptyContentNormalizationProcessor processor; + const auto status = processor.process(req); + + EXPECT_TRUE(status.ok()); + const auto& result = std::get(req.input); + EXPECT_EQ(result[0]["content"].as_string().value_or(""), "Hello, world!"); +} + +TEST(EmptyContentNormalizationProcessorTest, RawPromptInputRejected) { + InputRequest req; + req.input = std::string("raw prompt"); + EmptyContentNormalizationProcessor processor; + const auto status = processor.process(req); + + EXPECT_EQ(status.code(), absl::StatusCode::kInternal); +} diff --git a/src/test/llm/input_processing/input_processing_integration_test.cpp b/src/test/llm/input_processing/input_processing_integration_test.cpp index c2f78b79cb..609eca6297 100644 --- a/src/test/llm/input_processing/input_processing_integration_test.cpp +++ b/src/test/llm/input_processing/input_processing_integration_test.cpp @@ -342,20 +342,23 @@ TEST_P(InputProcessingIntegrationTest, GenerationConfigFields_SurviveFullPipelin } TEST_P(InputProcessingIntegrationTest, TextArrayContent_FlattenedByNormalizationProcessor) { - // Non-VLM path: content array with two text parts must be flattened by - // TextContentNormalizationProcessor so both strings appear in the final prompt. - auto result = runPipeline(textArrayJson(), /*isVLM=*/false); - - ASSERT_TRUE(result.parseStatus.ok()) << result.parseStatus.message(); - ASSERT_TRUE(result.processStatus.ok()) << result.processStatus.message(); - + // A text-only content array must be flattened by TextContentNormalizationProcessor + // so both strings appear in the final prompt. This holds on both the LM path and the + // VLM path used without images (VLM is a superset of LM). // TextContentNormalizationProcessor joins the two parts with \n before the template runs. const std::string expected = std::string(SMOL_DEFAULT_SYSTEM) + "<|im_start|>user\nFirst part.\nSecond part.<|im_end|>\n" "<|im_start|>assistant\n"; - EXPECT_EQ(result.req.promptText, expected); - EXPECT_TRUE(result.req.inputImages.empty()); + + for (bool isVLM : {false, true}) { + auto result = runPipeline(textArrayJson(), isVLM); + + ASSERT_TRUE(result.parseStatus.ok()) << "isVLM=" << isVLM << ": " << result.parseStatus.message(); + ASSERT_TRUE(result.processStatus.ok()) << "isVLM=" << isVLM << ": " << result.processStatus.message(); + EXPECT_EQ(result.req.promptText, expected) << "isVLM=" << isVLM; + EXPECT_TRUE(result.req.inputImages.empty()) << "isVLM=" << isVLM; + } } // --------------------------------------------------------------------------- diff --git a/src/test/llm/input_processing/text_content_normalization_processor_test.cpp b/src/test/llm/input_processing/text_content_normalization_processor_test.cpp index a1b814ab7a..2b1840f0de 100644 --- a/src/test/llm/input_processing/text_content_normalization_processor_test.cpp +++ b/src/test/llm/input_processing/text_content_normalization_processor_test.cpp @@ -80,8 +80,9 @@ TEST(TextContentNormalizationProcessorTest, MultipleTextPartsJoinedWithNewline) EXPECT_EQ(result[0]["content"].as_string().value_or(""), "first\nsecond"); } -TEST(TextContentNormalizationProcessorTest, NonTextPartsIgnored) { - // image_url entries alongside text: only text parts should contribute to combined string. +TEST(TextContentNormalizationProcessorTest, MixedContentArrayLeftUntouched) { + // image_url entries alongside text: the array is NOT text-only, so it must be + // left untouched for ImageDecodingProcessor to handle downstream. ov::genai::ChatHistory history; ov::AnyMap msg = {{"role", std::string("user")}}; ov::genai::JsonContainer parts = ov::genai::JsonContainer::from_json_string( @@ -95,5 +96,6 @@ TEST(TextContentNormalizationProcessorTest, NonTextPartsIgnored) { EXPECT_TRUE(status.ok()); const auto& result = std::get(req.input); - EXPECT_EQ(result[0]["content"].as_string().value_or(""), "describe this"); + ASSERT_TRUE(result[0]["content"].is_array()); + EXPECT_EQ(result[0]["content"].size(), 2u); } From c6cc30496940f2835f987b75f862711e71cbb57d Mon Sep 17 00:00:00 2001 From: mzegla Date: Wed, 8 Jul 2026 14:45:07 +0200 Subject: [PATCH 2/2] review --- src/llm/BUILD | 4 ++-- src/llm/apis/openai_completions.cpp | 2 +- src/llm/io_processing/input_processor.cpp | 4 ++-- ..._content_array_normalization_processor.cpp} | 6 +++--- ..._content_array_normalization_processor.hpp} | 2 +- .../text_content_normalization_processor.cpp | 16 ++++++++-------- src/test/http_openai_handler_test.cpp | 2 +- ...ent_array_normalization_processor_test.cpp} | 18 +++++++++--------- 8 files changed, 27 insertions(+), 27 deletions(-) rename src/llm/io_processing/input_processors/{empty_content_normalization_processor.cpp => empty_content_array_normalization_processor.cpp} (84%) rename src/llm/io_processing/input_processors/{empty_content_normalization_processor.hpp => empty_content_array_normalization_processor.hpp} (94%) rename src/test/llm/input_processing/{empty_content_normalization_processor_test.cpp => empty_content_array_normalization_processor_test.cpp} (83%) diff --git a/src/llm/BUILD b/src/llm/BUILD index 120f813846..d6b91852b1 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -155,14 +155,14 @@ ovms_cc_library( "io_processing/input_processors/chat_template_processor.hpp", "io_processing/input_processors/chat_template_adapter.hpp", "io_processing/chat_template/caps.hpp", - "io_processing/input_processors/empty_content_normalization_processor.hpp", + "io_processing/input_processors/empty_content_array_normalization_processor.hpp", "io_processing/input_processors/raw_prompt_extractor.hpp", "io_processing/input_processors/text_content_normalization_processor.hpp", "io_processing/input_processors/tokenization_processor.hpp"], srcs = ["io_processing/input_processors/image_decoding_processor.cpp", "io_processing/input_processors/chat_template_processor.cpp", "io_processing/input_processors/chat_template_adapter.cpp", - "io_processing/input_processors/empty_content_normalization_processor.cpp", + "io_processing/input_processors/empty_content_array_normalization_processor.cpp", "io_processing/input_processors/text_content_normalization_processor.cpp", "io_processing/input_processors/tokenization_processor.cpp"], deps = [ diff --git a/src/llm/apis/openai_completions.cpp b/src/llm/apis/openai_completions.cpp index d978bcbf13..d526a85358 100644 --- a/src/llm/apis/openai_completions.cpp +++ b/src/llm/apis/openai_completions.cpp @@ -196,7 +196,7 @@ absl::Status OpenAIChatCompletionsHandler::parseMessages(std::optionalvalue.IsArray()) { // Empty content arrays are accepted and preserved as-is. The - // EmptyContentNormalizationProcessor converts them to null before + // EmptyContentArrayNormalizationProcessor converts them to null before // downstream processing. for (const auto& v : member->value.GetArray()) { if (!v.IsObject()) { diff --git a/src/llm/io_processing/input_processor.cpp b/src/llm/io_processing/input_processor.cpp index bb88e6e003..504fdb51b0 100644 --- a/src/llm/io_processing/input_processor.cpp +++ b/src/llm/io_processing/input_processor.cpp @@ -23,7 +23,7 @@ #include "../../config.hpp" #include "../../logging.hpp" #include "input_processors/chat_template_processor.hpp" -#include "input_processors/empty_content_normalization_processor.hpp" +#include "input_processors/empty_content_array_normalization_processor.hpp" #include "input_processors/image_decoding_processor.hpp" #include "input_processors/chat_template_adapter.hpp" #include "input_processors/raw_prompt_extractor.hpp" @@ -40,7 +40,7 @@ InputProcessor::InputProcessor(InputProcessorContext& context, if (isChatPath) { // Normalize empty content arrays to null before any content-aware processor runs. - processors.emplace_back(std::make_unique()); + processors.emplace_back(std::make_unique()); // Flatten text-only content arrays for both LM and VLM. Arrays that contain // images (or other modalities) are left untouched for ImageDecodingProcessor. diff --git a/src/llm/io_processing/input_processors/empty_content_normalization_processor.cpp b/src/llm/io_processing/input_processors/empty_content_array_normalization_processor.cpp similarity index 84% rename from src/llm/io_processing/input_processors/empty_content_normalization_processor.cpp rename to src/llm/io_processing/input_processors/empty_content_array_normalization_processor.cpp index 0fb2b9e713..bdbfbad3ea 100644 --- a/src/llm/io_processing/input_processors/empty_content_normalization_processor.cpp +++ b/src/llm/io_processing/input_processors/empty_content_array_normalization_processor.cpp @@ -14,16 +14,16 @@ // limitations under the License. //***************************************************************************** -#include "empty_content_normalization_processor.hpp" +#include "empty_content_array_normalization_processor.hpp" #include namespace ovms { -absl::Status EmptyContentNormalizationProcessor::process(InputRequest& req) { +absl::Status EmptyContentArrayNormalizationProcessor::process(InputRequest& req) { if (!std::holds_alternative(req.input)) { return absl::Status(absl::StatusCode::kInternal, - "EmptyContentNormalizationProcessor received input that is not a ChatHistory"); + "EmptyContentArrayNormalizationProcessor received input that is not a ChatHistory"); } ov::genai::ChatHistory& chatHistory = std::get(req.input); for (size_t i = 0; i < chatHistory.size(); i++) { diff --git a/src/llm/io_processing/input_processors/empty_content_normalization_processor.hpp b/src/llm/io_processing/input_processors/empty_content_array_normalization_processor.hpp similarity index 94% rename from src/llm/io_processing/input_processors/empty_content_normalization_processor.hpp rename to src/llm/io_processing/input_processors/empty_content_array_normalization_processor.hpp index 467e0e47b1..69c0cbc82c 100644 --- a/src/llm/io_processing/input_processors/empty_content_normalization_processor.hpp +++ b/src/llm/io_processing/input_processors/empty_content_array_normalization_processor.hpp @@ -23,7 +23,7 @@ namespace ovms { // Runs for all chat paths (LM and VLM) and must execute before ImageDecodingProcessor // and TextContentNormalizationProcessor so downstream processors and chat templates // see a null content instead of an empty array. -class EmptyContentNormalizationProcessor : public BaseInputProcessor { +class EmptyContentArrayNormalizationProcessor : public BaseInputProcessor { public: absl::Status process(InputRequest& req) override; }; diff --git a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp index b12e32e494..2180d5faae 100644 --- a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp +++ b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp @@ -34,24 +34,24 @@ absl::Status TextContentNormalizationProcessor::process(InputRequest& req) { } // Only flatten arrays that contain exclusively text parts. Arrays with // images (or other modalities) are left untouched for ImageDecodingProcessor. + // Single pass: build the combined string while scanning, and bail out on the + // first non-text part without touching the message. + std::string combined; bool allText = true; for (size_t j = 0; j < content.size(); j++) { - if (content[j]["type"].as_string().value_or("") != "text") { + const auto part = content[j]; + if (part["type"].as_string().value_or("") != "text") { allText = false; break; } - } - if (!allText) { - continue; - } - std::string combined; - for (size_t j = 0; j < content.size(); j++) { - const auto part = content[j]; if (!combined.empty()) { combined += "\n"; } combined += part["text"].as_string().value_or(""); } + if (!allText) { + continue; + } chatHistory[i]["content"] = combined; } return absl::OkStatus(); diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index 99d94a6534..f8be827c4e 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -2339,7 +2339,7 @@ TEST_F(HttpOpenAIHandlerParsingTest, ParsingMessagesEmptyContentArrayPreservesAr ASSERT_FALSE(doc.HasParseError()); std::shared_ptr apiHandler = std::make_shared(doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer); // Empty content arrays are accepted and preserved as-is. The - // EmptyContentNormalizationProcessor converts them to null downstream. + // EmptyContentArrayNormalizationProcessor converts them to null downstream. ASSERT_EQ(apiHandler->parseMessages(), absl::OkStatus()); auto& chatHistory = apiHandler->getChatHistory(); ASSERT_EQ(chatHistory.size(), 1u); diff --git a/src/test/llm/input_processing/empty_content_normalization_processor_test.cpp b/src/test/llm/input_processing/empty_content_array_normalization_processor_test.cpp similarity index 83% rename from src/test/llm/input_processing/empty_content_normalization_processor_test.cpp rename to src/test/llm/input_processing/empty_content_array_normalization_processor_test.cpp index 45da44b516..4badd84b1d 100644 --- a/src/test/llm/input_processing/empty_content_normalization_processor_test.cpp +++ b/src/test/llm/input_processing/empty_content_array_normalization_processor_test.cpp @@ -18,7 +18,7 @@ #include #include -#include "../../../llm/io_processing/input_processors/empty_content_normalization_processor.hpp" +#include "../../../llm/io_processing/input_processors/empty_content_array_normalization_processor.hpp" #include "../../../llm/io_processing/input_request.hpp" using namespace ovms; @@ -33,14 +33,14 @@ static InputRequest makeChatRequest(ov::genai::ChatHistory chatHistory) { // Tests ------------------------------------------------------------------ -TEST(EmptyContentNormalizationProcessorTest, EmptyArrayConvertedToNull) { +TEST(EmptyContentArrayNormalizationProcessorTest, EmptyArrayConvertedToNull) { ov::genai::ChatHistory history; ov::AnyMap msg = {{"role", std::string("user")}}; msg["content"] = ov::genai::JsonContainer::from_json_string("[]"); history.push_back(msg); InputRequest req = makeChatRequest(history); - EmptyContentNormalizationProcessor processor; + EmptyContentArrayNormalizationProcessor processor; const auto status = processor.process(req); EXPECT_TRUE(status.ok()); @@ -49,7 +49,7 @@ TEST(EmptyContentNormalizationProcessorTest, EmptyArrayConvertedToNull) { EXPECT_TRUE(result[0]["content"].is_null()); } -TEST(EmptyContentNormalizationProcessorTest, NonEmptyArrayPreserved) { +TEST(EmptyContentArrayNormalizationProcessorTest, NonEmptyArrayPreserved) { ov::genai::ChatHistory history; ov::AnyMap msg = {{"role", std::string("user")}}; msg["content"] = ov::genai::JsonContainer::from_json_string( @@ -57,7 +57,7 @@ TEST(EmptyContentNormalizationProcessorTest, NonEmptyArrayPreserved) { history.push_back(msg); InputRequest req = makeChatRequest(history); - EmptyContentNormalizationProcessor processor; + EmptyContentArrayNormalizationProcessor processor; const auto status = processor.process(req); EXPECT_TRUE(status.ok()); @@ -66,12 +66,12 @@ TEST(EmptyContentNormalizationProcessorTest, NonEmptyArrayPreserved) { EXPECT_EQ(result[0]["content"].size(), 1u); } -TEST(EmptyContentNormalizationProcessorTest, StringContentPassedThrough) { +TEST(EmptyContentArrayNormalizationProcessorTest, StringContentPassedThrough) { ov::genai::ChatHistory history; history.push_back({{"role", "user"}, {"content", "Hello, world!"}}); InputRequest req = makeChatRequest(history); - EmptyContentNormalizationProcessor processor; + EmptyContentArrayNormalizationProcessor processor; const auto status = processor.process(req); EXPECT_TRUE(status.ok()); @@ -79,10 +79,10 @@ TEST(EmptyContentNormalizationProcessorTest, StringContentPassedThrough) { EXPECT_EQ(result[0]["content"].as_string().value_or(""), "Hello, world!"); } -TEST(EmptyContentNormalizationProcessorTest, RawPromptInputRejected) { +TEST(EmptyContentArrayNormalizationProcessorTest, RawPromptInputRejected) { InputRequest req; req.input = std::string("raw prompt"); - EmptyContentNormalizationProcessor processor; + EmptyContentArrayNormalizationProcessor processor; const auto status = processor.process(req); EXPECT_EQ(status.code(), absl::StatusCode::kInternal);