diff --git a/src/llm/BUILD b/src/llm/BUILD index 0ac3de44b8..d6b91852b1 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -155,12 +155,14 @@ ovms_cc_library( "io_processing/input_processors/chat_template_processor.hpp", "io_processing/input_processors/chat_template_adapter.hpp", "io_processing/chat_template/caps.hpp", + "io_processing/input_processors/empty_content_array_normalization_processor.hpp", "io_processing/input_processors/raw_prompt_extractor.hpp", "io_processing/input_processors/text_content_normalization_processor.hpp", "io_processing/input_processors/tokenization_processor.hpp"], srcs = ["io_processing/input_processors/image_decoding_processor.cpp", "io_processing/input_processors/chat_template_processor.cpp", "io_processing/input_processors/chat_template_adapter.cpp", + "io_processing/input_processors/empty_content_array_normalization_processor.cpp", "io_processing/input_processors/text_content_normalization_processor.cpp", "io_processing/input_processors/tokenization_processor.cpp"], deps = [ diff --git a/src/llm/apis/openai_completions.cpp b/src/llm/apis/openai_completions.cpp index 2ac285e61e..d526a85358 100644 --- a/src/llm/apis/openai_completions.cpp +++ b/src/llm/apis/openai_completions.cpp @@ -195,10 +195,9 @@ absl::Status OpenAIChatCompletionsHandler::parseMessages(std::optionalvalue.IsArray()) { - // Validate the content array and check whether it contains images. - if (member->value.GetArray().Size() == 0) { - return absl::InvalidArgumentError("Invalid message structure - content array is empty"); - } + // Empty content arrays are accepted and preserved as-is. The + // EmptyContentArrayNormalizationProcessor converts them to null before + // downstream processing. for (const auto& v : member->value.GetArray()) { if (!v.IsObject()) { return absl::InvalidArgumentError("Invalid message structure - content array should contain objects"); diff --git a/src/llm/io_processing/input_processor.cpp b/src/llm/io_processing/input_processor.cpp index 97cfd26d35..504fdb51b0 100644 --- a/src/llm/io_processing/input_processor.cpp +++ b/src/llm/io_processing/input_processor.cpp @@ -23,6 +23,7 @@ #include "../../config.hpp" #include "../../logging.hpp" #include "input_processors/chat_template_processor.hpp" +#include "input_processors/empty_content_array_normalization_processor.hpp" #include "input_processors/image_decoding_processor.hpp" #include "input_processors/chat_template_adapter.hpp" #include "input_processors/raw_prompt_extractor.hpp" @@ -37,22 +38,25 @@ InputProcessor::InputProcessor(InputProcessorContext& context, // Chat template already adds special tokens; completions path needs them added by the tokenizer. const bool addSpecialTokens = !isChatPath; - if (context.config.isVLM && isChatPath) { - const auto& settings = Config::instance().getServerSettings(); - processors.emplace_back(std::make_unique( - settings.allowedLocalMediaPath, - settings.allowedMediaDomains)); - } + if (isChatPath) { + // Normalize empty content arrays to null before any content-aware processor runs. + processors.emplace_back(std::make_unique()); - if (!context.config.isVLM && isChatPath) { + // Flatten text-only content arrays for both LM and VLM. Arrays that contain + // images (or other modalities) are left untouched for ImageDecodingProcessor. processors.emplace_back(std::make_unique()); - } - if (isChatPath && context.chatTemplateCaps.needsWorkarounds()) { - processors.emplace_back(std::make_unique(context.chatTemplateCaps)); - } + if (context.config.isVLM) { + const auto& settings = Config::instance().getServerSettings(); + processors.emplace_back(std::make_unique( + settings.allowedLocalMediaPath, + settings.allowedMediaDomains)); + } + + if (context.chatTemplateCaps.needsWorkarounds()) { + processors.emplace_back(std::make_unique(context.chatTemplateCaps)); + } - if (isChatPath) { #if (PYTHON_DISABLE == 0) // Select the path at construction time. If !useMinja but templateProcessor is null // (shouldn't happen on a properly initialized servable), fall back to the native path. diff --git a/src/llm/io_processing/input_processors/empty_content_array_normalization_processor.cpp b/src/llm/io_processing/input_processors/empty_content_array_normalization_processor.cpp new file mode 100644 index 0000000000..bdbfbad3ea --- /dev/null +++ b/src/llm/io_processing/input_processors/empty_content_array_normalization_processor.cpp @@ -0,0 +1,38 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "empty_content_array_normalization_processor.hpp" + +#include + +namespace ovms { + +absl::Status EmptyContentArrayNormalizationProcessor::process(InputRequest& req) { + if (!std::holds_alternative(req.input)) { + return absl::Status(absl::StatusCode::kInternal, + "EmptyContentArrayNormalizationProcessor received input that is not a ChatHistory"); + } + ov::genai::ChatHistory& chatHistory = std::get(req.input); + for (size_t i = 0; i < chatHistory.size(); i++) { + const auto content = chatHistory[i]["content"]; + if (content.is_array() && content.size() == 0) { + chatHistory[i]["content"] = ov::genai::JsonContainer(nullptr); + } + } + return absl::OkStatus(); +} + +} // namespace ovms diff --git a/src/llm/io_processing/input_processors/empty_content_array_normalization_processor.hpp b/src/llm/io_processing/input_processors/empty_content_array_normalization_processor.hpp new file mode 100644 index 0000000000..69c0cbc82c --- /dev/null +++ b/src/llm/io_processing/input_processors/empty_content_array_normalization_processor.hpp @@ -0,0 +1,31 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include "../base_input_processor.hpp" + +namespace ovms { + +// Replaces empty content arrays ("content": []) in ChatHistory messages with null. +// Runs for all chat paths (LM and VLM) and must execute before ImageDecodingProcessor +// and TextContentNormalizationProcessor so downstream processors and chat templates +// see a null content instead of an empty array. +class EmptyContentArrayNormalizationProcessor : public BaseInputProcessor { +public: + absl::Status process(InputRequest& req) override; +}; + +} // namespace ovms diff --git a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp index 2818b0b8ba..2180d5faae 100644 --- a/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp +++ b/src/llm/io_processing/input_processors/text_content_normalization_processor.cpp @@ -32,17 +32,26 @@ absl::Status TextContentNormalizationProcessor::process(InputRequest& req) { if (!content.is_array()) { continue; } + // Only flatten arrays that contain exclusively text parts. Arrays with + // images (or other modalities) are left untouched for ImageDecodingProcessor. + // Single pass: build the combined string while scanning, and bail out on the + // first non-text part without touching the message. std::string combined; + bool allText = true; for (size_t j = 0; j < content.size(); j++) { const auto part = content[j]; if (part["type"].as_string().value_or("") != "text") { - continue; + allText = false; + break; } if (!combined.empty()) { combined += "\n"; } combined += part["text"].as_string().value_or(""); } + if (!allText) { + continue; + } chatHistory[i]["content"] = combined; } return absl::OkStatus(); diff --git a/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp b/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp index 0899a13660..7280214920 100644 --- a/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp +++ b/src/llm/io_processing/input_processors/text_content_normalization_processor.hpp @@ -19,9 +19,10 @@ namespace ovms { -// Normalizes text-only content arrays in ChatHistory messages to plain strings. -// Parts are joined with "\n" for backward compatibility with LM chat templates. -// Active when: !config.isVLM && input is ChatHistory variant. +// Flattens text-only content arrays in ChatHistory messages to plain strings. +// Parts are joined with "\n" for backward compatibility with chat templates. +// Runs for both LM and VLM chat paths: arrays that contain images (or other +// non-text modalities) are left untouched for ImageDecodingProcessor. // Must run before ChatTemplateProcessor. class TextContentNormalizationProcessor : public BaseInputProcessor { public: diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index a600872f62..f8be827c4e 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -2325,7 +2325,7 @@ TEST_F(HttpOpenAIHandlerParsingTest, ParsingMessagesWithInvalidContentTypeFails) EXPECT_EQ(apiHandler->parseMessages(), absl::InvalidArgumentError("Unsupported content type")); } -TEST_F(HttpOpenAIHandlerParsingTest, ParsingMessagesEmptyContentArrayFails) { +TEST_F(HttpOpenAIHandlerParsingTest, ParsingMessagesEmptyContentArrayPreservesArray) { std::string json = R"({ "model": "llama", "messages": [ @@ -2338,7 +2338,14 @@ TEST_F(HttpOpenAIHandlerParsingTest, ParsingMessagesEmptyContentArrayFails) { doc.Parse(json.c_str()); ASSERT_FALSE(doc.HasParseError()); std::shared_ptr apiHandler = std::make_shared(doc, ovms::Endpoint::CHAT_COMPLETIONS, std::chrono::system_clock::now(), *tokenizer); - EXPECT_EQ(apiHandler->parseMessages(), absl::InvalidArgumentError("Invalid message structure - content array is empty")); + // Empty content arrays are accepted and preserved as-is. The + // EmptyContentArrayNormalizationProcessor converts them to null downstream. + ASSERT_EQ(apiHandler->parseMessages(), absl::OkStatus()); + auto& chatHistory = apiHandler->getChatHistory(); + ASSERT_EQ(chatHistory.size(), 1u); + auto content = chatHistory[0]["content"]; + EXPECT_TRUE(content.is_array()); + EXPECT_EQ(content.size(), 0u); } TEST_F(HttpOpenAIHandlerParsingTest, ParsingMessagesWithNoContentFieldAddsEmptyStringToChatHistory) { diff --git a/src/test/llm/input_processing/empty_content_array_normalization_processor_test.cpp b/src/test/llm/input_processing/empty_content_array_normalization_processor_test.cpp new file mode 100644 index 0000000000..4badd84b1d --- /dev/null +++ b/src/test/llm/input_processing/empty_content_array_normalization_processor_test.cpp @@ -0,0 +1,89 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include + +#include +#include + +#include "../../../llm/io_processing/input_processors/empty_content_array_normalization_processor.hpp" +#include "../../../llm/io_processing/input_request.hpp" + +using namespace ovms; + +// Helpers ---------------------------------------------------------------- + +static InputRequest makeChatRequest(ov::genai::ChatHistory chatHistory) { + InputRequest req; + req.input = std::move(chatHistory); + return req; +} + +// Tests ------------------------------------------------------------------ + +TEST(EmptyContentArrayNormalizationProcessorTest, EmptyArrayConvertedToNull) { + ov::genai::ChatHistory history; + ov::AnyMap msg = {{"role", std::string("user")}}; + msg["content"] = ov::genai::JsonContainer::from_json_string("[]"); + history.push_back(msg); + + InputRequest req = makeChatRequest(history); + EmptyContentArrayNormalizationProcessor processor; + const auto status = processor.process(req); + + EXPECT_TRUE(status.ok()); + const auto& result = std::get(req.input); + EXPECT_FALSE(result[0]["content"].is_array()); + EXPECT_TRUE(result[0]["content"].is_null()); +} + +TEST(EmptyContentArrayNormalizationProcessorTest, NonEmptyArrayPreserved) { + ov::genai::ChatHistory history; + ov::AnyMap msg = {{"role", std::string("user")}}; + msg["content"] = ov::genai::JsonContainer::from_json_string( + R"([{"type":"text","text":"hello"}])"); + history.push_back(msg); + + InputRequest req = makeChatRequest(history); + EmptyContentArrayNormalizationProcessor processor; + const auto status = processor.process(req); + + EXPECT_TRUE(status.ok()); + const auto& result = std::get(req.input); + ASSERT_TRUE(result[0]["content"].is_array()); + EXPECT_EQ(result[0]["content"].size(), 1u); +} + +TEST(EmptyContentArrayNormalizationProcessorTest, StringContentPassedThrough) { + ov::genai::ChatHistory history; + history.push_back({{"role", "user"}, {"content", "Hello, world!"}}); + + InputRequest req = makeChatRequest(history); + EmptyContentArrayNormalizationProcessor processor; + const auto status = processor.process(req); + + EXPECT_TRUE(status.ok()); + const auto& result = std::get(req.input); + EXPECT_EQ(result[0]["content"].as_string().value_or(""), "Hello, world!"); +} + +TEST(EmptyContentArrayNormalizationProcessorTest, RawPromptInputRejected) { + InputRequest req; + req.input = std::string("raw prompt"); + EmptyContentArrayNormalizationProcessor processor; + const auto status = processor.process(req); + + EXPECT_EQ(status.code(), absl::StatusCode::kInternal); +} diff --git a/src/test/llm/input_processing/input_processing_integration_test.cpp b/src/test/llm/input_processing/input_processing_integration_test.cpp index c2f78b79cb..609eca6297 100644 --- a/src/test/llm/input_processing/input_processing_integration_test.cpp +++ b/src/test/llm/input_processing/input_processing_integration_test.cpp @@ -342,20 +342,23 @@ TEST_P(InputProcessingIntegrationTest, GenerationConfigFields_SurviveFullPipelin } TEST_P(InputProcessingIntegrationTest, TextArrayContent_FlattenedByNormalizationProcessor) { - // Non-VLM path: content array with two text parts must be flattened by - // TextContentNormalizationProcessor so both strings appear in the final prompt. - auto result = runPipeline(textArrayJson(), /*isVLM=*/false); - - ASSERT_TRUE(result.parseStatus.ok()) << result.parseStatus.message(); - ASSERT_TRUE(result.processStatus.ok()) << result.processStatus.message(); - + // A text-only content array must be flattened by TextContentNormalizationProcessor + // so both strings appear in the final prompt. This holds on both the LM path and the + // VLM path used without images (VLM is a superset of LM). // TextContentNormalizationProcessor joins the two parts with \n before the template runs. const std::string expected = std::string(SMOL_DEFAULT_SYSTEM) + "<|im_start|>user\nFirst part.\nSecond part.<|im_end|>\n" "<|im_start|>assistant\n"; - EXPECT_EQ(result.req.promptText, expected); - EXPECT_TRUE(result.req.inputImages.empty()); + + for (bool isVLM : {false, true}) { + auto result = runPipeline(textArrayJson(), isVLM); + + ASSERT_TRUE(result.parseStatus.ok()) << "isVLM=" << isVLM << ": " << result.parseStatus.message(); + ASSERT_TRUE(result.processStatus.ok()) << "isVLM=" << isVLM << ": " << result.processStatus.message(); + EXPECT_EQ(result.req.promptText, expected) << "isVLM=" << isVLM; + EXPECT_TRUE(result.req.inputImages.empty()) << "isVLM=" << isVLM; + } } // --------------------------------------------------------------------------- diff --git a/src/test/llm/input_processing/text_content_normalization_processor_test.cpp b/src/test/llm/input_processing/text_content_normalization_processor_test.cpp index a1b814ab7a..2b1840f0de 100644 --- a/src/test/llm/input_processing/text_content_normalization_processor_test.cpp +++ b/src/test/llm/input_processing/text_content_normalization_processor_test.cpp @@ -80,8 +80,9 @@ TEST(TextContentNormalizationProcessorTest, MultipleTextPartsJoinedWithNewline) EXPECT_EQ(result[0]["content"].as_string().value_or(""), "first\nsecond"); } -TEST(TextContentNormalizationProcessorTest, NonTextPartsIgnored) { - // image_url entries alongside text: only text parts should contribute to combined string. +TEST(TextContentNormalizationProcessorTest, MixedContentArrayLeftUntouched) { + // image_url entries alongside text: the array is NOT text-only, so it must be + // left untouched for ImageDecodingProcessor to handle downstream. ov::genai::ChatHistory history; ov::AnyMap msg = {{"role", std::string("user")}}; ov::genai::JsonContainer parts = ov::genai::JsonContainer::from_json_string( @@ -95,5 +96,6 @@ TEST(TextContentNormalizationProcessorTest, NonTextPartsIgnored) { EXPECT_TRUE(status.ok()); const auto& result = std::get(req.input); - EXPECT_EQ(result[0]["content"].as_string().value_or(""), "describe this"); + ASSERT_TRUE(result[0]["content"].is_array()); + EXPECT_EQ(result[0]["content"].size(), 2u); }