From 07bd921cfbf28ede38b7ba3555550a40a4b2a03c Mon Sep 17 00:00:00 2001 From: umerkhan Date: Tue, 8 Sep 2026 00:51:08 +0530 Subject: [PATCH] FIX: rejoin words with the configured word_split_separator WordLevelConverter.convert_async splits the prompt on word_split_separator but the default join_words always rejoined with a space, so a custom separator was silently replaced in the output. EmojiConverter and BinAsciiConverter both expose the parameter, so "alpha,beta" converted with separator "," came back space-delimited. Rejoin with the separator the words were split on. A None separator splits on arbitrary whitespace, which has no single representation to restore, so it keeps joining with a space. The default separator is " ", so output is unchanged for every caller that does not set a custom separator. Co-Authored-By: Claude Opus 5 (1M context) --- pyrit/converter/word_level_converter.py | 8 +++- .../converter/test_word_level_converter.py | 40 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/pyrit/converter/word_level_converter.py b/pyrit/converter/word_level_converter.py index 2c173216b0..8778ce7982 100644 --- a/pyrit/converter/word_level_converter.py +++ b/pyrit/converter/word_level_converter.py @@ -82,13 +82,19 @@ def join_words(self, words: list[str]) -> str: """ Provide a way for subclasses to override the default behavior of joining words. + Words are rejoined with the same separator they were split on, so a custom + ``word_split_separator`` survives the round trip. A ``None`` separator splits on + arbitrary whitespace, which has no single representation to restore, so those + words are joined with a space. + Args: words (list[str]): List of words to join. Returns: str: The joined string. """ - return " ".join(words) + separator = " " if self._word_split_separator is None else self._word_split_separator + return separator.join(words) async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ diff --git a/tests/unit/converter/test_word_level_converter.py b/tests/unit/converter/test_word_level_converter.py index 3b92854ed8..a7c59c97e9 100644 --- a/tests/unit/converter/test_word_level_converter.py +++ b/tests/unit/converter/test_word_level_converter.py @@ -30,6 +30,19 @@ async def convert_word_async(self, word: str) -> str: return word.upper() +class SeparatorWordLevelConverter(WordLevelConverter): + """Exposes word_split_separator, mirroring converters like EmojiConverter.""" + + def __init__(self, *, word_split_separator=" ", word_selection_strategy=None): + super().__init__( + word_selection_strategy=word_selection_strategy, + word_split_separator=word_split_separator, + ) + + async def convert_word_async(self, word: str) -> str: + return word.upper() + + class TestWordLevelConverter: async def test_convert_async_all_mode(self): converter = SimpleWordLevelConverter() @@ -120,3 +133,30 @@ async def test_default_is_all_words(self): assert isinstance(converter._word_selection_strategy, AllWordsSelectionStrategy) result = await converter.convert_async(prompt="test prompt") assert result.output_text == "TEST PROMPT" + + +class TestWordLevelConverterSeparator: + @pytest.mark.parametrize("separator", [",", "|", "-", "::"]) + async def test_custom_separator_is_preserved(self, separator): + converter = SeparatorWordLevelConverter(word_split_separator=separator) + prompt = separator.join(["alpha", "beta", "gamma"]) + result = await converter.convert_async(prompt=prompt) + assert result.output_text == separator.join(["ALPHA", "BETA", "GAMMA"]) + + async def test_default_space_separator_unchanged(self): + converter = SeparatorWordLevelConverter() + result = await converter.convert_async(prompt="alpha beta gamma") + assert result.output_text == "ALPHA BETA GAMMA" + + async def test_none_separator_joins_with_space(self): + converter = SeparatorWordLevelConverter(word_split_separator=None) + result = await converter.convert_async(prompt="alpha\tbeta\ngamma") + assert result.output_text == "ALPHA BETA GAMMA" + + async def test_custom_separator_preserved_with_partial_selection(self): + converter = SeparatorWordLevelConverter( + word_split_separator=",", + word_selection_strategy=WordIndexSelectionStrategy(indices=[0, 2]), + ) + result = await converter.convert_async(prompt="alpha,beta,gamma") + assert result.output_text == "ALPHA,beta,GAMMA"