Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions cpp/src/gandiva/precompiled/string_ops.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3099,6 +3099,19 @@ static char mappings[] = {'0', '1', '2', '3', '0', '1', '2', '0', '0',
// 4. If the string have too few letters in the word that you can't assign three
// numbers, append with zeros until there are three numbers. If you have four or more
// numbers, retain only the first three.
//
// The mappings table below is defined only for the 26 ASCII letters, so the
// classification and case-folding used to index it must stay ASCII-only. The
// locale-sensitive isalpha/toupper let bytes 0x80-0xFF count as letters under a
// non-C locale, and toupper on such a byte yields an index past mappings.
static FORCE_INLINE bool is_ascii_letter(char c) {
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
}

static FORCE_INLINE char ascii_toupper(char c) {
return (c >= 'a' && c <= 'z') ? static_cast<char>(c - ('a' - 'A')) : c;
}

FORCE_INLINE
const char* soundex_utf8(gdv_int64 context, const char* in, gdv_int32 in_len,
bool in_validity, bool* out_valid, int32_t* out_len) {
Expand All @@ -3125,9 +3138,9 @@ const char* soundex_utf8(gdv_int64 context, const char* in, gdv_int32 in_len,

int start_idx = 0;
for (int i = 0; i < in_len; ++i) {
if (isalpha(static_cast<unsigned char>(in[i])) > 0) {
if (is_ascii_letter(in[i])) {
// Retain the first letter
ret[0] = toupper(static_cast<unsigned char>(in[i]));
ret[0] = ascii_toupper(in[i]);
start_idx = i + 1;
break;
}
Expand All @@ -3143,8 +3156,8 @@ const char* soundex_utf8(gdv_int64 context, const char* in, gdv_int32 in_len,
soundex[0] = '\0';
// Replace consonants with digits and special letters with 0
for (int i = start_idx; i < in_len; i++) {
if (isalpha(static_cast<unsigned char>(in[i])) > 0) {
c = toupper(static_cast<unsigned char>(in[i])) - 65;
if (is_ascii_letter(in[i])) {
c = ascii_toupper(in[i]) - 'A';
if (mappings[c] != soundex[si - 1]) {
soundex[si] = mappings[c];
si++;
Expand Down
48 changes: 48 additions & 0 deletions cpp/src/gandiva/precompiled/string_ops_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,12 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>

#include <cctype>
#include <clocale>
#include <cstring>
#include <limits>
#include <memory>
#include <string>

#include "gandiva/execution_context.h"
#include "gandiva/precompiled/types.h"
Expand Down Expand Up @@ -3046,6 +3049,51 @@ TEST(TestStringOps, TestSoundex) {
EXPECT_EQ(validity, true);
}

TEST(TestStringOps, TestSoundexNonAsciiNoOverread) {
gandiva::ExecutionContext ctx;
auto ctx_ptr = reinterpret_cast<int64_t>(&ctx);
int32_t out_len = 0;
bool validity = false;
const char* out;

// ASCII input is unaffected by the fix.
out = soundex_utf8(ctx_ptr, "Robert", 6, true, &validity, &out_len);
EXPECT_EQ(std::string(out, out_len), "R163");

// The mappings table has 26 entries (A-Z). Under a non-C locale isalpha
// accepts bytes 0x80-0xFF and toupper leaves them past 'Z', which the old
// code used to index the table out of bounds. Find such a byte if the
// platform provides a locale for it, then confirm it is now treated as a
// separator instead of over-reading.
const char* candidates[] = {"en_US.UTF-8", "en_US.ISO8859-1", "C.UTF-8",
"de_DE.ISO8859-1", ""};
std::string saved = setlocale(LC_CTYPE, nullptr);
int trigger = -1;
for (const char* loc : candidates) {
if (setlocale(LC_CTYPE, loc) == nullptr) {
continue;
}
for (int b = 0x80; b < 0x100; ++b) {
if (isalpha(b) && (toupper(b) - 'A' < 0 || toupper(b) - 'A' > 25)) {
trigger = b;
break;
}
}
if (trigger >= 0) {
break;
}
}

if (trigger >= 0) {
char buf[] = {'R', 'o', 'b', static_cast<char>(trigger), 'e', 'r', 't'};
out = soundex_utf8(ctx_ptr, buf, static_cast<int32_t>(sizeof(buf)), true, &validity,
&out_len);
EXPECT_EQ(std::string(out, out_len), "R163");
EXPECT_EQ(validity, true);
}
setlocale(LC_CTYPE, saved.c_str());
}

TEST(TestStringOps, TestInstr) {
std::string s1 = "hello world!";
auto s1_len = static_cast<int32_t>(s1.size());
Expand Down
Loading