Skip to content

Adding support for endianness in TypedBuffer codecs #233

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
avalerio-tkd merged 4 commits into from
Mar 19, 2026
Merged

Adding support for endianness in TypedBuffer codecs #233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e0c093b
- Adding endianness protection to the codecs.
avalerio-tkd Mar 19, 2026
c1f393f
- Reimplemented BytesToString more efficiently.
avalerio-tkd Mar 19, 2026
2ed0c92
- Added unitests for write_le and read_le functions with expected buf…
avalerio-tkd Mar 19, 2026
be0c4a5
- Updated comments on code duplication.
avalerio-tkd Mar 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 114 additions & 12 deletions src/common/bytes_utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@

inline constexpr size_t kSizePrefixBytes = sizeof(uint32_t);

// Utility functions for little-endian number reading and writing.
// Utility functions for little-endian number reading and writing (vectors and spans)

inline void append_u32_le(std::vector<uint8_t>& out, uint32_t v) {
const size_t offset = out.size();
Expand Down Expand Up @@ -81,13 +81,6 @@ inline void write_u32_le_at(std::vector<uint8_t>& buf, size_t offset, uint32_t v
buf[offset + 3] = static_cast<uint8_t>((v >> 24) & 0xFF);
}

inline void write_u32_le(uint8_t* p, uint32_t v) {
p[0] = static_cast<uint8_t>(v);
p[1] = static_cast<uint8_t>(v >> 8);
p[2] = static_cast<uint8_t>(v >> 16);
p[3] = static_cast<uint8_t>(v >> 24);
}

inline uint32_t read_u32_le(const std::vector<uint8_t>& in, size_t offset) {
return static_cast<uint32_t>(in[offset]) |
(static_cast<uint32_t>(in[offset + 1]) << 8) |
Expand All @@ -102,10 +95,114 @@ inline uint32_t read_u32_le(tcb::span<const uint8_t> in, size_t offset) {
(static_cast<uint32_t>(in[offset + 3]) << 24);
}

// Utility functions for reading and writing with templated types.

template <class T>
inline T read_le(const uint8_t* p) {
Comment thread
argmarco-tkd marked this conversation as resolved.
if constexpr (std::is_same_v<T, int32_t>) {
const uint32_t v =
(static_cast<uint32_t>(p[0]) ) |
(static_cast<uint32_t>(p[1]) << 8) |
(static_cast<uint32_t>(p[2]) << 16) |
(static_cast<uint32_t>(p[3]) << 24);
return static_cast<int32_t>(v);
} else if constexpr (std::is_same_v<T, int64_t>) {
const uint64_t v =
(static_cast<uint64_t>(p[0]) ) |
(static_cast<uint64_t>(p[1]) << 8) |
(static_cast<uint64_t>(p[2]) << 16) |
(static_cast<uint64_t>(p[3]) << 24) |
(static_cast<uint64_t>(p[4]) << 32) |
(static_cast<uint64_t>(p[5]) << 40) |
(static_cast<uint64_t>(p[6]) << 48) |
(static_cast<uint64_t>(p[7]) << 56);
return static_cast<int64_t>(v);
} else if constexpr (std::is_same_v<T, float>) {
const uint32_t bits =
(static_cast<uint32_t>(p[0]) ) |
(static_cast<uint32_t>(p[1]) << 8) |
(static_cast<uint32_t>(p[2]) << 16) |
(static_cast<uint32_t>(p[3]) << 24);
float value;
std::memcpy(&value, &bits, sizeof(value));
return value;
} else if constexpr (std::is_same_v<T, double>) {
const uint64_t bits =
(static_cast<uint64_t>(p[0]) ) |
(static_cast<uint64_t>(p[1]) << 8) |
(static_cast<uint64_t>(p[2]) << 16) |
(static_cast<uint64_t>(p[3]) << 24) |
(static_cast<uint64_t>(p[4]) << 32) |
(static_cast<uint64_t>(p[5]) << 40) |
(static_cast<uint64_t>(p[6]) << 48) |
(static_cast<uint64_t>(p[7]) << 56);
double value;
std::memcpy(&value, &bits, sizeof(value));
return value;
} else {
throw InvalidInputException("read_le<T>: unsupported type");
}
}

template <class T>
inline void write_le(const T& value, uint8_t* p) {
if constexpr (std::is_same_v<T, int32_t>) {
const uint32_t v = static_cast<uint32_t>(value);
p[0] = static_cast<uint8_t>( v & 0xFF);
p[1] = static_cast<uint8_t>((v >> 8) & 0xFF);
p[2] = static_cast<uint8_t>((v >> 16) & 0xFF);
p[3] = static_cast<uint8_t>((v >> 24) & 0xFF);
} else if constexpr (std::is_same_v<T, int64_t>) {
const uint64_t v = static_cast<uint64_t>(value);
p[0] = static_cast<uint8_t>( v & 0xFF);
p[1] = static_cast<uint8_t>((v >> 8) & 0xFF);
p[2] = static_cast<uint8_t>((v >> 16) & 0xFF);
p[3] = static_cast<uint8_t>((v >> 24) & 0xFF);
p[4] = static_cast<uint8_t>((v >> 32) & 0xFF);
p[5] = static_cast<uint8_t>((v >> 40) & 0xFF);
p[6] = static_cast<uint8_t>((v >> 48) & 0xFF);
p[7] = static_cast<uint8_t>((v >> 56) & 0xFF);
} else if constexpr (std::is_same_v<T, float>) {
uint32_t bits;
std::memcpy(&bits, &value, sizeof(bits));
p[0] = static_cast<uint8_t>( bits & 0xFF);
p[1] = static_cast<uint8_t>((bits >> 8) & 0xFF);
p[2] = static_cast<uint8_t>((bits >> 16) & 0xFF);
p[3] = static_cast<uint8_t>((bits >> 24) & 0xFF);
} else if constexpr (std::is_same_v<T, double>) {
uint64_t bits;
std::memcpy(&bits, &value, sizeof(bits));
p[0] = static_cast<uint8_t>( bits & 0xFF);
p[1] = static_cast<uint8_t>((bits >> 8) & 0xFF);
p[2] = static_cast<uint8_t>((bits >> 16) & 0xFF);
p[3] = static_cast<uint8_t>((bits >> 24) & 0xFF);
p[4] = static_cast<uint8_t>((bits >> 32) & 0xFF);
p[5] = static_cast<uint8_t>((bits >> 40) & 0xFF);
p[6] = static_cast<uint8_t>((bits >> 48) & 0xFF);
p[7] = static_cast<uint8_t>((bits >> 56) & 0xFF);
} else {
throw InvalidInputException("write_le<T>: unsupported type");
}
}

// Utility functions for little-endian number reading and writing UINT32 values.
//
// Since UINT32 functions are called heavily in tight loops and hot execution paths to read/write sizes,
// we keep them separate from other functions so they easy to identify in the library code and can be
// optimatized separately. This results in some code duplication.

inline void write_u32_le(uint8_t* p, uint32_t v) {
p[0] = static_cast<uint8_t>(v);
p[1] = static_cast<uint8_t>(v >> 8);
p[2] = static_cast<uint8_t>(v >> 16);
p[3] = static_cast<uint8_t>(v >> 24);
}

inline uint32_t read_u32_le(const uint8_t* p) {
uint32_t v;
std::memcpy(&v, p, sizeof(v));
return v;
return static_cast<uint32_t>(p[0]) |
(static_cast<uint32_t>(p[1]) << 8) |
(static_cast<uint32_t>(p[2]) << 16) |
(static_cast<uint32_t>(p[3]) << 24);
Comment thread
argmarco-tkd marked this conversation as resolved.
}

// Utility functions for splitting and joining byte vectors.
Expand Down Expand Up @@ -303,7 +400,12 @@ inline std::string AddStringAttribute(
return value;
}

// Helper function to convert string to binary data
// Helper function to convert string to binary data and vice versa

inline std::vector<uint8_t> StringToBytes(const std::string& str) {
return std::vector<uint8_t>(str.begin(), str.end());
}

inline std::string BytesToString(tcb::span<const uint8_t> span) {
return std::string(reinterpret_cast<const char*>(span.data()), span.size());
}
Comment thread
argmarco-tkd marked this conversation as resolved.
202 changes: 201 additions & 1 deletion src/common/bytes_utils_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
#include "bytes_utils.h"
#include "exceptions.h"

#include <array>
#include <cmath>
#include <vector>
#include <variant>
#include <gtest/gtest.h>
Expand Down Expand Up @@ -284,4 +286,202 @@ TEST(BytesUtils, StringToBytes_PreservesRawBytesAndNulls) {
static_cast<uint8_t>('Z')};

EXPECT_EQ(expected, result);
}
}

TEST(BytesUtils, BytesToString_ConvertsAsciiEmptyAndRawBytes) {
{
const std::vector<uint8_t> bytes = {'d', 'b', 'p', 's'};
const std::string result = BytesToString(tcb::span<const uint8_t>(bytes));
EXPECT_EQ(result, "dbps");
}

{
const std::vector<uint8_t> bytes;
const std::string result = BytesToString(tcb::span<const uint8_t>(bytes));
EXPECT_TRUE(result.empty());
}

{
const std::vector<uint8_t> bytes = {
static_cast<uint8_t>('D'),
static_cast<uint8_t>('B'),
static_cast<uint8_t>('P'),
static_cast<uint8_t>('S'),
static_cast<uint8_t>(0x00),
static_cast<uint8_t>('X'),
static_cast<uint8_t>('Y'),
static_cast<uint8_t>(0xFF),
static_cast<uint8_t>(0x80),
static_cast<uint8_t>(0x00),
static_cast<uint8_t>('Z')};
const std::string result = BytesToString(tcb::span<const uint8_t>(bytes));
const std::string expected = std::string{
'D', 'B', 'P', 'S', '\0', 'X', 'Y',
static_cast<char>(0xFF), static_cast<char>(0x80), '\0', 'Z'};
EXPECT_EQ(result.size(), expected.size());
EXPECT_EQ(result, expected);
}
}

TEST(BytesUtils, ReadU32Le_FromPointer_DecodesLittleEndianBytes) {
const std::array<uint8_t, 4> bytes = {0x78, 0x56, 0x34, 0x12};
const uint32_t value = read_u32_le(bytes.data());

EXPECT_EQ(value, 0x12345678u);
}

TEST(BytesUtils, ReadLeWriteLe_Int32_RoundTrip) {
const int32_t original = -2147483000;
std::array<uint8_t, sizeof(int32_t)> bytes{};
Comment thread
argmarco-tkd marked this conversation as resolved.

write_le<int32_t>(original, bytes.data());
const int32_t decoded = read_le<int32_t>(bytes.data());

EXPECT_EQ(decoded, original);
}

TEST(BytesUtils, ReadLeWriteLe_Int64_RoundTrip) {
const int64_t original = -9223372036854000000LL;
std::array<uint8_t, sizeof(int64_t)> bytes{};

write_le<int64_t>(original, bytes.data());
const int64_t decoded = read_le<int64_t>(bytes.data());

EXPECT_EQ(decoded, original);
}

TEST(BytesUtils, ReadLeWriteLe_Float_RoundTrip) {
const float original = -12345.625f;
std::array<uint8_t, sizeof(float)> bytes{};

write_le<float>(original, bytes.data());
const float decoded = read_le<float>(bytes.data());

EXPECT_FLOAT_EQ(decoded, original);
}

TEST(BytesUtils, ReadLeWriteLe_Double_RoundTrip) {
const double original = 9876543210.125;
std::array<uint8_t, sizeof(double)> bytes{};

write_le<double>(original, bytes.data());
const double decoded = read_le<double>(bytes.data());

EXPECT_DOUBLE_EQ(decoded, original);
}

TEST(BytesUtils, WriteReadU32Le_OffsetRoundTrip_VerifiesBytesAndGuards) {
constexpr size_t kPrefix = 3u;
constexpr size_t kValueSize = sizeof(uint32_t);
constexpr size_t kSuffix = 5u;
std::array<uint8_t, kPrefix + kValueSize + kSuffix> bytes;
bytes.fill(0xDD);

const uint32_t original = 0xD3A5C79Eu;
write_u32_le(bytes.data() + kPrefix, original);
const uint32_t decoded = read_u32_le(bytes.data() + kPrefix);

for (size_t i = 0; i < kPrefix; ++i) {
EXPECT_EQ(bytes[i], 0xDD);
}
for (size_t i = 0; i < kSuffix; ++i) {
EXPECT_EQ(bytes[kPrefix + kValueSize + i], 0xDD);
}
EXPECT_EQ(bytes[kPrefix + 0], 0x9E);
EXPECT_EQ(bytes[kPrefix + 1], 0xC7);
EXPECT_EQ(bytes[kPrefix + 2], 0xA5);
EXPECT_EQ(bytes[kPrefix + 3], 0xD3);
EXPECT_EQ(decoded, original);
}

TEST(BytesUtils, WriteReadLeInt32_OffsetRoundTrip_VerifiesBytesAndGuards) {
constexpr size_t kPrefix = 3u;
constexpr size_t kValueSize = sizeof(int32_t);
constexpr size_t kSuffix = 5u;
std::array<uint8_t, kPrefix + kValueSize + kSuffix> bytes;
bytes.fill(0xDD);

const int32_t original = 0x6E91A2F3;
write_le<int32_t>(original, bytes.data() + kPrefix);
const int32_t decoded = read_le<int32_t>(bytes.data() + kPrefix);

for (size_t i = 0; i < kPrefix; ++i) {
EXPECT_EQ(bytes[i], 0xDD);
}
for (size_t i = 0; i < kSuffix; ++i) {
EXPECT_EQ(bytes[kPrefix + kValueSize + i], 0xDD);
}
EXPECT_EQ(bytes[kPrefix + 0], 0xF3);
EXPECT_EQ(bytes[kPrefix + 1], 0xA2);
EXPECT_EQ(bytes[kPrefix + 2], 0x91);
EXPECT_EQ(bytes[kPrefix + 3], 0x6E);
EXPECT_EQ(decoded, original);
}

TEST(BytesUtils, WriteReadLeInt64_OffsetRoundTrip_VerifiesBytesAndGuards) {
constexpr size_t kPrefix = 3u;
constexpr size_t kValueSize = sizeof(int64_t);
constexpr size_t kSuffix = 5u;
std::array<uint8_t, kPrefix + kValueSize + kSuffix> bytes;
bytes.fill(0xDD);

const int64_t original = 0x0102030405060708LL;
write_le<int64_t>(original, bytes.data() + kPrefix);
const int64_t decoded = read_le<int64_t>(bytes.data() + kPrefix);

for (size_t i = 0; i < kPrefix; ++i) {
EXPECT_EQ(bytes[i], 0xDD);
}
for (size_t i = 0; i < kSuffix; ++i) {
EXPECT_EQ(bytes[kPrefix + kValueSize + i], 0xDD);
}
EXPECT_EQ(bytes[kPrefix + 0], 0x08);
EXPECT_EQ(bytes[kPrefix + 1], 0x07);
EXPECT_EQ(bytes[kPrefix + 2], 0x06);
EXPECT_EQ(bytes[kPrefix + 3], 0x05);
EXPECT_EQ(bytes[kPrefix + 4], 0x04);
EXPECT_EQ(bytes[kPrefix + 5], 0x03);
EXPECT_EQ(bytes[kPrefix + 6], 0x02);
EXPECT_EQ(bytes[kPrefix + 7], 0x01);
EXPECT_EQ(decoded, original);
}

TEST(BytesUtils, WriteReadLeFloat_OffsetRoundTrip_VerifiesBytesAndGuards) {
constexpr size_t kPrefix = 3u;
constexpr size_t kValueSize = sizeof(float);
constexpr size_t kSuffix = 5u;
std::array<uint8_t, kPrefix + kValueSize + kSuffix> bytes;
bytes.fill(0xDD);

constexpr float kOriginal = -3.1415927f;
write_le<float>(kOriginal, bytes.data() + kPrefix);
const float decoded = read_le<float>(bytes.data() + kPrefix);

for (size_t i = 0; i < kPrefix; ++i) {
EXPECT_EQ(bytes[i], 0xDD);
}
for (size_t i = 0; i < kSuffix; ++i) {
EXPECT_EQ(bytes[kPrefix + kValueSize + i], 0xDD);
}
EXPECT_FLOAT_EQ(decoded, kOriginal);
}

TEST(BytesUtils, WriteReadLeDouble_OffsetRoundTrip_VerifiesBytesAndGuards) {
constexpr size_t kPrefix = 3u;
constexpr size_t kValueSize = sizeof(double);
constexpr size_t kSuffix = 5u;
std::array<uint8_t, kPrefix + kValueSize + kSuffix> bytes;
bytes.fill(0xDD);

constexpr double kOriginal = -3.141592653589793;
write_le<double>(kOriginal, bytes.data() + kPrefix);
const double decoded = read_le<double>(bytes.data() + kPrefix);

for (size_t i = 0; i < kPrefix; ++i) {
EXPECT_EQ(bytes[i], 0xDD);
}
for (size_t i = 0; i < kSuffix; ++i) {
EXPECT_EQ(bytes[kPrefix + kValueSize + i], 0xDD);
}
EXPECT_DOUBLE_EQ(decoded, kOriginal);
}
Loading
Loading