From f6b73c6cb54a204067b15ffe0595b78c6e74283e Mon Sep 17 00:00:00 2001 From: KredeGC Date: Sat, 19 Jul 2025 19:00:31 +0200 Subject: [PATCH 1/2] Move checksum logic into trait Make endian swap constexpr --- README.md | 22 +++++++ generate.bat | 2 +- include/bitstream/bitstream.h | 1 + include/bitstream/stream/bit_reader.h | 29 --------- include/bitstream/stream/bit_writer.h | 43 +------------ include/bitstream/traits/checksum_trait.h | 78 +++++++++++++++++++++++ include/bitstream/utility/crc.h | 15 ++--- include/bitstream/utility/endian.h | 36 +++++++++-- include/bitstream/utility/parameter.h | 7 +- include/bitstream/utility/platform.h | 11 ++++ src/test/serialize_checksum_test.cpp | 61 ++++++++++++++++++ src/test/serialize_string_test.cpp | 8 +-- src/test/stream_test.cpp | 24 ------- 13 files changed, 214 insertions(+), 123 deletions(-) create mode 100644 include/bitstream/traits/checksum_trait.h create mode 100644 include/bitstream/utility/platform.h create mode 100644 src/test/serialize_checksum_test.cpp diff --git a/README.md b/README.md index 87c884b..f61e15e 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Based on [Glenn Fiedler's articles](https://gafferongames.com/post/reading_and_w * [Half-precision float - half_precision](#half-precision-float---half_precision) * [Bounded float - bounded_range](#bounded-float---bounded_range) * [Quaternion - smallest_three\](#quaternion---smallest_threeq-bitsperelement) + * [Checksum\](#checksumversion) * [Extensibility](#extensibility) * [Adding new serializables types](#adding-new-serializables-types) * [Unified serialization](#unified-serialization) @@ -443,6 +444,27 @@ bool status_write = writer.serialize>(in_value); bool status_read = reader.serialize>(out_value); ``` +## Checksum\ +A trait that creates a checksum based on the 32-bit number given.
+If the checksum that was written does not match when reading, it returns false. +Must be called before anything else is serizalized, and again once everything is fully serialized. +When reading you can omit the last serialize, as it is a noop. + +The call signature can be seen below: +```cpp +bool serialize>(); +``` +As well as a short example of its usage: +```cpp +bool status_write = writer.serialize>(); +// Serialize some stuff... +status_write = writer.serialize>(); + +bool status_read = reader.serialize>(); +// Deserialize some stuff... +status_read = reader.serialize>(); // Last deserialize on read is optional (a noop) +``` + # Extensibility The library is made with extensibility in mind. The `bit_writer` and `bit_reader` use a template trait specialization of the given type to deduce how to serialize and deserialize the object. diff --git a/generate.bat b/generate.bat index bed3bd6..b80cc62 100644 --- a/generate.bat +++ b/generate.bat @@ -1,2 +1,2 @@ -call vendor\bin\premake\premake5.exe vs2019 +call vendor\bin\premake\premake5.exe vs2022 pause \ No newline at end of file diff --git a/include/bitstream/bitstream.h b/include/bitstream/bitstream.h index edab2cf..264e28f 100644 --- a/include/bitstream/bitstream.h +++ b/include/bitstream/bitstream.h @@ -15,6 +15,7 @@ // Traits #include "traits/array_traits.h" #include "traits/bool_trait.h" +#include "traits/checksum_trait.h" #include "traits/enum_trait.h" #include "traits/float_trait.h" #include "traits/integral_traits.h" diff --git a/include/bitstream/stream/bit_reader.h b/include/bitstream/stream/bit_reader.h index 7614dcc..a3169e4 100644 --- a/include/bitstream/stream/bit_reader.h +++ b/include/bitstream/stream/bit_reader.h @@ -106,35 +106,6 @@ namespace bitstream */ [[nodiscard]] uint32_t get_total_bits() const noexcept { return m_Policy.get_total_bits(); } - /** - * @brief Reads the first 32 bits of the buffer and compares it to a checksum of the @p protocol_version and the rest of the buffer - * @param protocol_version A unique version number - * @return Whether the checksum matches what was written - */ - [[nodiscard]] bool serialize_checksum(uint32_t protocol_version) noexcept - { - BS_ASSERT(get_num_bits_serialized() == 0); - - BS_ASSERT(can_serialize_bits(32U)); - - uint32_t num_bytes = (get_total_bits() - 1U) / 8U + 1U; - const uint32_t* buffer = m_Policy.get_buffer(); - - // Generate checksum to compare against - uint32_t generated_checksum = utility::crc_uint32(reinterpret_cast(&protocol_version), reinterpret_cast(buffer + 1), num_bytes - 4); - - // Advance the reader by the size of the checksum (32 bits / 1 word) - m_WordIndex++; - - BS_ASSERT(m_Policy.extend(32U)); - - // Read the checksum - uint32_t checksum = *buffer; - - // Compare the checksum - return generated_checksum == checksum; - } - /** * @brief Pads the buffer up to the given number of bytes * @param num_bytes The byte number to pad to diff --git a/include/bitstream/stream/bit_writer.h b/include/bitstream/stream/bit_writer.h index c70bf57..bfaf6d2 100644 --- a/include/bitstream/stream/bit_writer.h +++ b/include/bitstream/stream/bit_writer.h @@ -126,46 +126,6 @@ namespace bitstream return get_num_bits_serialized(); } - /** - * @brief Instructs the writer that you intend to use `serialize_checksum()` later on, and to reserve the first 32 bits. - * @return Returns false if anything has already been written to the buffer or if there's no space to write the checksum - */ - [[nodiscard]] bool prepend_checksum() noexcept - { - BS_ASSERT(get_num_bits_serialized() == 0); - - BS_ASSERT(m_Policy.extend(32U)); - - // Advance the reader by the size of the checksum (32 bits / 1 word) - m_WordIndex++; - - return true; - } - - /** - * @brief Writes a checksum of the @p protocol_version and the rest of the buffer as the first 32 bits - * @param protocol_version A unique version number - * @return The number of bytes written to the buffer - */ - uint32_t serialize_checksum(uint32_t protocol_version) noexcept - { - uint32_t num_bits = flush(); - - BS_ASSERT(num_bits > 32U); - - // Copy protocol version to buffer - uint32_t* buffer = m_Policy.get_buffer(); - *buffer = protocol_version; - - // Generate checksum of version + data - uint32_t checksum = utility::crc_uint32(reinterpret_cast(buffer), get_num_bytes_serialized()); - - // Put checksum at beginning - *buffer = checksum; - - return num_bits; - } - /** * @brief Pads the buffer up to the given number of bytes with zeros * @param num_bytes The byte number to pad to @@ -337,7 +297,8 @@ namespace bitstream * @param writer The writer to copy into * @return Returns false if writing would overflow the buffer */ - [[nodiscard]] bool serialize_into(bit_writer& writer) const noexcept + template + [[nodiscard]] bool serialize_into(bit_writer& writer) const noexcept { uint8_t* buffer = reinterpret_cast(m_Policy.get_buffer()); uint32_t num_bits = get_num_bits_serialized(); diff --git a/include/bitstream/traits/checksum_trait.h b/include/bitstream/traits/checksum_trait.h new file mode 100644 index 0000000..1a6282e --- /dev/null +++ b/include/bitstream/traits/checksum_trait.h @@ -0,0 +1,78 @@ +#pragma once +#include "../utility/assert.h" +#include "../utility/crc.h" +#include "../utility/meta.h" +#include "../utility/parameter.h" + +#include "../stream/serialize_traits.h" + +namespace bitstream +{ + /** + * @brief Type for checksums + * @tparam Version A unique version number + */ + template + struct checksum; + + /** + * @brief A trait used to serialize a checksum of the @p Version and the rest of the buffer as the first 32 bits. + * This should be called both first and last when reading and writing to a buffer. + * @tparam Version A unique version number + */ + template + struct serialize_traits> + { + constexpr static uint32_t protocol_version = utility::to_big_endian32_const(Version); + constexpr static uint32_t protocol_size = sizeof(uint32_t); + + template + typename utility::is_writing_t + static serialize(Stream& writer) noexcept + { + if (writer.get_num_bits_serialized() == 0) + return writer.pad_to_size(4); + + uint32_t num_bits = writer.flush(); + + BS_ASSERT(num_bits >= 32U); + + // Get buffer info + uint8_t* byte_buffer = writer.get_buffer(); + uint32_t num_bytes = writer.get_num_bytes_serialized(); + + // Generate checksum of version + data + uint32_t generated_checksum = utility::crc_uint32(protocol_version, byte_buffer + protocol_size, writer.get_num_bytes_serialized() - protocol_size); + + // Put checksum at beginning + uint32_t* buffer = reinterpret_cast(byte_buffer); + *buffer = utility::to_big_endian32(generated_checksum); + + return true; + } + + template + typename utility::is_reading_t + static serialize(Stream& reader) noexcept + { + if (reader.get_num_bits_serialized() > 0) + return true; + + BS_ASSERT(reader.can_serialize_bits(32U)); + + // Get buffer info + const uint8_t* byte_buffer = reader.get_buffer(); + uint32_t num_bytes = (reader.get_total_bits() - 1U) / 8U + 1U; + + // Generate checksum to compare against + uint32_t generated_checksum = utility::crc_uint32(protocol_version, byte_buffer + protocol_size, num_bytes - protocol_size); + + // Read the checksum + uint32_t given_checksum; + BS_ASSERT(reader.serialize_bits(given_checksum, 32U)); + + // Compare the checksum + return generated_checksum == given_checksum; + } + }; +} \ No newline at end of file diff --git a/include/bitstream/utility/crc.h b/include/bitstream/utility/crc.h index d62d6d6..d795547 100644 --- a/include/bitstream/utility/crc.h +++ b/include/bitstream/utility/crc.h @@ -22,22 +22,15 @@ namespace bitstream::utility return table; }(); - inline constexpr uint32_t crc_uint32(const uint8_t* bytes, uint32_t size) + inline uint32_t crc_uint32(uint32_t checksum, const uint8_t* bytes, uint32_t size) { uint32_t result = 0xFFFFFFFF; - for (uint32_t i = 0; i < size; i++) - result = CHECKSUM_TABLE[(result & 0xFF) ^ *(bytes + i)] ^ (result >> 8); - - return ~result; - } - - inline constexpr uint32_t crc_uint32(const uint8_t* checksum, const uint8_t* bytes, uint32_t size) - { - uint32_t result = 0xFFFFFFFF; + uint8_t checksum_table[4]{}; + std::memcpy(&checksum_table, &checksum, sizeof(uint32_t)); for (uint32_t i = 0; i < 4; i++) - result = CHECKSUM_TABLE[(result & 0xFF) ^ *(checksum + i)] ^ (result >> 8); + result = CHECKSUM_TABLE[(result & 0xFF) ^ *(checksum_table + i)] ^ (result >> 8); for (uint32_t i = 0; i < size; i++) result = CHECKSUM_TABLE[(result & 0xFF) ^ *(bytes + i)] ^ (result >> 8); diff --git a/include/bitstream/utility/endian.h b/include/bitstream/utility/endian.h index f33ad42..807a61b 100644 --- a/include/bitstream/utility/endian.h +++ b/include/bitstream/utility/endian.h @@ -1,5 +1,7 @@ #pragma once +#include "platform.h" + #include #if defined(__cpp_lib_endian) && __cpp_lib_endian >= 201907L @@ -63,23 +65,43 @@ namespace bitstream::utility #endif // defined(BS_LITTLE_ENDIAN) } - inline uint32_t endian_swap32(uint32_t value) + constexpr inline uint32_t endian_swap32_const(uint32_t value) { -#if defined(_WIN32) - return _byteswap_ulong(value); -#elif defined(__linux__) - return __builtin_bswap32(value); -#else const uint32_t first = (value << 24) & 0xFF000000; const uint32_t second = (value << 8) & 0x00FF0000; const uint32_t third = (value >> 8) & 0x0000FF00; const uint32_t fourth = (value >> 24) & 0x000000FF; return first | second | third | fourth; + } + + BS_CONSTEXPR inline uint32_t endian_swap32(uint32_t value) + { + if BS_CONST_EVALUATED() + { + return endian_swap32_const(value); + } + else + { +#if defined(_WIN32) + return _byteswap_ulong(value); +#elif defined(__linux__) + return __builtin_bswap32(value); +#else + return endian_swap32_const(value); #endif // _WIN32 || __linux__ + } + } + + constexpr inline uint32_t to_big_endian32_const(uint32_t value) + { + if constexpr (little_endian()) + return endian_swap32_const(value); + else + return value; } - inline uint32_t to_big_endian32(uint32_t value) + BS_CONSTEXPR inline uint32_t to_big_endian32(uint32_t value) { if constexpr (little_endian()) return endian_swap32(value); diff --git a/include/bitstream/utility/parameter.h b/include/bitstream/utility/parameter.h index e19d282..1ee560e 100644 --- a/include/bitstream/utility/parameter.h +++ b/include/bitstream/utility/parameter.h @@ -1,16 +1,11 @@ #pragma once #include "assert.h" +#include "platform.h" #include #include -#ifdef __cpp_constexpr_dynamic_alloc -#define BS_CONSTEXPR constexpr -#else // __cpp_constexpr_dynamic_alloc -#define BS_CONSTEXPR -#endif // __cpp_constexpr_dynamic_alloc - namespace bitstream { #ifdef BS_DEBUG_BREAK diff --git a/include/bitstream/utility/platform.h b/include/bitstream/utility/platform.h new file mode 100644 index 0000000..8c8501f --- /dev/null +++ b/include/bitstream/utility/platform.h @@ -0,0 +1,11 @@ +#pragma once + +#include + +#if defined(__cpp_lib_is_constant_evaluated) && __cpp_lib_is_constant_evaluated >= 201811L +# define BS_CONST_EVALUATED() (std::is_constant_evaluated()) +# define BS_CONSTEXPR constexpr +#else // __cpp_lib_is_constant_evaluated +# define BS_CONST_EVALUATED() constexpr (false) +# define BS_CONSTEXPR +#endif // __cpp_lib_is_constant_evaluated \ No newline at end of file diff --git a/src/test/serialize_checksum_test.cpp b/src/test/serialize_checksum_test.cpp new file mode 100644 index 0000000..747dcc2 --- /dev/null +++ b/src/test/serialize_checksum_test.cpp @@ -0,0 +1,61 @@ +#include "../shared/assert.h" +#include "../shared/test.h" + +#include +#include + +#include + +namespace bitstream::test::traits +{ + // Checksums are stored in the first 4 bytes of the buffer + // They should only be used when done writing since they flush the buffer + // It is important to serialize the checksum both at the start and end when writing + // When reading you only need to serialize at the start as the end is a noop + + BS_ADD_TEST(test_serialize_checksum_empty) + { + // Test checksum + using protocol_version = checksum<0xBEEFCAFE>; + + // Write a checksum + byte_buffer<16> buffer; + fixed_bit_writer writer(buffer); + + BS_TEST_ASSERT(writer.serialize()); + BS_TEST_ASSERT(writer.serialize()); + uint32_t num_bits = writer.flush(); + + // Read the checksum and validate + fixed_bit_reader reader(buffer, num_bits); + + BS_TEST_ASSERT(reader.serialize()); + // When reading the last checksum check does nothing + } + + BS_ADD_TEST(test_serialize_checksum) + { + // Test checksum + using protocol_version = checksum<0xDEADBEEF>; + uint32_t value = 5; + + // Write some initial values and finish with a checksum + byte_buffer<16> buffer; + fixed_bit_writer writer(buffer); + + BS_TEST_ASSERT(writer.serialize()); // Must be called both before and after + BS_TEST_ASSERT(writer.serialize_bits(value, 3)); + BS_TEST_ASSERT(writer.serialize()); + uint32_t num_bits = writer.flush(); + + // Read the checksum and validate + uint32_t out_value; + fixed_bit_reader reader(buffer, num_bits); + + BS_TEST_ASSERT(reader.serialize()); + BS_TEST_ASSERT(reader.serialize_bits(out_value, 3)); + BS_TEST_ASSERT(reader.serialize()); // When reading the last checksum check does nothing + + BS_TEST_ASSERT(out_value == value); + } +} \ No newline at end of file diff --git a/src/test/serialize_string_test.cpp b/src/test/serialize_string_test.cpp index b956fdd..2db6e75 100644 --- a/src/test/serialize_string_test.cpp +++ b/src/test/serialize_string_test.cpp @@ -105,7 +105,7 @@ namespace bitstream::test::traits // Write a char array, but make sure the word count isn't whole byte_buffer<32> buffer; - bit_writer writer(buffer); + fixed_bit_writer writer(buffer); BS_TEST_ASSERT(writer.serialize_bits(padding, 26)); BS_TEST_ASSERT(writer.serialize(value, 32U)); @@ -116,7 +116,7 @@ namespace bitstream::test::traits // Read the array back and validate uint32_t out_padding; char8_t out_value[32]; - bit_reader reader(buffer, num_bits); + fixed_bit_reader reader(buffer, num_bits); BS_TEST_ASSERT(reader.serialize_bits(out_padding, 26)); BS_TEST_ASSERT(reader.serialize(out_value, 32U)); @@ -143,7 +143,7 @@ namespace bitstream::test::traits // Write a string, but make sure the word count isn't whole byte_buffer<32> buffer; - bit_writer writer(buffer); + fixed_bit_writer writer(buffer); BS_TEST_ASSERT(writer.serialize_bits(padding, 26)); BS_TEST_ASSERT(writer.serialize(value, 32U)); @@ -154,7 +154,7 @@ namespace bitstream::test::traits // Read the array back and validate uint32_t out_padding; std::u8string out_value; - bit_reader reader(buffer, num_bits); + fixed_bit_reader reader(buffer, num_bits); BS_TEST_ASSERT(reader.serialize_bits(out_padding, 26)); BS_TEST_ASSERT(reader.serialize(out_value, 32U)); diff --git a/src/test/stream_test.cpp b/src/test/stream_test.cpp index b0bf25a..c3d4a84 100644 --- a/src/test/stream_test.cpp +++ b/src/test/stream_test.cpp @@ -74,30 +74,6 @@ namespace bitstream::test::stream BS_TEST_ASSERT(out_value3 == in_value3); } - BS_ADD_TEST(test_serialize_checksum) - { - // Test checksum - uint32_t protocol_version = 0xDEADBEEF; - uint32_t value = 5; - - // Write some initial values and finish with a checksum - byte_buffer<16> buffer; - fixed_bit_writer writer(buffer); - - BS_TEST_ASSERT(writer.prepend_checksum()); - BS_TEST_ASSERT(writer.serialize_bits(value, 3)); - uint32_t num_bits = writer.serialize_checksum(protocol_version); - - // Read the checksum and validate - uint32_t out_value; - fixed_bit_reader reader(buffer, num_bits); - - BS_TEST_ASSERT(reader.serialize_checksum(protocol_version)); - BS_TEST_ASSERT(reader.serialize_bits(out_value, 3)); - - BS_TEST_ASSERT(out_value == value); - } - BS_ADD_TEST(test_serialize_padding_small) { // Test padding From b7c6820da6e1ab9e2aa9ce0228630879e3712216 Mon Sep 17 00:00:00 2001 From: KredeGC Date: Sat, 19 Jul 2025 19:02:06 +0200 Subject: [PATCH 2/2] Fix missing include --- include/bitstream/utility/crc.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/bitstream/utility/crc.h b/include/bitstream/utility/crc.h index d795547..1195e37 100644 --- a/include/bitstream/utility/crc.h +++ b/include/bitstream/utility/crc.h @@ -2,6 +2,7 @@ #include #include +#include namespace bitstream::utility {