From 7628ddfaf772cca6d2bd4e54623df912042f4171 Mon Sep 17 00:00:00 2001 From: Andreas Kling Date: Tue, 15 Apr 2025 17:58:42 +0200 Subject: [PATCH] AK: Remove endianness override from Utf16View Utf16View is now always in "host" endian mode. This makes it smaller and less branchy for everyone! --- AK/String.cpp | 25 ++-------------- AK/StringBuilder.cpp | 24 ++------------- AK/Utf16View.cpp | 41 +++++-------------------- AK/Utf16View.h | 17 ++++------- Tests/AK/TestUtf16.cpp | 68 ------------------------------------------ 5 files changed, 16 insertions(+), 159 deletions(-) diff --git a/AK/String.cpp b/AK/String.cpp index a344f46c093..e20a91eaa5c 100644 --- a/AK/String.cpp +++ b/AK/String.cpp @@ -100,32 +100,11 @@ ErrorOr String::from_utf16(Utf16View const& utf16) String result; - auto utf8_length = [&]() { - switch (utf16.endianness()) { - case Endianness::Host: - return simdutf::utf8_length_from_utf16(utf16.char_data(), utf16.length_in_code_units()); - case Endianness::Big: - return simdutf::utf8_length_from_utf16be(utf16.char_data(), utf16.length_in_code_units()); - case Endianness::Little: - return simdutf::utf8_length_from_utf16le(utf16.char_data(), utf16.length_in_code_units()); - } - VERIFY_NOT_REACHED(); - }(); + auto utf8_length = simdutf::utf8_length_from_utf16(utf16.char_data(), utf16.length_in_code_units()); TRY(result.replace_with_new_string(utf8_length, [&](Bytes buffer) -> ErrorOr { - [[maybe_unused]] auto result = [&]() { - switch (utf16.endianness()) { - case Endianness::Host: - return simdutf::convert_utf16_to_utf8(utf16.char_data(), utf16.length_in_code_units(), reinterpret_cast(buffer.data())); - case Endianness::Big: - return simdutf::convert_utf16be_to_utf8(utf16.char_data(), utf16.length_in_code_units(), reinterpret_cast(buffer.data())); - case Endianness::Little: - return simdutf::convert_utf16le_to_utf8(utf16.char_data(), utf16.length_in_code_units(), reinterpret_cast(buffer.data())); - } - VERIFY_NOT_REACHED(); - }(); + [[maybe_unused]] auto result = simdutf::convert_utf16_to_utf8(utf16.char_data(), utf16.length_in_code_units(), reinterpret_cast(buffer.data())); ASSERT(result == buffer.size()); - return {}; })); diff --git a/AK/StringBuilder.cpp b/AK/StringBuilder.cpp index 513469db422..eed182e4630 100644 --- a/AK/StringBuilder.cpp +++ b/AK/StringBuilder.cpp @@ -259,17 +259,7 @@ ErrorOr StringBuilder::try_append(Utf16View const& utf16_view) auto uninitialized_data_pointer = static_cast(m_buffer.end_pointer()); // Fast path. - auto result = [&]() { - switch (remaining_view.endianness()) { - case Endianness::Host: - return simdutf::convert_utf16_to_utf8_with_errors(remaining_view.char_data(), remaining_view.length_in_code_units(), uninitialized_data_pointer); - case Endianness::Big: - return simdutf::convert_utf16be_to_utf8_with_errors(remaining_view.char_data(), remaining_view.length_in_code_units(), uninitialized_data_pointer); - case Endianness::Little: - return simdutf::convert_utf16le_to_utf8_with_errors(remaining_view.char_data(), remaining_view.length_in_code_units(), uninitialized_data_pointer); - } - VERIFY_NOT_REACHED(); - }(); + auto result = simdutf::convert_utf16_to_utf8_with_errors(remaining_view.char_data(), remaining_view.length_in_code_units(), uninitialized_data_pointer); if (result.error == simdutf::SUCCESS) { auto bytes_just_written = result.count; m_buffer.set_size(m_buffer.size() + bytes_just_written); @@ -281,17 +271,7 @@ ErrorOr StringBuilder::try_append(Utf16View const& utf16_view) ASSERT(first_invalid_code_unit < remaining_view.length_in_code_units()); // Unfortunately, `simdutf` does not tell us how many bytes it just wrote in case of an error, so we have to calculate it ourselves. - auto bytes_just_written = [&]() { - switch (remaining_view.endianness()) { - case Endianness::Host: - return simdutf::utf8_length_from_utf16(remaining_view.char_data(), first_invalid_code_unit); - case Endianness::Big: - return simdutf::utf8_length_from_utf16be(remaining_view.char_data(), first_invalid_code_unit); - case Endianness::Little: - return simdutf::utf8_length_from_utf16le(remaining_view.char_data(), first_invalid_code_unit); - } - VERIFY_NOT_REACHED(); - }(); + auto bytes_just_written = simdutf::utf8_length_from_utf16(remaining_view.char_data(), first_invalid_code_unit); do { auto code_unit = remaining_view.code_unit_at(first_invalid_code_unit++); diff --git a/AK/Utf16View.cpp b/AK/Utf16View.cpp index 3ae3c50a5f4..11b91e77861 100644 --- a/AK/Utf16View.cpp +++ b/AK/Utf16View.cpp @@ -185,7 +185,7 @@ size_t Utf16View::length_in_code_points() const u16 Utf16View::code_unit_at(size_t index) const { VERIFY(index < length_in_code_units()); - return host_code_unit(m_code_units[index], m_endianness); + return host_code_unit(m_code_units[index], Endianness::Host); } u32 Utf16View::code_point_at(size_t index) const @@ -296,31 +296,12 @@ bool Utf16View::starts_with(Utf16View const& needle) const bool Utf16View::validate() const { - switch (m_endianness) { - case Endianness::Host: - return simdutf::validate_utf16(char_data(), length_in_code_units()); - case Endianness::Big: - return simdutf::validate_utf16be(char_data(), length_in_code_units()); - case Endianness::Little: - return simdutf::validate_utf16le(char_data(), length_in_code_units()); - } - VERIFY_NOT_REACHED(); + return simdutf::validate_utf16(char_data(), length_in_code_units()); } bool Utf16View::validate(size_t& valid_code_units) const { - auto result = [&]() { - switch (m_endianness) { - case Endianness::Host: - return simdutf::validate_utf16_with_errors(char_data(), length_in_code_units()); - case Endianness::Big: - return simdutf::validate_utf16be_with_errors(char_data(), length_in_code_units()); - case Endianness::Little: - return simdutf::validate_utf16le_with_errors(char_data(), length_in_code_units()); - } - VERIFY_NOT_REACHED(); - }(); - + auto result = simdutf::validate_utf16_with_errors(char_data(), length_in_code_units()); valid_code_units = result.count; return result.error == simdutf::SUCCESS; } @@ -330,16 +311,8 @@ size_t Utf16View::calculate_length_in_code_points() const // FIXME: simdutf's code point length method assumes valid UTF-16, whereas Utf16View uses U+FFFD as a replacement // for invalid code points. If we change Utf16View to only accept valid encodings as an invariant, we can // remove this branch. - if (validate()) [[likely]] { - switch (m_endianness) { - case Endianness::Host: - return simdutf::count_utf16(char_data(), length_in_code_units()); - case Endianness::Big: - return simdutf::count_utf16be(char_data(), length_in_code_units()); - case Endianness::Little: - return simdutf::count_utf16le(char_data(), length_in_code_units()); - } - } + if (validate()) [[likely]] + return simdutf::count_utf16(char_data(), length_in_code_units()); size_t code_points = 0; for ([[maybe_unused]] auto code_point : *this) @@ -397,11 +370,11 @@ u32 Utf16CodePointIterator::operator*() const // W2 as its 10 low-order bits. // 5) Add 0x10000 to U' to obtain the character value U. Terminate. - auto code_unit = host_code_unit(*m_ptr, m_endianness); + auto code_unit = host_code_unit(*m_ptr, Endianness::Host); if (Utf16View::is_high_surrogate(code_unit)) { if (m_remaining_code_units > 1) { - auto next_code_unit = host_code_unit(*(m_ptr + 1), m_endianness); + auto next_code_unit = host_code_unit(*(m_ptr + 1), Endianness::Host); if (Utf16View::is_low_surrogate(next_code_unit)) return Utf16View::decode_surrogate_pair(code_unit, next_code_unit); diff --git a/AK/Utf16View.h b/AK/Utf16View.h index 176c67fbd8e..f5b3f367751 100644 --- a/AK/Utf16View.h +++ b/AK/Utf16View.h @@ -51,16 +51,14 @@ public: size_t length_in_code_units() const; private: - Utf16CodePointIterator(u16 const* ptr, size_t length, Endianness endianness) + Utf16CodePointIterator(u16 const* ptr, size_t length) : m_ptr(ptr) , m_remaining_code_units(length) - , m_endianness(endianness) { } u16 const* m_ptr { nullptr }; size_t m_remaining_code_units { 0 }; - Endianness m_endianness { Endianness::Host }; }; class Utf16View { @@ -74,18 +72,16 @@ public: Utf16View() = default; ~Utf16View() = default; - explicit Utf16View(ReadonlySpan code_units, Endianness endianness = Endianness::Host) + explicit Utf16View(ReadonlySpan code_units) : m_code_units(code_units) - , m_endianness(endianness) { } template - Utf16View(char16_t const (&code_units)[Size], Endianness endianness = Endianness::Host) + Utf16View(char16_t const (&code_units)[Size]) : m_code_units( reinterpret_cast(&code_units[0]), code_units[Size - 1] == u'\0' ? Size - 1 : Size) - , m_endianness(endianness) { } @@ -104,10 +100,8 @@ public: size_t length_in_code_units() const { return m_code_units.size(); } size_t length_in_code_points() const; - Endianness endianness() const { return m_endianness; } - - Utf16CodePointIterator begin() const { return { begin_ptr(), m_code_units.size(), m_endianness }; } - Utf16CodePointIterator end() const { return { end_ptr(), 0, m_endianness }; } + Utf16CodePointIterator begin() const { return { begin_ptr(), m_code_units.size() }; } + Utf16CodePointIterator end() const { return { end_ptr(), 0 }; } u16 const* data() const { return m_code_units.data(); } char16_t const* char_data() const { return reinterpret_cast(data()); } @@ -142,7 +136,6 @@ private: ReadonlySpan m_code_units; NO_UNIQUE_ADDRESS mutable Optional m_length_in_code_points; - Endianness m_endianness { Endianness::Host }; }; } diff --git a/Tests/AK/TestUtf16.cpp b/Tests/AK/TestUtf16.cpp index 01f7fa973e5..befa9bec19b 100644 --- a/Tests/AK/TestUtf16.cpp +++ b/Tests/AK/TestUtf16.cpp @@ -357,71 +357,3 @@ TEST_CASE(starts_with) EXPECT(!emoji.starts_with(u"a")); EXPECT(!emoji.starts_with(u"🙃")); } - -TEST_CASE(big_endian) -{ - auto string = MUST(AK::utf8_to_utf16("säk😀"sv, AK::Endianness::Big)); - Utf16View view { string, AK::Endianness::Big }; - { - EXPECT(view.validate()); - EXPECT_EQ(MUST(view.to_utf8()), "säk😀"sv); - - EXPECT_EQ(view.length_in_code_units(), 5u); - EXPECT_EQ(view.length_in_code_points(), 4u); - - EXPECT_EQ(view.code_unit_at(0), 0x73u); - EXPECT_EQ(view.code_unit_at(1), 0xe4u); - EXPECT_EQ(view.code_unit_at(2), 0x6bu); - EXPECT_EQ(view.code_unit_at(3), 0xd83d); - EXPECT_EQ(view.code_unit_at(4), 0xde00u); - - EXPECT_EQ(view.code_point_at(0), 0x73u); - EXPECT_EQ(view.code_point_at(1), 0xe4u); - EXPECT_EQ(view.code_point_at(2), 0x6bu); - EXPECT_EQ(view.code_point_at(3), 0x1f600u); - EXPECT_EQ(view.code_point_at(4), 0xde00u); - } - { - Utf16Data data; - MUST(code_point_to_utf16(data, 's', AK::Endianness::Big)); - MUST(code_point_to_utf16(data, 0xe4, AK::Endianness::Big)); - MUST(code_point_to_utf16(data, 'k', AK::Endianness::Big)); - MUST(code_point_to_utf16(data, 0x1f600, AK::Endianness::Big)); - EXPECT_EQ(data, to_array({ 0x7300, 0xe400, 0x6b00, 0x3dd8, 0x00de })); - EXPECT_EQ(data, string); - } -} - -TEST_CASE(little_endian) -{ - auto string = MUST(AK::utf8_to_utf16("säk😀"sv, AK::Endianness::Little)); - Utf16View view { string, AK::Endianness::Little }; - { - EXPECT(view.validate()); - EXPECT_EQ(MUST(view.to_utf8()), "säk😀"sv); - - EXPECT_EQ(view.length_in_code_units(), 5u); - EXPECT_EQ(view.length_in_code_points(), 4u); - - EXPECT_EQ(view.code_unit_at(0), 0x73u); - EXPECT_EQ(view.code_unit_at(1), 0xe4u); - EXPECT_EQ(view.code_unit_at(2), 0x6bu); - EXPECT_EQ(view.code_unit_at(3), 0xd83d); - EXPECT_EQ(view.code_unit_at(4), 0xde00u); - - EXPECT_EQ(view.code_point_at(0), 0x73u); - EXPECT_EQ(view.code_point_at(1), 0xe4u); - EXPECT_EQ(view.code_point_at(2), 0x6bu); - EXPECT_EQ(view.code_point_at(3), 0x1f600u); - EXPECT_EQ(view.code_point_at(4), 0xde00u); - } - { - Utf16Data data; - MUST(code_point_to_utf16(data, 's', AK::Endianness::Little)); - MUST(code_point_to_utf16(data, 0xe4, AK::Endianness::Little)); - MUST(code_point_to_utf16(data, 'k', AK::Endianness::Little)); - MUST(code_point_to_utf16(data, 0x1f600, AK::Endianness::Little)); - EXPECT_EQ(data, to_array({ 0x73, 0xe4, 0x6b, 0xd83d, 0xde00 })); - EXPECT_EQ(data, string); - } -}