diff --git a/CHANGELOG.md b/CHANGELOG.md index e8c08c93..116d9e20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,6 +90,7 @@ - Dev: The timer for `StreamerMode` is now destroyed on the correct thread. (#5571) - Dev: Cleanup some parts of the `magic_enum` adaptation for Qt. (#5587) - Dev: Refactored `static`s in headers to only be present once in the final app. (#5588) +- Dev: Refactored legacy Unicode zero-width-joiner replacement. (#5594) - Dev: The JSON output when copying a message (SHIFT + right-click) is now more extensive. (#5600) ## 2.5.1 diff --git a/src/providers/recentmessages/Impl.cpp b/src/providers/recentmessages/Impl.cpp index 6971b554..410a34aa 100644 --- a/src/providers/recentmessages/Impl.cpp +++ b/src/providers/recentmessages/Impl.cpp @@ -1,22 +1,13 @@ #include "providers/recentmessages/Impl.hpp" #include "common/Env.hpp" -#include "common/QLogging.hpp" #include "messages/MessageBuilder.hpp" #include "providers/twitch/IrcMessageHandler.hpp" -#include "providers/twitch/TwitchChannel.hpp" -#include "util/FormatTime.hpp" +#include "util/Helpers.hpp" #include #include -namespace { - -// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -const auto &LOG = chatterinoRecentMessages; - -} // namespace - namespace chatterino::recentmessages::detail { // Parse the IRC messages returned in JSON form into Communi messages @@ -33,11 +24,7 @@ std::vector parseRecentMessages( for (const auto &jsonMessage : jsonMessages) { - auto content = jsonMessage.toString(); - - // For explanation of why this exists, see src/providers/twitch/TwitchChannel.hpp, - // where these constants are defined - content.replace(COMBINED_FIXER, ZERO_WIDTH_JOINER); + auto content = unescapeZeroWidthJoiner(jsonMessage.toString()); auto *message = Communi::IrcMessage::fromData(content.toUtf8(), nullptr); diff --git a/src/providers/twitch/IrcMessageHandler.cpp b/src/providers/twitch/IrcMessageHandler.cpp index 0e4571cf..279584c3 100644 --- a/src/providers/twitch/IrcMessageHandler.cpp +++ b/src/providers/twitch/IrcMessageHandler.cpp @@ -702,15 +702,8 @@ void IrcMessageHandler::handlePrivMessage(Communi::IrcPrivateMessage *message, } } - // This is for compatibility with older Chatterino versions. Twitch didn't use - // to allow ZERO WIDTH JOINER unicode character, so Chatterino used ESCAPE_TAG - // instead. - // See https://github.com/Chatterino/chatterino2/issues/3384 and - // https://mm2pl.github.io/emoji_rfc.pdf for more details - this->addMessage( - message, chan, - message->content().replace(COMBINED_FIXER, ZERO_WIDTH_JOINER), - twitchServer, false, message->isAction()); + this->addMessage(message, chan, unescapeZeroWidthJoiner(message->content()), + twitchServer, false, message->isAction()); if (message->tags().contains(u"pinned-chat-paid-amount"_s)) { @@ -915,10 +908,9 @@ void IrcMessageHandler::handleWhisperMessage(Communi::IrcMessage *ircMessage) auto *c = getApp()->getTwitch()->getWhispersChannel().get(); - MessageBuilder builder( - c, ircMessage, args, - ircMessage->parameter(1).replace(COMBINED_FIXER, ZERO_WIDTH_JOINER), - false); + MessageBuilder builder(c, ircMessage, args, + unescapeZeroWidthJoiner(ircMessage->parameter(1)), + false); if (builder.isIgnored()) { diff --git a/src/providers/twitch/TwitchChannel.hpp b/src/providers/twitch/TwitchChannel.hpp index 37b6d363..20f43079 100644 --- a/src/providers/twitch/TwitchChannel.hpp +++ b/src/providers/twitch/TwitchChannel.hpp @@ -27,24 +27,6 @@ namespace chatterino { -// This is for compatibility with older Chatterino versions. Twitch didn't use -// to allow ZERO WIDTH JOINER unicode character, so Chatterino used ESCAPE_TAG -// instead. -// See https://github.com/Chatterino/chatterino2/issues/3384 and -// https://mm2pl.github.io/emoji_rfc.pdf for more details -const QString ZERO_WIDTH_JOINER = QString(QChar(0x200D)); - -// Here be MSVC: Do NOT replace with "\U" literal, it will fail silently. -namespace { - const QChar ESCAPE_TAG_CHARS[2] = {QChar::highSurrogate(0xE0002), - QChar::lowSurrogate(0xE0002)}; -} -const QString ESCAPE_TAG = QString(ESCAPE_TAG_CHARS, 2); - -const static QRegularExpression COMBINED_FIXER( - QString("(? #include +namespace { + +const QString ZERO_WIDTH_JOINER = QStringLiteral("\u200D"); + +// Note: \U requires /utf-8 for MSVC +// See https://mm2pl.github.io/emoji_rfc.pdf +const QRegularExpression ESCAPE_TAG_REGEX( + QStringLiteral("(?> makeConditionedOptional(bool condition, return std::nullopt; } +/// @brief Unescapes zero width joiners (ZWJ; U+200D) from Twitch messages +/// +/// Older Chatterino versions escape ZWJ with an ESCAPE TAG (U+E0002), following +/// https://mm2pl.github.io/emoji_rfc.pdf. This function unescapes all tags with +/// a ZWJ. See also: https://github.com/Chatterino/chatterino2/issues/3384. +QString unescapeZeroWidthJoiner(QString escaped); + } // namespace chatterino diff --git a/tests/src/Helpers.cpp b/tests/src/Helpers.cpp index 4327bf51..c2cf5694 100644 --- a/tests/src/Helpers.cpp +++ b/tests/src/Helpers.cpp @@ -2,6 +2,8 @@ #include "Test.hpp" +#include + using namespace chatterino; using namespace helpers::detail; @@ -500,3 +502,57 @@ TEST(Helpers, parseDurationToSeconds) << c.output; } } + +TEST(Helpers, unescapeZeroWidthJoiner) +{ + struct TestCase { + QStringView input; + QStringView output; + }; + + std::vector tests{ + {u"foo bar", u"foo bar"}, + {u"", u""}, + {u"a", u"a"}, + {u"\U000E0002", u"\u200D"}, + {u"foo\U000E0002bar", u"foo\u200Dbar"}, + {u"foo \U000E0002 bar", u"foo \u200D bar"}, + {u"\U0001F468\U000E0002\U0001F33E", u"\U0001F468\u200D\U0001F33E"}, + // don't replace ZWJ + {u"\U0001F468\u200D\U0001F33E", u"\U0001F468\u200D\U0001F33E"}, + // only replace the first escape tag in sequences + { + u"\U0001F468\U000E0002\U000E0002\U0001F33E", + u"\U0001F468\u200D\U000E0002\U0001F33E", + }, + { + u"\U0001F468\U000E0002\U000E0002\U000E0002\U0001F33E", + u"\U0001F468\u200D\U000E0002\U000E0002\U0001F33E", + }, + }; + + // sanity check that the compiler supports unicode string literals + static_assert( + [] { + constexpr std::span zwj = u"\u200D"; + static_assert(zwj.size() == 2); + static_assert(zwj[0] == u'\x200D'); + static_assert(zwj[1] == u'\0'); + + constexpr std::span escapeTag = u"\U000E0002"; + static_assert(escapeTag.size() == 3); + static_assert(escapeTag[0] == u'\xDB40'); + static_assert(escapeTag[1] == u'\xDC02'); + static_assert(escapeTag[2] == u'\0'); + + return true; + }(), + "The compiler must support Unicode string literals"); + + for (const auto &c : tests) + { + const auto actual = unescapeZeroWidthJoiner(c.input.toString()); + + EXPECT_EQ(actual, c.output); + } +}