refactor: move zero width replacement to a function (#5594)

src/providers/recentmessages/Impl.cpp

@@ -1,22 +1,13 @@
 #include "providers/recentmessages/Impl.hpp"
 
 #include "common/Env.hpp"
-#include "common/QLogging.hpp"
 #include "messages/MessageBuilder.hpp"
 #include "providers/twitch/IrcMessageHandler.hpp"
-#include "providers/twitch/TwitchChannel.hpp"
-#include "util/FormatTime.hpp"
+#include "util/Helpers.hpp"
 
 #include <QJsonArray>
 #include <QUrlQuery>
 
-namespace {
-
-// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
-const auto &LOG = chatterinoRecentMessages;
-
-}  // namespace
-
 namespace chatterino::recentmessages::detail {
 
 // Parse the IRC messages returned in JSON form into Communi messages
@@ -33,11 +24,7 @@ std::vector<Communi::IrcMessage *> parseRecentMessages(
 
     for (const auto &jsonMessage : jsonMessages)
     {
-        auto content = jsonMessage.toString();
-
-        // For explanation of why this exists, see src/providers/twitch/TwitchChannel.hpp,
-        // where these constants are defined
-        content.replace(COMBINED_FIXER, ZERO_WIDTH_JOINER);
+        auto content = unescapeZeroWidthJoiner(jsonMessage.toString());
 
         auto *message =
             Communi::IrcMessage::fromData(content.toUtf8(), nullptr);

src/providers/twitch/IrcMessageHandler.cpp

@@ -702,15 +702,8 @@ void IrcMessageHandler::handlePrivMessage(Communi::IrcPrivateMessage *message,
         }
     }
 
-    // This is for compatibility with older Chatterino versions. Twitch didn't use
-    // to allow ZERO WIDTH JOINER unicode character, so Chatterino used ESCAPE_TAG
-    // instead.
-    // See https://github.com/Chatterino/chatterino2/issues/3384 and
-    // https://mm2pl.github.io/emoji_rfc.pdf for more details
-    this->addMessage(
-        message, chan,
-        message->content().replace(COMBINED_FIXER, ZERO_WIDTH_JOINER),
-        twitchServer, false, message->isAction());
+    this->addMessage(message, chan, unescapeZeroWidthJoiner(message->content()),
+                     twitchServer, false, message->isAction());
 
     if (message->tags().contains(u"pinned-chat-paid-amount"_s))
     {
@@ -915,10 +908,9 @@ void IrcMessageHandler::handleWhisperMessage(Communi::IrcMessage *ircMessage)
 
     auto *c = getApp()->getTwitch()->getWhispersChannel().get();
 
-    MessageBuilder builder(
-        c, ircMessage, args,
-        ircMessage->parameter(1).replace(COMBINED_FIXER, ZERO_WIDTH_JOINER),
-        false);
+    MessageBuilder builder(c, ircMessage, args,
+                           unescapeZeroWidthJoiner(ircMessage->parameter(1)),
+                           false);
 
     if (builder.isIgnored())
     {

src/providers/twitch/TwitchChannel.hpp

@@ -27,24 +27,6 @@
 
 namespace chatterino {
 
-// This is for compatibility with older Chatterino versions. Twitch didn't use
-// to allow ZERO WIDTH JOINER unicode character, so Chatterino used ESCAPE_TAG
-// instead.
-// See https://github.com/Chatterino/chatterino2/issues/3384 and
-// https://mm2pl.github.io/emoji_rfc.pdf for more details
-const QString ZERO_WIDTH_JOINER = QString(QChar(0x200D));
-
-// Here be MSVC: Do NOT replace with "\U" literal, it will fail silently.
-namespace {
-    const QChar ESCAPE_TAG_CHARS[2] = {QChar::highSurrogate(0xE0002),
-                                       QChar::lowSurrogate(0xE0002)};
-}
-const QString ESCAPE_TAG = QString(ESCAPE_TAG_CHARS, 2);
-
-const static QRegularExpression COMBINED_FIXER(
-    QString("(?<!%1)%1").arg(ESCAPE_TAG),
-    QRegularExpression::UseUnicodePropertiesOption);
-
 enum class HighlightState;
 
 struct Emote;

src/util/Helpers.cpp

@@ -7,6 +7,18 @@
 #include <QRegularExpression>
 #include <QUuid>
 
+namespace {
+
+const QString ZERO_WIDTH_JOINER = QStringLiteral("\u200D");
+
+// Note: \U requires /utf-8 for MSVC
+// See https://mm2pl.github.io/emoji_rfc.pdf
+const QRegularExpression ESCAPE_TAG_REGEX(
+    QStringLiteral("(?<!\U000E0002)\U000E0002"),
+    QRegularExpression::UseUnicodePropertiesOption);
+
+}  // namespace
+
 namespace chatterino {
 
 namespace helpers::detail {
@@ -283,4 +295,10 @@ bool compareEmoteStrings(const QString &a, const QString &b)
     return k < 0;
 }
 
+QString unescapeZeroWidthJoiner(QString escaped)
+{
+    escaped.replace(ESCAPE_TAG_REGEX, ZERO_WIDTH_JOINER);
+    return escaped;
+}
+
 }  // namespace chatterino

src/util/Helpers.hpp

@@ -182,4 +182,11 @@ constexpr std::optional<std::decay_t<T>> makeConditionedOptional(bool condition,
     return std::nullopt;
 }
 
+/// @brief Unescapes zero width joiners (ZWJ; U+200D) from Twitch messages
+///
+/// Older Chatterino versions escape ZWJ with an ESCAPE TAG (U+E0002), following
+/// https://mm2pl.github.io/emoji_rfc.pdf. This function unescapes all tags with
+/// a ZWJ. See also: https://github.com/Chatterino/chatterino2/issues/3384.
+QString unescapeZeroWidthJoiner(QString escaped);
+
 }  // namespace chatterino