Update to Emoji v13 (2020) (#1555)

.github/workflows/test.yml

@@ -72,8 +72,8 @@ jobs:
       - name: Build (Ubuntu)
         if: startsWith(matrix.os, 'ubuntu')
         run: |
-            cmake -DBUILD_TESTS=On ..
+            cmake -DBUILD_TESTS=On -DCMAKE_BUILD_TYPE=Release ..
-            cmake --build .
+            cmake --build . --config Release
         working-directory: build-test
         shell: bash
 

CHANGELOG.md

@@ -51,6 +51,8 @@
 - Minor: Properly respect RECONNECT messages from Twitch (#2347)
 - Minor: Hide "Case-sensitive" column for user highlights. (#2404)
 - Minor: Added human-readable formatting to remaining timeout duration. (#2398)
+- Minor: Update emojis version to 13 (2020). (#1555)
+- Minor: Remove EmojiOne 2 and 3 due to license restrictions. (#1555)
 - Minor: Added `/streamlink` command. Usage: `/streamlink <channel>`. You can also use the command without arguments in any twitch channel to open it in streamlink. (#2443)
 - Bugfix: Fix crash occurring when pressing Escape in the Color Picker Dialog (#1843)
 - Bugfix: Fix bug where the "check user follow state" event could trigger a network request requesting the user to follow or unfollow a user. By itself its quite harmless as it just repeats to Twitch the same follow state we had, so no follows should have been lost by this but it meant there was a rogue network request that was fired that could cause a crash (#1906)

CMakeLists.txt

@@ -21,6 +21,17 @@ set(chatterino_SOURCES
 
     src/common/UsernameSet.cpp
     src/controllers/highlights/HighlightPhrase.cpp
+
+
+    src/providers/emoji/Emojis.cpp
+
+    src/messages/Emote.cpp
+    src/messages/Image.cpp
+    src/messages/ImageSet.cpp
+    src/util/RapidjsonHelpers.cpp
+
+    ${CMAKE_CURRENT_LIST_DIR}/resources/resources.qrc
+    ${CMAKE_CURRENT_LIST_DIR}/resources/resources_autogenerated.qrc
     )
 
 find_package(Qt5 5.9.0 REQUIRED COMPONENTS
@@ -28,8 +39,7 @@ find_package(Qt5 5.9.0 REQUIRED COMPONENTS
     )
 
 set(CMAKE_AUTOMOC ON)
-
+set(CMAKE_AUTORCC ON)
-# set(CMAKE_AUTOMOC ON)
 
 if (BUILD_TESTS)
     set(BUILD_TESTS OFF)
@@ -41,12 +51,13 @@ if (BUILD_TESTS)
         ${chatterino_SOURCES}
 
         tests/src/main.cpp
+        tests/src/Emojis.cpp
         tests/src/NetworkRequest.cpp
         tests/src/UsernameSet.cpp
         tests/src/HighlightPhrase.cpp
         )
 
-    target_compile_definitions(chatterino-test PRIVATE CHATTERINO_GIT_HASH="test" AB_CUSTOM_SETTINGS)
+    target_compile_definitions(chatterino-test PRIVATE CHATTERINO_GIT_HASH="test" AB_CUSTOM_SETTINGS CHATTERINO_TEST)
 
     target_link_libraries(chatterino-test Qt5::Core Qt5::Widgets Qt5::Network Qt5::Concurrent)
 

resources/resources_autogenerated.qrc

@@ -41,7 +41,6 @@
     <file>com.chatterino.chatterino.desktop</file>
     <file>contributors.txt</file>
     <file>emoji.json</file>
-    <file>emojidata.txt</file>
     <file>error.png</file>
     <file>examples/moving.gif</file>
     <file>examples/splitting.gif</file>

src/messages/Image.cpp

@@ -15,11 +15,16 @@
 #include "common/QLogging.hpp"
 #include "debug/AssertInGuiThread.hpp"
 #include "debug/Benchmark.hpp"
-#include "singletons/Emotes.hpp"
+#ifndef CHATTERINO_TEST
+#    include "singletons/Emotes.hpp"
+#endif
 #include "singletons/WindowManager.hpp"
+#include "singletons/helper/GifTimer.hpp"
 #include "util/DebugCount.hpp"
 #include "util/PostToThread.hpp"
 
+#include <queue>
+
 namespace chatterino {
 namespace detail {
     // Frames
@@ -38,10 +43,12 @@ namespace detail {
         {
             DebugCount::increase("animated images");
 
+#ifndef CHATTERINO_TEST
             this->gifTimerConnection_ =
                 getApp()->emotes->gifTimer.signal.connect([this] {
                     this->advance();
                 });
+#endif
         }
 
         auto totalLength =
@@ -56,9 +63,11 @@ namespace detail {
         }
         else
         {
+#ifndef CHATTERINO_TEST
             this->durationOffset_ = std::min<int>(
                 int(getApp()->emotes->gifTimer.position() % totalLength),
                 60000);
+#endif
         }
         this->processOffset();
     }
@@ -182,7 +191,9 @@ namespace detail {
             }
         }
 
+#ifndef CHATTERINO_TEST
         getApp()->windows->forceLayoutChannelViews();
+#endif
         loadedEventQueued = false;
     }
 
@@ -225,6 +236,13 @@ namespace detail {
 // IMAGE2
 Image::~Image()
 {
+    if (this->empty_)
+    {
+        // No data in this image, don't bother trying to release it
+        // The reason we do this check is that we keep a few (or one) static empty image around that are deconstructed at the end of the programs lifecycle, and we want to prevent the isGuiThread call to be called after the QApplication has been exited
+        return;
+    }
+
     // run destructor of Frames in gui thread
     if (!isGuiThread())
     {

src/messages/ImageSet.cpp

@@ -60,7 +60,9 @@ const ImagePtr &ImageSet::getImage3() const
 
 const std::shared_ptr<Image> &getImagePriv(const ImageSet &set, float scale)
 {
+#ifndef CHATTERINO_TEST
     scale *= getSettings()->emoteScale;
+#endif
 
     int quality = 1;
 

src/providers/emoji/Emojis.cpp

@@ -14,6 +14,12 @@
 
 namespace chatterino {
 namespace {
+
+    auto toneNames = std::map<QString, QString>{
+        {"1F3FB", "tone1"}, {"1F3FC", "tone2"}, {"1F3FD", "tone3"},
+        {"1F3FE", "tone4"}, {"1F3FF", "tone5"},
+    };
+
     void parseEmoji(const std::shared_ptr<EmojiData> &emojiData,
                     const rapidjson::Value &unparsedEmoji,
                     QString shortCode = QString())
@@ -24,9 +30,7 @@ namespace {
             bool apple;
             bool google;
             bool twitter;
-            bool emojione;
             bool facebook;
-            bool messenger;
         } capabilities;
 
         if (!shortCode.isEmpty())
@@ -49,9 +53,7 @@ namespace {
         rj::getSafe(unparsedEmoji, "has_img_apple", capabilities.apple);
         rj::getSafe(unparsedEmoji, "has_img_google", capabilities.google);
         rj::getSafe(unparsedEmoji, "has_img_twitter", capabilities.twitter);
-        rj::getSafe(unparsedEmoji, "has_img_emojione", capabilities.emojione);
         rj::getSafe(unparsedEmoji, "has_img_facebook", capabilities.facebook);
-        rj::getSafe(unparsedEmoji, "has_img_messenger", capabilities.messenger);
 
         if (capabilities.apple)
         {
@@ -65,18 +67,10 @@ namespace {
         {
             emojiData->capabilities.insert("Twitter");
         }
-        if (capabilities.emojione)
-        {
-            emojiData->capabilities.insert("EmojiOne 3");
-        }
         if (capabilities.facebook)
         {
             emojiData->capabilities.insert("Facebook");
         }
-        if (capabilities.messenger)
-        {
-            emojiData->capabilities.insert("Messenger");
-        }
 
         QStringList unicodeCharacters;
         if (!emojiData->nonQualifiedCode.isEmpty())
@@ -103,14 +97,38 @@ namespace {
 
         emojiData->value = QString::fromUcs4(unicodeBytes, numUnicodeBytes);
     }
+
+    // getToneNames takes a tones and returns their names in the same order
+    // The format of the tones is: "1F3FB-1F3FB" or "1F3FB"
+    // The output of the tone names is: "tone1-tone1" or "tone1"
+    QString getToneNames(const QString &tones)
+    {
+        auto toneParts = tones.split('-');
+        QStringList toneNameResults;
+        for (const auto &tonePart : toneParts)
+        {
+            auto toneNameIt = toneNames.find(tonePart);
+            if (toneNameIt == toneNames.end())
+            {
+                qDebug() << "Tone with key" << tonePart
+                         << "does not exist in tone names map";
+                continue;
+            }
+
+            toneNameResults.append(toneNameIt->second);
+        }
+
+        assert(!toneNameResults.isEmpty());
+
+        return toneNameResults.join('-');
+    }
+
 }  // namespace
 
 void Emojis::load()
 {
     this->loadEmojis();
 
-    this->loadEmojiOne2Capabilities();
-
     this->sortEmojis();
 
     this->loadEmojiSet();
@@ -118,11 +136,7 @@ void Emojis::load()
 
 void Emojis::loadEmojis()
 {
-    auto toneNames = std::map<std::string, QString>{
+    // Current version: https://github.com/iamcal/emoji-data/blob/v6.0.0/emoji.json (Emoji version 13 (2020))
-        {"1F3FB", "tone1"}, {"1F3FC", "tone2"}, {"1F3FD", "tone3"},
-        {"1F3FE", "tone4"}, {"1F3FF", "tone5"},
-    };
-
     QFile file(":/emoji.json");
     file.open(QFile::ReadOnly);
     QTextStream s1(&file);
@@ -158,22 +172,13 @@ void Emojis::loadEmojis()
             for (const auto &skinVariation :
                  unparsedEmoji["skin_variations"].GetObject())
             {
-                std::string tone = skinVariation.name.GetString();
+                auto toneName = getToneNames(skinVariation.name.GetString());
                 const auto &variation = skinVariation.value;
 
                 auto variationEmojiData = std::make_shared<EmojiData>();
 
-                auto toneNameIt = toneNames.find(tone);
-                if (toneNameIt == toneNames.end())
-                {
-                    qCDebug(chatterinoEmoji)
-                        << "Tone with key" << tone.c_str()
-                        << "does not exist in tone names map";
-                    continue;
-                }
-
                 parseEmoji(variationEmojiData, variation,
-                           emojiData->shortCodes[0] + "_" + toneNameIt->second);
+                           emojiData->shortCodes[0] + "_" + toneName);
 
                 this->emojiShortCodeToEmoji_.insert(
                     variationEmojiData->shortCodes[0], variationEmojiData);
@@ -189,41 +194,6 @@ void Emojis::loadEmojis()
     }
 }
 
-void Emojis::loadEmojiOne2Capabilities()
-{
-    QFile file(":/emojidata.txt");
-    file.open(QFile::ReadOnly);
-    QTextStream in(&file);
-
-    while (!in.atEnd())
-    {
-        // Line example: sunglasses 1f60e
-        QString line = in.readLine();
-
-        if (line.at(0) == '#')
-        {
-            // Ignore lines starting with # (comments)
-            continue;
-        }
-
-        QStringList parts = line.split(' ');
-        if (parts.length() < 2)
-        {
-            continue;
-        }
-
-        QString shortCode = parts[0];
-
-        auto emojiIt = this->emojiShortCodeToEmoji_.find(shortCode);
-        if (emojiIt != this->emojiShortCodeToEmoji_.end())
-        {
-            std::shared_ptr<EmojiData> emoji = *emojiIt;
-            emoji->capabilities.insert("EmojiOne 2");
-            continue;
-        }
-    }
-}
-
 void Emojis::sortEmojis()
 {
     for (auto &p : this->emojiFirstByte_)
@@ -242,45 +212,45 @@ void Emojis::sortEmojis()
 
 void Emojis::loadEmojiSet()
 {
+#ifndef CHATTERINO_TEST
     getSettings()->emojiSet.connect([=](const auto &emojiSet) {
+#else
+    const QString emojiSet = "twitter";
+#endif
         this->emojis.each([=](const auto &name,
                               std::shared_ptr<EmojiData> &emoji) {
             QString emojiSetToUse = emojiSet;
             // clang-format off
             static std::map<QString, QString> emojiSets = {
-                {"EmojiOne 2", "https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.6/assets/png/"},
+                // JSDELIVR
-                // {"EmojiOne 3", "https://cdn.jsdelivr.net/npm/emoji-datasource-emojione@4.0.4/img/emojione/64/"},
                 // {"Twitter", "https://cdn.jsdelivr.net/npm/emoji-datasource-twitter@4.0.4/img/twitter/64/"},
                 // {"Facebook", "https://cdn.jsdelivr.net/npm/emoji-datasource-facebook@4.0.4/img/facebook/64/"},
-                // {"Apple", "https://cdn.jsdelivr.net/npm/emoji-datasource-apple@4.0.4/img/apple/64/"},
+                // {"Apple", "https://cdn.jsdelivr.net/npm/emoji-datasource-apple@5.0.1/img/apple/64/"},
                 // {"Google", "https://cdn.jsdelivr.net/npm/emoji-datasource-google@4.0.4/img/google/64/"},
                 // {"Messenger", "https://cdn.jsdelivr.net/npm/emoji-datasource-messenger@4.0.4/img/messenger/64/"},
 
-                {"EmojiOne 3", "https://pajbot.com/static/emoji/img/emojione/64/"},
+                // OBRODAI
-                {"Twitter", "https://pajbot.com/static/emoji/img/twitter/64/"},
+                {"Twitter", "https://pajbot.com/static/emoji-v2/img/twitter/64/"},
-                {"Facebook", "https://pajbot.com/static/emoji/img/facebook/64/"},
+                {"Facebook", "https://pajbot.com/static/emoji-v2/img/facebook/64/"},
-                {"Apple", "https://pajbot.com/static/emoji/img/apple/64/"},
+                {"Apple", "https://pajbot.com/static/emoji-v2/img/apple/64/"},
-                {"Google", "https://pajbot.com/static/emoji/img/google/64/"},
+                {"Google", "https://pajbot.com/static/emoji-v2/img/google/64/"},
-                {"Messenger", "https://pajbot.com/static/emoji/img/messenger/64/"},
+
+                // Cloudflare+B2 bucket
+                // {"Twitter", "https://chatterino2-emoji-cdn.pajlada.se/file/c2-emojis/emojis-v1/twitter/64/"},
+                // {"Facebook", "https://chatterino2-emoji-cdn.pajlada.se/file/c2-emojis/emojis-v1/facebook/64/"},
+                // {"Apple", "https://chatterino2-emoji-cdn.pajlada.se/file/c2-emojis/emojis-v1/apple/64/"},
+                // {"Google", "https://chatterino2-emoji-cdn.pajlada.se/file/c2-emojis/emojis-v1/google/64/"},
             };
             // clang-format on
 
             if (emoji->capabilities.count(emojiSetToUse) == 0)
             {
-                emojiSetToUse = "EmojiOne 3";
+                emojiSetToUse = "Twitter";
             }
 
-            QString code = emoji->unifiedCode;
+            QString code = emoji->unifiedCode.toLower();
-            if (emojiSetToUse == "EmojiOne 2")
+            QString urlPrefix =
-            {
+                "https://pajbot.com/static/emoji-v2/img/twitter/64/";
-                if (!emoji->nonQualifiedCode.isEmpty())
-                {
-                    code = emoji->nonQualifiedCode;
-                }
-            }
-            code = code.toLower();
-            QString urlPrefix = "https://cdnjs.cloudflare.com/ajax/libs/"
-                                "emojione/2.2.6/assets/png/";
             auto it = emojiSets.find(emojiSetToUse);
             if (it != emojiSets.end())
             {
@@ -291,7 +261,9 @@ void Emojis::loadEmojiSet()
                 EmoteName{emoji->value}, ImageSet{Image::fromUrl({url}, 0.35)},
                 Tooltip{":" + emoji->shortCodes[0] + ":<br/>Emoji"}, Url{}});
         });
+#ifndef CHATTERINO_TEST
     });
+#endif
 }
 
 std::vector<boost::variant<EmotePtr, QString>> Emojis::parse(

src/providers/emoji/Emojis.hpp

@@ -48,7 +48,6 @@ public:
 
 private:
     void loadEmojis();
-    void loadEmojiOne2Capabilities();
     void sortEmojis();
     void loadEmojiSet();
 

src/singletons/Settings.hpp

@@ -171,7 +171,7 @@ public:
     BoolSetting animateEmotes = {"/emotes/enableGifAnimations", true};
     FloatSetting emoteScale = {"/emotes/scale", 1.f};
 
-    QStringSetting emojiSet = {"/emotes/emojiSet", "EmojiOne 2"};
+    QStringSetting emojiSet = {"/emotes/emojiSet", "Twitter"};
 
     BoolSetting stackBits = {"/emotes/stackBits", false};
 

src/widgets/settingspages/AboutPage.cpp

@@ -131,12 +131,10 @@ AboutPage::AboutPage()
             auto l = attributions.emplace<QVBoxLayout>();
 
             // clang-format off
-            l.emplace<QLabel>("EmojiOne 2 and 3 emojis provided by <a href=\"https://www.emojione.com/\">EmojiOne</a>")->setOpenExternalLinks(true);
             l.emplace<QLabel>("Twemoji emojis provided by <a href=\"https://github.com/twitter/twemoji\">Twitter's Twemoji</a>")->setOpenExternalLinks(true);
             l.emplace<QLabel>("Facebook emojis provided by <a href=\"https://facebook.com\">Facebook</a>")->setOpenExternalLinks(true);
             l.emplace<QLabel>("Apple emojis provided by <a href=\"https://apple.com\">Apple</a>")->setOpenExternalLinks(true);
             l.emplace<QLabel>("Google emojis provided by <a href=\"https://google.com\">Google</a>")->setOpenExternalLinks(true);
-            l.emplace<QLabel>("Messenger emojis provided by <a href=\"https://facebook.com\">Facebook</a>")->setOpenExternalLinks(true);
             l.emplace<QLabel>("Emoji datasource provided by <a href=\"https://www.iamcal.com/\">Cal Henderson</a>"
                               "(<a href=\"https://github.com/iamcal/emoji-data/blob/master/LICENSE\">show license</a>)")->setOpenExternalLinks(true);
             l.emplace<QLabel>("Twitch emote data provided by <a href=\"https://twitchemotes.com/\">twitchemotes.com</a> through the <a href=\"https://github.com/Chatterino/api\">Chatterino API</a>")->setOpenExternalLinks(true);

src/widgets/settingspages/GeneralPage.cpp

@@ -315,8 +315,12 @@ void GeneralPage::initLayout(GeneralPageView &layout)
         },
         false);
     layout.addDropdown("Emoji style",
-                       {"EmojiOne 2", "EmojiOne 3", "Twitter", "Facebook",
+                       {
-                        "Apple", "Google", "Messenger"},
+                           "Twitter",
+                           "Facebook",
+                           "Apple",
+                           "Google",
+                       },
                        s.emojiSet);
 
     layout.addTitle("Streamer Mode");

tests/src/Emojis.cpp

@@ -0,0 +1,49 @@
+#include "providers/emoji/Emojis.hpp"
+
+#include <gtest/gtest.h>
+#include <QDebug>
+#include <QString>
+
+using namespace chatterino;
+
+TEST(Emojis, ShortcodeParsing)
+{
+    Emojis emojis;
+
+    emojis.load();
+
+    struct TestCase {
+        QString input;
+        QString expectedOutput;
+    };
+
+    std::vector<TestCase> tests{
+        {
+            .input = "foo :penguin: bar",
+            .expectedOutput = "foo 🐧 bar",
+        },
+        {
+            .input = "foo :nonexistantcode: bar",
+            .expectedOutput = "foo :nonexistantcode: bar",
+        },
+        {
+            .input = ":male-doctor:",
+            .expectedOutput = "👨‍⚕️",
+        },
+    };
+
+    for (const auto &test : tests)
+    {
+        auto output = emojis.replaceShortCodes(test.input);
+
+        auto matches = output == test.expectedOutput;
+        if (!matches && !output.endsWith(QChar(0xFE0F)))
+        {
+            // Try to append 0xFE0F if needed
+            output = output.append(QChar(0xFE0F));
+        }
+
+        EXPECT_EQ(output, test.expectedOutput)
+            << "Input " << test.input.toStdString() << " failed";
+    }
+}

tests/src/main.cpp

@@ -22,33 +22,6 @@
 
 using namespace chatterino;
 
-void xd()
-{
-    std::mutex mut;
-    bool requestDone = false;
-    std::condition_variable requestDoneCondition;
-
-    EXPECT_TRUE(NetworkManager::workerThread.isRunning());
-
-    using namespace std::chrono_literals;
-
-    auto url = "http://localhost:8000/status/200";
-    NetworkRequest(url)
-        .onSuccess([&](NetworkResult result) -> Outcome {
-            qDebug() << "ON SUCCESS";
-            qDebug() << url;
-            return Success;
-        })
-        .onError([&](NetworkResult result) {
-            qDebug() << "ON ERROR";
-        })
-        .execute();
-
-    EXPECT_TRUE(NetworkManager::workerThread.isRunning());
-
-    EXPECT_TRUE(true);
-}
-
 int main(int argc, char **argv)
 {
     ::testing::InitGoogleTest(&argc, argv);