Experimental emoji sets implemented

Fixes #376

resources/resources.qrc

@@ -60,6 +60,7 @@
         <file>licenses/qt_lgpl-3.0.txt</file>
         <file>licenses/rapidjson.txt</file>
         <file>licenses/websocketpp.txt</file>
+        <file>emoji.json</file>
     </qresource>
     <qresource prefix="/qt/etc">
         <file>qt.conf</file>

src/providers/emoji/emojis.cpp

@@ -1,10 +1,197 @@
 #include "providers/emoji/emojis.hpp"
 
+#include "application.hpp"
+#include "debug/log.hpp"
+#include "singletons/settingsmanager.hpp"
+
 namespace chatterino {
 namespace providers {
 namespace emoji {
 
+namespace {
+
+void parseEmoji(const std::shared_ptr<EmojiData> &emojiData, const rapidjson::Value &unparsedEmoji,
+                QString shortCode = QString())
+{
+    static uint unicodeBytes[4];
+
+    struct {
+        bool apple;
+        bool google;
+        bool twitter;
+        bool emojione;
+        bool facebook;
+        bool messenger;
+    } capabilities;
+
+    if (!shortCode.isEmpty()) {
+        emojiData->shortCode = shortCode;
+    } else if (!rj::getSafe(unparsedEmoji, "short_name", emojiData->shortCode)) {
+        return;
+    }
+    rj::getSafe(unparsedEmoji, "non_qualified", emojiData->nonQualifiedCode);
+    rj::getSafe(unparsedEmoji, "unified", emojiData->unifiedCode);
+
+    rj::getSafe(unparsedEmoji, "has_img_apple", capabilities.apple);
+    rj::getSafe(unparsedEmoji, "has_img_google", capabilities.google);
+    rj::getSafe(unparsedEmoji, "has_img_twitter", capabilities.twitter);
+    rj::getSafe(unparsedEmoji, "has_img_emojione", capabilities.emojione);
+    rj::getSafe(unparsedEmoji, "has_img_facebook", capabilities.facebook);
+    rj::getSafe(unparsedEmoji, "has_img_messenger", capabilities.messenger);
+
+    if (capabilities.apple) {
+        emojiData->capabilities.insert("Apple");
+    }
+    if (capabilities.google) {
+        emojiData->capabilities.insert("Google");
+    }
+    if (capabilities.twitter) {
+        emojiData->capabilities.insert("Twitter");
+    }
+    if (capabilities.emojione) {
+        emojiData->capabilities.insert("EmojiOne 3");
+    }
+    if (capabilities.facebook) {
+        emojiData->capabilities.insert("Facebook");
+    }
+    if (capabilities.messenger) {
+        emojiData->capabilities.insert("Messenger");
+    }
+
+    QStringList unicodeCharacters;
+    if (!emojiData->nonQualifiedCode.isEmpty()) {
+        unicodeCharacters = emojiData->nonQualifiedCode.toLower().split('-');
+    } else {
+        unicodeCharacters = emojiData->unifiedCode.toLower().split('-');
+    }
+    if (unicodeCharacters.length() < 1) {
+        return;
+    }
+
+    int numUnicodeBytes = 0;
+
+    for (const QString &unicodeCharacter : unicodeCharacters) {
+        unicodeBytes[numUnicodeBytes++] = QString(unicodeCharacter).toUInt(nullptr, 16);
+    }
+
+    emojiData->value = QString::fromUcs4(unicodeBytes, numUnicodeBytes);
+}
+
+}  // namespace
+
 void Emojis::load()
+{
+    this->loadEmojis();
+
+    this->loadEmojiOne2Capabilities();
+}
+
+void Emojis::loadEmojis()
+{
+    QFile file(":/emoji.json");
+    file.open(QFile::ReadOnly);
+    QTextStream s1(&file);
+    QString data = s1.readAll();
+    rapidjson::Document root;
+    rapidjson::ParseResult result = root.Parse(data.toUtf8(), data.length());
+
+    if (result.Code() != rapidjson::kParseErrorNone) {
+        debug::Log("JSON parse error: {} ({})", rapidjson::GetParseError_En(result.Code()),
+                   result.Offset());
+        return;
+    }
+
+    for (const auto &unparsedEmoji : root.GetArray()) {
+        auto emojiData = std::make_shared<EmojiData>();
+        parseEmoji(emojiData, unparsedEmoji);
+
+        this->emojiShortCodeToEmoji.insert(emojiData->shortCode, emojiData);
+        this->shortCodes.push_back(emojiData->shortCode.toStdString());
+
+        this->emojiFirstByte[emojiData->value.at(0)].append(emojiData);
+
+        this->emojis.insert(emojiData->unifiedCode, emojiData);
+
+        if (unparsedEmoji.HasMember("skin_variations")) {
+            for (const auto &skinVariation : unparsedEmoji["skin_variations"].GetObject()) {
+                const auto &tone = skinVariation.name.GetString();
+                const auto &variation = skinVariation.value;
+
+                static std::map<QString, QString> toneNames{
+                    {"1F3FB", "tone1"},  //
+                    {"1F3FC", "tone2"},  //
+                    {"1F3FD", "tone3"},  //
+                    {"1F3FE", "tone4"},  //
+                    {"1F3FF", "tone5"},  //
+                };
+
+                auto variationEmojiData = std::make_shared<EmojiData>();
+
+                parseEmoji(variationEmojiData, variation,
+                           emojiData->shortCode + "_" + toneNames[tone]);
+
+                this->emojiShortCodeToEmoji.insert(variationEmojiData->shortCode,
+                                                   variationEmojiData);
+                this->shortCodes.push_back(variationEmojiData->shortCode.toStdString());
+
+                this->emojiFirstByte[variationEmojiData->value.at(0)].append(variationEmojiData);
+
+                this->emojis.insert(variationEmojiData->unifiedCode, variationEmojiData);
+            }
+        }
+    }
+
+    for (auto &p : this->emojiFirstByte) {
+        std::stable_sort(p.begin(), p.end(), [](const auto &lhs, const auto &rhs) {
+            return lhs->value.length() > rhs->value.length();
+        });
+    }
+
+    auto &p = this->shortCodes;
+    std::stable_sort(p.begin(), p.end(),
+                     [](const auto &lhs, const auto &rhs) { return lhs < rhs; });
+
+    auto app = getApp();
+
+    app->settings->emojiSet.connect([=](const auto &emojiSet, auto) {
+        this->emojis.each([=](const auto &name, std::shared_ptr<EmojiData> &emoji) {
+            QString emojiSetToUse = emojiSet;
+            // clang-format off
+            static std::map<QString, QString> emojiSets = {
+                {"EmojiOne 2", "https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.6/assets/png/"},
+                {"EmojiOne 3", "https://cdn.jsdelivr.net/npm/emoji-datasource-emojione@4.0.4/img/emojione/64/"},
+                {"Twitter", "https://cdn.jsdelivr.net/npm/emoji-datasource-twitter@4.0.4/img/twitter/64/"},
+                {"Facebook", "https://cdn.jsdelivr.net/npm/emoji-datasource-facebook@4.0.4/img/facebook/64/"},
+                {"Apple", "https://cdn.jsdelivr.net/npm/emoji-datasource-apple@4.0.4/img/apple/64/"},
+                {"Google", "https://cdn.jsdelivr.net/npm/emoji-datasource-google@4.0.4/img/google/64/"},
+                {"Messenger", "https://cdn.jsdelivr.net/npm/emoji-datasource-messenger@4.0.4/img/messenger/64/"},
+            };
+            // clang-format on
+
+            if (emoji->capabilities.count(emojiSetToUse) == 0) {
+                emojiSetToUse = "EmojiOne 3";
+            }
+
+            QString code = emoji->unifiedCode;
+            if (emojiSetToUse == "EmojiOne 2") {
+                if (!emoji->nonQualifiedCode.isEmpty()) {
+                    code = emoji->nonQualifiedCode;
+                }
+            }
+            code = code.toLower();
+            QString urlPrefix = "https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.6/assets/png/";
+            auto it = emojiSets.find(emojiSetToUse);
+            if (it != emojiSets.end()) {
+                urlPrefix = it->second;
+            }
+            QString url = urlPrefix + code + ".png";
+            emoji->emoteData.image1x = new messages::Image(url, 0.35, emoji->value,
+                                                           ":" + emoji->shortCode + ":<br/>Emoji");
+        });
+    });
+}
+
+void Emojis::loadEmojiOne2Capabilities()
 {
     QFile file(":/emojidata.txt");
     file.open(QFile::ReadOnly);
@@ -27,44 +214,13 @@ void Emojis::load()
         }
 
         QString shortCode = parts[0];
-        QString code = parts[1];
 
-        QStringList unicodeCharacters = code.split('-');
+        auto emojiIt = this->emojiShortCodeToEmoji.find(shortCode);
-        if (unicodeCharacters.length() < 1) {
+        if (emojiIt != this->emojiShortCodeToEmoji.end()) {
+            std::shared_ptr<EmojiData> emoji = *emojiIt;
+            emoji->capabilities.insert("EmojiOne 2");
             continue;
         }
-
-        int numUnicodeBytes = 0;
-
-        for (const QString &unicodeCharacter : unicodeCharacters) {
-            unicodeBytes[numUnicodeBytes++] = QString(unicodeCharacter).toUInt(nullptr, 16);
-        }
-
-        QString unicodeString = QString::fromUcs4(unicodeBytes, numUnicodeBytes);
-
-        QString url = "https://cdnjs.cloudflare.com/ajax/libs/"
-                      "emojione/2.2.6/assets/png/" +
-                      code + ".png";
-
-        EmojiData emojiData{
-            unicodeString,  //
-            code,           //
-            shortCode,      //
-            {new messages::Image(url, 0.35, unicodeString, ":" + shortCode + ":<br/>Emoji")},
-        };
-
-        this->emojiShortCodeToEmoji.insert(shortCode, emojiData);
-        this->shortCodes.push_back(shortCode.toStdString());
-
-        this->emojiFirstByte[emojiData.value.at(0)].append(emojiData);
-
-        this->emojis.insert(code, emojiData);
-    }
-
-    for (auto &p : this->emojiFirstByte) {
-        std::stable_sort(p.begin(), p.end(), [](const auto &lhs, const auto &rhs) {
-            return lhs.value.length() > rhs.value.length();
-        });
     }
 }
 
@@ -86,16 +242,16 @@ void Emojis::parse(std::vector<std::tuple<util::EmoteData, QString>> &parsedWord
             continue;
         }
 
-        const QVector<EmojiData> possibleEmojis = it.value();
+        const auto &possibleEmojis = it.value();
 
         int remainingCharacters = text.length() - i - 1;
 
-        EmojiData matchedEmoji;
+        std::shared_ptr<EmojiData> matchedEmoji;
 
         int matchedEmojiLength = 0;
 
-        for (const EmojiData &emoji : possibleEmojis) {
+        for (const std::shared_ptr<EmojiData> &emoji : possibleEmojis) {
-            int emojiExtraCharacters = emoji.value.length() - 1;
+            int emojiExtraCharacters = emoji->value.length() - 1;
             if (emojiExtraCharacters > remainingCharacters) {
                 // It cannot be this emoji, there's not enough space for it
                 continue;
@@ -103,8 +259,8 @@ void Emojis::parse(std::vector<std::tuple<util::EmoteData, QString>> &parsedWord
 
             bool match = true;
 
-            for (int j = 1; j < emoji.value.length(); ++j) {
+            for (int j = 1; j < emoji->value.length(); ++j) {
-                if (text.at(i + j) != emoji.value.at(j)) {
+                if (text.at(i + j) != emoji->value.at(j)) {
                     match = false;
 
                     break;
@@ -113,7 +269,7 @@ void Emojis::parse(std::vector<std::tuple<util::EmoteData, QString>> &parsedWord
 
             if (match) {
                 matchedEmoji = emoji;
-                matchedEmojiLength = emoji.value.length();
+                matchedEmojiLength = emoji->value.length();
 
                 break;
             }
@@ -136,7 +292,7 @@ void Emojis::parse(std::vector<std::tuple<util::EmoteData, QString>> &parsedWord
 
         // Push the emoji as a word to parsedWords
         parsedWords.push_back(
-            std::tuple<util::EmoteData, QString>(matchedEmoji.emoteData, QString()));
+            std::tuple<util::EmoteData, QString>(matchedEmoji->emoteData, QString()));
 
         lastParsedEmojiEndIndex = currentParsedEmojiEndIndex;
 
@@ -171,9 +327,9 @@ QString Emojis::replaceShortCodes(const QString &text)
 
         auto emojiData = emojiIt.value();
 
-        ret.replace(offset + match.capturedStart(), match.capturedLength(), emojiData.value);
+        ret.replace(offset + match.capturedStart(), match.capturedLength(), emojiData->value);
 
-        offset += emojiData.value.size() - match.capturedLength();
+        offset += emojiData->value.size() - match.capturedLength();
     }
 
     return ret;

src/providers/emoji/emojis.hpp

@@ -17,25 +17,38 @@ struct EmojiData {
     // actual byte-representation of the emoji (i.e. \154075\156150 which is :male:)
     QString value;
 
-    // what's used in the emoji-one url
+    // i.e. 204e-50a2
-    QString code;
+    QString unifiedCode;
+    QString nonQualifiedCode;
 
     // i.e. thinking
     QString shortCode;
 
+    std::set<QString> capabilities;
+
+    std::vector<EmojiData> variations;
+
     util::EmoteData emoteData;
 };
 
-using EmojiMap = util::ConcurrentMap<QString, EmojiData>;
+using EmojiMap = util::ConcurrentMap<QString, std::shared_ptr<EmojiData>>;
 
 class Emojis
 {
 public:
+    void initialize();
+
     EmojiMap emojis;
 
     std::vector<std::string> shortCodes;
 
     void load();
+
+private:
+    void loadEmojis();
+    void loadEmojiOne2Capabilities();
+
+public:
     QString replaceShortCodes(const QString &text);
 
     void parse(std::vector<std::tuple<util::EmoteData, QString>> &parsedWords, const QString &text);
@@ -45,10 +58,10 @@ private:
     QRegularExpression findShortCodesRegex{":([-+\\w]+):"};
 
     // shortCodeToEmoji maps strings like "sunglasses" to its emoji
-    QMap<QString, EmojiData> emojiShortCodeToEmoji;
+    QMap<QString, std::shared_ptr<EmojiData>> emojiShortCodeToEmoji;
 
     // Maps the first character of the emoji unicode string to a vector of possible emojis
-    QMap<QChar, QVector<EmojiData>> emojiFirstByte;
+    QMap<QChar, QVector<std::shared_ptr<EmojiData>>> emojiFirstByte;
 };
 
 }  // namespace emoji

src/singletons/settingsmanager.hpp

@@ -85,6 +85,8 @@ public:
     // 3 = 3x
     IntSetting preferredEmoteQuality = {"/emotes/preferredEmoteQuality", 0};
 
+    QStringSetting emojiSet = {"/emotes/emojiSet", "EmojiOne 2"};
+
     /// Links
     BoolSetting linksDoubleClickOnly = {"/links/doubleClickToOpen", false};
 

src/widgets/emotepopup.cpp

@@ -144,8 +144,8 @@ void EmotePopup::loadEmojis()
     builder.getMessage()->flags |= Message::DisableCompactEmotes;
 
     emojis.each([&builder](const QString &key, const auto &value) {
-        builder.append((new EmoteElement(value.emoteData, MessageElement::Flags::AlwaysShow))
+        builder.append((new EmoteElement(value->emoteData, MessageElement::Flags::AlwaysShow))
-                           ->setLink(Link(Link::Type::InsertText, ":" + value.shortCode + ":")));
+                           ->setLink(Link(Link::Type::InsertText, ":" + value->shortCode + ":")));
     });
     emojiChannel->addMessage(builder.getMessage());
 

src/widgets/settingspages/aboutpage.cpp

@@ -82,6 +82,21 @@ AboutPage::AboutPage()
             addLicense(*form, "Websocketpp", "https://www.zaphoyd.com/websocketpp/",
                        ":/licenses/websocketpp.txt");
         }
+
+        auto attributions = layout.emplace<QGroupBox>("Attributions...");
+        {
+            auto l = attributions.emplace<QVBoxLayout>();
+
+            // clang-format off
+            l.emplace<QLabel>("EmojiOne 2 and 3 emojis provided by <a href=\"https://www.emojione.com/\">EmojiOne</a>");
+            l.emplace<QLabel>("Twemoji emojis provided by <a href=\"https://github.com/twitter/twemoji\">Twitter's Twemoji</a>");
+            l.emplace<QLabel>("Facebook emojis provided by <a href=\"https://facebook.com\">Facebook</a>");
+            l.emplace<QLabel>("Apple emojis provided by <a href=\"https://apple.com\">Apple</a>");
+            l.emplace<QLabel>("Google emojis provided by <a href=\"https://google.com\">Google</a>");
+            l.emplace<QLabel>("Messenger emojis provided by <a href=\"https://facebook.com\">Facebook</a>");
+            l.emplace<QLabel>("Emoji datasource provided by <a href=\"https://www.iamcal.com/\">Cal Henderson</a>");
+            // clang-format on
+        }
     }
 
     layout->addStretch(1);

src/widgets/settingspages/appearancepage.cpp

@@ -142,10 +142,24 @@ AppearancePage::AppearancePage()
 
             scaleLabel->setText(QString::number(app->settings->emoteScale.getValue()));
         }
+
+        {
+            auto *combo = new QComboBox(this);
+            combo->addItems({"EmojiOne 2", "EmojiOne 3", "Twitter", "Facebook", "Apple", "Google",
+                             "Messenger"});
+
+            QObject::connect(combo, &QComboBox::currentTextChanged, [](const QString &str) {
+                getApp()->settings->emojiSet = str;  //
+            });
+
+            auto hbox = emotes.emplace<QHBoxLayout>().withoutMargin();
+            hbox.emplace<QLabel>("Emoji set");
+            hbox.append(combo);
+        }
     }
 
     layout->addStretch(1);
-}  // namespace settingspages
+}
 
 QLayout *AppearancePage::createThemeColorChanger()
 {

tools/update-emoji-data.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+wget https://raw.githubusercontent.com/iamcal/emoji-data/master/emoji.json -O ../resources/emoji.json