Add cached emotes fallback for failed provider fetches (#6125)

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unversioned
 
+- Minor: Added cached emotes fallback when fetching from a provider fails. (#6125)
 - Minor: Add an option for the reduced opacity of message history. (#6121)
 - Minor: Make paused chat indicator more visible, and fix its zoom behavior. (#6123)
 - Minor: Added WebSocket API for plugins. (#6076)

src/providers/bttv/BttvEmotes.cpp

@@ -11,8 +11,11 @@
 #include "providers/bttv/liveupdates/BttvLiveUpdateMessages.hpp"
 #include "providers/twitch/TwitchChannel.hpp"
 #include "singletons/Settings.hpp"
+#include "util/Helpers.hpp"
 
 #include <QJsonArray>
+#include <QLoggingCategory>
+#include <QStringLiteral>
 #include <QThread>
 
 namespace {
@@ -218,9 +221,19 @@ void BttvEmotes::loadEmotes()
         return;
     }
 
+    readProviderEmotesCache("global", "betterttv", [this](const auto &jsonDoc) {
+        auto emotes = this->global_.get();
+        auto pair = parseGlobalEmotes(jsonDoc.array(), *emotes);
+        if (pair.first)
+        {
+            this->setEmotes(std::make_shared<EmoteMap>(std::move(pair.second)));
+        }
+    });
+
     NetworkRequest(QString(globalEmoteApiUrl))
         .timeout(30000)
         .onSuccess([this](auto result) {
+            writeProviderEmotesCache("global", "betterttv", result.getData());
             auto emotes = this->global_.get();
             auto pair = parseGlobalEmotes(result.parseJsonArray(), *emotes);
             if (pair.first)
@@ -229,6 +242,10 @@ void BttvEmotes::loadEmotes()
                     std::make_shared<EmoteMap>(std::move(pair.second)));
             }
         })
+        .onError([](auto result) {
+            qCWarning(chatterinoBttv) << "Failed to fetch global BTTV emotes. "
+                                      << result.formatError();
+        })
         .execute();
 }
 
@@ -241,15 +258,16 @@ void BttvEmotes::loadChannel(std::weak_ptr<Channel> channel,
                              const QString &channelId,
                              const QString &channelDisplayName,
                              std::function<void(EmoteMap &&)> callback,
-                             bool manualRefresh)
+                             bool manualRefresh, bool cacheHit)
 {
     NetworkRequest(QString(bttvChannelEmoteApiUrl) + channelId)
         .timeout(20000)
-        .onSuccess([callback = std::move(callback), channel, channelDisplayName,
+        .onSuccess([callback = std::move(callback), channel, channelId,
-                    manualRefresh](auto result) {
+                    channelDisplayName, manualRefresh](auto result) {
             auto emotes =
                 parseChannelEmotes(result.parseJson(), channelDisplayName);
             bool hasEmotes = !emotes.empty();
+            writeProviderEmotesCache(channelId, "betterttv", result.getData());
             callback(std::move(emotes));
 
             if (auto shared = channel.lock(); manualRefresh)
@@ -265,7 +283,7 @@ void BttvEmotes::loadChannel(std::weak_ptr<Channel> channel,
                 }
             }
         })
-        .onError([channelId, channel, manualRefresh](auto result) {
+        .onError([channelId, channel, manualRefresh, cacheHit](auto result) {
             auto shared = channel.lock();
             if (!shared)
             {
@@ -291,6 +309,11 @@ void BttvEmotes::loadChannel(std::weak_ptr<Channel> channel,
                     QStringLiteral("Failed to fetch BetterTTV channel "
                                    "emotes. (Error: %1)")
                         .arg(errorString));
+                if (cacheHit)
+                {
+                    shared->addSystemMessage(
+                        "Using cached BetterTTV emotes as fallback.");
+                }
             }
         })
         .execute();

src/providers/bttv/BttvEmotes.hpp

@@ -47,7 +47,7 @@ public:
                             const QString &channelId,
                             const QString &channelDisplayName,
                             std::function<void(EmoteMap &&)> callback,
-                            bool manualRefresh);
+                            bool manualRefresh, bool cacheHit);
 
     /**
      * Adds an emote to the `channelEmoteMap`.

src/providers/ffz/FfzEmotes.cpp

@@ -9,6 +9,7 @@
 #include "providers/ffz/FfzUtil.hpp"
 #include "providers/twitch/TwitchChannel.hpp"
 #include "singletons/Settings.hpp"
+#include "util/Helpers.hpp"
 
 namespace {
 
@@ -237,15 +238,26 @@ void FfzEmotes::loadEmotes()
         return;
     }
 
+    readProviderEmotesCache("global", "frankerfacez", [this](auto jsonDoc) {
+        auto parsedSet = parseGlobalEmotes(jsonDoc.object());
+        this->setEmotes(std::make_shared<EmoteMap>(std::move(parsedSet)));
+    });
+
     QString url("https://api.frankerfacez.com/v1/set/global");
 
     NetworkRequest(url)
-
         .timeout(30000)
         .onSuccess([this](auto result) {
+            writeProviderEmotesCache("global", "frankerfacez",
+                                     result.getData());
             auto parsedSet = parseGlobalEmotes(result.parseJson());
             this->setEmotes(std::make_shared<EmoteMap>(std::move(parsedSet)));
         })
+        .onError([](auto result) {
+            qCWarning(chatterinoFfzemotes)
+                << "Failed to fetch global FFZ emotes. "
+                << result.formatError();
+        })
         .execute();
 }
 
@@ -260,7 +272,7 @@ void FfzEmotes::loadChannel(
     std::function<void(std::optional<EmotePtr>)> modBadgeCallback,
     std::function<void(std::optional<EmotePtr>)> vipBadgeCallback,
     std::function<void(FfzChannelBadgeMap &&)> channelBadgesCallback,
-    bool manualRefresh)
+    bool manualRefresh, bool cacheHit)
 {
     qCDebug(LOG) << "Reload FFZ Channel Emotes for channel" << channelID;
 
@@ -271,7 +283,9 @@ void FfzEmotes::loadChannel(
                     modBadgeCallback = std::move(modBadgeCallback),
                     vipBadgeCallback = std::move(vipBadgeCallback),
                     channelBadgesCallback = std::move(channelBadgesCallback),
-                    channel, manualRefresh](const auto &result) {
+                    channel, channelID, manualRefresh](const auto &result) {
+            writeProviderEmotesCache(channelID, "frankerfacez",
+                                     result.getData());
             const auto json = result.parseJson();
 
             auto emoteMap = parseChannelEmotes(json);
@@ -301,33 +315,39 @@ void FfzEmotes::loadChannel(
                 }
             }
         })
-        .onError([channelID, channel, manualRefresh](const auto &result) {
+        .onError(
-            auto shared = channel.lock();
+            [channelID, channel, manualRefresh, cacheHit](const auto &result) {
-            if (!shared)
+                auto shared = channel.lock();
-            {
+                if (!shared)
-                return;
-            }
-
-            if (result.status() == 404)
-            {
-                // User does not have any FFZ emotes
-                if (manualRefresh)
                 {
-                    shared->addSystemMessage(CHANNEL_HAS_NO_EMOTES);
+                    return;
                 }
-            }
+
-            else
+                if (result.status() == 404)
-            {
+                {
-                // TODO: Auto retry in case of a timeout, with a delay
+                    // User does not have any FFZ emotes
-                auto errorString = result.formatError();
+                    if (manualRefresh)
-                qCWarning(LOG) << "Error fetching FFZ emotes for channel"
+                    {
-                               << channelID << ", error" << errorString;
+                        shared->addSystemMessage(CHANNEL_HAS_NO_EMOTES);
-                shared->addSystemMessage(
+                    }
-                    QStringLiteral("Failed to fetch FrankerFaceZ channel "
+                }
-                                   "emotes. (Error: %1)")
+                else
-                        .arg(errorString));
+                {
-            }
+                    // TODO: Auto retry in case of a timeout, with a delay
-        })
+                    auto errorString = result.formatError();
+                    qCWarning(LOG) << "Error fetching FFZ emotes for channel"
+                                   << channelID << ", error" << errorString;
+                    shared->addSystemMessage(
+                        QStringLiteral("Failed to fetch FrankerFaceZ channel "
+                                       "emotes. (Error: %1)")
+                            .arg(errorString));
+                    if (cacheHit)
+                    {
+                        shared->addSystemMessage(
+                            "Using cached FrankerFaceZ emotes as fallback.");
+                    }
+                }
+            })
         .execute();
 }
 

src/providers/ffz/FfzEmotes.hpp

@@ -51,7 +51,7 @@ public:
         std::function<void(std::optional<EmotePtr>)> modBadgeCallback,
         std::function<void(std::optional<EmotePtr>)> vipBadgeCallback,
         std::function<void(FfzChannelBadgeMap &&)> channelBadgesCallback,
-        bool manualRefresh);
+        bool manualRefresh, bool cacheHit);
 
 private:
     Atomic<std::shared_ptr<const EmoteMap>> global_;

src/providers/seventv/SeventvEmotes.cpp

@@ -16,6 +16,8 @@
 
 #include <QJsonArray>
 #include <QJsonDocument>
+#include <QJsonObject>
+#include <QStringView>
 #include <QThread>
 
 #include <array>
@@ -230,11 +232,18 @@ void SeventvEmotes::loadGlobalEmotes()
         return;
     }
 
+    readProviderEmotesCache("global", "seventv", [this](auto jsonDoc) {
+        auto emoteMap = parseEmotes(jsonDoc.object()["emotes"].toArray(), true);
+        this->setGlobalEmotes(std::make_shared<EmoteMap>(std::move(emoteMap)));
+    });
+
     qCDebug(chatterinoSeventv) << "Loading 7TV Global Emotes";
 
     getApp()->getSeventvAPI()->getEmoteSet(
         u"global"_s,
         [this](const auto &json) {
+            writeProviderEmotesCache("global", "seventv",
+                                     QJsonDocument(json).toJson());
             QJsonArray parsedEmotes = json["emotes"].toArray();
 
             auto emoteMap = parseEmotes(parsedEmotes, true);
@@ -256,7 +265,8 @@ void SeventvEmotes::setGlobalEmotes(std::shared_ptr<const EmoteMap> emotes)
 
 void SeventvEmotes::loadChannelEmotes(
     const std::weak_ptr<Channel> &channel, const QString &channelId,
-    std::function<void(EmoteMap &&, ChannelInfo)> callback, bool manualRefresh)
+    std::function<void(EmoteMap &&, ChannelInfo)> callback, bool manualRefresh,
+    bool cacheHit)
 {
     qCDebug(chatterinoSeventv)
         << "Reloading 7TV Channel Emotes" << channelId << manualRefresh;
@@ -265,6 +275,8 @@ void SeventvEmotes::loadChannelEmotes(
         channelId,
         [callback = std::move(callback), channel, channelId,
          manualRefresh](const auto &json) {
+            writeProviderEmotesCache(channelId, "seventv",
+                                     QJsonDocument(json).toJson());
             const auto emoteSet = json["emote_set"].toObject();
             const auto parsedEmotes = emoteSet["emotes"].toArray();
 
@@ -312,7 +324,7 @@ void SeventvEmotes::loadChannelEmotes(
                 }
             }
         },
-        [channelId, channel, manualRefresh](const auto &result) {
+        [channelId, channel, manualRefresh, cacheHit](const auto &result) {
             auto shared = channel.lock();
             if (!shared)
             {
@@ -339,6 +351,11 @@ void SeventvEmotes::loadChannelEmotes(
                     QStringLiteral("Failed to fetch 7TV channel "
                                    "emotes. (Error: %1)")
                         .arg(errorString));
+                if (cacheHit)
+                {
+                    shared->addSystemMessage(
+                        "Using cached 7TV emotes as fallback.");
+                }
             }
         });
 }

src/providers/seventv/SeventvEmotes.hpp

@@ -108,7 +108,7 @@ public:
     static void loadChannelEmotes(
         const std::weak_ptr<Channel> &channel, const QString &channelId,
         std::function<void(EmoteMap &&, ChannelInfo)> callback,
-        bool manualRefresh);
+        bool manualRefresh, bool cacheHit);
 
     /**
      * Adds an emote to the `map` if it's valid.

src/providers/twitch/TwitchChannel.cpp

@@ -327,6 +327,17 @@ void TwitchChannel::refreshBTTVChannelEmotes(bool manualRefresh)
         return;
     }
 
+    bool cacheHit = readProviderEmotesCache(
+        this->roomId(), "betterttv",
+        [this, weak = weakOf<Channel>(this)](auto jsonDoc) {
+            if (auto shared = weak.lock())
+            {
+                auto emoteMap = bttv::detail::parseChannelEmotes(
+                    jsonDoc.object(), this->getLocalizedName());
+                this->setBttvEmotes(std::make_shared<const EmoteMap>(emoteMap));
+            }
+        });
+
     BttvEmotes::loadChannel(
         weakOf<Channel>(this), this->roomId(), this->getLocalizedName(),
         [this, weak = weakOf<Channel>(this)](auto &&emoteMap) {
@@ -335,7 +346,7 @@ void TwitchChannel::refreshBTTVChannelEmotes(bool manualRefresh)
                 this->setBttvEmotes(std::make_shared<const EmoteMap>(emoteMap));
             }
         },
-        manualRefresh);
+        manualRefresh, cacheHit);
 }
 
 void TwitchChannel::refreshFFZChannelEmotes(bool manualRefresh)
@@ -346,6 +357,12 @@ void TwitchChannel::refreshFFZChannelEmotes(bool manualRefresh)
         return;
     }
 
+    bool cacheHit = readProviderEmotesCache(
+        this->roomId(), "frankerfacez", [this](const auto &jsonDoc) {
+            auto emoteMap = ffz::detail::parseChannelEmotes(jsonDoc.object());
+            this->setFfzEmotes(std::make_shared<const EmoteMap>(emoteMap));
+        });
+
     FfzEmotes::loadChannel(
         weakOf<Channel>(this), this->roomId(),
         [this, weak = weakOf<Channel>(this)](auto &&emoteMap) {
@@ -376,7 +393,7 @@ void TwitchChannel::refreshFFZChannelEmotes(bool manualRefresh)
                     std::forward<decltype(channelBadges)>(channelBadges);
             }
         },
-        manualRefresh);
+        manualRefresh, cacheHit);
 }
 
 void TwitchChannel::refreshSevenTVChannelEmotes(bool manualRefresh)
@@ -387,6 +404,15 @@ void TwitchChannel::refreshSevenTVChannelEmotes(bool manualRefresh)
         return;
     }
 
+    bool cacheHit = readProviderEmotesCache(
+        this->roomId(), "seventv", [this](auto jsonDoc) {
+            const auto json = jsonDoc.object();
+            const auto emoteSet = json["emote_set"].toObject();
+            const auto parsedEmotes = emoteSet["emotes"].toArray();
+            auto emoteMap = seventv::detail::parseEmotes(parsedEmotes, false);
+            this->setSeventvEmotes(std::make_shared<const EmoteMap>(emoteMap));
+        });
+
     SeventvEmotes::loadChannelEmotes(
         weakOf<Channel>(this), this->roomId(),
         [this, weak = weakOf<Channel>(this)](auto &&emoteMap,
@@ -401,7 +427,7 @@ void TwitchChannel::refreshSevenTVChannelEmotes(bool manualRefresh)
                     channelInfo.twitchConnectionIndex;
             }
         },
-        manualRefresh);
+        manualRefresh, cacheHit);
 }
 
 void TwitchChannel::setBttvEmotes(std::shared_ptr<const EmoteMap> &&map)

src/singletons/Paths.cpp

@@ -39,7 +39,7 @@ QString Paths::cacheDirectory() const
     static const auto pathSetting = [] {
         QStringSetting cachePathSetting("/cache/path");
 
-        cachePathSetting.connect([](const auto &newPath, auto) {
+        cachePathSetting.connect([](const auto &newPath) {
             if (!newPath.isEmpty())
             {
                 QDir().mkpath(newPath);
@@ -59,6 +59,11 @@ QString Paths::cacheDirectory() const
     return path;
 }
 
+QString Paths::cacheFilePath(const QString &fileName) const
+{
+    return combinePath(this->cacheDirectory(), fileName);
+}
+
 void Paths::initAppFilePathHash()
 {
     this->applicationFilePathHash =

src/singletons/Paths.hpp

@@ -50,6 +50,11 @@ public:
 
     QString cacheDirectory() const;
 
+    /// Returns the full file path for a file in the cache directory
+    ///
+    /// e.g. cacheFilePath("foo") will return <cacheDirectory>/foo
+    QString cacheFilePath(const QString &fileName) const;
+
 private:
     void initAppFilePathHash();
     void initCheckPortable();

src/util/Helpers.cpp

@@ -1,12 +1,19 @@
 #include "util/Helpers.hpp"
 
 #include "Application.hpp"
+#include "common/QLogging.hpp"
 #include "providers/twitch/TwitchCommon.hpp"
+#include "singletons/Paths.hpp"
 
 #include <QDateTime>
 #include <QDirIterator>
+#include <QJsonObject>
 #include <QLocale>
+#include <QLoggingCategory>
 #include <QRegularExpression>
+#include <QStringBuilder>
+#include <QStringView>
+#include <QThreadPool>
 #include <QTimeZone>
 #include <QUuid>
 
@@ -386,4 +393,49 @@ void removeLastQS(QString &str)
 #endif
 }
 
+void writeProviderEmotesCache(const QString &id, const QString &provider,
+                              const QByteArray &bytes)
+{
+    QThreadPool::globalInstance()->start([bytes, id, provider]() {
+        auto cacheKey = id % "." % provider;
+        QFile responseCache(getApp()->getPaths().cacheFilePath(cacheKey));
+
+        if (responseCache.open(QIODevice::WriteOnly))
+        {
+            qCDebug(chatterinoCache)
+                << "Saved json response " << id << "." << provider;
+            responseCache.write(qCompress(bytes));
+        }
+    });
+}
+
+bool readProviderEmotesCache(const QString &id, const QString &provider,
+                             const std::function<void(QJsonDocument)> &callback)
+{
+    auto cacheKey = id % "." % provider;
+    QFile responseCache(getApp()->getPaths().cacheFilePath(cacheKey));
+
+    if (responseCache.open(QIODevice::ReadOnly))
+    {
+        QJsonParseError parseError;
+        auto doc = QJsonDocument::fromJson(qUncompress(responseCache.readAll()),
+                                           &parseError);
+
+        if (parseError.error != QJsonParseError::NoError)
+        {
+            qCWarning(chatterinoCache)
+                << "Emote cache " << id << "." << provider
+                << " parsing failed: " << parseError.errorString();
+        }
+
+        qCDebug(chatterinoCache)
+            << "Loaded emote cache: " << id << "." << provider;
+        callback(doc);
+        return true;
+    }
+
+    // If the API call fails, we need to know if loading cached emotes was successful
+    return false;
+}
+
 }  // namespace chatterino

src/util/Helpers.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <QColor>
+#include <QJsonDocument>
 #include <QLocale>
 #include <QString>
 
@@ -218,4 +219,11 @@ void removeFirstQS(QString &str);
 /// @param str The Qt string we want to remove 1 character from
 void removeLastQS(QString &str);
 
+void writeProviderEmotesCache(const QString &id, const QString &provider,
+                              const QByteArray &bytes);
+
+bool readProviderEmotesCache(
+    const QString &id, const QString &provider,
+    const std::function<void(QJsonDocument)> &callback);
+
 }  // namespace chatterino