diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2f3c37f9..23e175be 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -112,6 +112,7 @@
 - Dev: Refactored the SplitOverlay code. (#5082)
 - Dev: Refactored the TwitchBadges structure, making it less of a singleton. (#5096)
 - Dev: Refactored the ChatterinoBadges structure, making it less of a singleton. (#5103)
+- Dev: Refactored the ColorProvider class a bit. (#5112)
 - Dev: Moved the Network files to their own folder. (#5089)
 - Dev: Fixed deadlock and use-after-free in tests. (#4981)
 - Dev: Moved all `.clang-format` files to the root directory. (#5037)
diff --git a/src/controllers/highlights/HighlightModel.cpp b/src/controllers/highlights/HighlightModel.cpp
index 6c311e11..c83657c8 100644
--- a/src/controllers/highlights/HighlightModel.cpp
+++ b/src/controllers/highlights/HighlightModel.cpp
@@ -465,8 +465,6 @@ void HighlightModel::customRowSetData(const std::vector<QStandardItem *> &row,
                 const auto setColor = [&](auto &setting, ColorType ty) {
                     auto color = value.value<QColor>();
                     setting.setValue(color.name(QColor::HexArgb));
-                    const_cast<ColorProvider &>(ColorProvider::instance())
-                        .updateColor(ty, color);
                 };
 
                 if (rowIndex == HighlightRowIndexes::SelfHighlightRow)
diff --git a/src/controllers/highlights/UserHighlightModel.cpp b/src/controllers/highlights/UserHighlightModel.cpp
index 11a4523f..b28866e6 100644
--- a/src/controllers/highlights/UserHighlightModel.cpp
+++ b/src/controllers/highlights/UserHighlightModel.cpp
@@ -103,10 +103,6 @@ void UserHighlightModel::customRowSetData(
                     // Update the setting with the new value
                     getSettings()->selfMessageHighlightColor.setValue(
                         colorName);
-                    // Update the color provider with the new color to be used for future
-                    const_cast<ColorProvider &>(ColorProvider::instance())
-                        .updateColor(ColorType::SelfMessageHighlight,
-                                     QColor(colorName));
                 }
             }
         }
diff --git a/src/providers/colors/ColorProvider.cpp b/src/providers/colors/ColorProvider.cpp
index fbda9c6e..72039a1f 100644
--- a/src/providers/colors/ColorProvider.cpp
+++ b/src/providers/colors/ColorProvider.cpp
@@ -1,5 +1,6 @@
 #include "providers/colors/ColorProvider.hpp"
 
+#include "common/QLogging.hpp"
 #include "controllers/highlights/HighlightPhrase.hpp"
 #include "singletons/Settings.hpp"
 
@@ -14,24 +15,16 @@ const ColorProvider &ColorProvider::instance()
 }
 
 ColorProvider::ColorProvider()
-    : typeColorMap_()
-    , defaultColors_()
 {
     this->initTypeColorMap();
     this->initDefaultColors();
 }
 
-const std::shared_ptr<QColor> ColorProvider::color(ColorType type) const
+std::shared_ptr<QColor> ColorProvider::color(ColorType type) const
 {
     return this->typeColorMap_.at(type);
 }
 
-void ColorProvider::updateColor(ColorType type, QColor color)
-{
-    auto colorPtr = this->typeColorMap_.at(type);
-    *colorPtr = std::move(color);
-}
-
 QSet<QColor> ColorProvider::recentColors() const
 {
     QSet<QColor> retVal;
@@ -40,12 +33,12 @@ QSet<QColor> ColorProvider::recentColors() const
      * Currently, only colors used in highlight phrases are considered. This
      * may change at any point in the future.
      */
-    for (auto phrase : getSettings()->highlightedMessages)
+    for (const auto &phrase : getSettings()->highlightedMessages)
     {
         retVal.insert(*phrase.getColor());
     }
 
-    for (auto userHl : getSettings()->highlightedUsers)
+    for (const auto &userHl : getSettings()->highlightedUsers)
     {
         retVal.insert(*userHl.getColor());
     }
@@ -67,117 +60,74 @@ void ColorProvider::initTypeColorMap()
 {
     // Read settings for custom highlight colors and save them in map.
     // If no custom values can be found, set up default values instead.
+    // Set up a signal to the respective setting for updating the color when it's changed
+    auto initColor = [this](ColorType colorType, QStringSetting &setting,
+                            QColor fallbackColor) {
+        const auto &colorString = setting.getValue();
+        QColor color(colorString);
+        if (color.isValid())
+        {
+            this->typeColorMap_.insert({
+                colorType,
+                std::make_shared<QColor>(color),
+            });
+        }
+        else
+        {
+            this->typeColorMap_.insert({
+                colorType,
+                std::make_shared<QColor>(fallbackColor),
+            });
+        }
 
-    QString customColor = getSettings()->selfHighlightColor;
-    if (QColor(customColor).isValid())
-    {
-        this->typeColorMap_.insert(
-            {ColorType::SelfHighlight, std::make_shared<QColor>(customColor)});
-    }
-    else
-    {
-        this->typeColorMap_.insert(
-            {ColorType::SelfHighlight,
-             std::make_shared<QColor>(
-                 HighlightPhrase::FALLBACK_HIGHLIGHT_COLOR)});
-    }
+        setting.connect(
+            [this, colorType](const auto &colorString) {
+                QColor color(colorString);
+                if (color.isValid())
+                {
+                    // Update color based on the update from the setting
+                    *this->typeColorMap_.at(colorType) = color;
+                }
+                else
+                {
+                    qCWarning(chatterinoCommon)
+                        << "Updated"
+                        << static_cast<std::underlying_type_t<ColorType>>(
+                               colorType)
+                        << "to invalid color" << colorString;
+                }
+            },
+            false);
+    };
 
-    customColor = getSettings()->selfMessageHighlightColor;
-    if (QColor(customColor).isValid())
-    {
-        this->typeColorMap_.insert({ColorType::SelfMessageHighlight,
-                                    std::make_shared<QColor>(customColor)});
-    }
-    else
-    {
-        this->typeColorMap_.insert(
-            {ColorType::SelfMessageHighlight,
-             std::make_shared<QColor>(
-                 HighlightPhrase::FALLBACK_SELF_MESSAGE_HIGHLIGHT_COLOR)});
-    }
+    initColor(ColorType::SelfHighlight, getSettings()->selfHighlightColor,
+              HighlightPhrase::FALLBACK_HIGHLIGHT_COLOR);
 
-    customColor = getSettings()->subHighlightColor;
-    if (QColor(customColor).isValid())
-    {
-        this->typeColorMap_.insert(
-            {ColorType::Subscription, std::make_shared<QColor>(customColor)});
-    }
-    else
-    {
-        this->typeColorMap_.insert(
-            {ColorType::Subscription,
-             std::make_shared<QColor>(HighlightPhrase::FALLBACK_SUB_COLOR)});
-    }
+    initColor(ColorType::SelfMessageHighlight,
+              getSettings()->selfMessageHighlightColor,
+              HighlightPhrase::FALLBACK_SELF_MESSAGE_HIGHLIGHT_COLOR);
 
-    customColor = getSettings()->whisperHighlightColor;
-    if (QColor(customColor).isValid())
-    {
-        this->typeColorMap_.insert(
-            {ColorType::Whisper, std::make_shared<QColor>(customColor)});
-    }
-    else
-    {
-        this->typeColorMap_.insert(
-            {ColorType::Whisper,
-             std::make_shared<QColor>(
-                 HighlightPhrase::FALLBACK_HIGHLIGHT_COLOR)});
-    }
+    initColor(ColorType::Subscription, getSettings()->subHighlightColor,
+              HighlightPhrase::FALLBACK_SUB_COLOR);
 
-    customColor = getSettings()->redeemedHighlightColor;
-    if (QColor(customColor).isValid())
-    {
-        this->typeColorMap_.insert({ColorType::RedeemedHighlight,
-                                    std::make_shared<QColor>(customColor)});
-    }
-    else
-    {
-        this->typeColorMap_.insert(
-            {ColorType::RedeemedHighlight,
-             std::make_shared<QColor>(
-                 HighlightPhrase::FALLBACK_REDEEMED_HIGHLIGHT_COLOR)});
-    }
+    initColor(ColorType::Whisper, getSettings()->whisperHighlightColor,
+              HighlightPhrase::FALLBACK_HIGHLIGHT_COLOR);
 
-    customColor = getSettings()->firstMessageHighlightColor;
-    if (QColor(customColor).isValid())
-    {
-        this->typeColorMap_.insert({ColorType::FirstMessageHighlight,
-                                    std::make_shared<QColor>(customColor)});
-    }
-    else
-    {
-        this->typeColorMap_.insert(
-            {ColorType::FirstMessageHighlight,
-             std::make_shared<QColor>(
-                 HighlightPhrase::FALLBACK_FIRST_MESSAGE_HIGHLIGHT_COLOR)});
-    }
+    initColor(ColorType::RedeemedHighlight,
+              getSettings()->redeemedHighlightColor,
+              HighlightPhrase::FALLBACK_REDEEMED_HIGHLIGHT_COLOR);
 
-    customColor = getSettings()->elevatedMessageHighlightColor;
-    if (QColor(customColor).isValid())
-    {
-        this->typeColorMap_.insert({ColorType::ElevatedMessageHighlight,
-                                    std::make_shared<QColor>(customColor)});
-    }
-    else
-    {
-        this->typeColorMap_.insert(
-            {ColorType::ElevatedMessageHighlight,
-             std::make_shared<QColor>(
-                 HighlightPhrase::FALLBACK_ELEVATED_MESSAGE_HIGHLIGHT_COLOR)});
-    }
+    initColor(ColorType::FirstMessageHighlight,
+              getSettings()->firstMessageHighlightColor,
+              HighlightPhrase::FALLBACK_FIRST_MESSAGE_HIGHLIGHT_COLOR);
 
-    customColor = getSettings()->threadHighlightColor;
-    if (QColor(customColor).isValid())
-    {
-        this->typeColorMap_.insert({ColorType::ThreadMessageHighlight,
-                                    std::make_shared<QColor>(customColor)});
-    }
-    else
-    {
-        this->typeColorMap_.insert(
-            {ColorType::ThreadMessageHighlight,
-             std::make_shared<QColor>(
-                 HighlightPhrase::FALLBACK_THREAD_HIGHLIGHT_COLOR)});
-    }
+    initColor(ColorType::ElevatedMessageHighlight,
+              getSettings()->elevatedMessageHighlightColor,
+              HighlightPhrase::FALLBACK_ELEVATED_MESSAGE_HIGHLIGHT_COLOR);
+
+    initColor(ColorType::ThreadMessageHighlight,
+              getSettings()->threadHighlightColor,
+              HighlightPhrase::FALLBACK_THREAD_HIGHLIGHT_COLOR);
 }
 
 void ColorProvider::initDefaultColors()
diff --git a/src/providers/colors/ColorProvider.hpp b/src/providers/colors/ColorProvider.hpp
index 12745371..dfd9b4b8 100644
--- a/src/providers/colors/ColorProvider.hpp
+++ b/src/providers/colors/ColorProvider.hpp
@@ -36,9 +36,7 @@ public:
      * of already parsed predefined (self highlights, subscriptions,
      * and whispers) highlights.
      */
-    const std::shared_ptr<QColor> color(ColorType type) const;
-
-    void updateColor(ColorType type, QColor color);
+    std::shared_ptr<QColor> color(ColorType type) const;
 
     /**
      * @brief Return a set of recently used colors used anywhere in Chatterino.