chatterino2
Code Issues

optimize chatter list (#2814)

Browse Source 
* optimize chatter list

* changelog

* Fix tests

Co-authored-by: Rasmus Karlsson <rasmus.karlsson@pajlada.com>
This commit is contained in:
fourtf
2021-05-24 12:13:59 +02:00
committed by GitHub
parent 7659dc27ae
commit 3fddafb867
19 changed files with 371 additions and 532 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/CMakeLists.txt
+2 -2
View File
@@ -18,6 +18,8 @@ set(SOURCE_FILES main.cpp
common/ChannelChatters.hpp
common/ChatterinoSetting.cpp
common/ChatterinoSetting.hpp
common/ChatterSet.cpp
common/ChatterSet.hpp
common/CompletionModel.cpp
common/CompletionModel.hpp
common/Credentials.cpp
@@ -42,8 +44,6 @@ set(SOURCE_FILES main.cpp
common/NetworkResult.hpp
common/QLogging.cpp
common/QLogging.hpp
common/UsernameSet.cpp
common/UsernameSet.hpp
common/Version.cpp
common/Version.hpp
common/WindowDescriptors.cpp
src/common/ChannelChatters.cpp
+7 -4
View File
@@ -11,14 +11,15 @@ ChannelChatters::ChannelChatters(Channel &channel)
{
}
SharedAccessGuard<const UsernameSet> ChannelChatters::accessChatters() const
SharedAccessGuard<const ChatterSet> ChannelChatters::accessChatters() const
{
return this->chatters_.accessConst();
}
void ChannelChatters::addRecentChatter(const QString &user)
{
this->chatters_.access()->insert(user);
auto chatters = this->chatters_.access();
chatters->addRecentChatter(user);
}
void ChannelChatters::addJoinedUser(const QString &user)
@@ -66,9 +67,11 @@ void ChannelChatters::addPartedUser(const QString &user)
}
}
void ChannelChatters::setChatters(UsernameSet &&set)
void ChannelChatters::updateOnlineChatters(
const std::unordered_set<QString> &chatters)
{
this->chatters_.access()->merge(std::move(set));
auto chatters_ = this->chatters_.access();
chatters_->updateOnlineChatters(chatters);
}
const QColor ChannelChatters::getUserColor(const QString &user)
src/common/ChannelChatters.hpp
+5 -6
View File
@@ -1,11 +1,10 @@
#pragma once
#include "common/Channel.hpp"
#include "common/ChatterSet.hpp"
#include "common/UniqueAccess.hpp"
#include "common/UsernameSet.hpp"
#include "util/QStringHash.hpp"
#include "lrucache/lrucache.hpp"
#include "util/QStringHash.hpp"
#include <QRgb>
@@ -17,14 +16,14 @@ public:
ChannelChatters(Channel &channel);
virtual ~ChannelChatters() = default; // add vtable
SharedAccessGuard<const UsernameSet> accessChatters() const;
SharedAccessGuard<const ChatterSet> accessChatters() const;
void addRecentChatter(const QString &user);
void addJoinedUser(const QString &user);
void addPartedUser(const QString &user);
void setChatters(UsernameSet &&set);
const QColor getUserColor(const QString &user);
void setUserColor(const QString &user, const QColor &color);
void updateOnlineChatters(const std::unordered_set<QString> &chatters);
private:
static constexpr int maxChatterColorCount = 5000;
@@ -32,7 +31,7 @@ private:
Channel &channel_;
// maps 2 char prefix to set of names
UniqueAccess<UsernameSet> chatters_;
UniqueAccess<ChatterSet> chatters_;
UniqueAccess<cache::lru_cache<QString, QRgb>> chatterColors_;
// combines multiple joins/parts into one message
src/common/ChatterSet.cpp
+58
View File
@@ -0,0 +1,58 @@
#include "common/ChatterSet.hpp"
#include <tuple>
#include "debug/Benchmark.hpp"
namespace chatterino {
ChatterSet::ChatterSet()
: items(chatterLimit)
{
}
void ChatterSet::addRecentChatter(const QString &userName)
{
this->items.put(userName.toLower(), userName);
}
void ChatterSet::updateOnlineChatters(
const std::unordered_set<QString> &lowerCaseUsernames)
{
BenchmarkGuard bench("update online chatters");
// Create a new lru cache without the users that are not present anymore.
cache::lru_cache<QString, QString> tmp(chatterLimit);
for (auto &&chatter : lowerCaseUsernames)
{
if (this->items.exists(chatter))
tmp.put(chatter, this->items.get(chatter));
// Less chatters than the limit => try to preserve as many as possible.
else if (lowerCaseUsernames.size() < chatterLimit)
tmp.put(chatter, chatter);
}
this->items = std::move(tmp);
}
bool ChatterSet::contains(const QString &userName) const
{
return this->items.exists(userName.toLower());
}
std::vector<QString> ChatterSet::filterByPrefix(const QString &prefix) const
{
QString lowerPrefix = prefix.toLower();
std::vector<QString> result;
for (auto &&item : this->items)
{
if (item.first.startsWith(lowerPrefix))
result.push_back(item.second);
}
return result;
}
} // namespace chatterino
src/common/ChatterSet.hpp
+46
View File
@@ -0,0 +1,46 @@
#pragma once
#include <QString>
#include <functional>
#include <set>
#include <unordered_map>
#include <unordered_set>
#include "lrucache/lrucache.hpp"
#include "util/QStringHash.hpp"
namespace chatterino {
/// ChatterSet is a limited container that contains a list of recent chatters
/// that can be referenced by name.
class ChatterSet
{
public:
/// The limit of how many chatters can be saved for a channel.
static constexpr size_t chatterLimit = 2000;
ChatterSet();
/// Inserts a user name if it isn't contained. Doesn't replace the original
/// if the casing hasn't changed.
void addRecentChatter(const QString &userName);
/// Removes chatters that aren't online anymore. Adds chatters that aren't
/// in the list yet.
void updateOnlineChatters(
const std::unordered_set<QString> &lowerCaseUsernames);
/// Checks if a username is in the list.
bool contains(const QString &userName) const;
/// Get filtered usernames by a prefix for autocompletion. Contained items
/// are in mixed case if available.
std::vector<QString> filterByPrefix(const QString &prefix) const;
private:
// user name in lower case -> user name in normal case
cache::lru_cache<QString, QString> items;
};
using ChatterSet = ChatterSet;
} // namespace chatterino
src/common/CompletionModel.cpp
+20 -22
View File
@@ -1,8 +1,8 @@
#include "common/CompletionModel.hpp"
#include "Application.hpp"
#include "common/ChatterSet.hpp"
#include "common/Common.hpp"
#include "common/UsernameSet.hpp"
#include "controllers/accounts/AccountController.hpp"
#include "controllers/commands/CommandController.hpp"
#include "debug/Benchmark.hpp"
@@ -108,33 +108,31 @@ void CompletionModel::refresh(const QString &prefix, bool isFirstWord)
}
// Usernames
if (prefix.length() >= UsernameSet::PrefixLength)
QString usernamePostfix =
isFirstWord && getSettings()->mentionUsersWithComma ? ","
: QString();
if (prefix.startsWith("@"))
{
auto usernames = channel->accessChatters();
QString usernamePrefix = prefix;
QString usernamePostfix =
isFirstWord && getSettings()->mentionUsersWithComma ? ","
: QString();
usernamePrefix.remove(0, 1);
if (usernamePrefix.startsWith("@"))
auto chatters =
channel->accessChatters()->filterByPrefix(usernamePrefix);
for (const auto &name : chatters)
{
usernamePrefix.remove(0, 1);
for (const auto &name :
usernames->subrange(Prefix(usernamePrefix)))
{
addString("@" + name + usernamePostfix,
TaggedString::Type::Username);
}
addString("@" + name + usernamePostfix,
TaggedString::Type::Username);
}
else if (!getSettings()->userCompletionOnlyWithAt)
}
else if (!getSettings()->userCompletionOnlyWithAt)
{
auto chatters = channel->accessChatters()->filterByPrefix(prefix);
for (const auto &name : chatters)
{
for (const auto &name :
usernames->subrange(Prefix(usernamePrefix)))
{
addString(name + usernamePostfix,
TaggedString::Type::Username);
}
addString(name + usernamePostfix, TaggedString::Type::Username);
}
}
src/common/UniqueAccess.hpp
+2 -2
View File
@@ -64,7 +64,7 @@ public:
}
UniqueAccess(T &&element)
: element_(element)
: element_(std::move(element))
{
}
@@ -76,7 +76,7 @@ public:
UniqueAccess<T> &operator=(T &&element)
{
this->element_ = element;
this->element_ = std::move(element);
return *this;
}
src/common/UsernameSet.cpp
-182
View File
@@ -1,182 +0,0 @@
#include "UsernameSet.hpp"
#include <tuple>
namespace chatterino {
namespace {
std::pair<UsernameSet::Iterator, bool> findOrErase(
std::set<QString, CaseInsensitiveLess> &set, const QString &value)
{
if (!value.isLower())
{
auto iter = set.find(value);
if (iter != set.end())
{
if (QString::compare(*iter, value, Qt::CaseSensitive) != 0)
{
set.erase(iter);
}
else
{
return {iter, false};
}
}
}
return {set.end(), true};
}
} // namespace
//
// UsernameSet
//
UsernameSet::ConstIterator UsernameSet::begin() const
{
return this->items.begin();
}
UsernameSet::ConstIterator UsernameSet::end() const
{
return this->items.end();
}
UsernameSet::Range UsernameSet::subrange(const Prefix &prefix) const
{
auto it = this->firstKeyForPrefix.find(prefix);
if (it != this->firstKeyForPrefix.end())
{
auto start = this->items.find(it->second);
auto end = start;
while (end != this->items.end() && prefix.isStartOf(*end))
{
end++;
}
return {start, end};
}
return {this->items.end(), this->items.end()};
}
std::set<QString>::size_type UsernameSet::size() const
{
return this->items.size();
}
std::pair<UsernameSet::Iterator, bool> UsernameSet::insert(const QString &value)
{
auto pair = findOrErase(this->items, value);
if (!pair.second)
{
return pair;
}
this->insertPrefix(value);
return this->items.insert(value);
}
std::pair<UsernameSet::Iterator, bool> UsernameSet::insert(QString &&value)
{
auto pair = findOrErase(this->items, value);
if (!pair.second)
{
return pair;
}
this->insertPrefix(value);
return this->items.insert(std::move(value));
}
void UsernameSet::insertPrefix(const QString &value)
{
auto &string = this->firstKeyForPrefix[Prefix(value)];
if (string.isNull() || value.compare(string, Qt::CaseInsensitive) < 0)
string = value;
}
bool UsernameSet::contains(const QString &value) const
{
return this->items.count(value) == 1;
}
void UsernameSet::merge(UsernameSet &&set)
{
for (auto it = this->items.begin(); it != this->items.end();)
{
auto iter = set.items.find(*it);
if (iter == set.items.end())
{
it = this->items.erase(it);
}
else
{
++it;
}
}
this->items.merge(set.items);
this->firstKeyForPrefix = set.firstKeyForPrefix;
}
//
// Range
//
UsernameSet::Range::Range(ConstIterator start, ConstIterator end)
: start_(start)
, end_(end)
{
}
UsernameSet::ConstIterator UsernameSet::Range::begin()
{
return this->start_;
}
UsernameSet::ConstIterator UsernameSet::Range::end()
{
return this->end_;
}
//
// Prefix
//
Prefix::Prefix(const QString &string)
: first(string.size() >= 1 ? string[0].toLower() : '\0')
, second(string.size() >= 2 ? string[1].toLower() : '\0')
{
}
bool Prefix::operator==(const Prefix &other) const
{
return std::tie(this->first, this->second) ==
std::tie(other.first, other.second);
}
bool Prefix::operator!=(const Prefix &other) const
{
return !(*this == other);
}
bool Prefix::isStartOf(const QString &string) const
{
if (string.size() == 0)
{
return this->first == QChar('\0') && this->second == QChar('\0');
}
else if (string.size() == 1)
{
return this->first == string[0].toLower() &&
this->second == QChar('\0');
}
else
{
return this->first == string[0].toLower() &&
this->second == string[1].toLower();
}
}
} // namespace chatterino
src/common/UsernameSet.hpp
-89
View File
@@ -1,89 +0,0 @@
#pragma once
#include <QString>
#include <functional>
#include <set>
#include <unordered_map>
namespace chatterino {
class Prefix
{
public:
Prefix(const QString &string);
bool operator==(const Prefix &other) const;
bool operator!=(const Prefix &other) const;
bool isStartOf(const QString &string) const;
private:
QChar first;
QChar second;
friend struct std::hash<Prefix>;
};
} // namespace chatterino
namespace std {
template <>
struct hash<chatterino::Prefix> {
size_t operator()(const chatterino::Prefix &prefix) const
{
return (size_t(prefix.first.unicode()) << 16) |
size_t(prefix.second.unicode());
}
};
} // namespace std
namespace chatterino {
struct CaseInsensitiveLess {
bool operator()(const QString &lhs, const QString &rhs) const
{
return lhs.compare(rhs, Qt::CaseInsensitive) < 0;
}
};
class UsernameSet
{
public:
static constexpr int PrefixLength = 2;
using Iterator = std::set<QString>::iterator;
using ConstIterator = std::set<QString>::const_iterator;
class Range
{
public:
Range(ConstIterator start, ConstIterator end);
ConstIterator begin();
ConstIterator end();
private:
ConstIterator start_;
ConstIterator end_;
};
ConstIterator begin() const;
ConstIterator end() const;
Range subrange(const Prefix &prefix) const;
std::set<QString>::size_type size() const;
std::pair<Iterator, bool> insert(const QString &value);
std::pair<Iterator, bool> insert(QString &&value);
bool contains(const QString &value) const;
void merge(UsernameSet &&set);
private:
void insertPrefix(const QString &string);
std::set<QString, CaseInsensitiveLess> items;
std::unordered_map<Prefix, QString> firstKeyForPrefix;
};
} // namespace chatterino
src/providers/twitch/TwitchChannel.cpp
+4 -3
View File
@@ -119,13 +119,14 @@ namespace {
return messages;
}
std::pair<Outcome, UsernameSet> parseChatters(const QJsonObject &jsonRoot)
std::pair<Outcome, std::unordered_set<QString>> parseChatters(
const QJsonObject &jsonRoot)
{
static QStringList categories = {"broadcaster", "vips", "moderators",
"staff", "admins", "global_mods",
"viewers"};
auto usernames = UsernameSet();
auto usernames = std::unordered_set<QString>();
// parse json
QJsonObject jsonCategories = jsonRoot.value("chatters").toObject();
@@ -823,7 +824,7 @@ void TwitchChannel::refreshChatters()
auto pair = parseChatters(std::move(data));
if (pair.first)
{
this->setChatters(std::move(pair.second));
this->updateOnlineChatters(pair.second);
}
return pair.first;
src/providers/twitch/TwitchChannel.hpp
+1 -1
View File
@@ -4,9 +4,9 @@
#include "common/Atomic.hpp"
#include "common/Channel.hpp"
#include "common/ChannelChatters.hpp"
#include "common/ChatterSet.hpp"
#include "common/Outcome.hpp"
#include "common/UniqueAccess.hpp"
#include "common/UsernameSet.hpp"
#include "providers/twitch/ChannelPointReward.hpp"
#include "providers/twitch/TwitchEmotes.hpp"
#include "providers/twitch/api/Helix.hpp"
src/providers/twitch/TwitchMessageBuilder.cpp
+2 -2
View File
@@ -519,11 +519,11 @@ void TwitchMessageBuilder::addTextOrEmoji(const QString &string_)
if (this->twitchChannel != nullptr && getSettings()->findAllUsernames)
{
auto chatters = this->twitchChannel->accessChatters();
auto match = allUsernamesMentionRegex.match(string);
QString username = match.captured(1);
if (match.hasMatch() && chatters->contains(username))
if (match.hasMatch() &&
this->twitchChannel->accessChatters()->contains(username))
{
auto originalTextColor = textColor;