Make tests more platform agnostic (#4650)

Use QTemporaryDir to create the test directory
Add option to use httpbin over local docker
Increase delay for opening a connection

Co-authored-by: Rasmus Karlsson <rasmus.karlsson@pajlada.com>

tests/src/HighlightController.cpp

@@ -19,6 +19,7 @@
 #include <QDir>
 #include <QFile>
 #include <QString>
+#include <QTemporaryDir>
 
 using namespace chatterino;
 using ::testing::Exactly;
@@ -178,9 +179,9 @@ protected:
     void SetUp() override
     {
         // Write default settings to the mock settings json file
-        ASSERT_TRUE(QDir().mkpath("/tmp/c2-tests"));
+        this->settingsDir_ = std::make_unique<QTemporaryDir>();
 
-        QFile settingsFile("/tmp/c2-tests/settings.json");
+        QFile settingsFile(this->settingsDir_->filePath("settings.json"));
         ASSERT_TRUE(settingsFile.open(QIODevice::WriteOnly | QIODevice::Text));
         ASSERT_GT(settingsFile.write(DEFAULT_SETTINGS.toUtf8()), 0);
         ASSERT_TRUE(settingsFile.flush());
@@ -194,7 +195,7 @@ protected:
         EXPECT_CALL(*this->mockHelix, update).Times(Exactly(1));
 
         this->mockApplication = std::make_unique<MockApplication>();
-        this->settings = std::make_unique<Settings>("/tmp/c2-tests");
+        this->settings = std::make_unique<Settings>(this->settingsDir_->path());
         this->paths = std::make_unique<Paths>();
 
         this->controller = std::make_unique<HighlightController>();
@@ -206,16 +207,19 @@ protected:
 
     void TearDown() override
     {
-        ASSERT_TRUE(QDir("/tmp/c2-tests").removeRecursively());
         this->mockApplication.reset();
         this->settings.reset();
         this->paths.reset();
 
         this->controller.reset();
 
+        this->settingsDir_.reset();
+
         delete this->mockHelix;
     }
 
+    std::unique_ptr<QTemporaryDir> settingsDir_;
+
     std::unique_ptr<MockApplication> mockApplication;
     std::unique_ptr<Settings> settings;
     std::unique_ptr<Paths> paths;

tests/src/InputCompletion.cpp

@@ -18,6 +18,7 @@
 #include <QFile>
 #include <QModelIndex>
 #include <QString>
+#include <QTemporaryDir>
 
 namespace {
 
@@ -122,9 +123,9 @@ protected:
     void SetUp() override
     {
         // Write default settings to the mock settings json file
-        ASSERT_TRUE(QDir().mkpath("/tmp/c2-tests"));
+        this->settingsDir_ = std::make_unique<QTemporaryDir>();
 
-        QFile settingsFile("/tmp/c2-tests/settings.json");
+        QFile settingsFile(this->settingsDir_->filePath("settings.json"));
         ASSERT_TRUE(settingsFile.open(QIODevice::WriteOnly | QIODevice::Text));
         ASSERT_GT(settingsFile.write(DEFAULT_SETTINGS.toUtf8()), 0);
         ASSERT_TRUE(settingsFile.flush());
@@ -137,7 +138,7 @@ protected:
         EXPECT_CALL(*this->mockHelix, update).Times(Exactly(1));
 
         this->mockApplication = std::make_unique<MockApplication>();
-        this->settings = std::make_unique<Settings>("/tmp/c2-tests");
+        this->settings = std::make_unique<Settings>(this->settingsDir_->path());
         this->paths = std::make_unique<Paths>();
 
         this->mockApplication->accounts.initialize(*this->settings,
@@ -153,15 +154,18 @@ protected:
 
     void TearDown() override
     {
-        ASSERT_TRUE(QDir("/tmp/c2-tests").removeRecursively());
         this->mockApplication.reset();
         this->settings.reset();
         this->paths.reset();
         this->mockHelix.reset();
         this->completionModel.reset();
         this->channelPtr.reset();
+
+        this->settingsDir_.reset();
     }
 
+    std::unique_ptr<QTemporaryDir> settingsDir_;
+
     std::unique_ptr<MockApplication> mockApplication;
     std::unique_ptr<Settings> settings;
     std::unique_ptr<Paths> paths;
@@ -222,8 +226,14 @@ protected:
 
 TEST_F(InputCompletionTest, EmoteNameFiltering)
 {
+    // The completion doesn't guarantee an ordering for a specific category of emotes.
+    // This tests a specific implementation of the underlying std::unordered_map,
+    // so depending on the standard library used when compiling, this might yield
+    // different results.
+
     auto completion = queryEmoteCompletion(":feels");
     ASSERT_EQ(completion.size(), 3);
+    // all these matches are BTTV global emotes
     ASSERT_EQ(completion[0].displayName, "FeelsBirthdayMan");
     ASSERT_EQ(completion[1].displayName, "FeelsBadMan");
     ASSERT_EQ(completion[2].displayName, "FeelsGoodMan");
@@ -231,6 +241,7 @@ TEST_F(InputCompletionTest, EmoteNameFiltering)
     completion = queryEmoteCompletion(":)");
     ASSERT_EQ(completion.size(), 3);
     ASSERT_EQ(completion[0].displayName, ":)");  // Exact match with : prefix
+    // all these matches are Twitch global emotes
     ASSERT_EQ(completion[1].displayName, ":-)");
     ASSERT_EQ(completion[2].displayName, "B-)");
 

tests/src/NetworkRequest.cpp

@@ -12,8 +12,11 @@ using namespace chatterino;
 
 namespace {
 
-// Change to http://httpbin.org if you don't want to run the docker image yourself to test this
+#ifdef CHATTERINO_TEST_USE_PUBLIC_HTTPBIN
+const char *const HTTPBIN_BASE_URL = "http://httpbin.org";
+#else
 const char *const HTTPBIN_BASE_URL = "http://127.0.0.1:9051";
+#endif
 
 QString getStatusURL(int code)
 {

tests/src/TwitchPubSubClient.cpp

@@ -48,7 +48,7 @@ TEST(TwitchPubSubClient, ServerRespondsToPings)
 
     pubSub->listenToTopic("test");
 
-    std::this_thread::sleep_for(50ms);
+    std::this_thread::sleep_for(150ms);
 
     ASSERT_EQ(pubSub->diag.connectionsOpened, 1);
     ASSERT_EQ(pubSub->diag.connectionsClosed, 0);
@@ -211,7 +211,7 @@ TEST(TwitchPubSubClient, ExceedTopicLimitSingleStep)
         pubSub->listenToTopic("test");
     }
 
-    std::this_thread::sleep_for(50ms);
+    std::this_thread::sleep_for(150ms);
 
     ASSERT_EQ(pubSub->diag.connectionsOpened, 2);
     ASSERT_EQ(pubSub->diag.connectionsClosed, 0);
@@ -242,7 +242,7 @@ TEST(TwitchPubSubClient, ReceivedWhisper)
 
     pubSub->listenToTopic("whispers.123456");
 
-    std::this_thread::sleep_for(50ms);
+    std::this_thread::sleep_for(150ms);
 
     ASSERT_EQ(pubSub->diag.connectionsOpened, 1);
     ASSERT_EQ(pubSub->diag.connectionsClosed, 0);
@@ -324,7 +324,7 @@ TEST(TwitchPubSubClient, MissingToken)
 
     pubSub->listenToTopic("chat_moderator_actions.123456.123456");
 
-    std::this_thread::sleep_for(50ms);
+    std::this_thread::sleep_for(150ms);
 
     ASSERT_EQ(pubSub->diag.connectionsOpened, 1);
     ASSERT_EQ(pubSub->diag.connectionsClosed, 0);

tests/src/main.cpp

@@ -1,14 +1,10 @@
 #include "common/NetworkManager.hpp"
-#include "common/NetworkRequest.hpp"
-#include "common/NetworkResult.hpp"
-#include "common/Outcome.hpp"
-#include "common/QLogging.hpp"
-#include "providers/twitch/api/Helix.hpp"
 #include "singletons/Settings.hpp"
 
 #include <gtest/gtest.h>
 #include <QApplication>
 #include <QJsonArray>
+#include <QLoggingCategory>
 #include <QtConcurrent>
 #include <QTimer>
 
@@ -25,17 +21,23 @@ int main(int argc, char **argv)
 
 #ifdef SUPPORT_QT_NETWORK_TESTS
     QApplication app(argc, argv);
+    // make sure to always debug-log
+    QLoggingCategory::setFilterRules("*.debug=true");
 
     chatterino::NetworkManager::init();
 
     // Ensure settings are initialized before any tests are run
-    chatterino::Settings settings("/tmp/c2-empty-test");
+    QTemporaryDir settingsDir;
+    settingsDir.setAutoRemove(false);  // we'll remove it manually
+    qDebug() << "Settings directory:" << settingsDir.path();
+    chatterino::Settings settings(settingsDir.path());
 
-    QtConcurrent::run([&app] {
+    QtConcurrent::run([&app, &settingsDir]() mutable {
         auto res = RUN_ALL_TESTS();
 
         chatterino::NetworkManager::deinit();
 
+        settingsDir.remove();
         app.exit(res);
     });