From 8e6867a0ed431b6c7e9e9d9d257d26fd8b5de432 Mon Sep 17 00:00:00 2001
From: Lukas Holecek <hluk@email.cz>
Date: Wed, 9 Oct 2019 17:33:51 +0200
Subject: [PATCH] Tests: Add ability to rerun failed tests
Signed-off-by: Lukas Holecek <hluk@email.cz>
---
src/tests/tests.cpp | 33 +++++++++++++++++++++++++++++++--
utils/appveyor/after_build.bat | 3 ++-
utils/gitlab/test_gui-script.sh | 1 +
utils/travis/script-linux.sh | 1 +
utils/travis/script-osx.sh | 4 ++--
5 files changed, 37 insertions(+), 5 deletions(-)
diff --git a/src/tests/tests.cpp b/src/tests/tests.cpp
index 7719c299f..6a1c062d5 100644
--- a/src/tests/tests.cpp
+++ b/src/tests/tests.cpp
@@ -571,6 +571,7 @@ public:
QByteArray cleanup() override
{
+ addFailedTest();
return QByteArray();
}
@@ -604,7 +605,33 @@ public:
m_env.insert("COPYQ_TEST_ID", id);
}
+ int runTests(QObject *testObject, int argc = 0, char **argv = nullptr)
+ {
+ int exitCode = QTest::qExec(testObject, argc, argv);
+
+ const int maxRuns = m_env.value("COPYQ_TESTS_RERUN_FAILED", "0").toInt();
+ for (int runCounter = 0; exitCode != 0 && !m_failed.isEmpty() && runCounter < maxRuns; ++runCounter) {
+ qInfo() << QString("Rerunning %1 failed tests (%2/%3): %4")
+ .arg(m_failed.size())
+ .arg(runCounter + 1)
+ .arg(maxRuns)
+ .arg(m_failed.join(", "));
+ QStringList args = m_failed;
+ m_failed.clear();
+ args.prepend( QString::fromUtf8(argv[0]) );
+ exitCode = QTest::qExec(testObject, args);
+ }
+
+ return exitCode;
+ }
+
private:
+ void addFailedTest()
+ {
+ if ( QTest::currentTestFailed() )
+ m_failed.append( QString::fromUtf8(QTest::currentTestFunction()) );
+ }
+
void verifyConfiguration()
{
AppConfig appConfig;
@@ -652,6 +679,8 @@ private:
QProcessEnvironment m_env;
QString m_testId;
QVariantMap m_settings;
+
+ QStringList m_failed;
};
QString keyNameFor(QKeySequence::StandardKey standardKey)
@@ -3523,7 +3552,7 @@ int runTests(int argc, char *argv[])
if (onlyPlugins.isEmpty()) {
test->setupTest("CORE", QVariant());
- exitCode = QTest::qExec(&tc, argc, argv);
+ exitCode = test->runTests(&tc, argc, argv);
}
if (runPluginTests) {
@@ -3534,7 +3563,7 @@ int runTests(int argc, char *argv[])
std::unique_ptr<QObject> pluginTests( loader->tests(test) );
if ( pluginTests != nullptr ) {
test->setupTest(loader->id(), pluginTests->property("CopyQ_test_settings"));
- const int pluginTestsExitCode = QTest::qExec(pluginTests.get(), argc, argv);
+ const int pluginTestsExitCode = test->runTests(pluginTests.get(), argc, argv);
exitCode = qMax(exitCode, pluginTestsExitCode);
test->stopServer();
}
diff --git a/utils/appveyor/after_build.bat b/utils/appveyor/after_build.bat
index eb49df639..cbea4fb59 100644
--- a/utils/appveyor/after_build.bat
+++ b/utils/appveyor/after_build.bat
@@ -37,10 +37,11 @@ choco install -y InnoSetup || goto :error
"C:\ProgramData\chocolatey\bin\ISCC.exe" "/O%APPVEYOR_BUILD_FOLDER%" "/DAppVersion=%AppVersion%" "/DRoot=%Destination%" "/DSource=%Source%" "%Source%\Shared\copyq.iss" || goto :error
set QT_FORCE_STDERR_LOGGING=1
+set COPYQ_TESTS_RERUN_FAILED=1
"%Executable%" --help || goto :error
"%Executable%" --version || goto :error
"%Executable%" --info || goto :error
-"%Executable%" tests || "%Executable%" tests || goto :error
+"%Executable%" tests || goto :error
:error
exit /b %errorlevel%
diff --git a/utils/gitlab/test_gui-script.sh b/utils/gitlab/test_gui-script.sh
index c85ccc51c..1cda4fcbe 100755
--- a/utils/gitlab/test_gui-script.sh
+++ b/utils/gitlab/test_gui-script.sh
@@ -33,5 +33,6 @@ export COPYQ_LOG_FILE="$TESTS_LOG_DIR/copyq.log"
# Disable encryption tests because exporting GPG key asks for password.
export COPYQ_TESTS_SKIP_ITEMENCRYPT=1
+export COPYQ_TESTS_RERUN_FAILED=0
"$INSTALL_PREFIX/bin/copyq" tests
diff --git a/utils/travis/script-linux.sh b/utils/travis/script-linux.sh
index 36174ba33..b827bd4c0 100755
--- a/utils/travis/script-linux.sh
+++ b/utils/travis/script-linux.sh
@@ -45,6 +45,7 @@ sleep 8
rm -rf ~/.config/copyq.test
# Run tests.
+export COPYQ_TESTS_RERUN_FAILED=0
./copyq tests
cd "$root"
diff --git a/utils/travis/script-osx.sh b/utils/travis/script-osx.sh
index fa683125f..6074371d3 100755
--- a/utils/travis/script-osx.sh
+++ b/utils/travis/script-osx.sh
@@ -36,8 +36,8 @@ brew uninstall --force qt5
# Run tests (retry once on error).
export COPYQ_TESTS_SKIP_COMMAND_EDIT=1
export COPYQ_TESTS_SKIP_CONFIG_MOVE=1
-"$executable" tests ||
- "$executable" tests
+export COPYQ_TESTS_RERUN_FAILED=1
+"$executable" tests
# Print dependencies to let us further make sure that we don't depend on local libraries
otool -L $executable
--
GitLab