diff --git a/src/tests/tests.cpp b/src/tests/tests.cpp index 7719c299f8806e86ad4cc95294d94d7bcdb4cf27..6a1c062d59717db274eb13499bb86aa8d458ccf4 100644 --- a/src/tests/tests.cpp +++ b/src/tests/tests.cpp @@ -571,6 +571,7 @@ public: QByteArray cleanup() override { + addFailedTest(); return QByteArray(); } @@ -604,7 +605,33 @@ public: m_env.insert("COPYQ_TEST_ID", id); } + int runTests(QObject *testObject, int argc = 0, char **argv = nullptr) + { + int exitCode = QTest::qExec(testObject, argc, argv); + + const int maxRuns = m_env.value("COPYQ_TESTS_RERUN_FAILED", "0").toInt(); + for (int runCounter = 0; exitCode != 0 && !m_failed.isEmpty() && runCounter < maxRuns; ++runCounter) { + qInfo() << QString("Rerunning %1 failed tests (%2/%3): %4") + .arg(m_failed.size()) + .arg(runCounter + 1) + .arg(maxRuns) + .arg(m_failed.join(", ")); + QStringList args = m_failed; + m_failed.clear(); + args.prepend( QString::fromUtf8(argv[0]) ); + exitCode = QTest::qExec(testObject, args); + } + + return exitCode; + } + private: + void addFailedTest() + { + if ( QTest::currentTestFailed() ) + m_failed.append( QString::fromUtf8(QTest::currentTestFunction()) ); + } + void verifyConfiguration() { AppConfig appConfig; @@ -652,6 +679,8 @@ private: QProcessEnvironment m_env; QString m_testId; QVariantMap m_settings; + + QStringList m_failed; }; QString keyNameFor(QKeySequence::StandardKey standardKey) @@ -3523,7 +3552,7 @@ int runTests(int argc, char *argv[]) if (onlyPlugins.isEmpty()) { test->setupTest("CORE", QVariant()); - exitCode = QTest::qExec(&tc, argc, argv); + exitCode = test->runTests(&tc, argc, argv); } if (runPluginTests) { @@ -3534,7 +3563,7 @@ int runTests(int argc, char *argv[]) std::unique_ptr<QObject> pluginTests( loader->tests(test) ); if ( pluginTests != nullptr ) { test->setupTest(loader->id(), pluginTests->property("CopyQ_test_settings")); - const int pluginTestsExitCode = QTest::qExec(pluginTests.get(), argc, argv); + const int pluginTestsExitCode = test->runTests(pluginTests.get(), argc, argv); exitCode = qMax(exitCode, pluginTestsExitCode); test->stopServer(); } diff --git a/utils/appveyor/after_build.bat b/utils/appveyor/after_build.bat index eb49df639386ac4cee7890cf206ed9c14192f661..cbea4fb59be7c73dec507733fecea875193d24ab 100644 --- a/utils/appveyor/after_build.bat +++ b/utils/appveyor/after_build.bat @@ -37,10 +37,11 @@ choco install -y InnoSetup || goto :error "C:\ProgramData\chocolatey\bin\ISCC.exe" "/O%APPVEYOR_BUILD_FOLDER%" "/DAppVersion=%AppVersion%" "/DRoot=%Destination%" "/DSource=%Source%" "%Source%\Shared\copyq.iss" || goto :error set QT_FORCE_STDERR_LOGGING=1 +set COPYQ_TESTS_RERUN_FAILED=1 "%Executable%" --help || goto :error "%Executable%" --version || goto :error "%Executable%" --info || goto :error -"%Executable%" tests || "%Executable%" tests || goto :error +"%Executable%" tests || goto :error :error exit /b %errorlevel% diff --git a/utils/gitlab/test_gui-script.sh b/utils/gitlab/test_gui-script.sh index c85ccc51c506214da8e8e618dfbfe3c9f3a00279..1cda4fcbe71d2ab0e2dfaf274cabe39c31437479 100755 --- a/utils/gitlab/test_gui-script.sh +++ b/utils/gitlab/test_gui-script.sh @@ -33,5 +33,6 @@ export COPYQ_LOG_FILE="$TESTS_LOG_DIR/copyq.log" # Disable encryption tests because exporting GPG key asks for password. export COPYQ_TESTS_SKIP_ITEMENCRYPT=1 +export COPYQ_TESTS_RERUN_FAILED=0 "$INSTALL_PREFIX/bin/copyq" tests diff --git a/utils/travis/script-linux.sh b/utils/travis/script-linux.sh index 36174ba33f241cd340aeb33dcc2cc69979daba04..b827bd4c0694464ad54d6deda63b4aa2d4e8b370 100755 --- a/utils/travis/script-linux.sh +++ b/utils/travis/script-linux.sh @@ -45,6 +45,7 @@ sleep 8 rm -rf ~/.config/copyq.test # Run tests. +export COPYQ_TESTS_RERUN_FAILED=0 ./copyq tests cd "$root" diff --git a/utils/travis/script-osx.sh b/utils/travis/script-osx.sh index fa683125f8856c2ce26fa46135f1a3cb326202b7..6074371d3479b675fb7dc2a252515eafbaa5decc 100755 --- a/utils/travis/script-osx.sh +++ b/utils/travis/script-osx.sh @@ -36,8 +36,8 @@ brew uninstall --force qt5 # Run tests (retry once on error). export COPYQ_TESTS_SKIP_COMMAND_EDIT=1 export COPYQ_TESTS_SKIP_CONFIG_MOVE=1 -"$executable" tests || - "$executable" tests +export COPYQ_TESTS_RERUN_FAILED=1 +"$executable" tests # Print dependencies to let us further make sure that we don't depend on local libraries otool -L $executable