Teuchos Package Browser (Single Doxygen Collection) Version of the Day
Teuchos_UnitTestRepository.cpp
Go to the documentation of this file.
00001 // @HEADER
00002 // ***********************************************************************
00003 //
00004 //                    Teuchos: Common Tools Package
00005 //                 Copyright (2004) Sandia Corporation
00006 //
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
00038 //
00039 // ***********************************************************************
00040 // @HEADER
00041 
00042 
00043 #include "Teuchos_UnitTestRepository.hpp"
00044 #include "Teuchos_UnitTestBase.hpp"
00045 #include "Teuchos_TestingHelpers.hpp"
00046 #include "Teuchos_Array.hpp"
00047 #include "Teuchos_Assert.hpp"
00048 #include "Teuchos_VerboseObject.hpp"
00049 #include "Teuchos_CommandLineProcessor.hpp"
00050 #include "Teuchos_Assert.hpp"
00051 #include "Teuchos_Time.hpp"
00052 #include "Teuchos_StandardCatchMacros.hpp"
00053 
00054 
00055 namespace Teuchos {
00056 
00057 
00058 struct UnitTestData {
00059 
00060   const Teuchos::UnitTestBase * unitTest;
00061   std::string groupName;
00062   std::string testName;
00063   int insertionIndex;
00064 
00065   UnitTestData(
00066     Teuchos::UnitTestBase *unitTest_in,
00067     const std::string groupName_in,
00068     const std::string testName_in
00069     )
00070     : unitTest(unitTest_in), groupName(groupName_in), testName(testName_in),
00071       insertionIndex(insersionIndexCounter_++)
00072     {
00073 #ifdef TEUCHOS_DEBUG
00074       TEUCHOS_ASSERT(unitTest_in);
00075 #endif
00076     }
00077 
00078 private:
00079   UnitTestData(); // Not defined!
00080   static int insersionIndexCounter_;
00081 };
00082 
00083 
00084 int UnitTestData::insersionIndexCounter_ = 0;
00085 
00086 
00087 bool operator<(const UnitTestData &a, const UnitTestData &b)
00088 {
00089   if (a.groupName < b.groupName) {
00090     return true;
00091   }
00092   else if (a.groupName > b.groupName) {
00093     return false;
00094   }
00095   return a.insertionIndex < b.insertionIndex;
00096 }
00097 
00098 
00099 
00100 std::string getUnitTestName(const std::string groupName,
00101   const std::string testName)
00102 {
00103   std::ostringstream oss;
00104   oss << groupName<<"_"<<testName<<"_UnitTest";
00105   return oss.str();
00106 }
00107 
00108 
00109 enum EShowTestDetails {
00110   SHOW_TEST_DETAILS_ALL,
00111   SHOW_TEST_DETAILS_TEST_NAMES,
00112   SHOW_TEST_DETAILS_FINAL_RESULTS
00113 };
00114 
00115 
00116 bool strMatch( const std::string &fullMatchStr, const std::string &str )
00117 {
00118 
00119   const std::string::size_type npos = std::string::npos;
00120 
00121   const int strLen = str.length();
00122   const int fullMatchStrLen = fullMatchStr.length();
00123 
00124   if (fullMatchStrLen == 0) {
00125     return true;
00126   }
00127 
00128   const bool beginGlob = fullMatchStr[0] == '*';
00129   const bool endGlob = fullMatchStr[fullMatchStrLen-1] == '*';
00130 
00131   const int matchStrLen =
00132   fullMatchStrLen + (beginGlob ? -1 : 0) + (endGlob ? -1 : 0);
00133 
00134   if (matchStrLen == 0) {
00135     return true;
00136   }
00137 
00138   if (matchStrLen > strLen) {
00139     return false;
00140   }
00141 
00142   if (beginGlob && endGlob) {
00143     return str.find(fullMatchStr.substr(1, matchStrLen)) != npos;
00144   }
00145 
00146   if (endGlob) {
00147     return fullMatchStr.substr(0, matchStrLen) == str.substr(0, matchStrLen);
00148   }
00149 
00150   if (beginGlob) {
00151     return fullMatchStr.substr(1, matchStrLen) ==
00152       str.substr(strLen-matchStrLen, matchStrLen);
00153   }
00154 
00155   return fullMatchStr == str;
00156 
00157 }
00158 
00159 
00160 } // namespace Teuchos
00161 
00162 
00163 
00164 
00165 namespace Teuchos {
00166 
00167 
00168 // Implementation class
00169 
00170 
00171 class UnitTestRepository::InstanceData {
00172 public:
00173 
00174   typedef Teuchos::Array<UnitTestData> unitTests_t;
00175 
00176   unitTests_t unitTests;
00177   CommandLineProcessor clp;
00178   EShowTestDetails showTestDetails;
00179   bool globallyReduceUnitTestResult;
00180   bool showSrcLocation;
00181   bool showFailSrcLocation;
00182   bool noOp;
00183   std::string groupName;
00184   std::string testName;
00185   std::string notUnitTestName;
00186   int testCounter;
00187 
00188   InstanceData()
00189     :clp(false),
00190      showTestDetails(SHOW_TEST_DETAILS_TEST_NAMES),
00191      globallyReduceUnitTestResult(false),
00192      showSrcLocation(false),
00193      showFailSrcLocation(true),
00194      noOp(false),
00195      testCounter(0)
00196     {}
00197 
00198 };
00199 
00200 
00201 // public
00202 
00203 
00204 CommandLineProcessor& UnitTestRepository::getCLP()
00205 {
00206   return getData().clp;
00207 }
00208 
00209 
00210 void UnitTestRepository::setGloballyReduceTestResult(
00211   const bool globallyReduceUnitTestResult)
00212 {
00213   getData().globallyReduceUnitTestResult = globallyReduceUnitTestResult;
00214 }
00215 
00216 
00217 bool UnitTestRepository::getGloballyReduceTestResult()
00218 {
00219   return getData().globallyReduceUnitTestResult;
00220 }
00221 
00222 
00223 bool UnitTestRepository::runUnitTests(FancyOStream &out)
00224 {
00225 
00226   typedef InstanceData::unitTests_t unitTests_t;
00227 
00228   using std::setprecision;
00229 
00230   Time overallTimer("overallTimer", true);
00231   Time timer("timer");
00232 
00233   const int timerPrec = 3;
00234 
00235   out << "\n***\n*** Unit test suite ...\n***\n\n";
00236 
00237   InstanceData &data = getData();
00238 
00239   const bool showAll = data.showTestDetails == SHOW_TEST_DETAILS_ALL;
00240   const bool showTestNames = data.showTestDetails == SHOW_TEST_DETAILS_TEST_NAMES || showAll;
00241 
00242   showTestFailureLocation(data.showFailSrcLocation);
00243 
00244   bool success = true;
00245   int testCounter = 0;
00246   int numTestsRun = 0;
00247   int numTestsFailed = 0;
00248 
00249   Array<std::string> failedTests;
00250 
00251   try {
00252     
00253     out << "\nSorting tests by group name then by the order they were added ...";
00254     timer.start(true);
00255     std::sort( data.unitTests.begin(), data.unitTests.end() );
00256     timer.stop();
00257     out << " (time = "<<setprecision(timerPrec)<<timer.totalElapsedTime()<<")\n";
00258 
00259     out << "\nRunning unit tests ...\n\n";
00260     unitTests_t::iterator iter = data.unitTests.begin();
00261     for ( ; iter != data.unitTests.end(); ++iter, ++testCounter ) {
00262 
00263       const UnitTestData &utd = (*iter);
00264 
00265       const std::string unitTestName = getUnitTestName(utd.groupName, utd.testName);
00266 
00267       if (
00268         (
00269           strMatch(data.groupName, utd.groupName)
00270           &&
00271           strMatch(data.testName, utd.testName)
00272           )
00273         &&
00274         (
00275           data.notUnitTestName.length() == 0
00276           ||
00277           !strMatch(data.notUnitTestName, unitTestName)
00278           )
00279         )
00280       {
00281 
00282         ++numTestsRun;
00283 
00284         std::ostringstream testHeaderOSS;
00285         testHeaderOSS <<testCounter<<". "<<unitTestName<<" ... ";
00286         const std::string testHeader = testHeaderOSS.str();
00287 
00288         if (showAll)
00289           out <<"\n";
00290 
00291         if (showTestNames)
00292           out <<testHeader<<std::flush;
00293 
00294         {
00295 
00296           RCP<std::ostringstream> oss;
00297           RCP<FancyOStream> localOut;
00298           if (showAll) {
00299             out << "\n";
00300             localOut = rcpFromRef(out);
00301           }
00302           else {
00303             oss = rcp(new std::ostringstream);
00304             localOut = fancyOStream(rcp_implicit_cast<std::ostream>(oss));
00305           }
00306 
00307           OSTab tab(out);
00308 
00309           if (!data.noOp) {
00310 
00311             timer.start(true);
00312             const bool result = runUnitTestImpl(*utd.unitTest, *localOut);
00313             timer.stop();
00314 
00315             if (!result) {
00316 
00317               failedTests.push_back(testHeader);
00318               
00319               if (!showTestNames)
00320                 out <<testHeader<<"\n"<<std::flush;
00321               else if (!showAll)
00322                 out <<"\n";
00323               
00324               if (!is_null(oss))
00325                 out << oss->str();
00326               
00327               out
00328                 <<"[FAILED] "
00329                 <<" "<<setprecision(timerPrec)<<"("<<timer.totalElapsedTime()<< " sec)"
00330                 <<" "<<unitTestName<<"\n"
00331                 <<"Location: "<<utd.unitTest->unitTestFile()<<":"
00332                 <<utd.unitTest->unitTestFileLineNumber()<<"\n";
00333               
00334               if (!is_null(oss))
00335                 out << "\n";
00336               
00337               success = false;
00338               
00339               ++numTestsFailed;
00340               
00341             }
00342             else {
00343               
00344               if (showTestNames)
00345                 out << "[Passed] "
00346                     << setprecision(timerPrec)<<"("<<timer.totalElapsedTime()<<" sec)\n";
00347               
00348               if (showAll && data.showSrcLocation)
00349                 out
00350                   << "Location: "<<utd.unitTest->unitTestFile()<<":"
00351                   <<utd.unitTest->unitTestFileLineNumber()<<"\n";
00352               
00353             }
00354 
00355           }
00356           else {
00357 
00358             if (showTestNames)
00359               out << "[Not Run]\n";
00360             
00361           }
00362 
00363         }
00364    
00365       }
00366 
00367     }
00368 
00369     TEUCHOS_ASSERT_EQUALITY(testCounter, as<int>(data.unitTests.size()));
00370 
00371   }
00372   TEUCHOS_STANDARD_CATCH_STATEMENTS(true, out, success);
00373 
00374   if (failedTests.size()) {
00375     out << "\nThe following tests FAILED:\n";
00376     for (Teuchos_Ordinal i = 0; i < failedTests.size(); ++i)
00377       out << "    " << failedTests[i] << "\n";
00378   }
00379 
00380   overallTimer.stop();
00381   out << "\nTotal Time: " << setprecision(timerPrec)
00382       << overallTimer.totalElapsedTime() << " sec\n";
00383 
00384   out
00385     << "\nSummary: total = " << testCounter
00386     << ", run = " << numTestsRun;
00387 
00388   if (!data.noOp) {
00389     out
00390       << ", passed = " << (numTestsRun-numTestsFailed)
00391       << ", failed = " << numTestsFailed << "\n";
00392   }
00393   else {
00394     out
00395       << ", passed = ???"
00396       << ", failed = ???\n";
00397   }
00398     
00399   return success;
00400 
00401 }
00402 
00403 
00404 int UnitTestRepository::runUnitTestsFromMain( int argc, char* argv[] )
00405 {
00406 
00407   const RCP<FancyOStream> out = VerboseObjectBase::getDefaultOStream();
00408 
00409   CommandLineProcessor &clp = getData().clp;
00410   setUpCLP(outArg(clp));
00411   CommandLineProcessor::EParseCommandLineReturn parse_return =
00412     clp.parse(argc,argv);
00413   if ( parse_return != CommandLineProcessor::PARSE_SUCCESSFUL ) {
00414     *out << "\nEnd Result: TEST FAILED" << std::endl;
00415     return parse_return;
00416   }
00417 
00418   const bool success = runUnitTests(*out);
00419 
00420   if (success)
00421     *out << "\nEnd Result: TEST PASSED" << std::endl;
00422   else
00423     *out << "\nEnd Result: TEST FAILED" << std::endl;
00424 
00425   clp.printFinalTimerSummary(out.ptr());
00426 
00427   return (success ? 0 : 1);
00428 
00429 }
00430 
00431 
00432 void UnitTestRepository::addUnitTest( UnitTestBase *unitTest,
00433   const std::string groupName, const std::string testName_in )
00434 {
00435   InstanceData &data = getData();
00436   std::string testName = testName_in;
00437   data.unitTests.push_back(UnitTestData(unitTest, groupName, testName));
00438 }
00439 
00440 
00441 bool UnitTestRepository::verboseUnitTests()
00442 {
00443   return (getData().showTestDetails == SHOW_TEST_DETAILS_ALL);
00444 }
00445 
00446 
00447 // private:
00448 
00449 
00450 UnitTestRepository::UnitTestRepository()
00451 {}
00452 
00453 
00454 void UnitTestRepository::setUpCLP(const Ptr<CommandLineProcessor>& clp)
00455 {
00456 
00457   clp->addOutputSetupOptions(true);
00458 
00459   const int numShowTestDetails = 3;
00460   const EShowTestDetails showTestDetailsValues[numShowTestDetails] =
00461     { SHOW_TEST_DETAILS_ALL,
00462       SHOW_TEST_DETAILS_TEST_NAMES,
00463       SHOW_TEST_DETAILS_FINAL_RESULTS
00464     };
00465   const char* showTestDetailsNames[numShowTestDetails] =
00466     { "ALL",
00467       "TEST_NAMES",
00468       "FINAL_RESULTS"
00469     };
00470   clp->setOption(
00471     "show-test-details", &getData().showTestDetails,
00472     numShowTestDetails, showTestDetailsValues, showTestDetailsNames,
00473     "Level of detail to show in the tests"
00474     );
00475   clp->setOption(
00476     "details", &getData().showTestDetails,
00477     numShowTestDetails, showTestDetailsValues, showTestDetailsNames,
00478     "Short for --show-test-details"
00479     );
00480 
00481   clp->setOption(
00482     "show-src-location", "no-show-src-location", &getData().showSrcLocation,
00483     "If true, then the location of the unit test source code is shown."
00484     "  Only meaningfull if --show-test-details=ALL."
00485     );
00486 
00487   clp->setOption(
00488     "show-fail-src-location", "no-show-fail-src-location", &getData().showFailSrcLocation,
00489     "If true, then the location of every failed unit test check is printed."
00490     );
00491 
00492   clp->setOption(
00493     "globally-reduce-test-result", "no-globally-reduce-test-result",
00494     &getData().globallyReduceUnitTestResult,
00495     "If true, individual unit test pass/fail is globally reduced across MPI processes."
00496     );
00497 
00498   clp->setOption(
00499     "group-name", &getData().groupName,
00500     "If specified, selects only tests that match the group name glob." );
00501   clp->setOption(
00502     "group", &getData().groupName,
00503     "Short for --group-name." );
00504 
00505   clp->setOption(
00506     "test-name", &getData().testName,
00507     "If specified, selects only tests that match the test name glob." );
00508   clp->setOption(
00509     "test", &getData().testName,
00510     "Short for --test-name." );
00511 
00512   clp->setOption(
00513     "not-unit-test", &getData().notUnitTestName,
00514     "If specified, full unit tests with glob matches will *not* be run." );
00515 
00516   clp->setOption(
00517     "no-op", "do-op", &getData().noOp,
00518     "If --no-op, then only the names of the tests that would be run are run."
00519     );
00520   
00521 }
00522 
00523 
00524 UnitTestRepository::InstanceData& UnitTestRepository::getData()
00525 {
00526   static UnitTestRepository::InstanceData data;
00527   return data;
00528 }
00529 
00530 
00531 bool UnitTestRepository::runUnitTestImpl(const UnitTestBase &unitTest,
00532   FancyOStream &out)
00533 {
00534   const bool result = unitTest.runUnitTest(out);
00535   if (getData().globallyReduceUnitTestResult) {
00536     const int globalSum = GlobalMPISession::sum(result ? 0 : 1);
00537     if (globalSum == 0) {
00538       return true;
00539     }
00540     else {
00541       // Only print that there are failures on processes where the local
00542       // unit test actally passed.  On processes where the local unit test
00543       // fails, users already know that test failed so there is no need to
00544       // exlain it.
00545       if (result) {
00546         out << "NOTE: Global reduction shows failures on other processes!\n"
00547             << "(rerun with --output-to-root-rank-only=-1 to see output\n"
00548             << "from other processes to see what process failed!)\n";
00549       }
00550       else {
00551         // The test failed on the root process so the user already knows it failed!
00552       }
00553       // Determine what processes have failing tests
00554       const int numProcs = GlobalMPISession::getNProc();
00555       Array<int> passFailFlags(numProcs);
00556       GlobalMPISession::allGather( result ? 0 : 1, passFailFlags());
00557       Array<int> procsThatFailed;
00558       for ( int proc_k = 0; proc_k < numProcs; ++proc_k ) {
00559         if (passFailFlags[proc_k] != 0) {
00560           procsThatFailed.push_back(proc_k);
00561         }
00562       }
00563       // Print what processes have the failing tests.  If there is only one
00564       // processes, don't print anything.
00565       if (numProcs > 1) {
00566         if (procsThatFailed.size() == numProcs) {
00567           out << "NOTE: Unit test failed on all processes!\n";
00568           // NOTE: when all the processes are failing it is useless to print
00569           // out a list of all of the processes.
00570         }
00571         else {
00572           out << "NOTE: Unit test failed on processes = " << procsThatFailed << "\n"
00573               << "(rerun with --output-to-root-rank-only=<procID> to see output\n"
00574               << "from individual processes where the unit test is failing!)\n";
00575         }
00576       }
00577       return false;
00578     }
00579   }
00580   return result;
00581 }
00582 
00583 
00584 } // namespace Teuchos
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines