Teuchos Package Browser (Single Doxygen Collection) Version of the Day
Teuchos_UnitTestRepository.cpp
Go to the documentation of this file.
00001 // @HEADER
00002 // ***********************************************************************
00003 //
00004 //                    Teuchos: Common Tools Package
00005 //                 Copyright (2004) Sandia Corporation
00006 //
00007 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
00008 // license for use of this work by or on behalf of the U.S. Government.
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are
00012 // met:
00013 //
00014 // 1. Redistributions of source code must retain the above copyright
00015 // notice, this list of conditions and the following disclaimer.
00016 //
00017 // 2. Redistributions in binary form must reproduce the above copyright
00018 // notice, this list of conditions and the following disclaimer in the
00019 // documentation and/or other materials provided with the distribution.
00020 //
00021 // 3. Neither the name of the Corporation nor the names of the
00022 // contributors may be used to endorse or promote products derived from
00023 // this software without specific prior written permission.
00024 //
00025 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
00026 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
00027 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
00028 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
00029 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
00030 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
00031 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00032 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00033 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00034 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00035 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00036 //
00037 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
00038 //
00039 // ***********************************************************************
00040 // @HEADER
00041 
00042 
00043 #include "Teuchos_UnitTestRepository.hpp"
00044 #include "Teuchos_UnitTestBase.hpp"
00045 #include "Teuchos_TestingHelpers.hpp"
00046 #include "Teuchos_Array.hpp"
00047 #include "Teuchos_TestForException.hpp"
00048 #include "Teuchos_VerboseObject.hpp"
00049 #include "Teuchos_CommandLineProcessor.hpp"
00050 #include "Teuchos_Assert.hpp"
00051 #include "Teuchos_Time.hpp"
00052 #include "Teuchos_StandardCatchMacros.hpp"
00053 
00054 
00055 namespace Teuchos {
00056 
00057 
00058 struct UnitTestData {
00059 
00060   const Teuchos::UnitTestBase * unitTest;
00061   std::string groupName;
00062   std::string testName;
00063   int insertionIndex;
00064 
00065   UnitTestData(
00066     Teuchos::UnitTestBase *unitTest_in,
00067     const std::string groupName_in,
00068     const std::string testName_in
00069     )
00070     : unitTest(unitTest_in), groupName(groupName_in), testName(testName_in),
00071       insertionIndex(insersionIndexCounter_++)
00072     {
00073 #ifdef TEUCHOS_DEBUG
00074       TEUCHOS_ASSERT(unitTest_in);
00075 #endif
00076     }
00077 
00078 private:
00079   UnitTestData(); // Not defined!
00080   static int insersionIndexCounter_;
00081 };
00082 
00083 
00084 int UnitTestData::insersionIndexCounter_ = 0;
00085 
00086 
00087 bool operator<(const UnitTestData &a, const UnitTestData &b)
00088 {
00089   if (a.groupName < b.groupName) {
00090     return true;
00091   }
00092   else if (a.groupName > b.groupName) {
00093     return false;
00094   }
00095   return a.insertionIndex < b.insertionIndex;
00096 }
00097 
00098 
00099 
00100 std::string getUnitTestName(const std::string groupName,
00101   const std::string testName)
00102 {
00103   std::ostringstream oss;
00104   oss << groupName<<"_"<<testName<<"_UnitTest";
00105   return oss.str();
00106 }
00107 
00108 
00109 enum EShowTestDetails {
00110   SHOW_TEST_DETAILS_ALL,
00111   SHOW_TEST_DETAILS_TEST_NAMES,
00112   SHOW_TEST_DETAILS_FINAL_RESULTS
00113 };
00114 
00115 
00116 bool strMatch( const std::string &fullMatchStr, const std::string &str )
00117 {
00118 
00119   const std::string::size_type npos = std::string::npos;
00120 
00121   const int strLen = str.length();
00122   const int fullMatchStrLen = fullMatchStr.length();
00123 
00124   if (fullMatchStrLen == 0) {
00125     return true;
00126   }
00127 
00128   const bool beginGlob = fullMatchStr[0] == '*';
00129   const bool endGlob = fullMatchStr[fullMatchStrLen-1] == '*';
00130 
00131   const int matchStrLen =
00132   fullMatchStrLen + (beginGlob ? -1 : 0) + (endGlob ? -1 : 0);
00133 
00134   if (matchStrLen == 0) {
00135     return true;
00136   }
00137 
00138   if (matchStrLen > strLen) {
00139     return false;
00140   }
00141 
00142   if (beginGlob && endGlob) {
00143     return str.find(fullMatchStr.substr(1, matchStrLen)) != npos;
00144   }
00145 
00146   if (endGlob) {
00147     return fullMatchStr.substr(0, matchStrLen) == str.substr(0, matchStrLen);
00148   }
00149 
00150   if (beginGlob) {
00151     return fullMatchStr.substr(1, matchStrLen) ==
00152       str.substr(strLen-matchStrLen, matchStrLen);
00153   }
00154 
00155   return fullMatchStr == str;
00156 
00157 }
00158 
00159 
00160 } // namespace Teuchos
00161 
00162 
00163 
00164 
00165 namespace Teuchos {
00166 
00167 
00168 // Implementation class
00169 
00170 
00171 class UnitTestRepository::InstanceData {
00172 public:
00173 
00174   typedef Teuchos::Array<UnitTestData> unitTests_t;
00175 
00176   unitTests_t unitTests;
00177   CommandLineProcessor clp;
00178   EShowTestDetails showTestDetails;
00179   bool showSrcLocation;
00180   bool showFailSrcLocation;
00181   bool noOp;
00182   std::string groupName;
00183   std::string testName;
00184   std::string notUnitTestName;
00185   int testCounter;
00186 
00187   InstanceData()
00188     :clp(false),
00189      showTestDetails(SHOW_TEST_DETAILS_TEST_NAMES),
00190      showSrcLocation(false),
00191      showFailSrcLocation(true),
00192      noOp(false),
00193      testCounter(0)
00194     {}
00195 
00196 };
00197 
00198 
00199 // public
00200 
00201 
00202 CommandLineProcessor& UnitTestRepository::getCLP()
00203 {
00204   return getData().clp;
00205 }
00206 
00207 
00208 bool UnitTestRepository::runUnitTests(FancyOStream &out)
00209 {
00210 
00211   typedef InstanceData::unitTests_t unitTests_t;
00212 
00213   using std::setprecision;
00214 
00215   Time overallTimer("overallTimer", true);
00216   Time timer("timer");
00217 
00218   const int timerPrec = 3;
00219 
00220   out << "\n***\n*** Unit test suite ...\n***\n\n";
00221 
00222   InstanceData &data = getData();
00223 
00224   const bool showAll = data.showTestDetails == SHOW_TEST_DETAILS_ALL;
00225   const bool showTestNames = data.showTestDetails == SHOW_TEST_DETAILS_TEST_NAMES || showAll;
00226 
00227   showTestFailureLocation(data.showFailSrcLocation);
00228 
00229   bool success = true;
00230   int testCounter = 0;
00231   int numTestsRun = 0;
00232   int numTestsFailed = 0;
00233 
00234   Array<std::string> failedTests;
00235 
00236   try {
00237     
00238     out << "\nSorting tests by group name then by the order they where added ...";
00239     timer.start(true);
00240     std::sort( data.unitTests.begin(), data.unitTests.end() );
00241     timer.stop();
00242     out << " (time = "<<setprecision(timerPrec)<<timer.totalElapsedTime()<<")\n";
00243 
00244     out << "\nRunning unit tests ...\n\n";
00245     unitTests_t::iterator iter = data.unitTests.begin();
00246     for ( ; iter != data.unitTests.end(); ++iter, ++testCounter ) {
00247 
00248       const UnitTestData &utd = (*iter);
00249 
00250       const std::string unitTestName = getUnitTestName(utd.groupName, utd.testName);
00251 
00252       if (
00253         (
00254           strMatch(data.groupName, utd.groupName)
00255           &&
00256           strMatch(data.testName, utd.testName)
00257           )
00258         &&
00259         (
00260           data.notUnitTestName.length() == 0
00261           ||
00262           !strMatch(data.notUnitTestName, unitTestName)
00263           )
00264         )
00265       {
00266 
00267         ++numTestsRun;
00268 
00269         std::ostringstream testHeaderOSS;
00270         testHeaderOSS <<testCounter<<". "<<unitTestName<<" ... ";
00271         const std::string testHeader = testHeaderOSS.str();
00272 
00273         if (showAll)
00274           out <<"\n";
00275 
00276         if (showTestNames)
00277           out <<testHeader<<std::flush;
00278 
00279         {
00280 
00281           RCP<std::ostringstream> oss;
00282           RCP<FancyOStream> localOut;
00283           if (showAll) {
00284             out << "\n";
00285             localOut = rcpFromRef(out);
00286           }
00287           else {
00288             oss = rcp(new std::ostringstream);
00289             localOut = fancyOStream(rcp_implicit_cast<std::ostream>(oss));
00290           }
00291 
00292           OSTab tab(out);
00293 
00294           if (!data.noOp) {
00295 
00296             timer.start(true);
00297             const bool result = utd.unitTest->runUnitTest(*localOut);
00298             timer.stop();
00299 
00300             if (!result) {
00301 
00302               failedTests.push_back(testHeader);
00303               
00304               if (!showTestNames)
00305                 out <<testHeader<<"\n"<<std::flush;
00306               else if (!showAll)
00307                 out <<"\n";
00308               
00309               if (!is_null(oss))
00310                 out << oss->str();
00311               
00312               out
00313                 <<"[FAILED] "
00314                 <<" "<<setprecision(timerPrec)<<"("<<timer.totalElapsedTime()<< " sec)"
00315                 <<" "<<unitTestName<<"\n"
00316                 <<"Location: "<<utd.unitTest->unitTestFile()<<":"
00317                 <<utd.unitTest->unitTestFileLineNumber()<<"\n";
00318               
00319               if (!is_null(oss))
00320                 out << "\n";
00321               
00322               success = false;
00323               
00324               ++numTestsFailed;
00325               
00326             }
00327             else {
00328               
00329               if (showTestNames)
00330                 out << "[Passed] "
00331                     << setprecision(timerPrec)<<"("<<timer.totalElapsedTime()<<" sec)\n";
00332               
00333               if (showAll && data.showSrcLocation)
00334                 out
00335                   << "Location: "<<utd.unitTest->unitTestFile()<<":"
00336                   <<utd.unitTest->unitTestFileLineNumber()<<"\n";
00337               
00338             }
00339 
00340           }
00341           else {
00342 
00343             if (showTestNames)
00344               out << "[Not Run]\n";
00345             
00346           }
00347 
00348         }
00349    
00350       }
00351 
00352     }
00353 
00354     TEUCHOS_ASSERT_EQUALITY(testCounter, as<int>(data.unitTests.size()));
00355 
00356   }
00357   TEUCHOS_STANDARD_CATCH_STATEMENTS(true, out, success);
00358 
00359   if (failedTests.size()) {
00360     out << "\nThe following tests FAILED:\n";
00361     for (Teuchos_Ordinal i = 0; i < failedTests.size(); ++i)
00362       out << "    " << failedTests[i] << "\n";
00363   }
00364 
00365   overallTimer.stop();
00366   out << "\nTotal Time: " << setprecision(timerPrec)
00367       << overallTimer.totalElapsedTime() << " sec\n";
00368 
00369   out
00370     << "\nSummary: total = " << testCounter
00371     << ", run = " << numTestsRun;
00372 
00373   if (!data.noOp) {
00374     out
00375       << ", passed = " << (numTestsRun-numTestsFailed)
00376       << ", failed = " << numTestsFailed << "\n";
00377   }
00378   else {
00379     out
00380       << ", passed = ???"
00381       << ", failed = ???\n";
00382   }
00383     
00384   return success;
00385 
00386 }
00387 
00388 
00389 int UnitTestRepository::runUnitTestsFromMain( int argc, char* argv[] )
00390 {
00391 
00392   const RCP<FancyOStream> out = VerboseObjectBase::getDefaultOStream();
00393 
00394   CommandLineProcessor &clp = getData().clp;
00395   setUpCLP(outArg(clp));
00396   CommandLineProcessor::EParseCommandLineReturn parse_return =
00397     clp.parse(argc,argv);
00398   if ( parse_return != CommandLineProcessor::PARSE_SUCCESSFUL ) {
00399     *out << "\nEnd Result: TEST FAILED" << std::endl;
00400     return parse_return;
00401   }
00402 
00403   const bool success = runUnitTests(*out);
00404 
00405   if (success)
00406     *out << "\nEnd Result: TEST PASSED" << std::endl;
00407   else
00408     *out << "\nEnd Result: TEST FAILED" << std::endl;
00409 
00410   return (success ? 0 : 1);
00411 
00412 }
00413 
00414 
00415 void UnitTestRepository::addUnitTest( UnitTestBase *unitTest,
00416   const std::string groupName, const std::string testName_in )
00417 {
00418   InstanceData &data = getData();
00419   std::string testName = testName_in;
00420   data.unitTests.push_back(UnitTestData(unitTest, groupName, testName));
00421 }
00422 
00423 
00424 // private:
00425 
00426 
00427 UnitTestRepository::UnitTestRepository()
00428 {}
00429 
00430 
00431 void UnitTestRepository::setUpCLP(const Ptr<CommandLineProcessor>& clp)
00432 {
00433 
00434   clp->addOutputSetupOptions(true);
00435 
00436   const int numShowTestDetails = 3;
00437   const EShowTestDetails showTestDetailsValues[numShowTestDetails] =
00438     { SHOW_TEST_DETAILS_ALL,
00439       SHOW_TEST_DETAILS_TEST_NAMES,
00440       SHOW_TEST_DETAILS_FINAL_RESULTS
00441     };
00442   const char* showTestDetailsNames[numShowTestDetails] =
00443     { "ALL",
00444       "TEST_NAMES",
00445       "FINAL_RESULTS"
00446     };
00447   clp->setOption(
00448     "show-test-details", &getData().showTestDetails,
00449     numShowTestDetails, showTestDetailsValues, showTestDetailsNames,
00450     "Level of detail to show in the tests"
00451     );
00452   clp->setOption(
00453     "details", &getData().showTestDetails,
00454     numShowTestDetails, showTestDetailsValues, showTestDetailsNames,
00455     "Short for --details"
00456     );
00457 
00458   clp->setOption(
00459     "show-src-location", "no-show-src-location", &getData().showSrcLocation,
00460     "If true, then the location of the unit test source code is shown."
00461     "  Only meaningfull if --show-test-details=ALL."
00462     );
00463 
00464   clp->setOption(
00465     "show-fail-src-location", "no-show-fail-src-location", &getData().showFailSrcLocation,
00466     "If true, then the location of every failed unit test check is printed."
00467     );
00468 
00469   clp->setOption(
00470     "group-name", &getData().groupName,
00471     "If specified, selects only tests that match the group name glob." );
00472   clp->setOption(
00473     "group", &getData().groupName,
00474     "Short for --group-name." );
00475 
00476   clp->setOption(
00477     "test-name", &getData().testName,
00478     "If specified, selects only tests that match the test name glob." );
00479   clp->setOption(
00480     "test", &getData().testName,
00481     "Short for --test-name." );
00482 
00483   clp->setOption(
00484     "not-unit-test", &getData().notUnitTestName,
00485     "If specified, full unit tests with glob matches will *not* be run." );
00486 
00487   clp->setOption(
00488     "no-op", "do-op", &getData().noOp,
00489     "If --no-op, then only the names of the tests that would be run are run."
00490     );
00491   
00492 }
00493 
00494 
00495 UnitTestRepository::InstanceData& UnitTestRepository::getData()
00496 {
00497   static UnitTestRepository::InstanceData data;
00498   return data;
00499 }
00500 
00501 
00502 } // namespace Teuchos
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines