/*------------------------------------------------------------------------- * drawElements Quality Program Test Executor * ------------------------------------------ * * Copyright 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * *//*! * \file * \brief Test log compare utility. *//*--------------------------------------------------------------------*/ #include "xeTestLogParser.hpp" #include "xeTestResultParser.hpp" #include "deFilePath.hpp" #include "deString.h" #include "deThread.hpp" #include "deCommandLine.hpp" #include <vector> #include <string> #include <cstdio> #include <cstdlib> #include <fstream> #include <iostream> #include <set> #include <map> using std::vector; using std::string; using std::set; using std::map; enum OutputMode { OUTPUTMODE_ALL = 0, OUTPUTMODE_DIFF, OUTPUTMODE_LAST }; enum OutputFormat { OUTPUTFORMAT_TEXT = 0, OUTPUTFORMAT_CSV, OUTPUTFORMAT_LAST }; enum OutputValue { OUTPUTVALUE_STATUS_CODE = 0, OUTPUTVALUE_STATUS_DETAILS, OUTPUTVALUE_LAST }; namespace opt { DE_DECLARE_COMMAND_LINE_OPT(OutMode, OutputMode); DE_DECLARE_COMMAND_LINE_OPT(OutFormat, OutputFormat); DE_DECLARE_COMMAND_LINE_OPT(OutValue, OutputValue); static void registerOptions (de::cmdline::Parser& parser) { using de::cmdline::Option; using de::cmdline::NamedValue; static const NamedValue<OutputMode> s_outputModes[] = { { "all", OUTPUTMODE_ALL }, { "diff", OUTPUTMODE_DIFF } }; static const NamedValue<OutputFormat> s_outputFormats[] = { { "text", OUTPUTFORMAT_TEXT }, { "csv", OUTPUTFORMAT_CSV } }; static const NamedValue<OutputValue> s_outputValues[] = { { "code", OUTPUTVALUE_STATUS_CODE }, { "details", OUTPUTVALUE_STATUS_DETAILS } }; parser << Option<OutFormat> ("f", "format", "Output format", s_outputFormats, "csv") << Option<OutMode> ("m", "mode", "Output mode", s_outputModes, "all") << Option<OutValue> ("v", "value", "Value to extract", s_outputValues, "code"); } } // opt struct CommandLine { CommandLine (void) : outMode (OUTPUTMODE_ALL) , outFormat (OUTPUTFORMAT_CSV) , outValue (OUTPUTVALUE_STATUS_CODE) { } OutputMode outMode; OutputFormat outFormat; OutputValue outValue; vector<string> filenames; }; struct ShortBatchResult { vector<xe::TestCaseResultHeader> resultHeaders; map<string, int> resultMap; }; class ShortResultHandler : public xe::TestLogHandler { public: ShortResultHandler (ShortBatchResult& result) : m_result(result) { } void setSessionInfo (const xe::SessionInfo&) { // Ignored. } xe::TestCaseResultPtr startTestCaseResult (const char* casePath) { return xe::TestCaseResultPtr(new xe::TestCaseResultData(casePath)); } void testCaseResultUpdated (const xe::TestCaseResultPtr&) { // Ignored. } void testCaseResultComplete (const xe::TestCaseResultPtr& caseData) { xe::TestCaseResultHeader header; int caseNdx = (int)m_result.resultHeaders.size(); header.casePath = caseData->getTestCasePath(); header.caseType = xe::TESTCASETYPE_SELF_VALIDATE; header.statusCode = caseData->getStatusCode(); header.statusDetails = caseData->getStatusDetails(); if (header.statusCode == xe::TESTSTATUSCODE_LAST) { xe::TestCaseResult fullResult; xe::parseTestCaseResultFromData(&m_testResultParser, &fullResult, *caseData.get()); header = xe::TestCaseResultHeader(fullResult); } // Insert into result list & map. m_result.resultHeaders.push_back(header); m_result.resultMap[header.casePath] = caseNdx; } private: ShortBatchResult& m_result; xe::TestResultParser m_testResultParser; }; static void readLogFile (ShortBatchResult& batchResult, const char* filename) { std::ifstream in (filename, std::ifstream::binary|std::ifstream::in); ShortResultHandler resultHandler (batchResult); xe::TestLogParser parser (&resultHandler); deUint8 buf [1024]; int numRead = 0; for (;;) { in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf)); numRead = (int)in.gcount(); if (numRead <= 0) break; parser.parse(&buf[0], numRead); } in.close(); } class LogFileReader : public de::Thread { public: LogFileReader (ShortBatchResult& batchResult, const char* filename) : m_batchResult (batchResult) , m_filename (filename) { } void run (void) { readLogFile(m_batchResult, m_filename.c_str()); } private: ShortBatchResult& m_batchResult; std::string m_filename; }; static void computeCaseList (vector<string>& cases, const vector<ShortBatchResult>& batchResults) { // \todo [2012-07-10 pyry] Do proper case ordering (eg. handle missing cases nicely). set<string> addedCases; for (vector<ShortBatchResult>::const_iterator batchIter = batchResults.begin(); batchIter != batchResults.end(); batchIter++) { for (vector<xe::TestCaseResultHeader>::const_iterator caseIter = batchIter->resultHeaders.begin(); caseIter != batchIter->resultHeaders.end(); caseIter++) { if (addedCases.find(caseIter->casePath) == addedCases.end()) { cases.push_back(caseIter->casePath); addedCases.insert(caseIter->casePath); } } } } static void getTestResultHeaders (vector<xe::TestCaseResultHeader>& headers, const vector<ShortBatchResult>& batchResults, const char* casePath) { headers.resize(batchResults.size()); for (int ndx = 0; ndx < (int)batchResults.size(); ndx++) { const ShortBatchResult& batchResult = batchResults[ndx]; map<string, int>::const_iterator resultPos = batchResult.resultMap.find(casePath); if (resultPos != batchResult.resultMap.end()) headers[ndx] = batchResult.resultHeaders[resultPos->second]; else { headers[ndx].casePath = casePath; headers[ndx].caseType = xe::TESTCASETYPE_SELF_VALIDATE; headers[ndx].statusCode = xe::TESTSTATUSCODE_LAST; } } } static const char* getStatusCodeName (xe::TestStatusCode code) { if (code == xe::TESTSTATUSCODE_LAST) return "Missing"; else return xe::getTestStatusCodeName(code); } static bool runCompare (const CommandLine& cmdLine, std::ostream& dst) { vector<ShortBatchResult> results; vector<string> batchNames; bool compareOk = true; XE_CHECK(!cmdLine.filenames.empty()); try { // Read in batch results results.resize(cmdLine.filenames.size()); { std::vector<de::SharedPtr<LogFileReader> > readers; for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++) { readers.push_back(de::SharedPtr<LogFileReader>(new LogFileReader(results[ndx], cmdLine.filenames[ndx].c_str()))); readers.back()->start(); } for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++) { readers[ndx]->join(); // Use file name as batch name. batchNames.push_back(de::FilePath(cmdLine.filenames[ndx].c_str()).getBaseName()); } } // Compute unified case list. vector<string> caseList; computeCaseList(caseList, results); // Stats. int numCases = (int)caseList.size(); int numEqual = 0; if (cmdLine.outFormat == OUTPUTFORMAT_CSV) { dst << "TestCasePath"; for (vector<string>::const_iterator nameIter = batchNames.begin(); nameIter != batchNames.end(); nameIter++) dst << "," << *nameIter; dst << "\n"; } // Compare cases. for (vector<string>::const_iterator caseIter = caseList.begin(); caseIter != caseList.end(); caseIter++) { const string& caseName = *caseIter; vector<xe::TestCaseResultHeader> headers; bool allEqual = true; getTestResultHeaders(headers, results, caseName.c_str()); for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin()+1; iter != headers.end(); iter++) { if (iter->statusCode != headers[0].statusCode) { allEqual = false; break; } } if (allEqual) numEqual += 1; if (cmdLine.outMode == OUTPUTMODE_ALL || !allEqual) { if (cmdLine.outFormat == OUTPUTFORMAT_TEXT) { dst << caseName << "\n"; for (int ndx = 0; ndx < (int)headers.size(); ndx++) dst << " " << batchNames[ndx] << ": " << getStatusCodeName(headers[ndx].statusCode) << " (" << headers[ndx].statusDetails << ")\n"; dst << "\n"; } else if (cmdLine.outFormat == OUTPUTFORMAT_CSV) { dst << caseName; for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin(); iter != headers.end(); iter++) dst << "," << (cmdLine.outValue == OUTPUTVALUE_STATUS_CODE ? getStatusCodeName(iter->statusCode) : iter->statusDetails.c_str()); dst << "\n"; } } } compareOk = numEqual == numCases; if (cmdLine.outFormat == OUTPUTFORMAT_TEXT) { dst << " " << numEqual << " / " << numCases << " test case results match.\n"; dst << " Comparison " << (compareOk ? "passed" : "FAILED") << "!\n"; } } catch (const std::exception& e) { printf("%s\n", e.what()); compareOk = false; } return compareOk; } static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv) { de::cmdline::Parser parser; de::cmdline::CommandLine opts; XE_CHECK(argc >= 1); opt::registerOptions(parser); if (!parser.parse(argc-1, &argv[1], &opts, std::cerr) || opts.getArgs().empty()) { std::cout << argv[0] << ": [options] [filenames]\n"; parser.help(std::cout); return false; } cmdLine.outFormat = opts.getOption<opt::OutFormat>(); cmdLine.outMode = opts.getOption<opt::OutMode>(); cmdLine.outValue = opts.getOption<opt::OutValue>(); cmdLine.filenames = opts.getArgs(); return true; } int main (int argc, const char* const* argv) { CommandLine cmdLine; if (!parseCommandLine(cmdLine, argc, argv)) return -1; try { bool compareOk = runCompare(cmdLine, std::cout); return compareOk ? 0 : -1; } catch (const std::exception& e) { printf("FATAL ERROR: %s\n", e.what()); return -1; } }