/*-------------------------------------------------------------------------
* drawElements Quality Program Test Executor
* ------------------------------------------
*
* Copyright 2014 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*//*!
* \file
* \brief Test log compare utility.
*//*--------------------------------------------------------------------*/
#include "xeTestLogParser.hpp"
#include "xeTestResultParser.hpp"
#include "deFilePath.hpp"
#include "deString.h"
#include "deThread.hpp"
#include "deCommandLine.hpp"
#include <vector>
#include <string>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <set>
#include <map>
using std::vector;
using std::string;
using std::set;
using std::map;
enum OutputMode
{
OUTPUTMODE_ALL = 0,
OUTPUTMODE_DIFF,
OUTPUTMODE_LAST
};
enum OutputFormat
{
OUTPUTFORMAT_TEXT = 0,
OUTPUTFORMAT_CSV,
OUTPUTFORMAT_LAST
};
enum OutputValue
{
OUTPUTVALUE_STATUS_CODE = 0,
OUTPUTVALUE_STATUS_DETAILS,
OUTPUTVALUE_LAST
};
namespace opt
{
DE_DECLARE_COMMAND_LINE_OPT(OutMode, OutputMode);
DE_DECLARE_COMMAND_LINE_OPT(OutFormat, OutputFormat);
DE_DECLARE_COMMAND_LINE_OPT(OutValue, OutputValue);
static void registerOptions (de::cmdline::Parser& parser)
{
using de::cmdline::Option;
using de::cmdline::NamedValue;
static const NamedValue<OutputMode> s_outputModes[] =
{
{ "all", OUTPUTMODE_ALL },
{ "diff", OUTPUTMODE_DIFF }
};
static const NamedValue<OutputFormat> s_outputFormats[] =
{
{ "text", OUTPUTFORMAT_TEXT },
{ "csv", OUTPUTFORMAT_CSV }
};
static const NamedValue<OutputValue> s_outputValues[] =
{
{ "code", OUTPUTVALUE_STATUS_CODE },
{ "details", OUTPUTVALUE_STATUS_DETAILS }
};
parser << Option<OutFormat> ("f", "format", "Output format", s_outputFormats, "csv")
<< Option<OutMode> ("m", "mode", "Output mode", s_outputModes, "all")
<< Option<OutValue> ("v", "value", "Value to extract", s_outputValues, "code");
}
} // opt
struct CommandLine
{
CommandLine (void)
: outMode (OUTPUTMODE_ALL)
, outFormat (OUTPUTFORMAT_CSV)
, outValue (OUTPUTVALUE_STATUS_CODE)
{
}
OutputMode outMode;
OutputFormat outFormat;
OutputValue outValue;
vector<string> filenames;
};
struct ShortBatchResult
{
vector<xe::TestCaseResultHeader> resultHeaders;
map<string, int> resultMap;
};
class ShortResultHandler : public xe::TestLogHandler
{
public:
ShortResultHandler (ShortBatchResult& result)
: m_result(result)
{
}
void setSessionInfo (const xe::SessionInfo&)
{
// Ignored.
}
xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
{
return xe::TestCaseResultPtr(new xe::TestCaseResultData(casePath));
}
void testCaseResultUpdated (const xe::TestCaseResultPtr&)
{
// Ignored.
}
void testCaseResultComplete (const xe::TestCaseResultPtr& caseData)
{
xe::TestCaseResultHeader header;
int caseNdx = (int)m_result.resultHeaders.size();
header.casePath = caseData->getTestCasePath();
header.caseType = xe::TESTCASETYPE_SELF_VALIDATE;
header.statusCode = caseData->getStatusCode();
header.statusDetails = caseData->getStatusDetails();
if (header.statusCode == xe::TESTSTATUSCODE_LAST)
{
xe::TestCaseResult fullResult;
xe::parseTestCaseResultFromData(&m_testResultParser, &fullResult, *caseData.get());
header = xe::TestCaseResultHeader(fullResult);
}
// Insert into result list & map.
m_result.resultHeaders.push_back(header);
m_result.resultMap[header.casePath] = caseNdx;
}
private:
ShortBatchResult& m_result;
xe::TestResultParser m_testResultParser;
};
static void readLogFile (ShortBatchResult& batchResult, const char* filename)
{
std::ifstream in (filename, std::ifstream::binary|std::ifstream::in);
ShortResultHandler resultHandler (batchResult);
xe::TestLogParser parser (&resultHandler);
deUint8 buf [1024];
int numRead = 0;
for (;;)
{
in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
numRead = (int)in.gcount();
if (numRead <= 0)
break;
parser.parse(&buf[0], numRead);
}
in.close();
}
class LogFileReader : public de::Thread
{
public:
LogFileReader (ShortBatchResult& batchResult, const char* filename)
: m_batchResult (batchResult)
, m_filename (filename)
{
}
void run (void)
{
readLogFile(m_batchResult, m_filename.c_str());
}
private:
ShortBatchResult& m_batchResult;
std::string m_filename;
};
static void computeCaseList (vector<string>& cases, const vector<ShortBatchResult>& batchResults)
{
// \todo [2012-07-10 pyry] Do proper case ordering (eg. handle missing cases nicely).
set<string> addedCases;
for (vector<ShortBatchResult>::const_iterator batchIter = batchResults.begin(); batchIter != batchResults.end(); batchIter++)
{
for (vector<xe::TestCaseResultHeader>::const_iterator caseIter = batchIter->resultHeaders.begin(); caseIter != batchIter->resultHeaders.end(); caseIter++)
{
if (addedCases.find(caseIter->casePath) == addedCases.end())
{
cases.push_back(caseIter->casePath);
addedCases.insert(caseIter->casePath);
}
}
}
}
static void getTestResultHeaders (vector<xe::TestCaseResultHeader>& headers, const vector<ShortBatchResult>& batchResults, const char* casePath)
{
headers.resize(batchResults.size());
for (int ndx = 0; ndx < (int)batchResults.size(); ndx++)
{
const ShortBatchResult& batchResult = batchResults[ndx];
map<string, int>::const_iterator resultPos = batchResult.resultMap.find(casePath);
if (resultPos != batchResult.resultMap.end())
headers[ndx] = batchResult.resultHeaders[resultPos->second];
else
{
headers[ndx].casePath = casePath;
headers[ndx].caseType = xe::TESTCASETYPE_SELF_VALIDATE;
headers[ndx].statusCode = xe::TESTSTATUSCODE_LAST;
}
}
}
static const char* getStatusCodeName (xe::TestStatusCode code)
{
if (code == xe::TESTSTATUSCODE_LAST)
return "Missing";
else
return xe::getTestStatusCodeName(code);
}
static bool runCompare (const CommandLine& cmdLine, std::ostream& dst)
{
vector<ShortBatchResult> results;
vector<string> batchNames;
bool compareOk = true;
XE_CHECK(!cmdLine.filenames.empty());
try
{
// Read in batch results
results.resize(cmdLine.filenames.size());
{
std::vector<de::SharedPtr<LogFileReader> > readers;
for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
{
readers.push_back(de::SharedPtr<LogFileReader>(new LogFileReader(results[ndx], cmdLine.filenames[ndx].c_str())));
readers.back()->start();
}
for (int ndx = 0; ndx < (int)cmdLine.filenames.size(); ndx++)
{
readers[ndx]->join();
// Use file name as batch name.
batchNames.push_back(de::FilePath(cmdLine.filenames[ndx].c_str()).getBaseName());
}
}
// Compute unified case list.
vector<string> caseList;
computeCaseList(caseList, results);
// Stats.
int numCases = (int)caseList.size();
int numEqual = 0;
if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
{
dst << "TestCasePath";
for (vector<string>::const_iterator nameIter = batchNames.begin(); nameIter != batchNames.end(); nameIter++)
dst << "," << *nameIter;
dst << "\n";
}
// Compare cases.
for (vector<string>::const_iterator caseIter = caseList.begin(); caseIter != caseList.end(); caseIter++)
{
const string& caseName = *caseIter;
vector<xe::TestCaseResultHeader> headers;
bool allEqual = true;
getTestResultHeaders(headers, results, caseName.c_str());
for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin()+1; iter != headers.end(); iter++)
{
if (iter->statusCode != headers[0].statusCode)
{
allEqual = false;
break;
}
}
if (allEqual)
numEqual += 1;
if (cmdLine.outMode == OUTPUTMODE_ALL || !allEqual)
{
if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
{
dst << caseName << "\n";
for (int ndx = 0; ndx < (int)headers.size(); ndx++)
dst << " " << batchNames[ndx] << ": " << getStatusCodeName(headers[ndx].statusCode) << " (" << headers[ndx].statusDetails << ")\n";
dst << "\n";
}
else if (cmdLine.outFormat == OUTPUTFORMAT_CSV)
{
dst << caseName;
for (vector<xe::TestCaseResultHeader>::const_iterator iter = headers.begin(); iter != headers.end(); iter++)
dst << "," << (cmdLine.outValue == OUTPUTVALUE_STATUS_CODE ? getStatusCodeName(iter->statusCode) : iter->statusDetails.c_str());
dst << "\n";
}
}
}
compareOk = numEqual == numCases;
if (cmdLine.outFormat == OUTPUTFORMAT_TEXT)
{
dst << " " << numEqual << " / " << numCases << " test case results match.\n";
dst << " Comparison " << (compareOk ? "passed" : "FAILED") << "!\n";
}
}
catch (const std::exception& e)
{
printf("%s\n", e.what());
compareOk = false;
}
return compareOk;
}
static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
{
de::cmdline::Parser parser;
de::cmdline::CommandLine opts;
XE_CHECK(argc >= 1);
opt::registerOptions(parser);
if (!parser.parse(argc-1, &argv[1], &opts, std::cerr) ||
opts.getArgs().empty())
{
std::cout << argv[0] << ": [options] [filenames]\n";
parser.help(std::cout);
return false;
}
cmdLine.outFormat = opts.getOption<opt::OutFormat>();
cmdLine.outMode = opts.getOption<opt::OutMode>();
cmdLine.outValue = opts.getOption<opt::OutValue>();
cmdLine.filenames = opts.getArgs();
return true;
}
int main (int argc, const char* const* argv)
{
CommandLine cmdLine;
if (!parseCommandLine(cmdLine, argc, argv))
return -1;
try
{
bool compareOk = runCompare(cmdLine, std::cout);
return compareOk ? 0 : -1;
}
catch (const std::exception& e)
{
printf("FATAL ERROR: %s\n", e.what());
return -1;
}
}