/*-------------------------------------------------------------------------
* drawElements Quality Program Test Executor
* ------------------------------------------
*
* Copyright 2014 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*//*!
* \file
* \brief Command line test executor.
*//*--------------------------------------------------------------------*/
#include "xeBatchExecutor.hpp"
#include "xeTestCaseListParser.hpp"
#include "xeTcpIpLink.hpp"
#include "xeLocalTcpIpLink.hpp"
#include "xeTestResultParser.hpp"
#include "xeTestLogWriter.hpp"
#include "deDirectoryIterator.hpp"
#include "deCommandLine.hpp"
#include "deString.h"
#include <vector>
#include <string>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <memory>
#include <algorithm>
#include <iostream>
#include <sstream>
// Command line arguments.
namespace opt
{
DE_DECLARE_COMMAND_LINE_OPT(StartServer, std::string);
DE_DECLARE_COMMAND_LINE_OPT(Host, std::string);
DE_DECLARE_COMMAND_LINE_OPT(Port, int);
DE_DECLARE_COMMAND_LINE_OPT(CaseListDir, std::string);
DE_DECLARE_COMMAND_LINE_OPT(TestSet, std::vector<std::string>);
DE_DECLARE_COMMAND_LINE_OPT(ExcludeSet, std::vector<std::string>);
DE_DECLARE_COMMAND_LINE_OPT(ContinueFile, std::string);
DE_DECLARE_COMMAND_LINE_OPT(TestLogFile, std::string);
DE_DECLARE_COMMAND_LINE_OPT(InfoLogFile, std::string);
DE_DECLARE_COMMAND_LINE_OPT(Summary, bool);
// TargetConfiguration
DE_DECLARE_COMMAND_LINE_OPT(BinaryName, std::string);
DE_DECLARE_COMMAND_LINE_OPT(WorkingDir, std::string);
DE_DECLARE_COMMAND_LINE_OPT(CmdLineArgs, std::string);
static void parseCommaSeparatedList (const char* src, std::vector<std::string>* dst)
{
std::istringstream inStr (src);
std::string comp;
while (std::getline(inStr, comp, ','))
dst->push_back(comp);
}
void registerOptions (de::cmdline::Parser& parser)
{
using de::cmdline::Option;
using de::cmdline::NamedValue;
static const NamedValue<bool> s_yesNo[] =
{
{ "yes", true },
{ "no", false }
};
parser << Option<StartServer> ("s", "start-server", "Start local execserver", "")
<< Option<Host> ("c", "connect", "Connect to host", "127.0.0.1")
<< Option<Port> ("p", "port", "Select TCP port to use", "50016")
<< Option<CaseListDir> ("cd", "caselistdir", "Path to test case XML files", ".")
<< Option<TestSet> ("t", "testset", "Test set", parseCommaSeparatedList, "")
<< Option<ExcludeSet> ("e", "exclude", "Comma-separated list of exclude filters", parseCommaSeparatedList, "")
<< Option<ContinueFile> (DE_NULL, "continue", "Continue execution by initializing results from existing test log", "")
<< Option<TestLogFile> ("o", "out", "Output test log filename", "")
<< Option<InfoLogFile> ("i", "info", "Output info log filename", "")
<< Option<Summary> (DE_NULL, "summary", "Print summary at the end", s_yesNo, "yes")
<< Option<BinaryName> ("b", "binaryname", "Test binary path, relative to working directory", "")
<< Option<WorkingDir> ("wd", "workdir", "Working directory for test execution", "")
<< Option<CmdLineArgs> (DE_NULL, "cmdline", "Additional command line arguments for test binary", "");
}
} // opt
using std::vector;
using std::string;
struct CommandLine
{
CommandLine (void)
: port (0)
, summary (false)
{
}
xe::TargetConfiguration targetCfg;
std::string serverBin;
std::string host;
int port;
std::string caseListDir;
std::vector<std::string> testset;
std::vector<std::string> exclude;
std::string inFile;
std::string outFile;
std::string infoFile;
bool summary;
};
static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
{
de::cmdline::Parser parser;
de::cmdline::CommandLine opts;
XE_CHECK(argc >= 1);
opt::registerOptions(parser);
if (!parser.parse(argc-1, argv+1, &opts, std::cerr))
{
std::cout << argv[0] << " [options]\n";
parser.help(std::cout);
return false;
}
cmdLine.serverBin = opts.getOption<opt::StartServer>();
cmdLine.host = opts.getOption<opt::Host>();
cmdLine.port = opts.getOption<opt::Port>();
cmdLine.caseListDir = opts.getOption<opt::CaseListDir>();
cmdLine.testset = opts.getOption<opt::TestSet>();
cmdLine.exclude = opts.getOption<opt::ExcludeSet>();
cmdLine.inFile = opts.getOption<opt::ContinueFile>();
cmdLine.outFile = opts.getOption<opt::TestLogFile>();
cmdLine.infoFile = opts.getOption<opt::InfoLogFile>();
cmdLine.summary = opts.getOption<opt::Summary>();
cmdLine.targetCfg.binaryName = opts.getOption<opt::BinaryName>();
cmdLine.targetCfg.workingDir = opts.getOption<opt::WorkingDir>();
cmdLine.targetCfg.cmdLineArgs = opts.getOption<opt::CmdLineArgs>();
return true;
}
static bool checkCasePathPatternMatch (const char* pattern, const char* casePath, bool isTestGroup)
{
int ptrnPos = 0;
int casePos = 0;
for (;;)
{
char c = casePath[casePos];
char p = pattern[ptrnPos];
if (p == '*')
{
/* Recurse to rest of positions. */
int next = casePos;
for (;;)
{
if (checkCasePathPatternMatch(pattern+ptrnPos+1, casePath+next, isTestGroup))
return DE_TRUE;
if (casePath[next] == 0)
return DE_FALSE; /* No match found. */
else
next += 1;
}
DE_ASSERT(DE_FALSE);
}
else if (c == 0 && p == 0)
return true;
else if (c == 0)
{
/* Incomplete match is ok for test groups. */
return isTestGroup;
}
else if (c != p)
return false;
casePos += 1;
ptrnPos += 1;
}
DE_ASSERT(false);
return false;
}
static void readCaseList (xe::TestGroup* root, const char* filename)
{
xe::TestCaseListParser caseListParser;
std::ifstream in (filename, std::ios_base::binary);
deUint8 buf[1024];
XE_CHECK(in.good());
caseListParser.init(root);
for (;;)
{
in.read((char*)&buf[0], sizeof(buf));
int numRead = (int)in.gcount();
if (numRead > 0)
caseListParser.parse(&buf[0], numRead);
if (numRead < (int)sizeof(buf))
break; // EOF
}
}
static void readCaseLists (xe::TestRoot& root, const char* caseListDir)
{
de::DirectoryIterator iter(caseListDir);
for (; iter.hasItem(); iter.next())
{
de::FilePath item = iter.getItem();
if (item.getType() == de::FilePath::TYPE_FILE)
{
std::string baseName = item.getBaseName();
if (baseName.find("-cases.xml") == baseName.length()-10)
{
std::string packageName = baseName.substr(0, baseName.length()-10);
xe::TestGroup* package = root.createGroup(packageName.c_str(), "");
readCaseList(package, item.getPath());
}
}
}
}
static void addMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
{
for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
{
const xe::TestNode* child = group.getChild(childNdx);
const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP;
const std::string fullPath = child->getFullPath();
if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
{
if (isGroup)
{
// Recurse into group.
addMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
}
else
{
DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
testSet.add(child);
}
}
}
}
static void removeMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
{
for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
{
const xe::TestNode* child = group.getChild(childNdx);
const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP;
const std::string fullPath = child->getFullPath();
if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
{
if (isGroup)
{
// Recurse into group.
removeMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
}
else
{
DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
testSet.remove(child);
}
}
}
}
class BatchResultHandler : public xe::TestLogHandler
{
public:
BatchResultHandler (xe::BatchResult* batchResult)
: m_batchResult(batchResult)
{
}
void setSessionInfo (const xe::SessionInfo& sessionInfo)
{
m_batchResult->getSessionInfo() = sessionInfo;
}
xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
{
// \todo [2012-11-01 pyry] What to do with duplicate results?
if (m_batchResult->hasTestCaseResult(casePath))
return m_batchResult->getTestCaseResult(casePath);
else
return m_batchResult->createTestCaseResult(casePath);
}
void testCaseResultUpdated (const xe::TestCaseResultPtr&)
{
}
void testCaseResultComplete (const xe::TestCaseResultPtr&)
{
}
private:
xe::BatchResult* m_batchResult;
};
static void readLogFile (xe::BatchResult* batchResult, const char* filename)
{
std::ifstream in (filename, std::ifstream::binary|std::ifstream::in);
BatchResultHandler handler (batchResult);
xe::TestLogParser parser (&handler);
deUint8 buf [1024];
int numRead = 0;
for (;;)
{
in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
numRead = (int)in.gcount();
if (numRead <= 0)
break;
parser.parse(&buf[0], numRead);
}
in.close();
}
static void printBatchResultSummary (const xe::TestNode* root, const xe::TestSet& testSet, const xe::BatchResult& batchResult)
{
int countByStatusCode[xe::TESTSTATUSCODE_LAST];
std::fill(&countByStatusCode[0], &countByStatusCode[0]+DE_LENGTH_OF_ARRAY(countByStatusCode), 0);
for (xe::ConstTestNodeIterator iter = xe::ConstTestNodeIterator::begin(root); iter != xe::ConstTestNodeIterator::end(root); ++iter)
{
const xe::TestNode* node = *iter;
if (node->getNodeType() == xe::TESTNODETYPE_TEST_CASE && testSet.hasNode(node))
{
const xe::TestCase* testCase = static_cast<const xe::TestCase*>(node);
std::string fullPath;
xe::TestStatusCode statusCode = xe::TESTSTATUSCODE_PENDING;
testCase->getFullPath(fullPath);
// Parse result data if such exists.
if (batchResult.hasTestCaseResult(fullPath.c_str()))
{
xe::ConstTestCaseResultPtr resultData = batchResult.getTestCaseResult(fullPath.c_str());
xe::TestCaseResult result;
xe::TestResultParser parser;
xe::parseTestCaseResultFromData(&parser, &result, *resultData.get());
statusCode = result.statusCode;
}
countByStatusCode[statusCode] += 1;
}
}
printf("\nTest run summary:\n");
int totalCases = 0;
for (int code = 0; code < xe::TESTSTATUSCODE_LAST; code++)
{
if (countByStatusCode[code] > 0)
printf(" %20s: %5d\n", xe::getTestStatusCodeName((xe::TestStatusCode)code), countByStatusCode[code]);
totalCases += countByStatusCode[code];
}
printf(" %20s: %5d\n", "Total", totalCases);
}
static void writeInfoLog (const xe::InfoLog& log, const char* filename)
{
std::ofstream out(filename, std::ios_base::binary);
XE_CHECK(out.good());
out.write((const char*)log.getBytes(), log.getSize());
out.close();
}
static xe::CommLink* createCommLink (const CommandLine& cmdLine)
{
if (!cmdLine.serverBin.empty())
{
xe::LocalTcpIpLink* link = new xe::LocalTcpIpLink();
try
{
link->start(cmdLine.serverBin.c_str(), DE_NULL, cmdLine.port);
return link;
}
catch (...)
{
delete link;
throw;
}
}
else
{
de::SocketAddress address;
address.setFamily(DE_SOCKETFAMILY_INET4);
address.setProtocol(DE_SOCKETPROTOCOL_TCP);
address.setHost(cmdLine.host.c_str());
address.setPort(cmdLine.port);
xe::TcpIpLink* link = new xe::TcpIpLink();
try
{
link->connect(address);
return link;
}
catch (...)
{
delete link;
throw;
}
}
}
static void runExecutor (const CommandLine& cmdLine)
{
xe::TestRoot root;
// Read case list definitions.
readCaseLists(root, cmdLine.caseListDir.c_str());
// Build test set.
xe::TestSet testSet;
// Build test set.
for (vector<string>::const_iterator filterIter = cmdLine.testset.begin(); filterIter != cmdLine.testset.end(); ++filterIter)
addMatchingCases(root, testSet, filterIter->c_str());
// Remove excluded cases.
for (vector<string>::const_iterator filterIter = cmdLine.exclude.begin(); filterIter != cmdLine.exclude.end(); ++filterIter)
removeMatchingCases(root, testSet, filterIter->c_str());
// Initialize batch result.
xe::BatchResult batchResult;
xe::InfoLog infoLog;
// Read existing results from input file (if supplied).
if (!cmdLine.inFile.empty())
readLogFile(&batchResult, cmdLine.inFile.c_str());
// Initialize commLink.
std::auto_ptr<xe::CommLink> commLink(createCommLink(cmdLine));
xe::BatchExecutor executor(cmdLine.targetCfg, commLink.get(), &root, testSet, &batchResult, &infoLog);
executor.run();
commLink.reset();
if (!cmdLine.outFile.empty())
{
xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
printf("Test log written to %s\n", cmdLine.outFile.c_str());
}
if (!cmdLine.infoFile.empty())
{
writeInfoLog(infoLog, cmdLine.infoFile.c_str());
printf("Info log written to %s\n", cmdLine.infoFile.c_str());
}
if (cmdLine.summary)
printBatchResultSummary(&root, testSet, batchResult);
}
int main (int argc, const char* const* argv)
{
CommandLine cmdLine;
if (!parseCommandLine(cmdLine, argc, argv))
return -1;
try
{
runExecutor(cmdLine);
}
catch (const std::exception& e)
{
printf("%s\n", e.what());
return -1;
}
return 0;
}