# SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2015, ARM Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import argparse
import fnmatch as fnm
import json
import math
import numpy as np
import os
import re
import sys
import logging
from collections import defaultdict
from colors import TestColors
from results import Results
# By default compare all the possible combinations
DEFAULT_COMPARE = [(r'base_', r'test_')]
class Report(object):
def __init__(self, results_dir, compare=None, formats=['relative']):
self.results_json = results_dir + '/results.json'
self.results = {}
self.compare = []
# Setup logging
self._log = logging.getLogger('Report')
# Parse results (if required)
if not os.path.isfile(self.results_json):
Results(results_dir)
# Load results from file (if already parsed)
self._log.info('Load results from [%s]...',
self.results_json)
with open(self.results_json) as infile:
self.results = json.load(infile)
# Setup configuration comparisons
if compare is None:
compare = DEFAULT_COMPARE
self._log.warning('Comparing all the possible combination')
for (base_rexp, test_rexp) in compare:
self._log.info('Configured regexps for comparisions '
'(bases , tests): (%s, %s)',
base_rexp, test_rexp)
base_rexp = re.compile(base_rexp, re.DOTALL)
test_rexp = re.compile(test_rexp, re.DOTALL)
self.compare.append((base_rexp, test_rexp))
# Report all supported workload classes
self.__rtapp_report(formats)
self.__default_report(formats)
############################### REPORT RTAPP ###############################
def __rtapp_report(self, formats):
if 'rtapp' not in self.results.keys():
self._log.debug('No RTApp workloads to report')
return
self._log.debug('Reporting RTApp workloads')
# Setup lables depending on requested report
if 'absolute' in formats:
nrg_lable = 'Energy Indexes (Absolute)'
prf_lable = 'Performance Indexes (Absolute)'
self._log.info('')
self._log.info('Absolute comparisions:')
print ''
else:
nrg_lable = 'Energy Indexes (Relative)'
prf_lable = 'Performance Indexes (Relative)'
self._log.info('')
self._log.info('Relative comparisions:')
print ''
# Dump headers
print '{:13s} {:20s} |'\
' {:33s} | {:54s} |'\
.format('Test Id', 'Comparision',
nrg_lable, prf_lable)
print '{:13s} {:20s} |'\
' {:>10s} {:>10s} {:>10s} |'\
' {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} |'\
.format('', '',
'LITTLE', 'big', 'Total',
'PerfIndex', 'NegSlacks', 'EDP1', 'EDP2', 'EDP3')
# For each test
_results = self.results['rtapp']
for tid in sorted(_results.keys()):
new_test = True
# For each configuration...
for base_idx in sorted(_results[tid].keys()):
# Which matches at least on base regexp
for (base_rexp, test_rexp) in self.compare:
if not base_rexp.match(base_idx):
continue
# Look for a configuration which matches the test regexp
for test_idx in sorted(_results[tid].keys()):
if test_idx == base_idx:
continue
if new_test:
print '{:-<37s}+{:-<35s}+{:-<56s}+'\
.format('','', '')
self.__rtapp_reference(tid, base_idx)
new_test = False
if test_rexp.match(test_idx) == None:
continue
self.__rtapp_compare(tid, base_idx, test_idx, formats)
print ''
def __rtapp_reference(self, tid, base_idx):
_results = self.results['rtapp']
self._log.debug('Test %s: compare against [%s] base',
tid, base_idx)
res_line = '{0:12s}: {1:22s} | '.format(tid, base_idx)
# Dump all energy metrics
for cpus in ['LITTLE', 'big', 'Total']:
res_base = _results[tid][base_idx]['energy'][cpus]['avg']
# Dump absolute values
res_line += ' {0:10.3f}'.format(res_base)
res_line += ' |'
# If available, dump also performance results
if 'performance' not in _results[tid][base_idx].keys():
print res_line
return
for pidx in ['perf_avg', 'slack_pct', 'edp1', 'edp2', 'edp3']:
res_base = _results[tid][base_idx]['performance'][pidx]['avg']
self._log.debug('idx: %s, base: %s', pidx, res_base)
if pidx in ['perf_avg']:
res_line += ' {0:s}'.format(TestColors.rate(res_base))
continue
if pidx in ['slack_pct']:
res_line += ' {0:s}'.format(
TestColors.rate(res_base, positive_is_good = False))
continue
if 'edp' in pidx:
res_line += ' {0:10.2e}'.format(res_base)
continue
res_line += ' |'
print res_line
def __rtapp_compare(self, tid, base_idx, test_idx, formats):
_results = self.results['rtapp']
self._log.debug('Test %s: compare %s with %s',
tid, base_idx, test_idx)
res_line = '{0:12s}: {1:20s} | '.format(tid, test_idx)
# Dump all energy metrics
for cpus in ['LITTLE', 'big', 'Total']:
res_base = _results[tid][base_idx]['energy'][cpus]['avg']
res_test = _results[tid][test_idx]['energy'][cpus]['avg']
speedup_cnt = res_test - res_base
if 'absolute' in formats:
res_line += ' {0:10.2f}'.format(speedup_cnt)
else:
speedup_pct = 0
if res_base != 0:
speedup_pct = 100.0 * speedup_cnt / res_base
res_line += ' {0:s}'\
.format(TestColors.rate(
speedup_pct,
positive_is_good = False))
res_line += ' |'
# If available, dump also performance results
if 'performance' not in _results[tid][base_idx].keys():
print res_line
return
for pidx in ['perf_avg', 'slack_pct', 'edp1', 'edp2', 'edp3']:
res_base = _results[tid][base_idx]['performance'][pidx]['avg']
res_test = _results[tid][test_idx]['performance'][pidx]['avg']
self._log.debug('idx: %s, base: %s, test: %s',
pidx, res_base, res_test)
if pidx in ['perf_avg']:
res_line += ' {0:s}'.format(TestColors.rate(res_test))
continue
if pidx in ['slack_pct']:
res_line += ' {0:s}'.format(
TestColors.rate(res_test, positive_is_good = False))
continue
# Compute difference base-vs-test
if 'edp' in pidx:
speedup_cnt = res_base - res_test
if 'absolute':
res_line += ' {0:10.2e}'.format(speedup_cnt)
else:
res_line += ' {0:s}'.format(TestColors.rate(speedup_pct))
res_line += ' |'
print res_line
############################### REPORT DEFAULT #############################
def __default_report(self, formats):
# Build list of workload types which can be rendered using the default parser
wtypes = []
for supported_wtype in DEFAULT_WTYPES:
if supported_wtype in self.results.keys():
wtypes.append(supported_wtype)
if len(wtypes) == 0:
self._log.debug('No Default workloads to report')
return
self._log.debug('Reporting Default workloads')
# Setup lables depending on requested report
if 'absolute' in formats:
nrg_lable = 'Energy Indexes (Absolute)'
prf_lable = 'Performance Indexes (Absolute)'
self._log.info('')
self._log.info('Absolute comparisions:')
print ''
else:
nrg_lable = 'Energy Indexes (Relative)'
prf_lable = 'Performance Indexes (Relative)'
self._log.info('')
self._log.info('Relative comparisions:')
print ''
# Dump headers
print '{:9s} {:20s} |'\
' {:33s} | {:54s} |'\
.format('Test Id', 'Comparision',
nrg_lable, prf_lable)
print '{:9s} {:20s} |'\
' {:>10s} {:>10s} {:>10s} |'\
' {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} |'\
.format('', '',
'LITTLE', 'big', 'Total',
'Perf', 'CTime', 'EDP1', 'EDP2', 'EDP3')
# For each default test
for wtype in wtypes:
_results = self.results[wtype]
for tid in sorted(_results.keys()):
new_test = True
# For each configuration...
for base_idx in sorted(_results[tid].keys()):
# Which matches at least on base regexp
for (base_rexp, test_rexp) in self.compare:
if not base_rexp.match(base_idx):
continue
# Look for a configuration which matches the test regexp
for test_idx in sorted(_results[tid].keys()):
if test_idx == base_idx:
continue
if new_test:
print '{:-<37s}+{:-<35s}+{:-<56s}+'\
.format('','', '')
new_test = False
if not test_rexp.match(test_idx):
continue
self.__default_compare(wtype, tid, base_idx, test_idx, formats)
print ''
def __default_compare(self, wtype, tid, base_idx, test_idx, formats):
_results = self.results[wtype]
self._log.debug('Test %s: compare %s with %s',
tid, base_idx, test_idx)
res_comp = '{0:s} vs {1:s}'.format(test_idx, base_idx)
res_line = '{0:8s}: {1:22s} | '.format(tid, res_comp)
# Dump all energy metrics
for cpus in ['LITTLE', 'big', 'Total']:
# If either base of test have a 0 MAX energy, this measn that
# energy has not been collected
base_max = _results[tid][base_idx]['energy'][cpus]['max']
test_max = _results[tid][test_idx]['energy'][cpus]['max']
if base_max == 0 or test_max == 0:
res_line += ' {0:10s}'.format('NA')
continue
# Otherwise, report energy values
res_base = _results[tid][base_idx]['energy'][cpus]['avg']
res_test = _results[tid][test_idx]['energy'][cpus]['avg']
speedup_cnt = res_test - res_base
if 'absolute' in formats:
res_line += ' {0:10.2f}'.format(speedup_cnt)
else:
speedup_pct = 100.0 * speedup_cnt / res_base
res_line += ' {0:s}'\
.format(TestColors.rate(
speedup_pct,
positive_is_good = False))
res_line += ' |'
# If available, dump also performance results
if 'performance' not in _results[tid][base_idx].keys():
print res_line
return
for pidx in ['perf_avg', 'ctime_avg', 'edp1', 'edp2', 'edp3']:
res_base = _results[tid][base_idx]['performance'][pidx]['avg']
res_test = _results[tid][test_idx]['performance'][pidx]['avg']
self._log.debug('idx: %s, base: %s, test: %s',
pidx, res_base, res_test)
# Compute difference base-vs-test
speedup_cnt = 0
if res_base != 0:
if pidx in ['perf_avg']:
speedup_cnt = res_test - res_base
else:
speedup_cnt = res_base - res_test
# Compute speedup if required
speedup_pct = 0
if 'absolute' in formats:
if 'edp' in pidx:
res_line += ' {0:10.2e}'.format(speedup_cnt)
else:
res_line += ' {0:10.2f}'.format(speedup_cnt)
else:
if res_base != 0:
if pidx in ['perf_avg']:
# speedup_pct = 100.0 * speedup_cnt / res_base
speedup_pct = speedup_cnt
else:
speedup_pct = 100.0 * speedup_cnt / res_base
res_line += ' {0:s}'.format(TestColors.rate(speedup_pct))
res_line += ' |'
print res_line
# List of workload types which can be parsed using the default test parser
DEFAULT_WTYPES = ['perf_bench_messaging', 'perf_bench_pipe']
#vim :set tabstop=4 shiftwidth=4 expandtab