# SPDX-License-Identifier: Apache-2.0 # # Copyright (C) 2015, ARM Limited and contributors. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import argparse import fnmatch as fnm import json import math import numpy as np import os import re import sys import logging from collections import defaultdict from colors import TestColors class Results(object): def __init__(self, results_dir): self.results_dir = results_dir self.results_json = results_dir + '/results.json' self.results = {} # Setup logging self._log = logging.getLogger('Results') # Do nothing if results have been already parsed if os.path.isfile(self.results_json): return # Parse results self.base_wls = defaultdict(list) self.test_wls = defaultdict(list) self._log.info('Loading energy/perf data...') for test_idx in sorted(os.listdir(self.results_dir)): test_dir = self.results_dir + '/' + test_idx if not os.path.isdir(test_dir): continue test = TestFactory.get(test_idx, test_dir, self.results) test.parse() results_json = self.results_dir + '/results.json' self._log.info('Dump perf results on JSON file [%s]...', results_json) with open(results_json, 'w') as outfile: json.dump(self.results, outfile, indent=4, sort_keys=True) ################################################################################ # Tests processing base classes ################################################################################ class Test(object): def __init__(self, test_idx, test_dir, res): self.test_idx = test_idx self.test_dir = test_dir self.res = res match = TEST_DIR_RE.search(test_dir) if not match: self._log.error('Results folder not matching naming template') self._log.error('Skip parsing of test results [%s]', test_dir) return # Create required JSON entries wtype = match.group(1) if wtype not in res.keys(): res[wtype] = {} wload_idx = match.group(3) if wload_idx not in res[wtype].keys(): res[wtype][wload_idx] = {} conf_idx = match.group(2) if conf_idx not in res[wtype][wload_idx].keys(): res[wtype][wload_idx][conf_idx] = {} # Set the workload type for this test self.wtype = wtype self.wload_idx = wload_idx self.conf_idx = conf_idx # Energy metrics collected for all tests self.little = [] self.total = [] self.big = [] def parse(self): self._log.info('Processing results from wtype [%s]', self.wtype) # Parse test's run results for run_idx in sorted(os.listdir(self.test_dir)): # Skip all files which are not folders run_dir = os.path.join(self.test_dir, run_idx) if not os.path.isdir(run_dir): continue run = self.parse_run(run_idx, run_dir) self.collect_energy(run) self.collect_performance(run) # Report energy/performance stats over all runs self.res[self.wtype][self.wload_idx][self.conf_idx]\ ['energy'] = self.energy() self.res[self.wtype][self.wload_idx][self.conf_idx]\ ['performance'] = self.performance() def collect_energy(self, run): # Keep track of average energy of each run self.little.append(run.little_nrg) self.total.append(run.total_nrg) self.big.append(run.big_nrg) def energy(self): # Compute energy stats over all run return { 'LITTLE' : Stats(self.little).get(), 'big' : Stats(self.big).get(), 'Total' : Stats(self.total).get() } class TestFactory(object): @staticmethod def get(test_idx, test_dir, res): # Retrive workload class from results folder name match = TEST_DIR_RE.search(test_dir) if not match: self._log.error('Results folder not matching naming template') self._log.error('Skip parsing of test results [%s]', test_dir) return # Create workload specifi test class wtype = match.group(1) if wtype == 'rtapp': return RTAppTest(test_idx, test_dir, res) # Return a generi test parser return DefaultTest(test_idx, test_dir, res) class Energy(object): def __init__(self, nrg_file): # Set of exposed attributes self.little = 0.0 self.big = 0.0 self.total = 0.0 self._log.debug('Parse [%s]...', nrg_file) with open(nrg_file, 'r') as infile: nrg = json.load(infile) if 'LITTLE' in nrg: self.little = float(nrg['LITTLE']) if 'big' in nrg: self.big = float(nrg['big']) self.total = self.little + self.big self._log.debug('Energy LITTLE [%s], big [%s], Total [%s]', self.little, self.big, self.total) class Stats(object): def __init__(self, data): self.stats = {} self.stats['count'] = len(data) self.stats['min'] = min(data) self.stats['max'] = max(data) self.stats['avg'] = sum(data)/len(data) std = Stats.stdev(data) c99 = Stats.ci99(data, std) self.stats['std'] = std self.stats['c99'] = c99 def get(self): return self.stats @staticmethod def stdev(values): sum1 = 0 sum2 = 0 for value in values: sum1 += value sum2 += math.pow(value, 2) # print 'sum1: {}, sum2: {}'.format(sum1, sum2) avg = sum1 / len(values) var = (sum2 / len(values)) - (avg * avg) # print 'avg: {} var: {}'.format(avg, var) std = math.sqrt(var) return float(std) @staticmethod def ci99(values, std): count = len(values) ste = std / math.sqrt(count) c99 = 2.58 * ste return c99 ################################################################################ # Run processing base classes ################################################################################ class Run(object): def __init__(self, run_idx, run_dir): self.run_idx = run_idx self.nrg = None self._log.debug('Parse [%s]...', 'Run', run_dir) # Energy stats self.little_nrg = 0 self.total_nrg = 0 self.big_nrg = 0 nrg_file = run_dir + '/energy.json' if os.path.isfile(nrg_file): self.nrg = Energy(nrg_file) self.little_nrg = self.nrg.little self.total_nrg = self.nrg.total self.big_nrg = self.nrg.big ################################################################################ # RTApp workload parsing classes ################################################################################ class RTAppTest(Test): def __init__(self, test_idx, test_dir, res): super(RTAppTest, self).__init__(test_idx, test_dir, res) # RTApp specific performance metric self.slack_pct = [] self.perf_avg = [] self.edp1 = [] self.edp2 = [] self.edp3 = [] self.rtapp_run = {} def parse_run(self, run_idx, run_dir): return RTAppRun(run_idx, run_dir) def collect_performance(self, run): # Keep track of average performances of each run self.slack_pct.extend(run.slack_pct) self.perf_avg.extend(run.perf_avg) self.edp1.extend(run.edp1) self.edp2.extend(run.edp2) self.edp3.extend(run.edp3) # Keep track of performance stats for each run self.rtapp_run[run.run_idx] = { 'slack_pct' : Stats(run.slack_pct).get(), 'perf_avg' : Stats(run.perf_avg).get(), 'edp1' : Stats(run.edp1).get(), 'edp2' : Stats(run.edp2).get(), 'edp3' : Stats(run.edp3).get(), } def performance(self): # Dump per run rtapp stats prf_file = os.path.join(self.test_dir, 'performance.json') with open(prf_file, 'w') as ofile: json.dump(self.rtapp_run, ofile, indent=4, sort_keys=True) # Return oveall stats return { 'slack_pct' : Stats(self.slack_pct).get(), 'perf_avg' : Stats(self.perf_avg).get(), 'edp1' : Stats(self.edp1).get(), 'edp2' : Stats(self.edp2).get(), 'edp3' : Stats(self.edp3).get(), } class RTAppRun(Run): def __init__(self, run_idx, run_dir): # Call base class to parse energy data super(RTAppRun, self).__init__(run_idx, run_dir) # RTApp specific performance stats self.slack_pct = [] self.perf_avg = [] self.edp1 = [] self.edp2 = [] self.edp3 = [] rta = {} # Load run's performance of each task for task_idx in sorted(os.listdir(run_dir)): if not fnm.fnmatch(task_idx, 'rt-app-*.log'): continue # Parse run's performance results prf_file = run_dir + '/' + task_idx task = RTAppPerf(prf_file, self.nrg) # Keep track of average performances of each task self.slack_pct.append(task.prf['slack_pct']) self.perf_avg.append(task.prf['perf_avg']) self.edp1.append(task.prf['edp1']) self.edp2.append(task.prf['edp2']) self.edp3.append(task.prf['edp3']) # Keep track of performance stats for each task rta[task.name] = task.prf # Dump per task rtapp stats prf_file = os.path.join(run_dir, 'performance.json') with open(prf_file, 'w') as ofile: json.dump(rta, ofile, indent=4, sort_keys=True) class RTAppPerf(object): def __init__(self, perf_file, nrg): # Set of exposed attibutes self.prf = { 'perf_avg' : 0, 'perf_std' : 0, 'run_sum' : 0, 'slack_sum' : 0, 'slack_pct' : 0, 'edp1' : 0, 'edp2' : 0, 'edp3' : 0 } self._log.debug('Parse [%s]...', perf_file) # Load performance data for each RT-App task self.name = perf_file.split('-')[-2] self.data = np.loadtxt(perf_file, comments='#', unpack=False) # Max Slack (i.e. configured/expected slack): period - run max_slack = np.subtract( self.data[:,RTAPP_COL_C_PERIOD], self.data[:,RTAPP_COL_C_RUN]) # Performance Index: 100 * slack / max_slack perf = np.divide(self.data[:,RTAPP_COL_SLACK], max_slack) perf = np.multiply(perf, 100) self.prf['perf_avg'] = np.mean(perf) self.prf['perf_std'] = np.std(perf) self._log.debug('perf [%s]: %6.2f,%6.2f', self.name, self.prf['perf_avg'], self.prf['perf_std']) # Negative slacks nslacks = self.data[:,RTAPP_COL_SLACK] nslacks = nslacks[nslacks < 0] self._log.debug('Negative slacks: %s', nslacks) self.prf['slack_sum'] = -nslacks.sum() self._log.debug('Negative slack [%s] sum: %6.2f', self.name, self.prf['slack_sum']) # Slack over run-time self.prf['run_sum'] = np.sum(self.data[:,RTAPP_COL_RUN]) self.prf['slack_pct'] = 100 * self.prf['slack_sum'] / self.prf['run_sum'] self._log.debug('SlackPct [%s]: %6.2f %%', self.name, self.slack_pct) if nrg is None: return # Computing EDP self.prf['edp1'] = nrg.total * math.pow(self.prf['run_sum'], 1) self._log.debug('EDP1 [%s]: {%6.2f}', self.name, self.prf['edp1']) self.prf['edp2'] = nrg.total * math.pow(self.prf['run_sum'], 2) self._log.debug('EDP2 [%s]: %6.2f', self.name, self.prf['edp2']) self.prf['edp3'] = nrg.total * math.pow(self.prf['run_sum'], 3) self._log.debug('EDP3 [%s]: %6.2f', self.name, self.prf['edp3']) # Columns of the per-task rt-app log file RTAPP_COL_IDX = 0 RTAPP_COL_PERF = 1 RTAPP_COL_RUN = 2 RTAPP_COL_PERIOD = 3 RTAPP_COL_START = 4 RTAPP_COL_END = 5 RTAPP_COL_REL_ST = 6 RTAPP_COL_SLACK = 7 RTAPP_COL_C_RUN = 8 RTAPP_COL_C_PERIOD = 9 RTAPP_COL_WU_LAT = 10 ################################################################################ # Generic workload performance parsing class ################################################################################ class DefaultTest(Test): def __init__(self, test_idx, test_dir, res): super(DefaultTest, self).__init__(test_idx, test_dir, res) # Default performance metric self.ctime_avg = [] self.perf_avg = [] self.edp1 = [] self.edp2 = [] self.edp3 = [] def parse_run(self, run_idx, run_dir): return DefaultRun(run_idx, run_dir) def collect_performance(self, run): # Keep track of average performances of each run self.ctime_avg.append(run.ctime_avg) self.perf_avg.append(run.perf_avg) self.edp1.append(run.edp1) self.edp2.append(run.edp2) self.edp3.append(run.edp3) def performance(self): return { 'ctime_avg' : Stats(self.ctime_avg).get(), 'perf_avg' : Stats(self.perf_avg).get(), 'edp1' : Stats(self.edp1).get(), 'edp2' : Stats(self.edp2).get(), 'edp3' : Stats(self.edp3).get(), } class DefaultRun(Run): def __init__(self, run_idx, run_dir): # Call base class to parse energy data super(DefaultRun, self).__init__(run_idx, run_dir) # Default specific performance stats self.ctime_avg = 0 self.perf_avg = 0 self.edp1 = 0 self.edp2 = 0 self.edp3 = 0 # Load default performance.json prf_file = os.path.join(run_dir, 'performance.json') if not os.path.isfile(prf_file): self._log.warning('No performance.json found in %s', run_dir) return # Load performance report from JSON with open(prf_file, 'r') as infile: prf = json.load(infile) # Keep track of performance value self.ctime_avg = prf['ctime'] self.perf_avg = prf['performance'] # Compute EDP indexes if energy measurements are available if self.nrg is None: return # Computing EDP self.edp1 = self.nrg.total * math.pow(self.ctime_avg, 1) self.edp2 = self.nrg.total * math.pow(self.ctime_avg, 2) self.edp3 = self.nrg.total * math.pow(self.ctime_avg, 3) ################################################################################ # Globals ################################################################################ # Regexp to match the format of a result folder TEST_DIR_RE = re.compile( r'.*/([^:]*):([^:]*):([^:]*)' ) #vim :set tabstop=4 shiftwidth=4 expandtab