#!/usr/bin/env python
#
# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Sample Usage:
# $ python update_profiles.py 500000 git_master ALL --profdata-suffix 2019-04-15
#
# Additional/frequently-used arguments:
# -b BUG adds a 'Bug: <BUG>' to the commit message when adding the profiles.
# --do-not-merge adds a 'DO NOT MERGE' tag to the commit message to restrict
# automerge of profiles from release branches.
#
# Try '-h' for a full list of command line arguments.
import argparse
import os
import shutil
import subprocess
import sys
import tempfile
import zipfile
import utils
X20_BASE_LOCATION = '/google/data/ro/teams/android-pgo-data'
class Benchmark(object):
def __init__(self, name):
self.name = name
def x20_profile_location(self):
raise NotImplementedError()
def apct_job_name(self):
raise NotImplementedError()
def profdata_file(self, suffix=''):
profdata = os.path.join(self.name, '{}.profdata'.format(self.name))
if suffix:
profdata += '.' + suffix
return profdata
def profraw_files(self):
raise NotImplementedError()
def merge_profraws(self, profile_dir, output):
profraws = [os.path.join(profile_dir, p) for p in self.profraw_files(profile_dir)]
utils.run_llvm_profdata(profraws, output)
class NativeExeBenchmark(Benchmark):
def apct_job_name(self):
return 'pgo-collector'
def x20_profile_location(self):
return os.path.join(X20_BASE_LOCATION, 'raw')
def profraw_files(self, profile_dir):
if self.name == 'hwui':
return ['hwuimacro.profraw', 'hwuimacro_64.profraw',
'hwuimicro.profraw', 'hwuimicro_64.profraw',
'skia_nanobench.profraw', 'skia_nanobench_64.profraw']
elif self.name == 'hwbinder':
return ['hwbinder.profraw', 'hwbinder_64.profraw']
class APKBenchmark(Benchmark):
def apct_job_name(self):
return 'apk-pgo-collector'
def x20_profile_location(self):
return os.path.join(X20_BASE_LOCATION, 'apk-raw')
def profdata_file(self, suffix=''):
profdata = os.path.join('art', '{}_arm_arm64.profdata'.format(self.name))
if suffix:
profdata += '.' + suffix
return profdata
def profraw_files(self, profile_dir):
return os.listdir(profile_dir)
def BenchmarkFactory(benchmark_name):
if benchmark_name == 'dex2oat':
return APKBenchmark(benchmark_name)
elif benchmark_name in ['hwui', 'hwbinder']:
return NativeExeBenchmark(benchmark_name)
else:
raise RuntimeError('Unknown benchmark ' + benchmark_name)
def extract_profiles(benchmark, branch, build, output_dir):
# The APCT results are stored in
# <x20_profile_base>/<branch>/<build>/<apct_job_name>/<arbitrary_invocation_dir>/
#
# The PGO files are in _data_local_tmp_<id>.zip in the above directory.
profile_base = os.path.join(benchmark.x20_profile_location(), branch, build,
benchmark.apct_job_name())
invocation_dirs = os.listdir(profile_base)
if len(invocation_dirs) == 0:
raise RuntimeError('No invocations found in {}'.format(profile_base))
if len(invocation_dirs) > 1:
# TODO Add option to pick/select an invocation from the command line.
raise RuntimeError('More than one invocation found in {}'.format(profile_base))
profile_dir = os.path.join(profile_base, invocation_dirs[0])
zipfiles = [f for f in os.listdir(profile_dir) if f.startswith('_data_local_tmp')]
if len(zipfiles) != 1:
raise RuntimeError('Expected one zipfile in {}. Found {}'.format(profile_dir,
len(zipfiles)))
zipfile_name = os.path.join(profile_dir, zipfiles[0])
zip_ref = zipfile.ZipFile(zipfile_name)
zip_ref.extractall(output_dir)
zip_ref.close()
KNOWN_BENCHMARKS = ['ALL', 'dex2oat', 'hwui', 'hwbinder']
def parse_args():
"""Parses and returns command line arguments."""
parser = argparse.ArgumentParser()
parser.add_argument(
'build', metavar='BUILD',
help='Build number to pull from the build server.')
parser.add_argument(
'-b', '--bug', type=int,
help='Bug to reference in commit message.')
parser.add_argument(
'--use-current-branch', action='store_true',
help='Do not repo start a new branch for the update.')
parser.add_argument(
'--add-do-not-merge', action='store_true',
help='Add \'DO NOT MERGE\' to the commit message.')
parser.add_argument(
'--profdata-suffix', type=str, default='',
help='Suffix to append to merged profdata file')
parser.add_argument(
'branch', metavar='BRANCH',
help='Fetch profiles for BRANCH (e.g. git_qt-release)')
parser.add_argument(
'benchmark', metavar='BENCHMARK',
help='Update profiles for BENCHMARK. Choices are {}'.format(KNOWN_BENCHMARKS))
parser.add_argument(
'--skip-cleanup', '-sc',
action='store_true',
default=False,
help='Skip the cleanup, and leave intermediate files (in /tmp/pgo-profiles-*)')
return parser.parse_args()
def get_current_profile(benchmark):
profile = benchmark.profdata_file()
dirname, basename = os.path.split(profile)
old_profiles = [f for f in os.listdir(dirname) if f.startswith(basename)]
if len(old_profiles) == 0:
return ''
return os.path.join(dirname, old_profiles[0])
def main():
args = parse_args()
if args.benchmark == 'ALL':
worklist = KNOWN_BENCHMARKS[1:]
else:
worklist = [args.benchmark]
profiles_project = os.path.join(utils.android_build_top(), 'toolchain',
'pgo-profiles')
os.chdir(profiles_project)
if not args.use_current_branch:
branch_name = 'update-profiles-' + args.build
utils.check_call(['repo', 'start', branch_name, '.'])
for benchmark_name in worklist:
benchmark = BenchmarkFactory(benchmark_name)
# Existing profile file, which gets 'rm'-ed from 'git' down below.
current_profile = get_current_profile(benchmark)
# Extract profiles to a temporary directory. After extraction, we
# expect to find one subdirectory with profraw files under the temporary
# directory.
extract_dir = tempfile.mkdtemp(prefix='pgo-profiles-'+benchmark_name)
extract_profiles(benchmark, args.branch, args.build, extract_dir)
if len(os.listdir(extract_dir)) != 1:
raise RuntimeError("Expected one subdir under {}".format(extract_dir))
extract_subdir = os.path.join(extract_dir, os.listdir(extract_dir)[0])
# Merge profiles.
profdata = benchmark.profdata_file(args.profdata_suffix)
benchmark.merge_profraws(extract_subdir, profdata)
# Construct 'git' commit message.
message_lines = [
'Update PGO profiles for {}'.format(benchmark_name), '',
'The profiles are from build {}.'.format(args.build), ''
]
if args.add_do_not_merge:
message_lines[0] = '[DO NOT MERGE] ' + message_lines[0]
if args.bug:
message_lines.append('')
message_lines.append('Bug: http://b/{}'.format(args.bug))
message_lines.append('Test: Build (TH)')
message = '\n'.join(message_lines)
# Invoke git: Delete current profile, add new profile and commit these
# changes.
if current_profile:
utils.check_call(['git', 'rm', current_profile])
utils.check_call(['git', 'add', profdata])
utils.check_call(['git', 'commit', '-m', message])
if not args.skip_cleanup:
shutil.rmtree(extract_dir)
if __name__ == '__main__':
main()