# Copyright 2014 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import os.path
import tempfile
import subprocess
import time
import sys
import textwrap
import its.device
from its.device import ItsSession
def main():
"""Run all the automated tests, saving intermediate files, and producing
a summary/report of the results.
Script should be run from the top-level CameraITS directory.
Command line Arguments:
camera: the camera(s) to be tested. Use comma to separate multiple
camera Ids. Ex: "camera=0,1" or "camera=1"
scenes: the test scene(s) to be executed. Use comma to separate multiple
scenes. Ex: "scenes=scene0,scene1" or "scenes=0,1,sensor_fusion"
(sceneX can be abbreviated by X where X is a integer)
chart: [Experimental] another android device served as test chart
display. When this argument presents, change of test scene will
be handled automatically. Note that this argument requires
special physical/hardware setup to work and may not work on
all android devices.
"""
SKIP_RET_CODE = 101
# Not yet mandated tests
NOT_YET_MANDATED = {
"scene0":[
"test_jitter"
],
"scene1":[
"test_ae_precapture_trigger",
"test_crop_region_raw",
"test_ev_compensation_advanced",
"test_ev_compensation_basic",
"test_yuv_plus_jpeg"
],
"scene2":[],
"scene3":[],
"scene4":[],
"scene5":[],
"sensor_fusion":[]
}
all_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4", "scene5"]
auto_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4"]
scene_req = {
"scene0" : None,
"scene1" : "A grey card covering at least the middle 30% of the scene",
"scene2" : "A picture containing human faces",
"scene3" : "A chart containing sharp edges like ISO 12233",
"scene4" : "A specific test page of a circle covering at least the "
"middle 50% of the scene. See CameraITS.pdf section 2.3.4 "
"for more details",
"scene5" : "Capture images with a diffuser attached to the camera. See "
"CameraITS.pdf section 2.3.4 for more details",
"sensor_fusion" : "Rotating checkboard pattern. See "
"sensor_fusion/SensorFusion.pdf for detailed "
"instructions. Note that this test will be skipped "
"on devices not supporting REALTIME camera timestamp."
"If that is the case, no scene setup is required and "
"you can just answer Y when being asked if the scene "
"is okay"
}
scene_extra_args = {
"scene5" : ["doAF=False"]
}
camera_ids = []
scenes = []
chart_host_id = None
for s in sys.argv[1:]:
if s[:7] == "camera=" and len(s) > 7:
camera_ids = s[7:].split(',')
elif s[:7] == "scenes=" and len(s) > 7:
scenes = s[7:].split(',')
elif s[:6] == 'chart=' and len(s) > 6:
chart_host_id = s[6:]
auto_scene_switch = chart_host_id is not None
# Run through all scenes if user does not supply one
possible_scenes = auto_scenes if auto_scene_switch else all_scenes
if not scenes:
scenes = possible_scenes
else:
# Validate user input scene names
valid_scenes = True
temp_scenes = []
for s in scenes:
if s in possible_scenes:
temp_scenes.append(s)
else:
try:
# Try replace "X" to "sceneX"
scene_num = int(s)
scene_str = "scene" + s
if scene_str not in possible_scenes:
valid_scenes = False
break
temp_scenes.append(scene_str)
except ValueError:
valid_scenes = False
break
if not valid_scenes:
print "Unknown scene specifiied:", s
assert(False)
scenes = temp_scenes
# Initialize test results
results = {}
result_key = ItsSession.RESULT_KEY
for s in all_scenes:
results[s] = {result_key: ItsSession.RESULT_NOT_EXECUTED}
# Make output directories to hold the generated files.
topdir = tempfile.mkdtemp()
print "Saving output files to:", topdir, "\n"
device_id = its.device.get_device_id()
device_id_arg = "device=" + device_id
print "Testing device " + device_id
# user doesn't specify camera id, run through all cameras
if not camera_ids:
camera_ids_path = os.path.join(topdir, "camera_ids.txt")
out_arg = "out=" + camera_ids_path
cmd = ['python',
os.path.join(os.getcwd(),"tools/get_camera_ids.py"), out_arg,
device_id_arg]
retcode = subprocess.call(cmd,cwd=topdir)
assert(retcode == 0)
with open(camera_ids_path, "r") as f:
for line in f:
camera_ids.append(line.replace('\n', ''))
print "Running ITS on camera: %s, scene %s" % (camera_ids, scenes)
if auto_scene_switch:
print 'Waking up chart screen: ', chart_host_id
screen_id_arg = ('screen=%s' % chart_host_id)
cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools',
'wake_up_screen.py'), screen_id_arg]
retcode = subprocess.call(cmd)
assert retcode == 0
for camera_id in camera_ids:
# Loop capturing images until user confirm test scene is correct
camera_id_arg = "camera=" + camera_id
print "Preparing to run ITS on camera", camera_id
os.mkdir(os.path.join(topdir, camera_id))
for d in scenes:
os.mkdir(os.path.join(topdir, camera_id, d))
for scene in scenes:
tests = [(s[:-3],os.path.join("tests", scene, s))
for s in os.listdir(os.path.join("tests",scene))
if s[-3:] == ".py" and s[:4] == "test"]
tests.sort()
summary = "Cam" + camera_id + " " + scene + "\n"
numpass = 0
numskip = 0
num_not_mandated_fail = 0
numfail = 0
if scene_req[scene] != None:
out_path = os.path.join(topdir, camera_id, scene+".jpg")
out_arg = "out=" + out_path
if auto_scene_switch:
scene_arg = "scene=" + scene
cmd = ['python',
os.path.join(os.getcwd(), 'tools/load_scene.py'),
scene_arg, screen_id_arg]
else:
scene_arg = "scene=" + scene_req[scene]
extra_args = scene_extra_args.get(scene, [])
cmd = ['python',
os.path.join(os.getcwd(),"tools/validate_scene.py"),
camera_id_arg, out_arg,
scene_arg, device_id_arg] + extra_args
retcode = subprocess.call(cmd,cwd=topdir)
assert(retcode == 0)
print "Start running ITS on camera %s, %s" % (camera_id, scene)
# Run each test, capturing stdout and stderr.
for (testname,testpath) in tests:
if auto_scene_switch:
# Send an input event to keep the screen not dimmed.
# Since we are not using camera of chart screen, FOCUS event
# should does nothing but keep the screen from dimming.
# The "sleep after x minutes of inactivity" display setting
# determines how long this command can keep screen bright.
# Setting it to something like 30 minutes should be enough.
cmd = ('adb -s %s shell input keyevent FOCUS'
% chart_host_id)
subprocess.call(cmd.split())
cmd = ['python', os.path.join(os.getcwd(),testpath)] + \
sys.argv[1:] + [camera_id_arg]
outdir = os.path.join(topdir,camera_id,scene)
outpath = os.path.join(outdir,testname+"_stdout.txt")
errpath = os.path.join(outdir,testname+"_stderr.txt")
t0 = time.time()
with open(outpath,"w") as fout, open(errpath,"w") as ferr:
retcode = subprocess.call(
cmd,stderr=ferr,stdout=fout,cwd=outdir)
t1 = time.time()
test_failed = False
if retcode == 0:
retstr = "PASS "
numpass += 1
elif retcode == SKIP_RET_CODE:
retstr = "SKIP "
numskip += 1
elif retcode != 0 and testname in NOT_YET_MANDATED[scene]:
retstr = "FAIL*"
num_not_mandated_fail += 1
else:
retstr = "FAIL "
numfail += 1
test_failed = True
msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0)
print msg
msg_short = "%s %s [%.1fs]" % (retstr, testname, t1-t0)
if test_failed:
summary += msg_short + "\n"
if numskip > 0:
skipstr = ", %d test%s skipped" % (
numskip, "s" if numskip > 1 else "")
else:
skipstr = ""
test_result = "\n%d / %d tests passed (%.1f%%)%s" % (
numpass + num_not_mandated_fail, len(tests) - numskip,
100.0 * float(numpass + num_not_mandated_fail) /
(len(tests) - numskip)
if len(tests) != numskip else 100.0,
skipstr)
print test_result
if num_not_mandated_fail > 0:
msg = "(*) tests are not yet mandated"
print msg
summary_path = os.path.join(topdir, camera_id, scene, "summary.txt")
with open(summary_path, "w") as f:
f.write(summary)
passed = numfail == 0
results[scene][result_key] = (ItsSession.RESULT_PASS if passed
else ItsSession.RESULT_FAIL)
results[scene][ItsSession.SUMMARY_KEY] = summary_path
print "Reporting ITS result to CtsVerifier"
its.device.report_result(device_id, camera_id, results)
if auto_scene_switch:
print 'Shutting down chart screen: ', chart_host_id
screen_id_arg = ('screen=%s' % chart_host_id)
cmd = ['python', os.path.join(os.environ['CAMERA_ITS_TOP'], 'tools',
'turn_off_screen.py'), screen_id_arg]
retcode = subprocess.call(cmd)
assert retcode == 0
print "ITS tests finished. Please go back to CtsVerifier and proceed"
if __name__ == '__main__':
main()