From 3de0aefe89b698e4a50f814973232ccb1dc32dfc Mon Sep 17 00:00:00 2001 From: Chase Qi Date: Tue, 8 Sep 2015 18:03:15 +0800 Subject: ubuntu: upgrade lkp test definition * Unify python code style * Reform test code structure * Add COMMIT parameter to specify kernel commit hash * Add MONITORS parameter to control monitor process Change-Id: I7c0251671f75d8b215b04782d52629c19e6c4ef2 Signed-off-by: Chase Qi --- common/scripts/lkp.py | 154 +++++++++++++++++++++++++------------------------- ubuntu/lkp.yaml | 12 +++- 2 files changed, 88 insertions(+), 78 deletions(-) diff --git a/common/scripts/lkp.py b/common/scripts/lkp.py index 8d22001..f27c02e 100755 --- a/common/scripts/lkp.py +++ b/common/scripts/lkp.py @@ -16,8 +16,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # Author: Chase Qi # @@ -30,31 +29,32 @@ import pwd import shutil from subprocess import call -LKPPath = str(sys.argv[1]) -print 'LKP test suite path: %s' % (LKPPath) -WD = str(sys.argv[2]) -print 'Working directory: %s' % (WD) -LOOPS = int(sys.argv[3]) -JOB = str(sys.argv[4]) -print 'Going to run %s %s times' % (JOB, LOOPS) -HostName = platform.node() -KernelVersion = platform.release() -DIST = str.lower(platform.dist()[0]) +LKP_PATH = sys.argv[1] +WD = sys.argv[2] +JOB = sys.argv[3] +COMMIT = sys.argv[4] +LOOPS = int(sys.argv[5]) +MONITORS = sys.argv[6] +HOSTNAME = platform.node() +ROOTFS = str.lower(platform.dist()[0]) CONFIG = 'defconfig' +COMPILER = os.readlink('/usr/bin/gcc') +print 'Working directory: %s' % (WD) +print 'LKP test suite path: %s' % (LKP_PATH) +print 'About to run %s %s times' % (JOB, LOOPS) -def test_result(TestCommand, TestCaseID): +def test_result(test_command, test_case_id): # For each step of test run, print pass or fail to test log. - if call(TestCommand) == 0: - print '%s pass' % (TestCaseID) + if call(test_command) == 0: + print '%s pass' % (test_case_id) return True else: - print '%s fail' % (TestCaseID) + print '%s fail' % (test_case_id) return False def find_user(name): - # Create user 'lkp' if it doesn't exist. try: return pwd.getpwnam(name) except KeyError: @@ -69,8 +69,7 @@ else: print 'User lkp already exists.' if not os.path.exists('/home/lkp'): - call(['mkdir', '-p', '/home/lkp']) - + os.makedirs('/home/lkp') call(['chown', '-R', 'lkp:lkp', '/home/lkp']) f = open('/etc/apt/sources.list.d/multiverse.list', 'w') @@ -78,90 +77,93 @@ f.write('deb http://ports.ubuntu.com/ubuntu-ports/ vivid multiverse\n') f.close() call(['apt-get', 'update']) +# Setup test job. +SETUP_JOB = [LKP_PATH + '/bin/setup-local', + LKP_PATH + '/jobs/' + JOB + '.yaml'] +print 'Setup %s test with command: %s' % (JOB, SETUP_JOB) +if not test_result(SETUP_JOB, 'setup-' + JOB): + sys.exit(1) + # Split test job. if not os.path.exists(WD + '/' + JOB): os.makedirs(WD + '/' + JOB) -SplitJob = [LKPPath + '/sbin/split-job', '--output', WD + '/' + JOB, - LKPPath + '/jobs/' + JOB + '.yaml'] -print 'Splitting job %s with command: %s' % (JOB, SplitJob) -if not test_result(SplitJob, 'split-job-' + JOB): +if MONITORS == 'default': + MONITORS = '' +SPLIT_JOB = [LKP_PATH + '/sbin/split-job', MONITORS, '--kernel', COMMIT, + '--output', WD + '/' + JOB, LKP_PATH + '/jobs/' + JOB + '.yaml'] +print 'Splitting job %s with command: %s' % (JOB, SPLIT_JOB) +if not test_result(SPLIT_JOB, 'split-job-' + JOB): sys.exit(1) -# Setup test job. -SubTests = glob.glob(WD + '/' + JOB + '/*.yaml') -print 'Sub-tests of %s: %s' % (JOB, SubTests) -SetupLocal = [LKPPath + '/bin/setup-local', SubTests[0]] -print 'Set up %s test with command: %s' % (JOB, SetupLocal) -if not test_result(SetupLocal, 'setup-local-' + JOB): - sys.exit(1) - -# Delete test results from last lava-test-shell-run. +# Clean result root directory. if os.path.exists('/result/'): shutil.rmtree('/result/', ignore_errors=True) # Run tests. -for SubTest in SubTests: - COUNT = 1 - DONE = True - SubTestCaseID = os.path.basename(SubTest)[:-5] - ResultRoot = str('/'.join(['/result', JOB, - SubTestCaseID[int(len(JOB) + 1):], HostName, - DIST, CONFIG, KernelVersion])) - while (COUNT <= LOOPS): - # Use suffix for mutiple runs. +SUB_TESTS = glob.glob(WD + '/' + JOB + '/*.yaml') +for sub_test in SUB_TESTS: + count = 1 + done = True + sub_test_case_id = os.path.basename(sub_test)[:-46] + result_root = '/'.join(['/result', JOB, + sub_test_case_id[int(len(JOB) + 1):], + HOSTNAME, ROOTFS, CONFIG, COMPILER, COMMIT]) + while count <= LOOPS: + # Set suffix for mutiple runs. if LOOPS > 1: - SUFFIX = '-run' + str(COUNT) + suffix = '-run' + str(count) else: - SUFFIX = '' - - RunLocal = [LKPPath + '/bin/run-local', SubTest] - print 'Running test %s%s with command: %s' % (SubTestCaseID, SUFFIX, - RunLocal) - if not test_result(RunLocal, 'run-local-' + SubTestCaseID + SUFFIX): - DONE = False + suffix = '' + + lkp_run = [LKP_PATH + '/bin/run-local', + '--set', 'compiler: ' + COMPILER, sub_test] + print 'Running test %s%s with command: %s' % (sub_test_case_id, + suffix, lkp_run) + if not test_result(lkp_run, 'run-' + sub_test_case_id + suffix): + done = False break # For each run, decode JOB.json to pick up the scores produced by the # benchmark itself. - ResultFile = ResultRoot + '/' + str(COUNT - 1) + '/' + JOB + '.json' - if not os.path.isfile(ResultFile): - print '%s not found' % (ResultFile) + result_file = result_root + '/' + str(count - 1) + '/' + JOB + '.json' + if not os.path.isfile(result_file): + print '%s not found' % (result_file) else: - JsonData = open(ResultFile) - DICT = json.load(JsonData) - for item in DICT: - call(['lava-test-case', SubTestCaseID + '-' + item + SUFFIX, - '--result', 'pass', '--measurement', str(DICT[item][0])]) - JsonData.close() + json_data = open(result_file) + dict = json.load(json_data) + for item in dict: + call(['lava-test-case', sub_test_case_id + '-' + item + suffix, + '--result', 'pass', '--measurement', str(dict[item][0])]) + json_data.close() - COUNT = COUNT + 1 + count = count + 1 # For mutiple runs, if all runs are completed and results found, decode # avg.json. - if LOOPS > 1 and DONE: - AvgFile = ResultRoot + '/' + 'avg.json' - if not os.path.isfile(ResultFile): - print '%s not found' % (ResultFile) - elif not os.path.isfile(AvgFile): - print '%s not found' % (AvgFile) + if LOOPS > 1 and done: + avg_file = result_root + '/' + 'avg.json' + if not os.path.isfile(result_file): + print '%s not found' % (result_file) + elif not os.path.isfile(avg_file): + print '%s not found' % (avg_file) else: - JsonData = open(ResultFile) - AvgJsonData = open(AvgFile) - DICT = json.load(JsonData) - AvgDict = json.load(AvgJsonData) - for item in DICT: - if item in AvgDict: + json_data = open(result_file) + avg_json_data = open(avg_file) + dict = json.load(json_data) + avg_dict = json.load(avg_json_data) + for item in dict: + if item in avg_dict: call(['lava-test-case', - SubTestCaseID + '-' + item + '-avg', '--result', - 'pass', '--measurement', str(AvgDict[item])]) - JsonData.close() - AvgJsonData.close() + sub_test_case_id + '-' + item + '-avg', '--result', + 'pass', '--measurement', str(avg_dict[item])]) + json_data.close() + avg_json_data.close() # Compress and attach raw data. call(['tar', 'caf', 'lkp-result-' + JOB + '.tar.xz', '/result/' + JOB]) call(['lava-test-run-attach', 'lkp-result-' + JOB + '.tar.xz']) -if not DONE: +if not done: sys.exit(1) else: sys.exit(0) diff --git a/ubuntu/lkp.yaml b/ubuntu/lkp.yaml index 72bfdbb..4a4e4a5 100644 --- a/ubuntu/lkp.yaml +++ b/ubuntu/lkp.yaml @@ -12,6 +12,7 @@ metadata: - functional devices: - mustang + - juno install: git-repos: @@ -31,9 +32,16 @@ install: - xz-utils params: + # Used to specify jobs located in 'lkp-tests/jobs/'. + JOB: 'hackbench' + # Used to specify all 40 commit hash characters. + COMMIT: 'commit' + # Set LOOPS to the times to run in the same lava_test_shell. LOOPS: 1 - JOB: 'unixbench' + # Set MONITORS to '--no-defaults' to disable the default monitors. + MONITORS: 'default' run: steps: - - './common/scripts/lkp.py $(readlink -f lkp-tests) $(pwd) $LOOPS $JOB' + - './common/scripts/lkp.py $(readlink -f lkp-tests) $(pwd) + $JOB $COMMIT $LOOPS $MONITORS' -- cgit v1.2.3