diff options
| -rw-r--r-- | meta/lib/oeqa/recipetests/__init__.py | 2 | ||||
| -rw-r--r-- | meta/lib/oeqa/recipetests/base.py | 25 | ||||
| -rw-r--r-- | meta/lib/oeqa/recipetests/buildrecipe.py | 62 | ||||
| -rw-r--r-- | meta/lib/oeqa/utils/commands.py | 21 | ||||
| -rwxr-xr-x | scripts/test-recipe | 306 |
5 files changed, 416 insertions, 0 deletions
diff --git a/meta/lib/oeqa/recipetests/__init__.py b/meta/lib/oeqa/recipetests/__init__.py new file mode 100644 index 00000000000..3ad9513f40e --- /dev/null +++ b/meta/lib/oeqa/recipetests/__init__.py @@ -0,0 +1,2 @@ +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/meta/lib/oeqa/recipetests/base.py b/meta/lib/oeqa/recipetests/base.py new file mode 100644 index 00000000000..148c123544e --- /dev/null +++ b/meta/lib/oeqa/recipetests/base.py @@ -0,0 +1,25 @@ +import os +import shutil +from oeqa.selftest.base import oeSelfTest +import oeqa.utils.ftools as ftools + + +class RecipeTests(oeSelfTest): + + def __init__(self, methodName="runTest"): + super(RecipeTests, self).__init__(methodName) + self.testinc_path = os.path.join(self.builddir, 'conf/testrecipe.inc') + self.required_path = os.path.join(self.builddir, 'conf/required.inc') + self.testrecipe = os.getenv("TESTRECIPE") + + # Use a clean TMPDIR for each run + tmpdir_name = self.builddir + '/tmp_' + self.testrecipe + shutil.rmtree(tmpdir_name, ignore_errors=True) + + feature = 'TMPDIR = "%s"\n' % tmpdir_name + self.set_required_config(feature) + + # write to <builddir>/conf/requred.inc + def set_required_config(self, data): + self.log.debug("Writing to: %s\n%s\n" % (self.required_path, data)) + ftools.write_file(self.required_path, data) diff --git a/meta/lib/oeqa/recipetests/buildrecipe.py b/meta/lib/oeqa/recipetests/buildrecipe.py new file mode 100644 index 00000000000..c303b29589e --- /dev/null +++ b/meta/lib/oeqa/recipetests/buildrecipe.py @@ -0,0 +1,62 @@ +from oeqa.recipetests.base import RecipeTests +from oeqa.selftest.base import get_available_machines +from oeqa.utils.commands import bitbake, get_tasks_for_recipe +import logging + + +class BuildRecipeTests(RecipeTests): + + log = logging.getLogger('test-recipe.build-recipe') + + def test_build_recipe_for_all_major_architectures(self): + """ Build the recipe with all major architectures(qemux86, qemux86-64, qemuarm, qemuppc, qemumips) """ + + machines = get_available_machines() + qemu_machines = [m for m in machines if 'qemu' in m] + + # Build the recipe for all major architectures + for m in qemu_machines: + self.log.info('Building recipe "%s" for architecture "%s"' % (self.testrecipe, m)) + self.write_config('MACHINE = "%s"' % m) + bitbake(self.testrecipe) + + def test_rebuild_recipe_from_sstate_1(self): + """ Rebuild the recipe from sstate with sstate file for the recipe """ + bitbake(self.testrecipe) + bitbake('-c clean %s' % self.testrecipe) + bitbake(self.testrecipe) + + def test_rebuild_recipe_from_sstate_2(self): + """ Rebuild the recipe from sstate without sstate file for the recipe """ + bitbake(self.testrecipe) + bitbake('-c cleansstate %s' % self.testrecipe) + bitbake(self.testrecipe) + + def test_cleaning_operations_on_recipe(self): + """ Perform cleaning operations on the recipe(cleansstate, clean, cleanall) """ + + clean_tasks = ['cleansstate', 'clean', 'cleanall'] + + for task in clean_tasks: + bitbake(self.testrecipe) + self.log.info('Performing %s for recipe %s' % (task, self.testrecipe)) + bitbake('-c %s %s' % (task, self.testrecipe)) + + def test_force_major_tasks_on_recipe(self): + """ Force all major tasks on the recipe (bitbake -C <task> <recipe>) """ + major_tasks = ['unpack', 'patch', 'configure', 'compile', 'install', 'populate_sysroot', 'package', + 'package_write_rpm', 'package_write_deb', 'package_write_ipk'] + + feature = 'PACKAGE_CLASSES = "package_rpm package_deb package_ipk"\n' + self.write_config(feature) + + available_tasks = get_tasks_for_recipe(self.testrecipe) + + for task in major_tasks: + # Check if task is available for recipe + if task not in available_tasks: + self.log.warning('Task %s not available for recipe %s' % (task, self.testrecipe)) + continue + # Force task on recipe + self.log.info('Forcing task %s' % task) + bitbake('-C %s %s' % (task, self.testrecipe)) diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py index 08e2cbb906c..8ddb62d2bb1 100644 --- a/meta/lib/oeqa/utils/commands.py +++ b/meta/lib/oeqa/utils/commands.py @@ -153,6 +153,18 @@ def get_bb_var(var, target=None, postconfig=None): lastline = line return val + +def get_all_available_recipes(): + ret = bitbake('-s') + available_recipes = re.findall(r'\n(\S+)\s+:', ret.output) + + return available_recipes + + +def is_recipe_valid(recipe): + return recipe in get_all_available_recipes() + + def get_test_layer(): layers = get_bb_var("BBLAYERS").split() testlayer = None @@ -164,6 +176,15 @@ def get_test_layer(): break return testlayer + +def get_tasks_for_recipe(recipe): + """ Get available tasks for recipe """ + ret = bitbake('-c listtasks %s' % recipe) + tasks = re.findall('\ndo_(\S+)\s', ret.output) + + return tasks + + def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec='recipes-*/*'): os.makedirs(os.path.join(templayerdir, 'conf')) with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f: diff --git a/scripts/test-recipe b/scripts/test-recipe new file mode 100755 index 00000000000..8cb9409a10c --- /dev/null +++ b/scripts/test-recipe @@ -0,0 +1,306 @@ +#!/usr/bin/env python + +# Copyright (c) 2016, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# DESCRIPTION: Will run recipe tests on each provided recipe. +# +# USAGE: recipe-test --recipes <recipe1 recipe2> --tests all +# +# OPTIONS: --recipes <recipe1> +# --run-tests <test1 test2>| all +# --list-tests +# +# NOTE: tests are located in lib/oeqa/recipetests +# +# AUTHOR: Daniel Istrate <daniel.alexandrux.istrate@intel.com> +# + + +import argparse +import logging +import sys +import os +import time +import unittest + +sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib') +import scriptpath +scriptpath.add_bitbake_lib_path() +scriptpath.add_oe_lib_path() + +import oeqa.recipetests +from oeqa.utils.commands import get_test_layer, get_bb_var, bitbake, is_recipe_valid +from oeqa.selftest.base import oeSelfTest +import oeqa.utils.ftools as ftools +from oeqa.recipetests.base import RecipeTests + + +parser = argparse.ArgumentParser() +parser.add_argument('-r', '--recipes', nargs='+', default=False, dest='recipes', + help='recipe(s) to run tests against.') +parser.add_argument('-t', '--run-tests', dest='run_tests', default='all', nargs='*', + help='Run specified tests.') +parser.add_argument('-l', '--list-tests', required=False, action='store_true', dest="list_tests", default=False, + help='List all available tests.') + +if len(sys.argv) < 2: + parser.print_usage() + parser.exit(1) + +args = parser.parse_args() + + +def logger_create(): + log_file = 'test-recipe-' + time.strftime("%Y%m%d%H%M%S") + '.log' + if os.path.exists('test-recipe.log'): + os.remove('test-recipe.log') + os.symlink(log_file, 'test-recipe.log') + + log = logging.getLogger("test-recipe") + log.setLevel(logging.DEBUG) + + fh = logging.FileHandler(filename=log_file, mode='w') + fh.setLevel(logging.DEBUG) + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s', "%Y-%m-%d %H:%M:%S") + fh.setFormatter(formatter) + ch.setFormatter(formatter) + + log.addHandler(fh) + log.addHandler(ch) + + return log + +log = logger_create() + + +def preflight_check(): + + log.info("Checking that everything is in order before running the tests") + + if not os.environ.get("BUILDDIR"): + log.error("BUILDDIR isn't set. Did you forget to source your build environment setup script?") + return False + + builddir = os.environ.get("BUILDDIR") + if os.getcwd() != builddir: + log.info("Changing cwd to %s" % builddir) + os.chdir(builddir) + + if not "meta-selftest" in get_bb_var("BBLAYERS"): + log.error("You don't seem to have the meta-selftest layer in BBLAYERS") + return False + + log.info("Running bitbake -p") + bitbake("-p") + + return True + + +def add_include(): + builddir = os.environ.get("BUILDDIR") + if "#include added by test-recipe" \ + not in ftools.read_file(os.path.join(builddir, "conf/local.conf")): + log.info("Adding: \"include testrecipe.inc\" in local.conf") + ftools.append_file(os.path.join(builddir, "conf/local.conf"), \ + "\n#include added by test-recipe\ninclude required.inc\ninclude testrecipe.inc") + + if "#include added by test-recipe" \ + not in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")): + log.info("Adding: \"include bblayers.inc\" in bblayers.conf") + ftools.append_file(os.path.join(builddir, "conf/bblayers.conf"), \ + "\n#include added by test-recipe\ninclude bblayers.inc") + + +def remove_include(): + builddir = os.environ.get("BUILDDIR") + if builddir is None: + return + if "#include added by test-recipe" \ + in ftools.read_file(os.path.join(builddir, "conf/local.conf")): + log.info("Removing the include from local.conf") + ftools.remove_from_file(os.path.join(builddir, "conf/local.conf"), \ + "\n#include added by test-recipe\ninclude required.inc\ninclude testrecipe.inc") + + if "#include added by test-recipe" \ + in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")): + log.info("Removing the include from bblayers.conf") + ftools.remove_from_file(os.path.join(builddir, "conf/bblayers.conf"), \ + "\n#include added by test-recipe\ninclude bblayers.inc") + + +def remove_inc_files(): + try: + os.remove(os.path.join(os.environ.get("BUILDDIR"), "conf/testrecipe.inc")) + for root, _, files in os.walk(get_test_layer()): + for f in files: + if f == 'test_recipe.inc': + os.remove(os.path.join(root, f)) + except (AttributeError, OSError,) as e: # AttributeError may happen if BUILDDIR is not set + pass + + for incl_file in ['conf/bblayers.inc', 'conf/required.inc']: + try: + os.remove(os.path.join(os.environ.get("BUILDDIR"), incl_file)) + except: + pass + + +def get_tests_from_module(tmod): + tlist = [] + + try: + import importlib + modlib = importlib.import_module(tmod) + for mod in vars(modlib).values(): + if isinstance(mod, type(RecipeTests)) and issubclass(mod, RecipeTests) and mod is not RecipeTests: + for test in dir(mod): + if test.startswith('test_') and callable(vars(mod)[test]): + try: + test_description = vars(mod)[test].__doc__.split('\n')[0][:60].strip() + '...' + except: + test_description = None + tlist.append((mod.__module__.split('.')[-1], mod.__name__, test, test_description)) + + except: + pass + + return tlist + + +def list_recipe_tests(): + tmodules = set() + testlist = [] + prefix = 'oeqa.recipetests.' + + for tpath in oeqa.recipetests.__path__: + files = sorted([f for f in os.listdir(tpath) if f.endswith('.py') and not + f.startswith(('_', '__')) and f != 'base.py']) + for f in files: + tmodules.add(prefix + f.rstrip('.py')) + + # Get all the tests from modules + tmodules = sorted(list(tmodules)) + + for tmod in tmodules: + testlist += get_tests_from_module(tmod) + + print '%-10s\t%-20s\t%-50s\t%-80s' % ('module', 'class', 'name', 'description') + print '_' * 160 + for t in testlist: + print '%-10s\t%-20s\t%-50s\t%-80s' % t + print '_' * 160 + print 'Total found:\t %s' % len(testlist) + + +def get_recipe_tests(exclusive_modules=[]): + testslist = [] + prefix = 'oeqa.recipetests.' + + if exclusive_modules != ['all']: + for mod in exclusive_modules: + testslist.append(prefix + mod) + + if not testslist: + for testpath in oeqa.recipetests.__path__: + files = sorted([f for f in os.listdir(testpath) if f.endswith('.py') and not f.startswith('__') and f != 'base.py']) + for f in files: + module = prefix + f[:-3] + if module not in testslist: + testslist.append(module) + + return testslist + + +def main(): + + if args.list_tests: + list_recipe_tests() + return 0 + + if args.run_tests != 'all' and not args.recipes: + print 'Please specify the recipe(s) to tests against ( -r or --recipes).' + return 1 + + # Do we want to be able to test multiple recipes? + if not is_recipe_valid(args.recipes[0]): + print '"%s" is not a valid recipe. Make sure it shows up in "bitbake -s". Check your spelling.' % args.recipes[0] + return 1 + + if args.run_tests: + if not preflight_check(): + return 1 + + testslist = get_recipe_tests(exclusive_modules=(args.run_tests or [])) + + os.environ['TESTRECIPE'] = args.recipes[0] + log.info('Running tests for recipe "%s" ...' % args.recipes[0]) + + suite = unittest.TestSuite() + loader = unittest.TestLoader() + loader.sortTestMethodsUsing = None + runner = unittest.TextTestRunner(verbosity=2, resultclass=StampedResult) + # we need to do this here, otherwise just loading the tests + # will take 2 minutes (bitbake -e calls) + oeSelfTest.testlayer_path = get_test_layer() + + for test in testslist: + log.info("Loading tests from: %s" % test) + try: + suite.addTests(loader.loadTestsFromName(test)) + except AttributeError as e: + log.error("Failed to import %s" % test) + log.error(e) + return 1 + add_include() + + result = runner.run(suite) + + log.info("Finished") + + if result.wasSuccessful(): + return 0 + else: + return 1 + + +class StampedResult(unittest.TextTestResult): + """ + Custom TestResult that prints the time when a test starts. As test-recipe + can take a long time (ie a few hours) to run, timestamps help us understand + what tests are taking a long time to execute. + """ + def startTest(self, test): + import time + self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ") + super(StampedResult, self).startTest(test) + +if __name__ == '__main__': + + try: + ret = main() + except Exception: + ret = 1 + import traceback + traceback.print_exc(5) + finally: + remove_include() + remove_inc_files() + sys.exit(ret) |