diff options
| -rw-r--r-- | scripts/lib/devtool/standard.py | 2 | ||||
| -rw-r--r-- | scripts/lib/recipetool/create_perl.py | 347 |
2 files changed, 348 insertions, 1 deletions
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py index aca74b1fc66..ce5d23bb4a7 100644 --- a/scripts/lib/devtool/standard.py +++ b/scripts/lib/devtool/standard.py @@ -159,7 +159,7 @@ def add(args, config, basepath, workspace): tempdir = tempfile.mkdtemp(prefix='devtool') try: try: - stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) + stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool -d --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True) except bb.process.ExecutionError as e: if e.exitcode == 15: raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line') diff --git a/scripts/lib/recipetool/create_perl.py b/scripts/lib/recipetool/create_perl.py new file mode 100644 index 00000000000..27c03fd7763 --- /dev/null +++ b/scripts/lib/recipetool/create_perl.py @@ -0,0 +1,347 @@ +# Recipe creation tool - create handler for perl modules +# +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only +# + +import collections +import email +import imp +import glob +import itertools +import json +import logging +import os +from pathlib import Path +import re +import sys +import subprocess +import yaml +from recipetool.create import RecipeHandler + +logger = logging.getLogger('recipetool') + +tinfoil = None + + +def tinfoil_init(instance): + global tinfoil + tinfoil = instance + + +class PerlRecipeHandler(RecipeHandler): + base_pkgdeps = ['perl-core'] + excluded_pkgdeps = ['perl-dbg'] + + bbvar_map = { + 'Name': 'PN', + 'Version': 'PV', + 'Home-page': 'HOMEPAGE', + 'Summary': 'SUMMARY', + 'Description': 'DESCRIPTION', + 'License': 'LICENSE', + 'Requires': 'RDEPENDS_${PN}', + 'Provides': 'RPROVIDES_${PN}', + 'Obsoletes': 'RREPLACES_${PN}', + } + + cpan_mirror = { + 'https://cpan.metacpan.org/CPAN', '${CPAN_MIRROR)', + } + + # http://search.cpan.org/perldoc?CPAN::Meta::Spec + # not mapped: + # ssleay Original SSLeay License + # sun Sun Internet Standards Source License (SISSL) + # open_source Other Open Source Initiative (OSI) approved license + # unrestricted Not an OSI approved license, but not restricted + # + # restricted mapped to Proprietary + metajson_license_map = { + 'agpl_3': 'AGPL-3.0', + 'apache_1_1': 'Apache-1.1', + 'apache_2_0': 'Apache-2.0', + 'artistic_1': 'Artistic-1.0', + 'artistic_2': 'Artistic-2.0', + 'bsd': 'BSD-3-Clause', + 'freebsd': 'BSD-2-Clause', + 'gfdl_1_2': 'GFDL-1.2', + 'gfdl_1_3': 'GFDL-1.3', + 'gpl_1': 'GPL-1.0', + 'gpl_2': 'GPL-2.0', + 'gpl_3': 'GPL-3.0', + 'lgpl_2_1': 'LGPL-2.1', + 'lgpl_3_0': 'LGPL-3.0', + 'mit': 'MIT', + 'mozilla_1_0': 'MPL-1.0', + 'mozilla_1_1': 'MPL-1.1', + 'openss': 'OpenSSL', + 'perl_5': 'Artistic-1.0 | GPL-1.0+', + 'qpl_1_0': 'QPL-1.0', + 'zlib': 'Zlib', + 'restricted': 'Proprietary', + 'unknown': 'UNKNOWN', + } + + # http://module-build.sourceforge.net/META-spec-v1.4.html + # + # NOTE: This specification is significantly less precise + # about license types. Due diligence is required. + # + # not mapped: + # open_source Other Open Source Initiative (OSI) approved license + # unrestricted Not an OSI approved license, but not restricted + # + # restricted mapped to Proprietary + metayaml_license_map = { + 'apache': 'Apache-2.0', + 'apache_1_1': 'Apache-1.1', + 'artistic': 'Artistic-1.0', + 'artistic_2': 'Artistic-2.0', + 'bsd': 'BSD-2-Clause', + 'gpl': 'GPL-2.0 | GPL-3.0', + 'lgpl': 'LGPL-2.0 | LGPL-2.1 | LGPL-3.0', + 'mozilla': 'MPL-1.0 | MPL-1.1', + 'perl': 'Artistic-1.0 | GPL-1.0+', + 'restricted': 'Proprietary', + } + + def __init__(self): + self.use_metajson = False + self.use_metayaml = False + self.uses_makemaker = False + self.is_app = False + self.module_files = [] + self.test_files = [] + self.deps = set() + self.deps_ptest = set() + self.unmet_deps = set() + self.unmet_deps_ptest = set() + logger.debug("PerlRecipeHandler.__init__") + pass + + def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): + logger.debug("PerlRecipeHandler.process") + + if 'buildsystem' in handled: + logger.debug("buildsystem in handled") + return False + + if RecipeHandler.checkfiles(srctree, ['META.json']): + self.use_metajson = True + logger.debug("META.json found in source") + self.parse_metajson(srctree, classes, lines_before, lines_after, handled, extravalues) + pass + elif RecipeHandler.checkfiles(srctree, ['META.yml']): + self.use_metayaml = True + logger.debug("No META.json found in source. Falling back on META.yml found in source") + self.parse_metayaml(srctree, handled) + pass + elif RecipeHandler.checkfiles(srctree, ['Makefile.PL', 'Makefile.pl']): + logger.debug("No META.json or META.yml found in source. Falling back on Makefile.pl found in source") + self.uses_makemaker = True + elif RecipeHandler.checkfiles(srctree, ['Build.PL', 'Build.pl']): + logger.debug("No META.json, META.yml or Makefile.pl found in source. Falling back on Build.pl found in source") + self.uses_makemaker = False + else: + logger.debug("No META.json, META.yml, Makefile.pl or Build.pl found in source") + return + + if self.uses_makemaker: + classes.append('cpan') + logger.debug('inherit cpan') + else: + classes.append('cpan_build') + logger.debug('inherit cpan_build') + + if self.enable_ptest(srctree): + classes.append('ptest-perl') + logger.debug('inherit ptest-perl') + + for root, dirs, files in os.walk(srctree, topdown=True): + dirs[:] = [d for d in dirs if d not in set('/t')] + logger.debug("dirs: {}", dirs) + files[:] = [f for f in files if f.endswith(tuple(['.pm', '.xs']))] + logger.debug("files: {}", files) + for f in [os.path.join(root, file_) for file_ in files]: + self.module_files.append(f) + + if self.module_files: + for module_file in self.module_files: + logger.debug('module_file:') + deps, unmet_deps = self.scan_perl_dependencies(module_file) + for dep in deps: + self.deps.add(dep) + for unmet in unmet_deps: + self.unmet_deps.add(unmet) + if self.deps: + lines_after.append('#') + lines_after.append('# Runtime dependencies have been detected') + lines_after.append('# An attempt has been made to map them properly') + lines_after.append('#') + if self.unmet_deps: + lines_after.append('# Unmet dependencies have been detected:') + for unmet in sorted(self.unmet_deps): + lines_after.append('# {}'.format(unmet)) + if self.deps: + lines_after.append('RDEPENDS_${PN} += " \\') + for dep in sorted(self.deps): + lines_after.append( " {} \\".format(dep)) + lines_after.append('"') + lines_after.append('') + + if self.test_files: + for test_file in self.test_files: + logger.debug('test_file:') + deps, unmet_deps = self.scan_perl_dependencies(test_file) + for dep in deps: + if dep not in self.deps: + self.deps_ptest.add(dep) + for unmet in unmet_deps: + if unmet not in self.unmet_deps: + self.unmet_deps_ptest.add(unmet) + if self.deps_ptest: + lines_after.append('#') + lines_after.append('# ptest dependencies have been detected') + lines_after.append('# An attempt has been made to map them properly') + lines_after.append('#') + if self.unmet_deps_ptest: + lines_after.append('# Unmet dependencies for ptest have been detected:') + for unmet in sorted(self.unmet_deps_ptest): + lines_after.append('# {}'.format(unmet)) + lines_after.append('#') + if self.deps_ptest: + lines_after.append('RDEPENDS_${PN}-ptest += " \\') + for dep in sorted(self.deps_ptest): + lines_after.append( " {} \\".format(dep)) + lines_after.append('"') + lines_after.append('') + + lines_after.append('BBCLASSEXTEND = "native nativesdk"') + + # Done editing the recipe + handled.append('buildsystem') + + def parse_metajson(self, srctree, classes, lines_before, lines_after, handled, extravalues): + with open(os.path.join(srctree, "META.json"), 'r') as stream: + try: + metajson = json.load(stream) + try: + for build_requires in metajson["prereqs"]["build"]["requires"]: + if "ExtUtils::MakeMaker" in build_requires: + self.uses_makemaker = True + except KeyError as e: + for configure_requires in metajson["prereqs"]["configure"]["requires"]: + if "ExtUtils::MakeMaker" in configure_requires: + self.uses_makemaker = True + self._handle_license(handled, lines_before, lines_after, metajson['license'], self.metajson_license_map) + except (json.JSONDecodeError, KeyError) as e: + logger.error(e) + return + + def parse_metayaml(self, srctree, handled): + with open(os.path.join(srctree, "META.yml"), 'r') as stream: + try: + metayaml = yaml.safe_load(stream) + for build_requires in metayaml["build_requires"]: + if "ExtUtils::MakeMaker" in build_requires: + self.uses_makemaker = True + self._handle_license(handled, lines_before, lines_after, metayaml['license'], self.metayaml_license_map) + except yaml.YAMLError as e: + logger.error(e) + return + + def enable_ptest(self, srctree): + root = os.path.join(srctree, r't') + test_files = [os.path.join(root, file_) for file_ in os.listdir(root) if file_.endswith('.t')] + if test_files: + for test_file in test_files: + self.test_files.append(test_file) + return True + return False + + def _handle_license(self, handled, lines_before, lines_after, licsrc, licmap): + logger.debug('_handle_license') + logger.debug('licsrc: %s', licsrc) + try: + for license in licsrc: + logger.debug('license detected: %s', license) + mapped = licmap.get(license) + logger.debug('mapped license: %s', mapped) + if mapped: + handled.append('license') + # lines_after.append('LICENSE = "{}"'.format(mapped)) + else: + logger.debug('Failed to map license "%s"', license) + except Exception as e: + logger.error(e) + pass + + def scan_perl_dependencies(self, paths): + import bb.providers + logger.debug('paths: %s', paths) + deps = set() + unmapped_deps = set() + try: + dep_output = self.run_command(['scandeps.pl', '-B', paths]) + except (OSError, subprocess.CalledProcessError): + pass + else: + for line in dep_output.splitlines(): + logger.debug('line: {}', line) + # for when scandeps.pl returns a comment and not a match + if line.startswith('#') or not '=>' in line: + continue + try: + dep, min_version = line.split('=>', 1) + except ValueError as e: + logger.debug('Cannot split line: {}', line) + raise + dep.rstrip() + core_module_naming, lib_module_naming, app_naming = self.debianize(dep) + core_module_mapped = tinfoil.get_runtime_providers(core_module_naming) + lib_module_mapped = tinfoil.get_runtime_providers(lib_module_naming) + if core_module_mapped: + deps.add(core_module_naming) + elif lib_module_mapped: + deps.add(lib_module_naming) + else: + # If it didn't map to core module, assume it is an unmet lib module + logger.debug('unmapped dependency: %s', lib_module_naming) + unmapped_deps.add(lib_module_naming) + + # TODO: remove deps that are provided by this package + return deps, unmapped_deps + + def debianize(self, perl_name): + perl_name = perl_name.lower() + perl_name = perl_name.rstrip() + perl_name = perl_name.replace("_",'-') + perl_name = perl_name.replace("::",'-') + perl_name = perl_name.replace("'", '') + try: + core_module_naming = "perl-module-{}".format(perl_name) + lib_module_naming = "lib{}-perl".format(perl_name) + app_naming = perl_name + return core_module_naming, lib_module_naming, app_naming + except Exception as e: + logger.error(e) + raise + + @classmethod + def run_command(cls, cmd, **popenargs): + if 'stderr' not in popenargs: + popenargs['stderr'] = subprocess.STDOUT + try: + return subprocess.check_output(cmd, **popenargs).decode('utf-8') + except OSError as exc: + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) + raise + except subprocess.CalledProcessError as exc: + logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) + raise + +def register_recipe_handlers(handlers): + # We need to make sure this is ahead of the makefile fallback handler + handlers.append((PerlRecipeHandler(), 80)) |