diff options
| -rw-r--r-- | meta/classes/create-spdx-3.0.bbclass | 728 |
1 files changed, 224 insertions, 504 deletions
diff --git a/meta/classes/create-spdx-3.0.bbclass b/meta/classes/create-spdx-3.0.bbclass index b0aef80db16..ffe34969a89 100644 --- a/meta/classes/create-spdx-3.0.bbclass +++ b/meta/classes/create-spdx-3.0.bbclass @@ -11,7 +11,7 @@ DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx" CVE_PRODUCT ??= "${BPN}" CVE_VERSION ??= "${PV}" -SPDXDIR ??= "${WORKDIR}/spdx" +SPDXDIR ??= "${WORKDIR}/spdx-3.0" SPDXDEPLOY = "${SPDXDIR}/deploy" SPDXWORK = "${SPDXDIR}/work" SPDXIMAGEWORK = "${SPDXDIR}/image-work" @@ -64,21 +64,74 @@ def get_doc_namespace(d, doc): namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) -def create_annotation(d, comment): +def generate_creationInfo(d, document): + """ + Generate the creationInfo and its elements for a document + """ from datetime import datetime, timezone + import oe.spdx3 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - annotation = oe.spdx.SPDXAnnotation() - annotation.annotationDate = creation_time - annotation.annotationType = "OTHER" - annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) - annotation.comment = comment - return annotation + + document.creationInfo = oe.spdx3.SPDX3CreationInfo() + document.creationInfo.specVersion = "3.0.0" + document.creationInfo.created = creation_time + document.creationInfo.dataLicense = "https://spdx.org/licenses/CC0-1.0" + + tool = oe.spdx3.SPDX3Tool() + tool.name = "OpenEmbedded Core create-spdx.bbclass" + tool.spdxId = "spdx-" + d.getVar("PN") + ":SPDXRef-Actor-" + tool.name.replace(" ", "") + tool.creationInfo = document.creationInfo + document.element.append(tool) + document.creationInfo.createdUsing.append(tool) + + organization = oe.spdx3.SPDX3Organization() + organization.name = d.getVar("SPDX_ORG") + organization.spdxId = "spdx-" + d.getVar("PN") + ":SPDXRef-Actor-" + organization.name.replace(" ", "") + organization.creationInfo = document.creationInfo + document.element.append(organization) + document.creationInfo.createdBy.append(organization) + + person = oe.spdx3.SPDX3Person() + person.name = "Person: N/A ()" + person.spdxId = "spdx-" + d.getVar("PN") + ":SPDXRef-Actor-" + person.name.replace(" ", "") + document.creationInfo.createdBy.append(person) + document.element.append(person) + +def get_supplier(d, doc=None): + """ + Get the supplier of a document or create it. + """ + import oe.spdx3 + + supplier = d.getVar("SPDX_SUPPLIER") + agentName = supplier.split(": ")[1] + agentType = supplier.split(": ")[0] + + if doc: + for element in doc.element: + if(isinstance(element, oe.spdx3.SPDX3Agent) and element.name == agentName): + return element + + if(agentType == "Organization"): + agent = oe.spdx3.SPDX3Organization() + elif(agentType == "Person"): + agent = oe.spdx3.SPDX3Person() + else: + raise KeyError("%r is not a valid SPDX agent type" % agentType) + + agent.name = agentName + agent.spdxId = "spdx-" + d.getVar("PN") + ":SPDXRef-Actor-" + agent.name.replace(" ", "") + agent.creationInfo = doc.creationInfo + + return agent def recipe_spdx_is_native(d, recipe): - return any(a.annotationType == "OTHER" and - a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and - a.comment == "isNative" for a in recipe.annotations) + return False +# TODO: find a better way to mark native recipes +# return any(a.annotationType == "OTHER" and +# a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and +# a.comment == "isNative" for a in recipe.annotations) def is_work_shared_spdx(d): return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR')) @@ -113,7 +166,7 @@ def convert_license_to_spdx(lic, document, d, existing={}): if name in extracted: return - extracted_info = oe.spdx.SPDXExtractedLicensingInfo() + extracted_info = oe.spdx.SPDX3ExtractedLicensingInfo() extracted_info.name = name extracted_info.licenseId = ident extracted_info.extractedText = None @@ -202,8 +255,7 @@ def process_sources(d): def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): from pathlib import Path - import oe.spdx - import hashlib + import oe.spdx3 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") if source_date_epoch: @@ -223,11 +275,18 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv filename = str(filepath.relative_to(topdir)) if not filepath.is_symlink() and filepath.is_file(): - spdx_file = oe.spdx.SPDXFile() - spdx_file.SPDXID = get_spdxid(file_counter) - for t in get_types(filepath): - spdx_file.fileTypes.append(t) - spdx_file.fileName = filename + spdx_file = oe.spdx3.SPDX3File() + spdx_file.name = filename + spdx_file.spdxId = get_spdxid(file_counter) + spdx_file.primaryPurpose = None + spdx_file.additionalPurpose = [] + types = get_types(filepath) + for t in types: + if t in oe.spdx3.SPDX3SoftwarePurpose: + if spdx_file.primaryPurpose == None: + spdx_file.primaryPurpose = t + else: + spdx_file.additionalPurpose.append(t) if archive is not None: with filepath.open("rb") as f: @@ -245,42 +304,37 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv sha1 = bb.utils.sha1_file(filepath) sha1s.append(sha1) - spdx_file.checksums.append(oe.spdx.SPDXChecksum( - algorithm="SHA1", - checksumValue=sha1, - )) - spdx_file.checksums.append(oe.spdx.SPDXChecksum( - algorithm="SHA256", - checksumValue=bb.utils.sha256_file(filepath), - )) - - if "SOURCE" in spdx_file.fileTypes: - extracted_lics = extract_licenses(filepath) - if extracted_lics: - spdx_file.licenseInfoInFiles = extracted_lics - - doc.files.append(spdx_file) - doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) - spdx_pkg.hasFiles.append(spdx_file.SPDXID) - spdx_files.append(spdx_file) + hashSha1 = oe.spdx3.SPDX3Hash() + hashSha1.algorithm = "sha1" + hashSha1.hashValue = sha1 + spdx_file.verifiedUsing.append(hashSha1) - file_counter += 1 + hashSha256 = oe.spdx3.SPDX3Hash() + hashSha256.algorithm = "sha256" + hashSha256.hashValue = bb.utils.sha256_file(filepath) + spdx_file.verifiedUsing.append(hashSha256) + + # TODO: Rework when License Profile implemented + #if "SOURCE" in spdx_file.fileTypes: + # extracted_lics = extract_licenses(filepath) + # if extracted_lics: + # spdx_file.licenseInfoInFiles = extracted_lics - sha1s.sort() - verifier = hashlib.sha1() - for v in sha1s: - verifier.update(v.encode("utf-8")) - spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest() + doc.element.append(spdx_file) + + doc.add_relationship(spdx_pkg, "contains", spdx_file) + + spdx_files.append(spdx_file) + file_counter += 1 return spdx_files def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources): from pathlib import Path - import hashlib import oe.packagedata - import oe.spdx + import oe.spdx3 debug_search_paths = [ Path(d.getVar('PKGD')), @@ -299,15 +353,15 @@ def add_package_sources_from_debug(d, package_doc, spdx_package, package, packag continue for pkg_file in package_files: - if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): + if file_path.lstrip("/") == pkg_file.name.lstrip("/"): break else: bb.fatal("No package file found for %s in %s; SPDX found: %s" % (str(file_path), package, - " ".join(p.fileName for p in package_files))) + " ".join(p.name for p in package_files))) continue for debugsrc in file_data["debugsrc"]: - ref_id = "NOASSERTION" + ref_id = None for search in debug_search_paths: if debugsrc.startswith("/usr/src/kernel"): debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') @@ -320,24 +374,32 @@ def add_package_sources_from_debug(d, package_doc, spdx_package, package, packag if file_sha256 in sources: source_file = sources[file_sha256] - - doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace) + doc_ref = package_doc.find_external_map(source_file.doc.documentNamespace) if doc_ref is None: - doc_ref = oe.spdx.SPDXExternalDocumentRef() - doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name - doc_ref.spdxDocument = source_file.doc.documentNamespace - doc_ref.checksum.algorithm = "SHA1" - doc_ref.checksum.checksumValue = source_file.doc_sha1 - package_doc.externalDocumentRefs.append(doc_ref) - - ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID) + doc_ref = oe.spdx3.SPDX3ExternalMap() + doc_ref.externalId = "DocumentRef-dependency-" + source_file.doc.name + doc_ref.verifiedUsing = oe.spdx3.SPDX3Hash() + doc_ref.verifiedUsing.algorithm = "sha1" + doc_ref.verifiedUsing.hashValue = source_file.doc_sha1 + doc_ref.definingDocument = source_file.doc.documentNamespace + + package_doc.imports.append(doc_ref) + + ref_id = "%s:%s" % (doc_ref.externalId, source_file.file.spdxId) else: bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) break else: bb.debug(1, "Debug source %s not found" % debugsrc) - package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc) + relation_id = package_doc.add_relationship(ref_id, "generates", pkg_file) + comment = oe.spdx3.SPDX3Annotation() + comment.subject = relation_id + comment.annotationType = "other" + comment.statement = "debugsrc" + package_doc.element.append(comment) + + return add_package_sources_from_debug[vardepsexclude] += "STAGING_KERNEL_DIR" @@ -345,7 +407,7 @@ def collect_dep_recipes(d, doc, spdx_recipe): import json from pathlib import Path import oe.sbom - import oe.spdx + import oe.spdx3 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) spdx_deps_file = Path(d.getVar("SPDXDEPS")) @@ -362,10 +424,10 @@ def collect_dep_recipes(d, doc, spdx_recipe): if not dep_recipe_path: bb.fatal("Cannot find any SPDX file for recipe %s, %s" % (dep_pn, dep_hashfn)) - spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) + spdx_dep_doc, spdx_dep_pkg, spdx_dep_sha1 = oe.sbom.search_doc(dep_recipe_path, ["Package"]) - for pkg in spdx_dep_doc.packages: - if pkg.name == dep_pn: + for pkg in spdx_dep_pkg['Package']: + if pkg["name"] == dep_pn: spdx_dep_recipe = pkg break else: @@ -373,19 +435,15 @@ def collect_dep_recipes(d, doc, spdx_recipe): dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe)) - dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() - dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name - dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace - dep_recipe_ref.checksum.algorithm = "SHA1" - dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 - - doc.externalDocumentRefs.append(dep_recipe_ref) + dep_recipe_ref = oe.spdx3.SPDX3ExternalMap() + dep_recipe_ref.externalId = "DocumentRef-%s" % spdx_dep_doc["name"] + hashSha1 = oe.spdx3.SPDX3Hash() + hashSha1.algorithm = "sha1" + hashSha1.hashValue = spdx_dep_sha1 + dep_recipe_ref.verifiedUsing.append(hashSha1) - doc.add_relationship( - "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID), - "BUILD_DEPENDENCY_OF", - spdx_recipe - ) + doc.imports.append(dep_recipe_ref) + doc.add_relationship("%s:%s" % (dep_recipe_ref.externalId, spdx_dep_recipe["spdxId"]), "buildDependency", spdx_recipe) return dep_recipes @@ -393,24 +451,35 @@ collect_dep_recipes[vardepsexclude] = "SSTATE_ARCHS" def collect_dep_sources(d, dep_recipes): import oe.sbom + import oe.spdx3 sources = {} for dep in dep_recipes: # Don't collect sources from native recipes as they # match non-native sources also. - if recipe_spdx_is_native(d, dep.recipe): - continue - recipe_files = set(dep.recipe.hasFiles) - - for spdx_file in dep.doc.files: - if spdx_file.SPDXID not in recipe_files: - continue + if hasattr(dep.doc, "element"): + for element in dep.doc.element: + if isinstance(element, oe.spdx3.SPDX3Annotation) \ + and element.subject == dep.recipe.spdxId \ + and element.statement == "isNative": + continue - if "SOURCE" in spdx_file.fileTypes: - for checksum in spdx_file.checksums: - if checksum.algorithm == "SHA256": - sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) - break + recipe_files = [] + + if hasattr(dep.doc, "element"): + for element in dep.doc.element: + if isinstance(element, oe.spdx3.SPDX3Relationship) and element._from == dep.recipe.spdxId and element.relationshipType == "contains": + recipe_files = element.to + + for element in dep.doc.element: + if isinstance(element, oe.spdx3.SPDX3File) \ + and element.spdxId not in recipe_files \ + and (element.primaryPurpose == "source" or "source" in element.additionalPurpose): + for checksum in element.verifiedUsing: + if algorithm in checksum.properties() \ + and checksum.algorithm == "sha256": + sources[checksum.hashValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) + break return sources @@ -418,16 +487,16 @@ def add_download_packages(d, doc, recipe): import os.path from bb.fetch2 import decodeurl, CHECKSUM_LIST import bb.process - import oe.spdx + import oe.spdx3 import oe.sbom for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()): f = bb.fetch2.FetchData(src_uri, d) for name in f.names: - package = oe.spdx.SPDXPackage() + package = oe.spdx3.SPDX3Package() package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1) - package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1) + package.spdxId = oe.sbom.get_download_spdxid(d, download_idx + 1) if f.type == "file": continue @@ -443,42 +512,28 @@ def add_download_packages(d, doc, recipe): if f.method.supports_checksum(f): for checksum_id in CHECKSUM_LIST: - if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS: + if checksum_id not in oe.spdx3.SPDX3HashAlgorithm: continue expected_checksum = getattr(f, "%s_expected" % checksum_id) if expected_checksum is None: continue - c = oe.spdx.SPDXChecksum() + c = oe.spdx3.SPDX3Hash() c.algorithm = checksum_id.upper() - c.checksumValue = expected_checksum - package.checksums.append(c) + c.hashValue = expected_checksum + package.verifiedUsing.append(c) package.downloadLocation = uri - doc.packages.append(package) - doc.add_relationship(doc, "DESCRIBES", package) - # In the future, we might be able to do more fancy dependencies, - # but this should be sufficient for now - doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) + doc.element.append(package) -def collect_direct_deps(d, dep_task): - current_task = "do_" + d.getVar("BB_CURRENTTASK") - pn = d.getVar("PN") + doc.add_relationship(doc, "describes", package) + doc.add_relationship(package, "buildDependency", recipe) - taskdepdata = d.getVar("BB_TASKDEPDATA", False) - for this_dep in taskdepdata.values(): - if this_dep[0] == pn and this_dep[1] == current_task: - break - else: - bb.fatal(f"Unable to find this {pn}:{current_task} in taskdepdata") +def collect_direct_deps(d, dep_task): deps = set() - for dep_name in this_dep[3]: - dep_data = taskdepdata[dep_name] - if dep_data[1] == dep_task and dep_data[0] != pn: - deps.add((dep_data[0], dep_data[7])) return sorted(deps) @@ -509,9 +564,8 @@ do_collect_spdx_deps[deptask] = "do_create_spdx" do_collect_spdx_deps[dirs] = "${SPDXDIR}" python do_create_spdx() { - from datetime import datetime, timezone import oe.sbom - import oe.spdx + import oe.spdx3 import uuid from pathlib import Path from contextlib import contextmanager @@ -538,36 +592,34 @@ python do_create_spdx() { include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" - pkg_arch = d.getVar("SSTATE_PKGARCH") - - creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - doc = oe.spdx.SPDXDocument() + doc = oe.spdx3.SPDX3SpdxDocument() doc.name = "recipe-" + d.getVar("PN") doc.documentNamespace = get_doc_namespace(d, doc) - doc.creationInfo.created = creation_time - doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." - doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] - doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") - doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) - doc.creationInfo.creators.append("Person: N/A ()") - - recipe = oe.spdx.SPDXPackage() + generate_creationInfo(d, doc) + + recipe = oe.spdx3.SPDX3Package() + recipe.spdxId = oe.sbom.get_recipe_spdxid(d) recipe.name = d.getVar("PN") - recipe.versionInfo = d.getVar("PV") - recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) - recipe.supplier = d.getVar("SPDX_SUPPLIER") + recipe.packageVersion = d.getVar("PV") + recipe.suppliedBy.append(get_supplier(d, doc)) + if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): - recipe.annotations.append(create_annotation(d, "isNative")) + comment = oe.spdx3.SPDX3Annotation() + comment.annotationType = "other" + comment.subject = recipe.spdxId + comment.statement = "isNative" + + doc.element.append(comment) homepage = d.getVar("HOMEPAGE") if homepage: - recipe.homepage = homepage - - license = d.getVar("LICENSE") - if license: - recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) + recipe.homePage = homepage +# TODO: Rework when License Profile implemented +# license = d.getVar("LICENSE") +# if license: +# recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) summary = d.getVar("SUMMARY") if summary: @@ -581,26 +633,11 @@ python do_create_spdx() { for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split(): recipe.annotations.append(create_annotation(d, var + "=" + d.getVar(var))) - # Some CVEs may be patched during the build process without incrementing the version number, - # so querying for CVEs based on the CPE id can lead to false positives. To account for this, - # save the CVEs fixed by patches to source information field in the SPDX. - patched_cves = oe.cve_check.get_patched_cves(d) - patched_cves = list(patched_cves) - patched_cves = ' '.join(patched_cves) - if patched_cves: - recipe.sourceInfo = "CVEs fixed: " + patched_cves - - cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) - if cpe_ids: - for cpe_id in cpe_ids: - cpe = oe.spdx.SPDXExternalReference() - cpe.referenceCategory = "SECURITY" - cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type" - cpe.referenceLocator = cpe_id - recipe.externalRefs.append(cpe) - - doc.packages.append(recipe) - doc.add_relationship(doc, "DESCRIBES", recipe) + # TODO: CVE handling + + doc.element.append(recipe) + + doc.add_relationship(doc, "describes", recipe) add_download_packages(d, doc, recipe) @@ -615,7 +652,7 @@ python do_create_spdx() { recipe, spdx_workdir, lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter), - lambda filepath: ["SOURCE"], + lambda filepath: ["source"], ignore_dirs=[".git"], ignore_top_level_dirs=["temp"], archive=archive, @@ -626,17 +663,13 @@ python do_create_spdx() { dep_recipes = collect_dep_recipes(d, doc, recipe) - doc_sha1 = oe.sbom.write_doc(d, doc, pkg_arch, "recipes", indent=get_json_indent(d)) + doc_sha1 = oe.sbom.write_doc(d, doc, doc, d.getVar("SSTATE_PKGARCH"), "recipes", indent=get_json_indent(d)) dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) - recipe_ref = oe.spdx.SPDXExternalDocumentRef() - recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name - recipe_ref.spdxDocument = doc.documentNamespace - recipe_ref.checksum.algorithm = "SHA1" - recipe_ref.checksum.checksumValue = doc_sha1 + #TODO: references sources = collect_dep_sources(d, dep_recipes) - found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} +# found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} if not recipe_spdx_is_native(d, recipe): bb.build.exec_func("read_subpackage_metadata", d) @@ -646,42 +679,41 @@ python do_create_spdx() { if not oe.packagedata.packaged(package, d): continue - package_doc = oe.spdx.SPDXDocument() + doc = oe.spdx3.SPDX3SpdxDocument() pkg_name = d.getVar("PKG:%s" % package) or package - package_doc.name = pkg_name - package_doc.documentNamespace = get_doc_namespace(d, package_doc) - package_doc.creationInfo.created = creation_time - package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." - package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] - package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") - package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) - package_doc.creationInfo.creators.append("Person: N/A ()") - package_doc.externalDocumentRefs.append(recipe_ref) + doc.name = pkg_name + doc.documentNamespace = get_doc_namespace(d, doc) + generate_creationInfo(d, doc) + + # TODO: Rework when License Profile implemented + # package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] + # package_doc.externalDocumentRefs.append(recipe_ref) package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE") - spdx_package = oe.spdx.SPDXPackage() + spdx_package = oe.spdx3.SPDX3Package() - spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) + spdx_package.spdxId = oe.sbom.get_package_spdxid(pkg_name) spdx_package.name = pkg_name - spdx_package.versionInfo = d.getVar("PV") - spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) - spdx_package.supplier = d.getVar("SPDX_SUPPLIER") + spdx_package.packageVersion = d.getVar("PV") + # TODO: Rework when License Profile implemented + #spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) + spdx_package.suppliedBy = [ d.getVar("SPDX_SUPPLIER") ] - package_doc.packages.append(spdx_package) + doc.element.append(spdx_package) - package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) - package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) + doc.add_relationship(recipe, "generates", spdx_package) + doc.add_relationship(doc, "describes", spdx_package) - package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst") + package_archive = deploy_dir_spdx / "packages" / (doc.name + ".tar.zst") with optional_tarfile(package_archive, archive_packaged) as archive: package_files = add_package_files( d, - package_doc, + doc, spdx_package, pkgdest / package, lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), - lambda filepath: ["BINARY"], + lambda filepath: ["executable"], ignore_top_level_dirs=['CONTROL', 'DEBIAN'], archive=archive, ) @@ -689,9 +721,9 @@ python do_create_spdx() { if archive is not None: spdx_package.packageFileName = str(package_archive.name) - add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources) + add_package_sources_from_debug(d, doc, spdx_package, package, package_files, sources) - oe.sbom.write_doc(d, package_doc, pkg_arch, "packages", indent=get_json_indent(d)) + oe.sbom.write_doc(d, doc, doc, d.getVar("SSTATE_PKGARCH"), "packages", indent=get_json_indent(d)) } do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS" # NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source @@ -749,127 +781,11 @@ def collect_package_providers(d): collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" python do_create_runtime_spdx() { - from datetime import datetime, timezone - import oe.sbom - import oe.spdx - import oe.packagedata - from pathlib import Path - - deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) - spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) - is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) - - creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - providers = collect_package_providers(d) - pkg_arch = d.getVar("SSTATE_PKGARCH") - package_archs = d.getVar("SSTATE_ARCHS").split() - package_archs.reverse() - - if not is_native: - bb.build.exec_func("read_subpackage_metadata", d) - - dep_package_cache = {} - - pkgdest = Path(d.getVar("PKGDEST")) - for package in d.getVar("PACKAGES").split(): - localdata = bb.data.createCopy(d) - pkg_name = d.getVar("PKG:%s" % package) or package - localdata.setVar("PKG", pkg_name) - localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) - - if not oe.packagedata.packaged(package, localdata): - continue - - pkg_spdx_path = oe.sbom.doc_path(deploy_dir_spdx, pkg_name, pkg_arch, "packages") - - package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) - - for p in package_doc.packages: - if p.name == pkg_name: - spdx_package = p - break - else: - bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path)) - - runtime_doc = oe.spdx.SPDXDocument() - runtime_doc.name = "runtime-" + pkg_name - runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) - runtime_doc.creationInfo.created = creation_time - runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." - runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] - runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") - runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) - runtime_doc.creationInfo.creators.append("Person: N/A ()") - - package_ref = oe.spdx.SPDXExternalDocumentRef() - package_ref.externalDocumentId = "DocumentRef-package-" + package - package_ref.spdxDocument = package_doc.documentNamespace - package_ref.checksum.algorithm = "SHA1" - package_ref.checksum.checksumValue = package_doc_sha1 - - runtime_doc.externalDocumentRefs.append(package_ref) - - runtime_doc.add_relationship( - runtime_doc.SPDXID, - "AMENDS", - "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID) - ) - - deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") - seen_deps = set() - for dep, _ in deps.items(): - if dep in seen_deps: - continue - - if dep not in providers: - continue - - (dep, dep_hashfn) = providers[dep] - - if not oe.packagedata.packaged(dep, localdata): - continue - - dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) - dep_pkg = dep_pkg_data["PKG"] - - if dep in dep_package_cache: - (dep_spdx_package, dep_package_ref) = dep_package_cache[dep] - else: - dep_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, dep_pkg, dep_hashfn) - if not dep_path: - bb.fatal("No SPDX file found for package %s, %s" % (dep_pkg, dep_hashfn)) - - spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) - - for pkg in spdx_dep_doc.packages: - if pkg.name == dep_pkg: - dep_spdx_package = pkg - break - else: - bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path)) - - dep_package_ref = oe.spdx.SPDXExternalDocumentRef() - dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name - dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace - dep_package_ref.checksum.algorithm = "SHA1" - dep_package_ref.checksum.checksumValue = spdx_dep_sha1 - - dep_package_cache[dep] = (dep_spdx_package, dep_package_ref) - - runtime_doc.externalDocumentRefs.append(dep_package_ref) - - runtime_doc.add_relationship( - "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID), - "RUNTIME_DEPENDENCY_OF", - "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID) - ) - seen_deps.add(dep) - - oe.sbom.write_doc(d, runtime_doc, pkg_arch, "runtime", spdx_deploy, indent=get_json_indent(d)) + # TODO: implement for SPDX3 + return } -do_create_runtime_spdx[vardepsexclude] += "OVERRIDES SSTATE_ARCHS" +do_create_runtime_spdx[vardepsexclude] += "OVERRIDES" addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work SSTATETASKS += "do_create_runtime_spdx" @@ -950,209 +866,13 @@ POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spd POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx" python image_combine_spdx() { - import os - import oe.sbom - from pathlib import Path - from oe.rootfs import image_list_installed_packages - - image_name = d.getVar("IMAGE_NAME") - image_link_name = d.getVar("IMAGE_LINK_NAME") - imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) - img_spdxid = oe.sbom.get_image_spdxid(image_name) - packages = image_list_installed_packages(d) - - combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages, Path(d.getVar("SPDXIMAGEWORK"))) - - def make_image_link(target_path, suffix): - if image_link_name: - link = imgdeploydir / (image_link_name + suffix) - if link != target_path: - link.symlink_to(os.path.relpath(target_path, link.parent)) - - spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") - make_image_link(spdx_tar_path, ".spdx.tar.zst") + return } python sdk_host_combine_spdx() { - sdk_combine_spdx(d, "host") + return } python sdk_target_combine_spdx() { - sdk_combine_spdx(d, "target") + return } - -def sdk_combine_spdx(d, sdk_type): - import oe.sbom - from pathlib import Path - from oe.sdk import sdk_list_installed_packages - - sdk_name = d.getVar("TOOLCHAIN_OUTPUTNAME") + "-" + sdk_type - sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) - sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name) - sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") - combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages, Path(d.getVar('SPDXSDKWORK'))) - -def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx_workdir): - import os - import oe.spdx - import oe.sbom - import io - import json - from datetime import timezone, datetime - from pathlib import Path - import tarfile - import bb.compress.zstd - - providers = collect_package_providers(d) - package_archs = d.getVar("SSTATE_ARCHS").split() - package_archs.reverse() - - creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) - source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") - - doc = oe.spdx.SPDXDocument() - doc.name = rootfs_name - doc.documentNamespace = get_doc_namespace(d, doc) - doc.creationInfo.created = creation_time - doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." - doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] - doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") - doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) - doc.creationInfo.creators.append("Person: N/A ()") - - image = oe.spdx.SPDXPackage() - image.name = d.getVar("PN") - image.versionInfo = d.getVar("PV") - image.SPDXID = rootfs_spdxid - image.supplier = d.getVar("SPDX_SUPPLIER") - - doc.packages.append(image) - - for name in sorted(packages.keys()): - if name not in providers: - bb.fatal("Unable to find SPDX provider for '%s'" % name) - - pkg_name, pkg_hashfn = providers[name] - - pkg_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, pkg_name, pkg_hashfn) - if not pkg_spdx_path: - bb.fatal("No SPDX file found for package %s, %s" % (pkg_name, pkg_hashfn)) - - pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) - - for p in pkg_doc.packages: - if p.name == name: - pkg_ref = oe.spdx.SPDXExternalDocumentRef() - pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name - pkg_ref.spdxDocument = pkg_doc.documentNamespace - pkg_ref.checksum.algorithm = "SHA1" - pkg_ref.checksum.checksumValue = pkg_doc_sha1 - - doc.externalDocumentRefs.append(pkg_ref) - doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) - break - else: - bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) - - runtime_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "runtime-" + name, pkg_hashfn) - if not runtime_spdx_path: - bb.fatal("No runtime SPDX document found for %s, %s" % (name, pkg_hashfn)) - - runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) - - runtime_ref = oe.spdx.SPDXExternalDocumentRef() - runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name - runtime_ref.spdxDocument = runtime_doc.documentNamespace - runtime_ref.checksum.algorithm = "SHA1" - runtime_ref.checksum.checksumValue = runtime_doc_sha1 - - # "OTHER" isn't ideal here, but I can't find a relationship that makes sense - doc.externalDocumentRefs.append(runtime_ref) - doc.add_relationship( - image, - "OTHER", - "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), - comment="Runtime dependencies for %s" % name - ) - - image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json") - - with image_spdx_path.open("wb") as f: - doc.to_json(f, sort_keys=True, indent=get_json_indent(d)) - - num_threads = int(d.getVar("BB_NUMBER_THREADS")) - - visited_docs = set() - - index = {"documents": []} - - spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") - with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f: - with tarfile.open(fileobj=f, mode="w|") as tar: - def collect_spdx_document(path): - nonlocal tar - nonlocal deploy_dir_spdx - nonlocal source_date_epoch - nonlocal index - - if path in visited_docs: - return - - visited_docs.add(path) - - with path.open("rb") as f: - doc, sha1 = oe.sbom.read_doc(f) - f.seek(0) - - if doc.documentNamespace in visited_docs: - return - - bb.note("Adding SPDX document %s" % path) - visited_docs.add(doc.documentNamespace) - info = tar.gettarinfo(fileobj=f) - - info.name = doc.name + ".spdx.json" - info.uid = 0 - info.gid = 0 - info.uname = "root" - info.gname = "root" - - if source_date_epoch is not None and info.mtime > int(source_date_epoch): - info.mtime = int(source_date_epoch) - - tar.addfile(info, f) - - index["documents"].append({ - "filename": info.name, - "documentNamespace": doc.documentNamespace, - "sha1": sha1, - }) - - for ref in doc.externalDocumentRefs: - ref_path = oe.sbom.doc_find_by_namespace(deploy_dir_spdx, package_archs, ref.spdxDocument) - if not ref_path: - bb.fatal("Cannot find any SPDX file for document %s" % ref.spdxDocument) - collect_spdx_document(ref_path) - - collect_spdx_document(image_spdx_path) - - index["documents"].sort(key=lambda x: x["filename"]) - - index_str = io.BytesIO(json.dumps( - index, - sort_keys=True, - indent=get_json_indent(d), - ).encode("utf-8")) - - info = tarfile.TarInfo() - info.name = "index.json" - info.size = len(index_str.getvalue()) - info.uid = 0 - info.gid = 0 - info.uname = "root" - info.gname = "root" - - tar.addfile(info, fileobj=index_str) - -combine_spdx[vardepsexclude] += "BB_NUMBER_THREADS SSTATE_ARCHS" |