diff options
Diffstat (limited to 'bitbake/lib/bb/ui/buildinfohelper.py')
| -rw-r--r-- | bitbake/lib/bb/ui/buildinfohelper.py | 175 |
1 files changed, 134 insertions, 41 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index e1b59c3e878..bf032ae8356 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py @@ -37,7 +37,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] =\ django.setup() from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText -from orm.models import Target_Image_File, BuildArtifact, TargetArtifactFile +from orm.models import Target_Image_File, TargetKernelFile, TargetSDKFile from orm.models import Variable, VariableHistory from orm.models import Package, Package_File, Target_Installed_Package, Target_File from orm.models import Task_Dependency, Package_Dependency @@ -128,6 +128,15 @@ class ORMWrapper(object): """ return target.get_similar_target_with_image_files() + def get_similar_target_with_sdk_files(self, target): + return target.get_similar_target_with_sdk_files() + + def clone_image_artifacts(self, target_from, target_to): + target_to.clone_image_artifacts_from(target_from) + + def clone_sdk_artifacts(self, target_from, target_to): + target_to.clone_sdk_artifacts_from(target_from) + def _timestamp_to_datetime(self, secs): """ Convert timestamp in seconds to Python datetime @@ -682,35 +691,22 @@ class ORMWrapper(object): logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg) def save_target_image_file_information(self, target_obj, file_name, file_size): - Target_Image_File.objects.create( target = target_obj, - file_name = file_name, - file_size = file_size) + Target_Image_File.objects.create(target=target_obj, + file_name=file_name, file_size=file_size) - def save_target_artifact_file(self, target_obj, file_name, file_size): + def save_target_kernel_file(self, target_obj, file_name, file_size): """ - Save artifact file information for a Target target_obj. - - Note that this doesn't include SDK artifacts, only images and - related files (e.g. bzImage). + Save kernel file (bzImage, modules*) information for a Target target_obj. """ - TargetArtifactFile.objects.create(target=target_obj, + TargetKernelFile.objects.create(target=target_obj, file_name=file_name, file_size=file_size) - def save_artifact_information(self, build_obj, file_name, file_size): + def save_target_sdk_file(self, target_obj, file_name, file_size): """ - TODO this is currently used to save SDK artifacts to the database, - but will be replaced in future once SDK artifacts are associated with - Target objects (as they eventually should be) + Save SDK artifacts to the database, associating them with a + Target object. """ - # do not update artifacts found in other builds - if BuildArtifact.objects.filter(file_name = file_name).count() > 0: - return - - # do not update artifact if already a target artifact file for that path - if TargetArtifactFile.objects.filter(file_name = file_name).count() > 0: - return - - BuildArtifact.objects.create(build=build_obj, file_name=file_name, + TargetSDKFile.objects.create(target=target_obj, file_name=file_name, file_size=file_size) def create_logmessage(self, log_information): @@ -1085,11 +1081,6 @@ class BuildInfoHelper(object): return self.brbe - def update_artifact_image_file(self, event): - evdata = BuildInfoHelper._get_data_from_event(event) - for artifact_path in evdata.keys(): - self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path]) - def update_build_information(self, event, errors, warnings, taskfailures): if 'build' in self.internal_state: self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) @@ -1568,9 +1559,9 @@ class BuildInfoHelper(object): return image_files - def scan_build_artifacts(self): + def scan_image_artifacts(self): """ - Scan for build artifacts in DEPLOY_DIR_IMAGE and associate them + Scan for built image artifacts in DEPLOY_DIR_IMAGE and associate them with a Target object in self.internal_state['targets']. We have two situations to handle: @@ -1615,7 +1606,7 @@ class BuildInfoHelper(object): # filter out anything which isn't an image target image_targets = [target for target in targets if target.is_image] - for target in image_targets: + for image_target in image_targets: # this is set to True if we find at least one file relating to # this target; if this remains False after the scan, we copy the # files from the most-recent Target with the same target + machine @@ -1627,7 +1618,7 @@ class BuildInfoHelper(object): # 'defaultpkgname-<MACHINE>-<BUILDNAME>'; # we need to change it to # <TARGET>-<MACHINE>-<BUILDNAME> - real_image_name = re.sub(r'^defaultpkgname', target.target, + real_image_name = re.sub(r'^defaultpkgname', image_target.target, image_name) image_license_manifest_path = os.path.join( @@ -1651,7 +1642,7 @@ class BuildInfoHelper(object): # note that the artifact will only be saved against this # build if it hasn't been already - self.orm_wrapper.save_target_artifact_file(target, + self.orm_wrapper.save_target_kernel_file(image_target, artifact_path, artifact_size) # store the license manifest path on the target @@ -1659,8 +1650,8 @@ class BuildInfoHelper(object): license_path = os.path.join(license_directory, real_image_name, 'license.manifest') - self.orm_wrapper.update_target_set_license_manifest(target, - license_path) + self.orm_wrapper.update_target_set_license_manifest( + image_target, license_path) # scan the directory for image files relating to this build # (via real_image_name); note that we don't have to set @@ -1675,7 +1666,7 @@ class BuildInfoHelper(object): for image_file in image_files: self.orm_wrapper.save_target_image_file_information( - target, image_file['path'], image_file['size']) + image_target, image_file['path'], image_file['size']) if not has_files: # copy image files and build artifacts from the @@ -1683,13 +1674,115 @@ class BuildInfoHelper(object): # same target + machine as this Target; also copy the license # manifest path, as that is not treated as an artifact and needs # to be set separately - most_recent = \ - self.orm_wrapper.get_similar_target_with_image_files(target) + similar_target = \ + self.orm_wrapper.get_similar_target_with_image_files( + image_target) - if most_recent: + if similar_target: logger.info('image artifacts for target %s cloned from ' \ - 'target %s' % (target.pk, most_recent.pk)) - target.clone_artifacts_from(most_recent) + 'target %s' % (image_target.pk, similar_target.pk)) + self.orm_wrapper.clone_image_artifacts(similar_target, + image_target) + + def _get_sdk_targets(self): + """ + Return targets which could generate SDK artifacts, i.e. + "do_populate_sdk" and "do_populate_sdk_ext". + """ + return [target for target in self.internal_state['targets'] \ + if target.task in ['populate_sdk', 'populate_sdk_ext']] + + def scan_sdk_artifacts(self, event): + """ + Note that we have to intercept an SDKArtifactInfo event from + toaster.bbclass (via toasterui) to get hold of the SDK variables we + need to be able to scan for files accurately: this is because + variables like TOOLCHAIN_OUTPUTNAME have reset to None by the time + BuildCompleted is fired by bitbake, so we have to get those values + while the build is still in progress. + + For populate_sdk_ext, this runs twice, with two different + TOOLCHAIN_OUTPUTNAME settings, each of which will capture some of the + files in the SDK output directory. + """ + sdk_vars = BuildInfoHelper._get_data_from_event(event) + toolchain_outputname = sdk_vars['TOOLCHAIN_OUTPUTNAME'] + + # targets which might have created SDK artifacts + sdk_targets = self._get_sdk_targets() + + # location of SDK artifacts + tmpdir = self.server.runCommand(['getVariable', 'TMPDIR'])[0] + sdk_dir = os.path.join(tmpdir, 'deploy', 'sdk') + + # all files in the SDK directory + artifacts = [] + for dir_path, _, filenames in os.walk(sdk_dir): + for filename in filenames: + full_path = os.path.join(dir_path, filename) + if not os.path.islink(full_path): + artifacts.append(full_path) + + for sdk_target in sdk_targets: + # find files in the SDK directory which haven't already been + # recorded against a Target and whose basename matches + # TOOLCHAIN_OUTPUTNAME + for artifact_path in artifacts: + basename = os.path.basename(artifact_path) + + toolchain_match = basename.startswith(toolchain_outputname) + + # files which match the name of the target which produced them; + # for example, + # poky-glibc-x86_64-core-image-sato-i586-toolchain-ext-2.1+snapshot.sh + target_match = re.search(sdk_target.target, basename) + + # targets which produce "*-nativesdk-*" files + is_ext_sdk_target = sdk_target.task in \ + ['do_populate_sdk_ext', 'populate_sdk_ext'] + + # SDK files which don't match the target name, i.e. + # x86_64-nativesdk-libc.* + # poky-glibc-x86_64-buildtools-tarball-i586-buildtools-nativesdk-standalone-2.1+snapshot* + is_ext_sdk_file = re.search('-nativesdk-', basename) + + file_from_target = (toolchain_match and target_match) or \ + (is_ext_sdk_target and is_ext_sdk_file) + + if file_from_target: + # don't record the file if it's already been added to this + # target + matching_files = TargetSDKFile.objects.filter( + target=sdk_target, file_name=artifact_path) + + if matching_files.count() == 0: + artifact_size = os.stat(artifact_path).st_size + + self.orm_wrapper.save_target_sdk_file( + sdk_target, artifact_path, artifact_size) + + def clone_required_sdk_artifacts(self): + """ + If an SDK target doesn't have any SDK artifacts, this means that + the postfuncs of populate_sdk or populate_sdk_ext didn't fire, which + in turn means that the targets of this build didn't generate any new + artifacts. + + In this case, clone SDK artifacts for targets in the current build + from existing targets for this build. + """ + sdk_targets = self._get_sdk_targets() + for sdk_target in sdk_targets: + # only clone for SDK targets which have no TargetSDKFiles yet + if sdk_target.targetsdkfile_set.all().count() == 0: + similar_target = \ + self.orm_wrapper.get_similar_target_with_sdk_files( + sdk_target) + if similar_target: + logger.info('SDK artifacts for target %s cloned from ' \ + 'target %s' % (sdk_target.pk, similar_target.pk)) + self.orm_wrapper.clone_sdk_artifacts(similar_target, + sdk_target) def close(self, errorcode): if self.brbe is not None: |