path: root/scripts/buildhistory-diff

#!/usr/bin/env python3

# Report significant differences in the buildhistory repository since a specific revision
#
# Copyright (C) 2013 Intel Corporation
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
#
# SPDX-License-Identifier: GPL-2.0-only
#

import sys
import os
import argparse
import shlex
import difflib

import bb.utils

# Ensure PythonGit is installed (buildhistory_analysis needs it)
try:
    import git
    import gitdb
except ImportError:
    print("Please install GitPython (python3-git) 0.3.4 or later in order to use this script")
    sys.exit(1)

scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])))
lib_path = scripts_path + '/lib'
sys.path = sys.path + [lib_path]
import scriptpath
scriptpath.add_oe_lib_path()

import oe.buildhistory_analysis as ba

def get_args_parser():
    description = "Reports significant differences in the buildhistory repository."

    parser = argparse.ArgumentParser(description=description,
                                     usage="""
    %(prog)s [options] [from-revision [to-revision]]
    (if not specified, from-revision defaults to build-minus-1, and to-revision defaults to HEAD)""")

    default_dir = os.path.join(os.environ.get('BUILDDIR', '.'), 'buildhistory')

    parser.add_argument('-p', '--buildhistory-dir',
                        action='store',
                        dest='buildhistory_dir',
                        default=default_dir,
                        help="Specify path to buildhistory directory (defaults to buildhistory/ under cwd)")
    parser.add_argument('-v', '--report-version',
                        action='store_true',
                        dest='report_ver',
                        default=False,
                        help="Report changes in PKGE/PKGV/PKGR even when the values are still the default (PE/PV/PR)")
    parser.add_argument('-a', '--report-all',
                        action='store_true',
                        dest='report_all',
                        default=False,
                        help="Report all changes, not just the default significant ones")
    parser.add_argument('-s', '---signatures',
                        action='store_true',
                        dest='sigs',
                        default=False,
                        help="Report list of signatures differing instead of output")
    parser.add_argument('-S', '--signatures-with-diff',
                        action='store_true',
                        dest='sigsdiff',
                        default=False,
                        help="Report on actual signature differences instead of output (requires signature data to have been generated, either by running the actual tasks or using bitbake -S)")
    parser.add_argument('-e', '--exclude-path',
                        action='append',
                        help="Exclude path from the output")
    parser.add_argument('-c', '--colour',
                        choices=('yes', 'no', 'auto'),
                        default="auto",
                        help="Whether to colourise (defaults to auto)")
    parser.add_argument('revisions',
                        default = ['build-minus-1', 'HEAD'],
                        nargs='*',
                        help=argparse.SUPPRESS)
    return parser

list_order_fields = ['PACKAGES']

colours = {
    'colour_default': '',
    'colour_add':     '',
    'colour_remove':  '',
}

def init_colours(use_colours):
    global colours
    if use_colours:
        colours = {
            'colour_default': '\033[0m',
            'colour_add':     '\033[1;32m',
            'colour_remove':  '\033[1;31m',
        }
    else:
        colours = {
            'colour_default': '',
            'colour_add':     '',
            'colour_remove':  '',
        }


def str_pretty(chg):
    if '/image-files/' in chg.path:
        prefix = '%s: ' % chg.path.split('/image-files/')[0]
    else:
        prefix = '%s: ' % chg.path

    def pkglist_combine(depver):
        pkglist = []
        for k,v in depver.items():
            if v:
                pkglist.append("%s (%s)" % (k,v))
            else:
                pkglist.append(k)
        return pkglist

    def detect_renamed_dirs(aitems, bitems):
        adirs = set(map(os.path.dirname, aitems))
        bdirs = set(map(os.path.dirname, bitems))
        files_ab = [(name, sorted(os.path.basename(item) for item in aitems if os.path.dirname(item) == name)) \
                            for name in adirs - bdirs]
        files_ba = [(name, sorted(os.path.basename(item) for item in bitems if os.path.dirname(item) == name)) \
                            for name in bdirs - adirs]
        renamed_dirs = []
        for dir1, files1 in files_ab:
            rename = False
            for dir2, files2 in files_ba:
                if files1 == files2 and not rename:
                    renamed_dirs.append((dir1,dir2))
                    # Make sure that we don't use this (dir, files) pair again.
                    files_ba.remove((dir2,files2))
                    # If a dir has already been found to have a rename, stop and go no further.
                    rename = True

        # remove files that belong to renamed dirs from aitems and bitems
        for dir1, dir2 in renamed_dirs:
            aitems = [item for item in aitems if os.path.dirname(item) not in (dir1, dir2)]
            bitems = [item for item in bitems if os.path.dirname(item) not in (dir1, dir2)]
        return sorted(renamed_dirs), sorted(aitems), sorted(bitems)

    if chg.fieldname in ba.list_fields or chg.fieldname in list_order_fields:
        renamed_dirs = []
        changed_order = False
        if chg.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
            (depvera, depverb) = ba.compare_pkg_lists(chg.oldvalue, chg.newvalue)
            aitems = pkglist_combine(depvera)
            bitems = pkglist_combine(depverb)
        else:
            if chg.fieldname == 'FILELIST':
                aitems = shlex.split(chg.oldvalue)
                bitems = shlex.split(chg.newvalue)
                renamed_dirs, aitems, bitems = detect_renamed_dirs(aitems, bitems)
            else:
                aitems = chg.oldvalue.split()
                bitems = chg.newvalue.split()

        removed = list(sorted(set(aitems) - set(bitems)))
        added = list(sorted(set(bitems) - set(aitems)))

        if not removed and not added and chg.fieldname in ['RPROVIDES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RREPLACES', 'RCONFLICTS']:
            depvera = bb.utils.explode_dep_versions2(chg.oldvalue, sort=False)
            depverb = bb.utils.explode_dep_versions2(chg.newvalue, sort=False)
            for i, j in zip(depvera.items(), depverb.items()):
                if i[0] != j[0]:
                    changed_order = True
                    break

        lines = []
        if renamed_dirs:
            for dfrom, dto in renamed_dirs:
                lines.append('directory renamed {colour_remove}{}{colour_default} -> {colour_add}{}{colour_default}'.format(dfrom, dto, **colours))
        if removed or added:
            if removed and not bitems:
                lines.append('removed all items "{colour_remove}{}{colour_default}"'.format(' '.join(removed), **colours))
            else:
                if removed:
                    lines.append('removed "{colour_remove}{value}{colour_default}"'.format(value=' '.join(removed), **colours))
                if added:
                    lines.append('added "{colour_add}{value}{colour_default}"'.format(value=' '.join(added), **colours))
        else:
            lines.append('changed order')

        if not (removed or added or changed_order):
            out = ''
        else:
            out = '%s: %s' % (chg.fieldname, ', '.join(lines))

    elif chg.fieldname in ba.numeric_fields:
        aval = int(chg.oldvalue or 0)
        bval = int(chg.newvalue or 0)
        if aval != 0:
            percentchg = ((bval - aval) / float(aval)) * 100
        else:
            percentchg = 100
        out = '{} changed from {colour_remove}{}{colour_default} to {colour_add}{}{colour_default} ({}{:.0f}%)'.format(chg.fieldname, chg.oldvalue or "''", chg.newvalue or "''", '+' if percentchg > 0 else '', percentchg, **colours)
    elif chg.fieldname in ba.defaultval_map:
        out = '{} changed from {colour_remove}{}{colour_default} to {colour_add}{}{colour_default}'.format(chg.fieldname, chg.oldvalue, chg.newvalue, **colours)
        if chg.fieldname == 'PKG' and '[default]' in chg.newvalue:
            out += ' - may indicate debian renaming failure'
    elif chg.fieldname in ['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm']:
        if chg.oldvalue and chg.newvalue:
            out = '%s changed:\n  ' % chg.fieldname
        elif chg.newvalue:
            out = '%s added:\n  ' % chg.fieldname
        elif chg.oldvalue:
            out = '%s cleared:\n  ' % chg.fieldname
        alines = chg.oldvalue.splitlines()
        blines = chg.newvalue.splitlines()
        diff = difflib.unified_diff(alines, blines, chg.fieldname, chg.fieldname, lineterm='')
        out += '\n  '.join(list(diff)[2:])
        out += '\n  --'
    elif chg.fieldname in ['installed-package-names.txt', 'files-in-image.txt'] or '/image-files/' in chg.path or chg.fieldname == "sysroot":
        if chg.filechanges or (chg.oldvalue and chg.newvalue):
            fieldname = chg.fieldname
            if '/image-files/' in chg.path:
                fieldname = os.path.join('/' + chg.path.split('/image-files/')[1], chg.fieldname)
                out = 'Changes to %s:\n  ' % fieldname
            else:
                prefix = 'Changes to %s ' % chg.path
                out = '(%s):\n  ' % chg.fieldname
            if chg.filechanges:
                out += '\n  '.join(['%s' % i for i in chg.filechanges])
            else:
                alines = chg.oldvalue.splitlines()
                blines = chg.newvalue.splitlines()
                diff = difflib.unified_diff(alines, blines, fieldname, fieldname, lineterm='')
                out += '\n  '.join(list(diff))
                out += '\n  --'
        else:
            out = ''
    else:
        out = '{} changed from "{colour_remove}{}{colour_default}" to "{colour_add}{}{colour_default}"'.format(chg.fieldname, chg.oldvalue, chg.newvalue, **colours)

    return '%s%s' % (prefix, out) if out else ''


def main():

    parser = get_args_parser()
    args = parser.parse_args()

    if len(args.revisions) > 2:
        sys.stderr.write('Invalid argument(s) specified: %s\n\n' % ' '.join(args.revisions[2:]))
        parser.print_help()

        sys.exit(1)

    if not os.path.exists(args.buildhistory_dir):
        sys.stderr.write('Buildhistory directory "%s" does not exist\n\n' % args.buildhistory_dir)
        parser.print_help()
        sys.exit(1)

    if len(args.revisions) == 1:
        if '..'  in args.revisions[0]:
            fromrev, torev = args.revisions[0].split('..')
        else:
            fromrev, torev = args.revisions[0], 'HEAD'
    elif len(args.revisions) == 2:
        fromrev, torev = args.revisions

    init_colours({"yes": True, "no": False, "auto": sys.stdout.isatty()}[args.colour])

    try:
        changes = ba.process_changes(args.buildhistory_dir, fromrev, torev,
                                  args.report_all, args.report_ver, args.sigs,
                                  args.sigsdiff, args.exclude_path)
    except gitdb.exc.BadObject as e:
        if not args.revisions:
            sys.stderr.write("Unable to find previous build revision in buildhistory repository\n\n")
            parser.print_help()
        else:
            sys.stderr.write('Specified git revision "%s" is not valid\n' % e.args[0])
        sys.exit(1)

    for chg in changes:
        out = str_pretty(chg)
        if out:
            print(out)

    sys.exit(0)

if __name__ == "__main__":
    main()