diff options
| -rw-r--r-- | bitbake/lib/bb/runqueue.py | 31 | ||||
| -rw-r--r-- | bitbake/lib/bb/siggen.py | 4 |
2 files changed, 28 insertions, 7 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 2ad8aad98e9..13b0d5a1e55 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py @@ -36,6 +36,7 @@ from bb import msg, data, event from bb import monitordisk import subprocess import pickle +from multiprocessing import Process bblogger = logging.getLogger("BitBake") logger = logging.getLogger("BitBake.RunQueue") @@ -1302,15 +1303,35 @@ class RunQueue: else: self.rqexe.finish() + def rq_dump_sigfn(self, fn, options): + bb_cache = bb.cache.NoCache(self.cooker.databuilder) + the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) + siggen = bb.parse.siggen + dataCaches = self.rqdata.dataCaches + siggen.dump_sigfn(fn, dataCaches, options) + def dump_signatures(self, options): - done = set() + fns = set() bb.note("Reparsing files to collect dependency data") - bb_cache = bb.cache.NoCache(self.cooker.databuilder) + for tid in self.rqdata.runtaskentries: fn = fn_from_tid(tid) - if fn not in done: - the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) - done.add(fn) + fns.add(fn) + + cpus = os.cpu_count() or 1 + + # We cannot use the real multiprocessing.Pool directly due to some local data + # that can't be pickled. This cheap multi-process solution is much faster than + # a single-thread, just less optimized as the real Pool. + while (fns): + batch = [] + process_num = min(len(fns), int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or cpus)) + for n in range(0, process_num): + p = Process(target=self.rq_dump_sigfn, args=(fns.pop(), options)) + batch.append(p) + p.start() + for p in batch: + p.join() bb.parse.siggen.dump_sigs(self.rqdata.dataCaches, options) diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index b20b9cf7719..ae50a18d672 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py @@ -307,8 +307,8 @@ class SignatureGeneratorBasic(SignatureGenerator): pass raise err - def dump_sigs(self, dataCaches, options): - for fn in self.taskdeps: + def dump_sigfn(self, fn, dataCaches, options): + if fn in self.taskdeps: for task in self.taskdeps[fn]: tid = fn + ":" + task (mc, _, _) = bb.runqueue.split_tid(tid) |