1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
|
#!/usr/bin/env python3
#
# Iterate over a set of configurations from json.conf, calling setup-config for each one, then running the build.
#
# Called with $1 - The 'nightly' target the autobuilder is running
# $2 - The target build directory to configure
# $3 - The poky branch name the build is running on
# $4 - The name of the repository the build is running on
# $5 - The directory to publish sstate into
# $6 - A build-appliance SRCREV to use
# $7 - Where to publish artefacts to (or None)
# $8 - URL back to this build (for the error reporting system)
#
import json
import os
import sys
import subprocess
import errno
import utils
if len(sys.argv) != 9:
print("Incorrect number of parameters, please call as %s <nightly-target> <target-builddir> <branch-name> <repo-name> <sstate-publish-dir> <build-app-srcrev> <publish-dir> <error-report-url>" % sys.argv[0])
sys.exit(1)
target = sys.argv[1]
builddir = sys.argv[2]
branchname = sys.argv[3]
reponame = sys.argv[4]
sstate_release = sys.argv[5]
buildappsrcrev = sys.argv[6]
publish = None
if sys.argv[7] != "None":
publish = sys.argv[7]
errorurl = None
if sys.argv[8] != "None":
errorurl = sys.argv[8]
scriptsdir = os.path.dirname(os.path.realpath(__file__))
ourconfig = utils.loadconfig()
testmode = False
if "ABHELPERTEST" in os.environ:
testmode = True
# Find out the number of steps this target has
maxsteps = 1
if target in ourconfig['overrides']:
for v in ourconfig['overrides'][target]:
if v.startswith("step"):
n = int(v[4:])
if n <= maxsteps:
continue
maxsteps = n
utils.printheader("Target task %s has %d steps" % (target, maxsteps))
finalret = 0
def flush():
sys.stdout.flush()
sys.stderr.flush()
lognum = 0
def logname(path):
global lognum
lognum += 1
return path + "/command.log.%s" % lognum
revision = "unknown"
report = utils.ErrorReport(ourconfig, target, builddir, branchname, revision)
errordir = utils.errorreportdir(builddir)
utils.mkdir(errordir)
def bitbakecmd(builddir, cmd, report, stepnum, oeenv=True):
global finalret
flush()
log = logname(builddir)
errordir = utils.errorreportdir(builddir)
try:
numreports = len(os.listdir(errordir))
except FileNotFoundError:
numreports = 0
if oeenv:
cmd = ". ./oe-init-build-env; %s" % cmd
if testmode:
print("Running %s" % cmd)
return
with subprocess.Popen(cmd, shell=True, cwd=builddir + "/..", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1) as p, open(log, 'wb') as f:
for line in p.stdout:
sys.stdout.buffer.write(line)
sys.stdout.flush()
f.write(line)
ret = p.wait()
if ret:
utils.printheader("ERROR: Command %s failed with exit code %d, see errors above." % (cmd, ret))
# No error report was written but the command failed so we should write one
try:
finalnumreports = len(os.listdir(errordir))
except FileNotFoundError:
finalnumreports = 0
if finalnumreports == numreports:
report.create(cmd, stepnum, log)
finalret += 1
def runcmd(cmd, *args, **kwargs):
if testmode:
print("Running %s" % cmd)
if "setup-config" not in cmd[0]:
return
subprocess.check_call(cmd, *args, **kwargs)
bh_path, remoterepo, remotebranch, baseremotebranch = utils.getbuildhistoryconfig(ourconfig, builddir, target, reponame, branchname)
if bh_path:
runcmd([os.path.join(scriptsdir, "buildhistory-init"), bh_path, remoterepo, remotebranch, baseremotebranch])
for stepnum in range(1, maxsteps + 1):
# Add any layers specified
layers = utils.getconfiglist("ADDLAYER", ourconfig, target, stepnum)
for layer in layers:
bitbakecmd(builddir, "bitbake-layers add-layer %s" % layer, report, stepnum)
flush()
# Generate the configuration files needed for this step
if utils.getconfigvar("WRITECONFIG", ourconfig, target, stepnum):
runcmd([scriptsdir + "/setup-config", target, str(stepnum - 1), builddir, branchname, reponame, sstate_release, buildappsrcrev])
# Execute the targets for this configuration
targets = utils.getconfigvar("BBTARGETS", ourconfig, target, stepnum)
if targets:
utils.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, targets))
bitbakecmd(builddir, "bitbake %s" % targets, report, stepnum)
# Execute the sanity targets for this configuration
sanitytargets = utils.getconfigvar("SANITYTARGETS", ourconfig, target, stepnum)
if sanitytargets:
utils.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, sanitytargets))
bitbakecmd(builddir, "checkvnc; DISPLAY=:1 bitbake %s" % sanitytargets, report, stepnum)
# Run any extra commands specified
cmds = utils.getconfiglist("EXTRACMDS", ourconfig, target, stepnum)
for cmd in cmds:
utils.printheader("Step %s/%s: Running command %s" % (stepnum, maxsteps, cmd))
bitbakecmd(builddir, cmd, report, stepnum)
cmds = utils.getconfiglist("EXTRAPLAINCMDS", ourconfig, target, stepnum)
for cmd in cmds:
utils.printheader("Step %s/%s: Running 'plain' command %s" % (stepnum, maxsteps, cmd))
bitbakecmd(builddir, cmd, report, stepnum, oeenv=False)
# Remove any layers we added in a reverse order
for layer in reversed(layers):
bitbakecmd(builddir, "bitbake-layers remove-layer %s" % layer, report, stepnum)
if publish:
utils.printheader("Running publish artefacts")
runcmd([scriptsdir + "/publish-artefacts", builddir, publish, target])
if errorurl and utils.getconfigvar("SENDERRORS", ourconfig, target, stepnum):
utils.printheader("Sending any error reports")
runcmd([scriptsdir + "/upload-error-reports", builddir, errorurl])
if finalret:
utils.printheader("There were %s failures" % finalret)
sys.exit(1)
sys.exit(0)
|
