path: root/scripts/poky-setup-autobuilder

#!/usr/bin/env python
#
# Poky Automated Build Server Installation Script. This script
# assists in setting up an autobuilder setup.
#
# For simplicity in this script, we assume that the build systems
# are setup with a user account which is shared by both the buildbot 
# "master" and the buildbot "slave" and that this script is being run
# as that user.
#
# The master is responsible for scheduling work for the slave to do, 
# the slave is the actual Poky builds. In reality there can be multiple 
# slaves, each on different machines serving one master.
#
# The autobuilder is assumed to have its own configuration in an SCM like the 
# poky-autobuilder repository. You probably shouldn't use the main git repo
# as that is not assured to be stable.
#
# To use this script, add a new user to your system, and read ../INSTALL
#
# The older version of this script dealt with using it behind a firewall through
# a proxy. I've included those notes in ../INSTALL
#
# Random notes on Prerequisites: 
#   For Fedora 10: "yum install mc wget python-devel python-twisted git cvs \
#                  subversion patch help2man diffstat texi2html texinfo gcc-c++"
#   For Ubuntu 9.04: "apt-get install mc git-core python-twisted cvs \
#                    subversion help2man patch texi2html texinfo diffstat gcc \
#                    gawk g++ mercurial"
##
# Copyright (C) 2010 Intel Corp.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

import optparse
import sys
import getpass
import struct
import fcntl
import os
import signal
import urllib2
import tarfile
import shutil
from socket import gethostname


#
# build-bot slave source does not contain the Makefile anymore, so I'm creating it here.
#
def makeMakefile(Dir):
    bbMakefile = """
# This is a simple makefile which lives in a buildmaster
# directory (next to the buildbot.tac file). It allows you to start/stop the
# master by doing 'make start' or 'make stop'.

# The 'reconfig' target will tell a buildmaster to reload its config file.

start:
	twistd --no_save -y buildbot.tac

stop:
	if [ -e twistd.pid ]; \\
	then kill `cat twistd.pid`; \\
	else echo "Nothing to stop."; \\
	fi

reconfig:
	if [ -e twistd.pid ]; \\
	then kill -HUP `cat twistd.pid`; \\
	else echo "Nothing to reconfig."; \\
	fi

log:
	if [ -e twistd.log ]; \\
	then tail -f twistd.log; \\
	else echo "Nothing to tail."; \\
	fi

"""
    print "Creating Makefile for starting the build master."        
    fout = open(Dir+"/Makefile", "wb")
    fout.write(bbMakefile)
    fout.close()


def downloadLocalTarball(buildtype):
    os.chdir(bbInstallerHome)
    if buildtype == "master":
        URL = bbMasterDownloadUrl
    elif buildtype == "slave":
        URL = bbSlaveDownloadUrl
    try:
        fileopener = urllib2.build_opener()
        rawfile = fileopener.open(URL)
        tarball = rawfile.read()
        bbInstallFile = "./" + URL.rpartition("/")[2]
        fout = open(bbInstallFile, "wb")
        fout.write(tarball)
        fout.close()
    except:
        print "Issues downloading " + URL + " were encountered. Exiting"
        sys.exit(1)
    try:
        print "Extracting " + bbInstallFile        
        tar =  tarfile.open(bbInstallFile)
        tar.extractall() 
        tar.close()
    except:
        print "Issues extracting the tarball from " + URL
        sys.exit(1)
    

def configureBot(buildtype):
    os.chdir(bbInstallerHome)
    if buildtype == "master":
        URL = bbMasterDownloadUrl
    elif buildtype == "slave":
        URL = bbSlaveDownloadUrl
    bbInstallFile = "./" + URL.rpartition("/")[2]
    try:
        bbSourceDir = bbInstallerHome + "/" + bbInstallFile.rpartition("/")[2].replace(".tar.gz", "")
        print "Configuring " + bbSourceDir        
        os.chdir(bbSourceDir)
        os.system ("echo 'export PYTHONPATH=" + bbSourceDir + "/lib/python2.6/site-packages/:$PYTHONPATH' >> "  
            + bbHome + "/.profile" )
        if buildtype == "master":
            bbInstallDir = bbMasterDir
        elif buildtype == "slave":
            bbInstallDir = bbSlaveDir
        cmd = "cd " + bbSourceDir + "; export PYTHONPATH=" + bbInstallDir + "/lib/python2.6/site-packages/:$PYTHONPATH; python ./setup.py install --prefix=" + bbInstallDir
        os.system (cmd) 
    except:
        print "Issues Configuring "
        sys.exit(1)


#BuildBot download parameters
bbVersion = "0.8.3"
bbMasterDownloadUrl = "http://buildbot.googlecode.com/files/buildbot-" + bbVersion + ".tar.gz"
bbSlaveDownloadUrl = "http://buildbot.googlecode.com/files/buildbot-slave-" + bbVersion + ".tar.gz"
bbConfigDownloadUrl = None
bbInstallerHome = os.getcwd()
bbInstallFile = ""
# We need the expanded path for ~ later
bbHome = os.path.expanduser('~')
bbType = None
bbSlaveDesc="Poky Autobuilder Example"

usage  = """%prog [options] master|slave|both   

Set up a poky autobuilder instance using BuildBot.
"""
parser = optparse.OptionParser(usage=usage)
parser.add_option( "--buildpasswd", help = "The BuildBot users password. If this is left empty, we'll prompt you for it.",
                       action = "store", dest = "bbBuildPasswd", default = "" )
parser.add_option( "--builduser", help = "The BuildBot users password. If this is left empty, we'll prompt you for it.",
                       action = "store", dest = "bbBuildUser", default = "" )
parser.add_option( "--gitrepo", help = "The Git Repo used to get as poky-autobuilder controler repo. It's generally safe to ignore this.",
                       action = "store", dest = "bbGitRepo", default = "" )
parser.add_option( "--masterdir", help = "The directory you want the master/slave installed to. If this is left empty, we'll use ~/poky-master.",
                       action = "store", dest = "bbMasterDir", default = "" )
parser.add_option( "--slavedir", help = "The directory you want the master/slave installed to. If this is left empty, we'll use ~/poky-slave.",
                       action = "store", dest = "bbSlaveDir", default = "" )
parser.add_option( "--outputdir", help = "The directory you want the build output to go to. If this is left empty, we'll use ~/poky-slave/output.",
                       action = "store", dest = "bbOutputDir", default = "" )
parser.add_option( "--sourcedir", help = "The directory you want the build source to go to. If this is left empty, we'll use ~/poky-slave/source.",
                       action = "store", dest = "bbSourceDir", default = "" )
parser.add_option( "--pstagedir", help = "The directory you want the package staging to occur in. If this is left empty, we'll use ~/poky-slave/pstaging.",
                       action = "store", dest = "bbPStagingDir", default = "" )
parser.add_option( "--controldir", help = "The directory you want to install poky autobuilder control scripts into. If this is left empty, we'll use ~/poky-autobuilder.",
                       action = "store", dest = "bbControlDir", default = "" )
parser.add_option( "--maxbuilds", help = "The max number of builds you want saved. If this is left empty, we'll set this to 1.",
                       action = "store", dest = "bbMaxBuilds", default = "1" )
parser.add_option( "--maxlogs", help = "The max number of logs you want saved. If this is left empty, we'll set this to 10.",
                       action = "store", dest = "bbMaxLogs", default = "10" )                        
parser.add_option( "--adminmail", help = "The administrator email address. If left unset we set it to current user @ localhost",
                       action = "store", dest = "bbAdminMail", default = "root@localhost" )                        

options, args = parser.parse_args( sys.argv )

# We need to decide if we're doing a master or slave install or both
if len(args) != 2:
    parser.error("""
    You must specify if you wish to create master, slave or both. 
    If you wish to have master and slave on different machines, you should not use 'both'.
    
    """ + usage )
elif len(args) == 2 and (args[1].lower() == "master" or 
    args[1].lower() == "slave" or args[1].lower() == "both"):
    bbType = args[1]


# I dislike having to reference options.*
# Let's dump these into variables
for attr in dir(options):
    vars()[attr] = getattr(options,attr)

if bbAdminMail == "":
    bbAdminMail = "root@localhost"

if bbGitRepo == "":
    bbGitRepo = "git://my.git.server/autobuilder-repo.git"

if bbBuildUser == "":
    bbBuildUser = raw_input("Please enter the account name that is allowed to kickstart builds and then hit Enter."
        + " \nThis is not the user the buildbot process runs under.\n")

if bbBuildPasswd == "":
    bbBuildPasswd = getpass.getpass(prompt="Please enter the password used for the account that is allowed to kickstart" 
        + " builds and then hit Enter. \nThis should not be the password of the user the buildbot process runs under. \n")

if bbMasterDir == "":
    bbMasterDir = bbHome + "/poky-master/"

if bbSlaveDir == "":
    bbSlaveDir = bbHome + "/poky-slave/"

if bbOutputDir == "":
    bbOutputDir = bbHome + "/poky-slave/output/"

if bbPStagingDir == "":
    bbPStagingDir = bbHome + "/poky-slave/pstaging/"

if bbType == "master" or bbType == "both":
    try:
        os.mkdir(bbMasterDir)
    except:
        print bbMasterDir + " already exists."
        pass

    print "Getting BuildBot Master Source"
    downloadLocalTarball("master")
    configureBot("master")
    print "Setting up the BuildBot Master"
    os.system ("echo 'export PYTHONPATH=" + bbMasterDir + "/lib/python2.6/site-packages/:$PYTHONPATH' >> "  
        + bbHome + "/.profile" )
    os.system ("cd " + bbMasterDir + "/bin; export PYTHONPATH=" + bbMasterDir 
        + "/lib/python2.6/site-packages/:$PYTHONPATH; ./buildbot create-master --log-count=" 
        + bbMaxLogs + " " + bbMasterDir)

    try:
        makeMakefile(bbMasterDir)        
    except:
        print "Something went wrong creating the makefile"
        sys.exit(1)

    try:    
        print "Creating the controller class directory in " + bbControlDir
        shutil.copytree(bbInstallerHome + "/class", bbControlDir + "/class")
    except OSError:
        print "Directory probably exists. Skipping"
        pass

    try:
        print "Creating the controller scripts directory in " + bbControlDir
        shutil.copytree(bbInstallerHome + "/scripts", bbControlDir + "/scripts")
    except OSError:
        print "Directory probably exists. Skipping"
        pass

    try:
        print "Creating a basic pokyABConfig.py in " + bbMasterDir
        shutil.copy(bbInstallerHome + "/scripts/pokyABConfig.py", bbMasterDir)
        print ""
        print "----------------------------------------------------------------"
        print "NOTE: This is a developers pokyABConfig.py. If you have need for"
        print "a more substantial pokyABConfig.py for a build server, see"
        print "pokyABConfig.py.example"
        print "----------------------------------------------------------------"
        print ""

    except OSError:
        print "Something went wrong. Skipping"
        pass

    # Create the poky master.cfg 
    print "Creating the master.cfg  in " + bbMasterDir
    bbMasterConfig = """
c = BuildmasterConfig = {}
from buildbot.buildslave import BuildSlave
c['slaves'] = [BuildSlave("builder1", "%s", max_builds=%s)]
c['slavePortnum'] = 9989

c['status'] = []
from buildbot.status.html import WebStatus
from buildbot.status.web.authz import Authz
from buildbot.status.web.auth import BasicAuth
users = [('%s', '%s')]
authz_cfg=Authz(auth=BasicAuth(users),
	forceBuild = 'auth',
	stopBuild = 'auth',
	cancelPendingBuild = True,
	pingBuilder = False)
c['status'].append(WebStatus(http_port=8010, authz=authz_cfg))
c['debugPassword'] = \"%s\"
c['buildbotURL'] = \"http://%s:8010/\"

import pokyABConfig
reload(pokyABConfig)
c['builders'] = pokyABConfig.poky_builders
c['change_source'] = pokyABConfig.poky_sources
c['schedulers'] = pokyABConfig.poky_sched
c['projectName'] = pokyABConfig.poky_projname
c['projectURL'] = pokyABConfig.poky_projurl

""" % (bbBuildPasswd, bbMaxBuilds, bbBuildUser, bbBuildPasswd, bbBuildPasswd, gethostname())

    fout = open(bbMasterDir+"/master.cfg", "wb")
    fout.write(bbMasterConfig)
    fout.close()
    print "Wrote master.cfg to " + bbMasterDir +"/master.cfg"
    os.mkdir(bbHome + "/poky-autobuilder")
    try:
        shutil.copytree(bbInstallerHome + "/scripts", bbHome + "/poky-autobuilder/scripts")
        os.system ("echo 'export PATH=" + bbHome + "/poky-autobuilder/scripts:$PATH' >> "  
        + bbHome + "/.profile" )
    except:
        print "Something went wrong in copying the poky-autobuilder directory contents"

if bbType == "slave" or bbType == "both":
    try:
        os.mkdir(bbSlaveDir) 
        os.mkdir(bbOutputDir) 
        os.mkdir(bbPStagingDir) 

    except:
        pass
    
    print "Getting BuildBot Slave Source"    
    downloadLocalTarball("slave")
    configureBot("slave")
    os.system ("echo 'export PYTHONPATH=" + bbSlaveDir + "/lib/python2.6/site-packages/:$PYTHONPATH' >> "  
        + bbHome + "/.profile" )
    os.system ("cd " + bbSlaveDir + "/bin; export PYTHONPATH=" + bbSlaveDir 
        + "/lib/python2.6/site-packages/:$PYTHONPATH; ./buildslave create-slave --umask=022 " + bbSlaveDir + " localhost:9989 "
        + "builder1" + " " + bbBuildPasswd)
    
    try:
        makeMakefile(bbSlaveDir)        
    except:
        print "Something went wrong creating the makefile"
        sys.exit(1)

    try:
        fout = open(bbSlaveDir+"/info/admin", "wb")
        fout.write(bbAdminMail)
        fout.close()
    except:
        print "Couldn't write the admin email to " + bbSlaveDir + "/info/admin"
        pass
         
    try:
        fout = open(bbSlaveDir+"/info/host", "wb")
        fout.write(bbSlaveDesc)
        fout.close()
    except:
        print "Couldn't write the builder description to" + bbSlaveDir + "/info/host"
        pass

    if bbControlDir != "":
        print "Creating repo"
        os.system ("git clone " + bbGitRepo + " " + bbControlDir)
    else:
        bbControlDir = bbHome + "/poky-autobuilder"

    bbPokyABDotFile = """
BHOME=%s
BUILDPASS=%s 
BMASTER=%s 
BSLAVE=%s 
ADMINADDR=%s 
SLAVEDESC="%s" 
CONTROLREPO=%s 
CONTROLPATH=%s 
BUILDOUTPUT=%s 
SOURCEDIR=%s 
PSTAGEDIR=%s 
""" % (bbHome, bbBuildPasswd, bbMasterDir, bbSlaveDir, bbAdminMail, bbSlaveDesc, bbGitRepo, bbControlDir, bbOutputDir, bbSourceDir, bbPStagingDir)
   
    fout = open(bbHome+"/.pokyautobuild", "wb")
    fout.write(bbPokyABDotFile)
    fout.close()


#
# Inform the user about the remaining configuration needed
# We should correct this for slave only/master only builds

print """
nstallation complete. Please review the output above for any errors.
Then edit the master.cfg file in %s and start the build master and
build slave by running 'make start' in %s and %s directories.
---------------------------------------------------------------------
If you are using the developers pokyABConfig.py you will need to:

mkdir %s/poky-nightly
ln -s %s/poky-incremental %s/poky-nightly
---------------------------------------------------------------------
To run the poky autobuilder:

source ~/.profile
cd <poky-master>; make start
cd <poky-slave>; make start

""" % (bbMasterDir, bbMasterDir, bbSlaveDir, bbSlaveDir, bbSlaveDir, bbSlaveDir)