Testing:TestBuilder

TestBuilder

Here is script collections I using for doing OpenEmbedded test-builds. You can try to get archive here: http://jay-home.ath.cx/oe/testbuilder-0.1.tar.gz

How to setup

1. You should copy testbuilder.conf.sample to testbuilder.conf and change variables according to your needs.
2. Then run setup-testbuilder. It will fetch bitbake and OE trees from git and make local.conf.
3. Run testbuilder to start build

Config file (testbuilder.conf.sample)

# Testbuilder config
# For first time setup look into end of file 
# for "Misc dirs" and "Setup options"

# You can prefix any word in LISTS with '!' do disable it

## DISTROS: Distributions to build
DISTROS="angstrom_2008 minimal"
# OE name of distribution (if needed)
DISTRO_NAME_angstrom_2008="angstrom-2008.1"

## BUILD_SETS: build-sets, should have unique names
# Build all machines in one TMPDIR
BUILD_SETS_minimal="minimal"
# Catch-all variable (will catch angstrom distro)
BUILD_SETS="armv4 armv5 armv7 mips32 i686"

## MACHINES: sets to build in one TMPDIR
# sets for angstrom distro
MACHINES_armv4="collie"
MACHINES_armv5="tosa akita qemuarm"
MACHINES_armv7="efikamx"
MACHINES_mips32="ben-nanonote"
MACHINES_i686="qemux86"

# Catch-all (will catch 'minimal' group)
MACHINES="qemux86 qemuarm"

## Or you can build every machine for angstrom-2008.1 in separate TMPDIRs
#BUILD_SETS_angstrom_2008=""
#MACHINES="tosa collie akita !efikamx !ben-nanonote !qemux86 !qemuarm"

## IMAGES: what images to build.
# Build console-image for 'minimal' group machines
IMAGES_minimal="console-image"
# Build console-image for set MACHINE_armv7
IMAGES_armv7="console-image x11-image"
# Catch-all (will catch all other machines)
IMAGES="console-image x11-image opie-image"

## STAGES: What to do with images:
# update_tree - update git tree before builds
# build_clean - do clean builds
# build_incremental - do incremental builds
STAGES="update_tree build_clean build_incremental"

## Other options

# Number of parallel bitbake threads
BB_NUMBER_THREADS=4
# Number of parallel make threads inside one bitbake thread (-j)
MAKE_NUMBER_THREADS=2

# Name for oestats-client
TB_BUILDER="username-testing-next"

# Misc dirs
TB_BASE=`pwd`
BB_DIR="${TB_BASE}/bitbake"
OE_DIR="${TB_BASE}/openembedded"
DL_DIR="${TB_BASE}/sources"
BLD_DIR="${TB_BASE}/build"

# testbuilder log file
TB_LOG="${TB_BASE}/tb.log"

## Setup options

# Which bitbake version to use
# use branch
BB_BRANCH="1.10"	# 1.8, 1.10, master
# or use specified release tag
#BB_TAG="1.10.1"	# 1.8.18

# Which OE branch to checkout
OE_BRANCH="testing-next"

TestBuilder setup script (setup-testbuilder)

#!/bin/sh

# TestBuilder setup script for OpenEmbedded
# Copyright (c) 2010 Yuri Bushmelev <jay4mail@gmail.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

TB_DIR=`dirname $0`
CFG_FILE="${TB_DIR}/testbuilder.conf"

. "${CFG_FILE}"

cd "${TB_BASE}"

## Checkout bitbake
echo "= Fetching bitbake"
git clone -n git://git.openembedded.org/bitbake "${BB_DIR}"
cd "${BB_DIR}"

if [ -n "${BB_BRANCH}" ]; then
	git checkout -b "${BB_BRANCH}" "origin/${BB_BRANCH}"
elif [ -n "${BB_TAG}" ]; then
	git checkout -b "${BB_TAG}" "${BB_TAG}"
else
	echo "You should specify BB_BRANCH or BB_TAG in ${CFG_FILE}"
fi

echo "= Fetching OE"
cd "${TB_BASE}"

git clone -n git://git.openembedded.org/openembedded "${OE_DIR}"
cd "${OE_DIR}"

if [ -n "${OE_BRANCH}" ]; then
	git checkout -b "${OE_BRANCH}" "origin/${OE_BRANCH}"
else
	echo "You should specify OE_BRANCH in ${CFG_FILE}"
fi

cd "${TB_BASE}"
mkdir -p "${BLD_DIR}/conf"

echo "= Creating bitbake config"
cat >> "${BLD_DIR}/conf/local.conf" << "EOF"
# Testbuilder bitbake local configuration file

DISTRO ?= "${@bb.fatal('Set DISTRO in your config')}"
MACHINE ?= "${@bb.fatal('Set MACHINE in your config')}"

DL_DIR ?= "${@bb.fatal('Set DL_DIR in your config')}"
TMPDIR = "${TMP_DIR}"

BBFILES ?= "${OE_DIR}/recipes/*/*.bb"

# ENABLE_BINARY_LOCALE_GENERATION = "0"
# GLIBC_GENERATE_LOCALES = "en_US.UTF-8 en_GB.UTF-8 de_DE.UTF-8 ru_RU.UTF-8"
# IMAGE_LINGUAS="en-us en-gb ru-ru"

# jffs2, tar(.gz|bz2), cpio(.gz), cramfs, ext2(.gz), ext3(.gz)
# squashfs, squashfs-lzma
IMAGE_FSTYPES = "jffs2 tar.gz"

# CCACHE = "${@bb.which(bb.data.getVar('PATH', d, 1), 'ccache') and 'ccache '}"
PARALLEL_MAKE = "-j ${MAKE_NUMBER_THREADS}"
BB_NUMBER_THREADS ?= "${@bb.fatal('Set BB_NUMBER_THREADS in your config')}"

INHERIT += "rm_work"
#INHERIT += "devshell"
#TERMCMD = ${SCREEN_TERMCMD}

BBINCLUDELOGS = "yes"

# OE stats client (make troubleshooting easier)
INHERIT += "oestats-client"
OESTATS_BUILDER = "${TB_BUILDER}"
OESTATS_SERVER = "tinderbox.openembedded.net"

EOF

echo "= All is done. Run ${TB_DIR}/testbuilder now"

TestBuilder itself (testbuilder)

#!/bin/sh

# TestBuilder for OpenEmbedded
# Copyright (c) 2010 Yuri Bushmelev <jay4mail@gmail.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.


TB_DIR=`dirname $0`
cd "${TB_DIR}"

CFG_FILE="${TB_DIR}/testbuilder.conf"
. "${CFG_FILE}"

# Sanity cleanup
ORIG_PATH=${PATH}
ORIG_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
ORIG_LANG=${LANG}
export ORIG_PATH ORIG_LD_LIBRARY_PATH ORIG_LANG

# clean LD_LIBRARY_PATH
unset LD_LIBRARY_PATH

# clean locale
#LANG=C

# include bitbake into PATH
PATH=${BB_DIR}/bin:${PATH}

BBPATH="${BLD_DIR}:${OE_DIR}"

#export LD_LIBRARY_PATH LANG PATH
export LD_LIBRARY_PATH LANG

BB_ENV_EXTRAWHITE="MACHINE DISTRO TB_BUILDER OE_DIR TMP_DIR DL_DIR MAKE_NUMBER_THREADS BB_NUMBER_THREADS"
export   BB_ENV_EXTRAWHITE BBPATH TB_BUILDER OE_DIR TMP_DIR DL_DIR MAKE_NUMBER_THREADS BB_NUMBER_THREADS


### Functions

# log message
tb_log() {
	local _dt=`date "+%F %X (%s)"`
	echo "[${_dt}] $@" >&2
	echo "[${_dt}] $@" >> $TB_LOG
}

# Do build of image-set $bbimages for machine-set $MLIST in $TMP_DIR
# tb_build_machines "${TMP_DIR}" "$MLIST" $bbimages
tb_build_machines() {
	local _TMP_DIR=$1; shift
	local _MLIST=$1; shift
	local _bbimages=$@
	local _machine
	local _rc
	
	for _machine in $_MLIST; do
		[ "${_machine}" != "${_machine#!}" ] && continue

		tb_log "${_machine} build started, images: ${_bbimages}"

		export MACHINE=$_machine
		export TMP_DIR=${_TMP_DIR}
		bitbake $_bbimages
		#echo $_bbimages
		_rc=$?
		tb_log "${_machine} build finished. Exit code:${_rc}."
	done
}


# update git tree
tb_update_tree() {
	local _branch=`git --git-dir=openembedded/.git branch | awk '/^*/{print $2;}'`

	# check current branch
	if [ "$_branch" != "testing-next" ]; then
		tb_log "Current branch (${_branch}) is not 'testing-next'!"
		#return 1
	fi

	# pull
	tb_log "Updating git tree ${OE_DIR}"
	git --git-dir="${OE_DIR}/.git" pull
}

### Main code 

#mv -f $TB_LOG "${TB_LOG}.old" 2>/dev/null

DO_BUILD_CLEAN=''
DO_BUILD_INCREMENTAL=''
for stage in $STAGES; do
[ "${stage}" != "${stage#!}" ] && continue
	case ${stage} in
	'build_clean')
		DO_BUILD_CLEAN=y
		;;
	'build_incremental')
		DO_BUILD_INCREMENTAL=y
		;;
	'update_tree')
		tb_update_tree
		;;
	*)
		tb_log "Unknown stage ${stage}."
		;;
	esac
done

for distro in $DISTROS; do
	[ "${distro}" != "${distro#!}" ] && continue

	# Get real distro name
	eval DISTRO="\$DISTRO_NAME_${distro}"
	[ -z "${DISTRO}" ] && DISTRO=${distro}
	export DISTRO

	# Get groups list
	eval BSLIST="\$BUILD_SETS_${distro}"
	[ -z "${BSLIST}" ] && BSLIST=${BUILD_SETS}

	# Empty $BUILD_SETS is exception - we should use $MACHINES 
	# to build every machine in separate TMPDIRs
	if [ -z "${BSLIST}" ]; then
		# Build every machine as separate group
		for machine in $MACHINES; do
			[ "${machine}" != "${machine#!}" ] && continue
			eval "MACHINE_${machine}=${machine}"
			BSLIST="${BSLIST} ${machine}"
		done
	fi

	tb_log "= Processing distro ${DISTRO}, groups: ${BSLIST}"

	for group in $BSLIST; do
		[ "${group}" != "${group#!}" ] && continue

		eval MLIST="\$MACHINES_${group}"
		[ -z "${MLIST}" ] && MLIST=${MACHINES}

		tb_log "== Processing group ${group}, machines: ${MLIST}"

		CLN_TMP_DIR="${BLD_DIR}/tmp"
		INC_TMP_DIR="${BLD_DIR}/${distro}/${group}"

		eval ILIST="\$IMAGES_${group}"
		[ -z "${ILIST}" ] && ILIST=${IMAGES}

		# collect images to build in $bbimages
		bbimages=''
		for image in $ILIST; do
			[ "${image}" != "${image#!}" ] && continue
			bbimages="${bbimages} ${image}"
		done

		# If there is no previous incremental builddir and we are requested
		# to do both incremental and clean builds
		# then skip clean build and do only incremental because it will be clean build then :)
		if [	"${DO_BUILD_INCREMENTAL}" = 'y' \
			-a "${DO_BUILD_CLEAN}" = 'y' \
			-a ! -d "${INC_TMP_DIR}/work" \
		]; then
			tb_log "Will do incremental build instead of clean to populate TMPDIR"
			DO_BUILD_CLEAN='skip-once'
		fi

		## Do clean builds
		case "${DO_BUILD_CLEAN}" in
		'y')
			mkdir -p "${CLN_TMP_DIR}"
	
			tb_log "== Cleaning ${CLN_TMP_DIR}"
			rm -rf ${CLN_TMP_DIR}/*
			
			tb_log "== Starting clean builds for machine-set {$MLIST}"
			tb_build_machines "${CLN_TMP_DIR}" "$MLIST" $bbimages

			tb_log "== Cleaning ${CLN_TMP_DIR} again"
			rm -rf ${CLN_TMP_DIR}/*
			;;
		'skip-once')
			# Re-enable skipped clean build
			DO_BUILD_CLEAN=y
			;;
		esac

		## Do incremental build
		case "${DO_BUILD_INCREMENTAL}" in
		'y')
			# Switch tmpdir to archive
			mkdir -p "${INC_TMP_DIR}"

			tb_log "== Starting incremental builds for machine-set {$MLIST}"
			tb_build_machines "${INC_TMP_DIR}" "$MLIST" $bbimages
			;;
		esac
	done
done
tb_log "All done."