#!/bin/sh
#
# Run test builds on one or more PCP machines
#

_usage()
{
    echo >&2 "Usage: `basename $0` [options] vm ..."
    echo >&2 "options:"
    echo >&2 "  -a          all except shutdown VM [implies -pmiqh]"
    echo >&2 "  -b branch   pcp branch to checkout [default $pcp_branch]"
    echo >&2 "  -c          check git repositories"
    echo >&2 "  -d          output debugging diagnostics"
    echo >&2 "  -e file     prerun script"
    echo >&2 "  -f          force shutdown"
    echo >&2 "  -g what     QA tests or groups to run, e.g. -g \"123 -g sanity -x remote\""
    if [ "$what" = all ]
    then
	echo >&2 "              [default <nothing> => all tests]"
    else
	echo >&2 "              [default $what]"
    fi
    echo >&2 "  -h          harvest QA failures"
    echo >&2 "  -i          install built pcp packages"
    echo >&2 "  -l          harvest daily.log/daily.qa.log after aborted run"
    echo >&2 "  -m          build (Makepkgs) for pcp"
    echo >&2 "  -p          git pull for pcp"
    echo >&2 "  -q          run qa"
    echo >&2 "  -r          rerun qa for failing cases"
    echo >&2 "  -s          shutdown VM"
    echo >&2 "  -t          tree for pcp [default $pcp_tree]"
    echo >&2 "default: -pmiqhs"
}

# $1 is "daily" (for build/install) else "daily.qa" (for qa)
#
_script_header()
{
    if [ "$1" = daily ]
    then
	action="Build"
    else
	action="QA"
    fi
    cat <<End-of-File
#!/bin/sh
#
# PCP $action Script from pcp-daily on `date`
#

[ -f \$HOME/.dailyrc ] && . \$HOME/.dailyrc
if [ $debug = true ]
then
    echo PATH=\$PATH
    echo MAKE=\$MAKE
    which \${MAKE:-make}
fi

tmp=/var/tmp/\$\$
trap "rm -f \$tmp.*; exit 0" 0 1 2 3 15
End-of-File
    if $lflag
    then
	:
    else
	cat <<End-of-File
rm -f \$HOME/$1.log
echo "--- starting $1 on \`date\` ---" >\$HOME/$1.log
if [ -x ./freespace ] 
then
    ./freespace -l >>\$HOME/$1.log
fi
End-of-File
    fi
    if [ "$1" = daily ]
    then
	# not QA, need git tree
	#
	if [ "$pcp_tree" = pcpfans ]
	then
	    cat <<End-of-File
if [ ! -d $build_dir ]
then
    git clone git://sourceware.org/git/pcpfans.git $build_dir
    echo "Info: clone pcpfans git tree" | tee -a \$HOME/daily.log
fi
End-of-File
	else
	    cat <<End-of-File
if [ ! -d $build_dir ]
then
    git clone ssh://bozo/home/kenj/git-mirror/pcp.git $build_dir
    echo "Info: clone $pcp_tree git tree" | tee -a \$HOME/daily.log
fi
End-of-File
	fi
	cat <<End-of-File
cd $build_dir
End-of-File
    fi
}

_script_git()
{
    cat <<End-of-File
git branch -a >\$tmp.branch
if grep " $pcp_branch\$" <\$tmp.branch >/dev/null
then
    :
else
    if grep " remotes/origin/$pcp_branch\$" <\$tmp.branch >/dev/null
    then
	git branch --track $pcp_branch remotes/origin/$pcp_branch
	echo "Info: pcp git branch $pcp_branch set up for remote tracking" | tee -a \$HOME/daily.log
    else
	echo "Error: pcp git branch $pcp_branch unknown!" | tee -a \$HOME/daily.log
	exit 1
    fi
fi
End-of-File
}

_script_git_check()
{
    cat <<End-of-File
git status -s | grep 'M ' >\$tmp.status
if [ -s \$tmp.status ]
then
    echo "Warning: modified pcp files ..." | tee -a \$HOME/daily.log
    sed -e 's/.*M  */    /' \$tmp.status | tee -a \$HOME/daily.log
else
    echo "Info: no modified pcp files" | tee -a \$HOME/daily.log
fi
End-of-File
}

_script_git_pull()
{
    cat <<End-of-File
# need to clean before pulling in case unwanted files no longer removed,
# e.g. in PCP 3.x -> PCP 4.0 migration of source base
# not needed any longer - 26 Mar 2013
#\${MAKE:-make} clean >>\$HOME/daily.log 2>&1
# only VERSION.pcp is expected to be modified, sometimes!
mv VERSION.pcp VERSION.tmp
git checkout -- VERSION.pcp
if git checkout $pcp_branch >>\$HOME/daily.log 2>&1
then
    echo "Info: pcp git checkout $pcp_branch" | tee -a \$HOME/daily.log
    rm VERSION.tmp
else
    echo "Error: pcp git checkout $pcp_branch failed!" | tee -a \$HOME/daily.log
    if [ -f VERSION.pcp ]
    then
	rm VERSION.tmp
    else
	mv VERSION.tmp VERSION.pcp
    fi
    exit 1
fi
# on some VMs, there is a problem with the shutdown that deletes
# some files which causes the subsequent Makepkgs to fail ... try
# to recover from this
git status -s | awk '\$1 == "D" { print \$2 }' | xargs git checkout
if git pull >>\$HOME/daily.log 2>&1
then
    echo "Info: pcp git pull" | tee -a \$HOME/daily.log
else
    echo "Error: pcp git pull failed!" | tee -a \$HOME/daily.log
    exit 1
fi
End-of-File
}

_script_make()
{
    host="$1"
    cat <<End-of-File
if [ -f qa/admin/list-packages ]
then
    qa/admin/list-packages -m >\$tmp.tmp
    if [ -s \$tmp.tmp ]
    then
	echo "Info: list-packages reports missing: \`cat \$tmp.tmp\`" | tee -a \$HOME/daily.log
    fi
fi
ok=false
# if needed, bump build number for pcp-daily
if [ ! -f VERSION.pcp.daily ]
then
    cp VERSION.pcp VERSION.pcp.daily
else
    diff VERSION.pcp VERSION.pcp.daily >\$tmp.tmp
    if grep -E 'PACKAGE_MAJOR|PACKAGE_MINOR|PACKAGE_REVISION' <\$tmp.tmp >/dev/null
    then
	# different PCP release, do an initial build
	cp VERSION.pcp VERSION.pcp.daily
    else
	# same PCP release, bump away
	awk -F= <VERSION.pcp.daily '
BEGIN	{ OFS = "=" }
\$1 == "PACKAGE_BUILD"	{ print "PACKAGE_BUILD",\$2+1; next }
			{ print }' \\
| sed -e 's/=\$//' >\$tmp.version
	cp \$tmp.version VERSION.pcp.daily
    fi
fi
cp VERSION.pcp.daily VERSION.pcp
# if we're using systemctl, then need to stop PCP services
# to prevent autorestart banging its head against the wall
# once the libraries are removed and the packages are being
# built (and installed)
#
if [ "$PCPQA_SYSTEMD" = yes ]
then
    echo "Info: stopping PCP services" | tee -a \$HOME/daily.log
    for svc in pmie pmlogger pmproxy pmcd
    do
	sudo systemctl stop \$svc
    done
fi
# simulate a first-time build
sudo rm -rf /usr/include/pcp
sudo rm -f /usr/lib*/libpcp[_.]*
# clean up any build turds
rm -f qa/qa_outfiles
if MAKE=\$MAKE ./Makepkgs --clean >>\$HOME/daily.log 2>&1
then
    ok=true
else
    # for Debian/Ubuntu, it may just be the signing of the packages that
    # failed
    #
    if tail -20 \$HOME/daily.log | grep 'dpkg-buildpackage: warning: Failed to sign' >/dev/null
    then
	echo "Warning: Makepkgs failed to sign dpkg's" | tee -a \$HOME/daily.log
    else
	echo "Error: Makepkgs failed!" | tee -a \$HOME/daily.log
	# restore build number
	rm -f VERSION.pcp
	git checkout -- VERSION.pcp
	exit 1
    fi
fi
# restore build number
rm -f VERSION.pcp
git checkout -- VERSION.pcp
# Look for errors/warnings in the build logs ...
#
# In the seds below are for "errors" and "warnings" that are:
# (a) out of out control (e.g. in flex/bison generated code,
#     Qt headers, non-PCP libraries, etc), or
# (b) apparently benign things we're no going to fix, or
# (c) stuff we've triaged and made a "no change needed" call
#
for log in Logs/*
do
    echo "--- \$log ---" >>\$HOME/daily.log
    cat \$log >>\$HOME/daily.log
    echo "--- end \$log ---" >>\$HOME/daily.log
done
scripts/cull-build-warnings $host <Logs/pcp >\$tmp.err
if [ -s \$tmp.err ]
then
    echo "--- start build errors/warnings (host=$host) ---" >>\$HOME/daily.log
    cat \$tmp.err >>\$HOME/daily.log
    echo "Warning: \`wc -l <\$tmp.err | sed -e 's/ //g'\` build errors/warnings" | tee -a \$HOME/daily.log
    echo "--- end build errors/warnings ---" >>\$HOME/daily.log
    ok=false
fi
\$ok && echo "Info: Makepkgs" | tee -a \$HOME/daily.log
End-of-File
}

_script_install()
{
    cat <<'End-of-File'
# extract buildversion
. ./VERSION.pcp
if [ -z "$PACKAGE_MAJOR" -o -z "$PACKAGE_MINOR" -o -z "$PACKAGE_REVISION" ]
then
    ls -l VERSION.pcp | tee -a $HOME/daily.log
    cat VERSION.pcp | tee -a $HOME/daily.log
    echo "Error: VERSION.pcp is damaged" | tee -a $HOME/daily.log
    exit 1
fi
buildversion=$PACKAGE_MAJOR.$PACKAGE_MINOR.$PACKAGE_REVISION
# find lsm ... in pcp-$buildversion for most build types, but in 
# build/deb/pcp-$buildversion for debian
lsm=''
[ -f pcp-$buildversion/pcp.lsm ] && lsm=pcp-$buildversion/pcp.lsm
[ -f build/deb/pcp-$buildversion/pcp.lsm ] && lsm=build/deb/pcp-$buildversion/pcp.lsm
if [ -z "$lsm" ]
then
    ls -l pcp.lsm* pcp-$buildversion/pcp.lsm* | tee -a $HOME/daily.log
    echo "Error: pcp.lsm not found" | tee -a $HOME/daily.log
    exit 1
fi
src_version=`sed -n <$lsm -e 's/[ 	]*$//' -e '/^Version:/{
s/Version:[ 	]*//
p
q
}'`
notfound=true
case `echo build/deb/pcp_[0-9]*.deb`
in
    *\ *)
	ls -l build/deb/pcp_[0-9]*.deb
	echo "Error: more than one deb pkg found" | tee -a $HOME/daily.log
	exit 1
	;;
    'build/deb/pcp_[0-9]*.deb')
	# no deb packages
	;;
    *)
	# Debian packaging ...
	apt_ver=`apt-get -v | sed -n -e '/^apt/{
s/apt //
s/ .*//p
}'`
	case "$apt_ver"
	in
	    #0.*|1.0.*)
	    *)
		# the old-school dpkg way
		#
		if yes Y | sudo dpkg -i --auto-deconfigure --force-depends build/deb/*.deb >>$HOME/daily.log 2>&1
		then
		    echo "Info: dpkg install" | tee -a $HOME/daily.log
		else
		    echo "Error: dpkg failed!" | tee -a $HOME/daily.log
		    exit 1
		fi
		;;
	    #*)
	    do-not-do-this-any-more)
		# this does not work reliably ...
		#
		# the new-school apt-get way
		#
		here=`pwd`
		cd build/deb
		if sudo apt-get --yes --reinstall install ./*.deb >>$HOME/daily.log 2>&1
		then
		    echo "Info: apt-get install" | tee -a $HOME/daily.log
		else
		    echo "Error: apt-get failed!" | tee -a $HOME/daily.log
		    exit 1
		fi
		cd $here
		;;
	esac
	notfound=false
	;;
esac
$notfound && case `echo pcp-$buildversion/build/rpm/pcp-[0-9]*.src.rpm`
in
    *\ *)
	ls -l pcp-$buildversion/build/rpm/pcp-[0-9]*.src.rpm | tee -a $HOME/daily.log
	echo "Error: more than one rpm pkg found" | tee -a $HOME/daily.log
	exit 1
	;;
    "pcp-$buildversion"'/build/rpm/pcp-[0-9]*.src.rpm')
	# no rpm source package, assume some other sort of packaging
	;;
    *)
	# RPM packaging ...
	# Notes may need to add these flags ...
	#  --nodeps	needed because perl-Spreadsheet-Read is not available
	#		e.g. Fedora 15
	#  --oldpackage	sometimes we do a downgrade on the QA machines
	#
	here=`pwd`
	if [ ! -d pcp-$buildversion/build/rpm ]
	then
	    echo "Error: pcp-$buildversion/build/rpm: no dir: rpm failed!" | tee -a $HOME/daily.log
	    exit 1
	fi
	cd pcp-$buildversion/build/rpm
	if sudo rpm -U --force `./cherrypick 2>$tmp.err` >$tmp.out 2>&1
	then
	    cat $tmp.out >>$HOME/daily.log
	    cat $tmp.err >>$HOME/daily.log
	    echo "Info: rpm install" | tee -a $HOME/daily.log
	else
	    cat $tmp.out >>$HOME/daily.log
	    cat $tmp.err >>$HOME/daily.log
	    grep -v 'is already installed' $tmp.out >$tmp.tmp
	    if [ -s $tmp.tmp ]
	    then
		echo "Error: rpm failed!" | tee -a $HOME/daily.log
		exit 1
	    else
		echo "Warning: rpm packages already installed" | tee -a $HOME/daily.log
	    fi
	fi
	cd $here
	notfound=false
	;;
esac
$notfound && if [ -f pcp-$buildversion/build/mac/pcp-[0-9]*[0-9].dmg ]
then
    # Mac OS X "pkg in a disk image" packaging
    here=`pwd`
    cd pcp-$buildversion/build/mac
    if ./cmdline-install >>$HOME/daily.log 2>&1
    then
	echo "Info: installer install" | tee -a $HOME/daily.log
    else
	echo "Error: installer failed!" | tee -a $HOME/daily.log
	exit 1
    fi
    cd $here
elif [ -f pcp-$buildversion/build/sun/pcp-$src_version ]
then
    # System V style pkgadd/pkgrm packaging ...
    here=`pwd`
    cd pcp-$buildversion/build/sun
    yes Y | sudo /usr/sbin/pkgrm pcp >>$HOME/daily.log 2>&1
    if yes Y | sudo /usr/sbin/pkgadd -d `pwd` pcp all >>$HOME/daily.log 2>&1
    then
	echo "Info: pkgadd install" | tee -a $HOME/daily.log
    else
	echo "Error: pkgadd failed!" | tee -a $HOME/daily.log
	exit 1
    fi
    cd $here
    sudo /usr/sbin/svcadm enable -s svc:/application/pcp/pmcd >>$HOME/daily.log 2>&1
    sudo /usr/sbin/svcadm enable -s svc:/application/pcp/pmlogger >>$HOME/daily.log 2>&1
elif [ -f pcp-$buildversion/build/tar/pcp-[0-9]*[0-9].tar.gz ]
then
    # tarball packaging ... (this has to be last, because some of the
    # other packaging regimes also create tarballs)
    here=`pwd`
    tarball=$here/pcp-$buildversion/build/tar/pcp-[0-9]*[0-9].tar.gz
    cd pcp-$buildversion/build/tar
    sudo ./preinstall >>$HOME/daily.log 2>&1
    cd /
    if sudo tar -zxpf $tarball >>$HOME/daily.log 2>&1
    then
	echo "Info: tar install" | tee -a $HOME/daily.log
    else
	echo "Error: tar install failed!" | tee -a $HOME/daily.log
	exit 1
    fi
    cd $here
    cd pcp-$buildversion/build/tar
    sudo ./postinstall >>$HOME/daily.log 2>&1
    cd $here
else
    echo "Don't know how to install packages" | tee -a $HOME/daily.log
    exit 1
fi
if [ ! -f /etc/pcp.env ]
then
    echo "Error: /etc/pcp.env not installed" | tee -a $HOME/daily.log
    exit 1
fi
. /etc/pcp.env
if [ -f pcp-$buildversion/build/sun/pcp-$src_version ]
then
    : pmcd started above
else
    if sudo $PCP_RC_DIR/pcp start >$tmp.out 2>&1
    then
	cat $tmp.out >>$HOME/daily.log
	echo "Info: pmcd started" | tee -a $HOME/daily.log
    else
	cat $tmp.out >>$HOME/daily.log
	echo "Error: pmcd start failed" | tee -a $HOME/daily.log
	exit 1
    fi
fi
running_version=`pmprobe -v pmcd.version | sed -e 's/"//g' -e 's/pmcd.version 1 //'`
echo "Source version: $src_version" >>$HOME/daily.log
echo "Running version: $running_version" >>$HOME/daily.log
if [ "$src_version" != "$running_version" ]
then
    pcp >>$HOME/daily.log
    echo "Error: version mismatch src=$src_version running=$running_version" | tee -a $HOME/daily.log
    exit 1
fi
# sample PMDA may have an updated PMNS, so re-install just in case
( cd $PCP_VAR_DIR/pmdas/sample; sudo ./Install </dev/null ) >>$HOME/daily.log
End-of-File
}

_script_qa()
{
    cat <<End-of-File
rm -f *.out.bad

# Policy for QA Farm: exclude all tests tagged BAD
#
qa_restrict="$check_what -x BAD"

echo "Info: QA \$qa_restrict starting"
./check -l \$qa_restrict >>\$HOME/daily.qa.log 2>&1
sts=\$?
# look for badness in daily.qa.log and report here (so it goes into any
# cron-based email)
awk <\$HOME/daily.qa.log '
/waiting for lock/		{ print }
/could not acquire lock/	{ print }
/error while loading shared libraries/	{ print }
\$1 == "Aborted!"		{ print }
\$1 == "Check!"			{ print }'
# get last block from check.log ... then get the summary on
# stdout (not needed in the daily.qa.log already there)
sed -e '/^    [0-9 ]*\$/d' <check.log \
| awk '
NF == 0	{ txt = ""; next }
	{ if (txt == "") txt = \$0
	  else txt = txt "\n" \$0
	}
END	{ print txt }' \
| awk '
\$1 == "Triaged:"	{ want = 1 }
\$1 == "Failures:"	{ want = 1 }
\$1 == "Passed"		{ want = 1 }
want == 1		{ print }'
if [ "\$sts" = 0 ]
then
    echo "Info: QA \$qa_restrict done" | tee -a \$HOME/daily.qa.log
else
    echo "Error: QA \$qa_restrict failed!" | tee -a \$HOME/daily.qa.log
fi
# if we're using systemctl, then need to start enabled PCP
# services so we leave things in a good state
#
if [ "$PCPQA_SYSTEMD" = yes ]
then
    systemctl list-unit-files >\$tmp.systemctl
    for svc in pmcd pmproxy pmie pmlogger_daily_report pmlogger
    do
	if grep "\$svc.service[ 	]*enabled" <\$tmp.systemctl >/dev/null
	then
	    echo "Info: restarting \$svc" | tee -a \$HOME/daily.qa.log
	    sudo systemctl reset-failed \$svc
	    sudo systemctl restart \$svc
	fi
    done
fi
date >>\$HOME/daily.qa.log
End-of-File
}

_script_qa_rerun()
{
    cat <<End-of-File
if ./recheck >>\$HOME/daily.qa.log 2>&1
then
    echo "Info: QA recheck" | tee -a \$HOME/daily.qa.log
else
    echo "Error: QA recheck failed!" | tee -a \$HOME/daily.qa.log
fi
End-of-File
}

_script_harvest()
{
    cat <<End-of-File
rm -f \$HOME/daily.harvest
touch \$HOME/daily.harvest
nbad=0
nfull=0
for bad in *.out.bad
do
    [ "\$bad" = '*.out.bad' ] && continue
    echo \$bad >>\$HOME/daily.harvest
    nbad=\`expr \$nbad + 1\`
    seq=\`echo \$bad | sed -e 's/\.out\.bad//'\`
    [ -f \$seq.out ] && echo \$seq.out >>\$HOME/daily.harvest
    for full in \$seq.full*
    do
	if [ -f "\$full" ]
	then
	    echo "\$full" >>\$HOME/daily.harvest
	    nfull=\`expr \$nfull + 1\`
	fi
    done
done
[ -s \$HOME/daily.harvest ] && cat \$HOME/daily.harvest >>\$HOME/daily.qa.log
echo "Info: harvest \$nbad *.bad files, \$nfull *.full files" | tee -a \$HOME/daily.qa.log
End-of-File
}

# Mainline starts here ...
#

tmp=/var/tmp/$$
sts=0
trap "rm -f $tmp.*; exit \$sts" 0 1 2 3 15

args="$*"
pcp_branch=main
pcp_tree=pcp
build_user=${USER-kenj}
build_dir=src/$pcp_tree
qa_user=${USER-kenj}
qa_dir=src/pcp/qa
cflag=false
fflag=false
hflag=false
iflag=false
lflag=false
mflag=false
pflag=false
qflag=false
rflag=false
what="-g sanity -g libpcp -g pdu -g pmda -g valgrind -g pmie"
what=all
sflag=false
options=false
work=false
efile=''
debug=false
while getopts "ab:B:cde:fg:hilmpqrst:T:?" c
do
    case $c
    in
	a)
	    options=true
	    work=true
	    pflag=true
	    mflag=true
	    iflag=true
	    qflag=true
	    hflag=true
	    ;;
	b)
	    pcp_branch="$OPTARG"
	    ;;
	c)
	    options=true
	    work=true
	    cflag=true
	    ;;
	d)
	    debug=true
	    ;;
	e)
	    efile="$OPTARG"
	    if [ -f "$efile" ]
	    then
		:
	    else
		echo "Error: $efile: cannot be found"
		sts=1
		exit
	    fi
	    ;;
	f)
	    fflag=true
	    ;;
	g)
	    what="$OPTARG"
	    ;;
	h)
	    options=true
	    work=true
	    hflag=true
	    ;;
	i)
	    options=true
	    work=true
	    iflag=true
	    ;;
	l)
	    options=true
	    work=true
	    lflag=true
	    ;;
	m)
	    options=true
	    work=true
	    mflag=true
	    ;;
	p)
	    options=true
	    work=true
	    pflag=true
	    ;;
	q)
	    options=true
	    work=true
	    qflag=true
	    ;;
	r)
	    options=true
	    work=true
	    rflag=true
	    ;;
	s)
	    options=true
	    sflag=true
	    ;;
	t)
	    pcp_tree="$OPTARG"
	    build_dir=src/$pcp_tree
	    ;;
	?)
	    _usage
	    sts=1
	    exit
	    ;;
    esac
done
shift `expr $OPTIND - 1`

if [ $# -eq 0 ]
then
    echo "Error: must specify at least one VM"
    sts=1
    exit
fi

if $options
then
    :
else
    # defaults
    #
    work=true
    pflag=true
    mflag=true
    iflag=true
    qflag=true
    hflag=true
    sflag=true
fi

today=`date +%Y-%m-%d`
mkdir -p $HOME/Logs/by-date/$today

rm -f $tmp.hosts

# some driver host specific things ...
#
case `hostname -s`
in
    bozo)
	echo "bozo" >>$tmp.hosts
	echo "bozo-vm" >>$tmp.hosts
	echo "oracle" >>$tmp.hosts
	# and open up the X server
	#
	# Ubuntu up to 15.04
	#DISPLAY=localhost:0.0 xhost + >/dev/null
	# Ubuntu 15.10
	DISPLAY=:0.0 xhost + >/dev/null
	;;
esac

if which virsh >/dev/null 2>&1
then
    # sudo here is odd ... after Ubuntu 17.10 upgrade, virsh run
    # from a cron-launched shell produces no output, sudo virsh does
    # produce output again ... sigh.
    #
    sudo virsh list --all \
    | sed >>$tmp.hosts \
        -e 1d \
        -e '/-----/d' \
        -e 's/^  *//' \
        -e 's/^[0-9][0-9]*[ 	]*//' \
        -e 's/^-[ 	]*//' \
	-e 's/[    ].*//' \
        -e '/^$/d'
fi

for target
do
    grep "^$target" <$tmp.hosts >$tmp.host
    if [ ! -s $tmp.host ]
    then
	echo "No host matches \"$target\" ... the hosts I have are ..."
	cat $tmp.hosts
	continue
    fi
    count=`wc -l <$tmp.host | sed -e 's/ //g'`
    if [ $count != 1 ]
    then
	echo "Warning: more than one host matches \"$target\" ... the matches I have are ..."
	cat $tmp.host
	echo "... using the first one."
    fi
    host=`sed -e 1q $tmp.host`
    echo "::: $host :::"

    check_what="$what"
    valgrind_opt=''
    [ X"$what" = Xall ] && check_what=''
    case "$host"
    in

	bozo|vm03)
	    # run both the valgrind and the non-valgrind variants of tests
	    #
	    valgrind_opt='export PCPQA_VALGRIND=both'
	    ;;

	oracle)
	    # just the sanity group ...
	    [ X"$what" = Xall ] && check_what='-g sanity'
	    ;;

	vm01)
	    # 739 segv's sometimes deep inside Python
	    # and 749 recently (Dec 2023) has been locking the whole VM
	    # up
	    # fixed? [Jan 2024]
	    #
	    #[ X"$what" = Xall ] && check_what='-X 739 -X 749'
	    ;;

	vm05)
	    # Exclude -x valgrind ... does not work on my Gentoo at the
	    # moment
	    #
	    [ X"$what" = Xall ] && check_what='-x valgrind'
	    ;;

	vm06)
	    # Only selected groups at this stage for FreeBSD
	    #	sanity trace libpcp kernel pmrep threads
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity -g trace -g libpcp -g kernel -g pmrep -g threads -g perl -x valgrind'
	    ;;

	vm09)
	    # Only selected groups at this stage for NetBSD
	    #	sanity trace libpcp kernel pmrep threads
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity -g trace -g libpcp -g kernel -g pmrep -g threads -x valgrind'
	    ;;

	vm10)
	    # Only selected groups at this stage for FreeBSD ...
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity -g trace -g libpcp -g kernel -g pmrep -g threads -g perl -g pmchart -x valgrind'
	    ;;

	vm15)
	    # 1123 loops forever lost in an X11 wasteland
	    [ X"$what" = Xall ] && check_what='-X 1123'
	    ;;

	vm16)
	    # this is PCP 4.0.0
	    # no -p or -m or -i flags and no -e processing
	    # group "remote" should be enough to exercise interoperability
	    [ X"$what" = Xall ] && check_what='-g remote'
	    qa_user=pcpqa
	    qa_dir=/var/lib/pcp/testsuite
	    echo "Info: this PCP 4.0.0 inter-op testing ... no new PCP builds or installs"
	    pflag=false
	    mflag=false
	    iflag=false
	    if [ -n "$efile" ]
	    then
		echo "Info: no -e $efile for vm16"
		efile=''
	    fi
	    #
	    # 003 fails parsing a current /proc/net/netstat
	    # 066 fails parsing a current /proc/net/netstat
	    # 116 fails because down-rev pmlc can't talk to current pmlogger
	    # 257 fails because the local 4.0.0 and remote current PMNS
	    #     are too out of whack
	    # 322 fails because down-rev pmlc can't talk to current pmlogger
	    # 365 fails parsing a current /proc/net/netstat
	    # 374 fails because down-rev pmlc can't talk to current pmlogger
	    #
	    [ X"$what" = Xall ] && check_what="$check_what -X 003 -X 066 -X 116 -X 257 -X 322 -X 365 -X 374"
	    ;;

	vm20)
	    # Run from testsuite, not git tree
	    #
	    qa_dir=/var/lib/pcp/testsuite
	    qa_user=pcpqa
	    ;;

	vm32)
	    # Only selected groups at this stage for FreeBSD ...
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity -g trace -g libpcp -g kernel -g pmrep -g threads -x valgrind -X 603'
	    ;;

	vm33)
	    # Only selected groups at this stage for OpenBSD
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity -g trace -g libpcp -g kernel -g pmcd -g pminfo -g pmlogger -g pmrep -g threads -g logutil -g dbpmda -g pmcheck -g pmlc'
	    ;;

	vm35)
	    # this is PCP 2.7.8, need to run as user pcpqa and just
	    # the sanity group and no -p or -m or -i flags
	    # and no -e processing
	    [ X"$what" = Xall ] && check_what='-g sanity'
	    qa_user=pcpqa
	    qa_dir=/var/lib/pcp/testsuite
	    if $pflag
	    then
		echo "Info: skip -p for vm35"
		pflag=false
	    fi
	    if $mflag
	    then
		echo "Info: skip -m for vm35"
		mflag=false
	    fi
	    if $iflag
	    then
		echo "Info: skip -i for vm35"
		iflag=false
	    fi
	    if [ -n "$efile" ]
	    then
		echo "Info: no -e $efile for vm35"
		efile=''
	    fi
	    ;;

	vm37)
	    # Only selected groups this stage for OpenBSD
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity -g trace -g libpcp -g kernel -g pmcd -g pminfo -g pmlogger -g pmrep -g threads -g logutil -g dbpmda -g pmcheck -g pmlc'
	    ;;

	vm38)
	    # Only -g sanity at this stage for NetBSD
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity'
	    ;;

    esac

    if which virsh >/dev/null 2>&1
    then
	# see note above re. sudo virsh
	#
	state=`sudo virsh list --all | grep "$host" | awk '{print $3}'`
	case "$state"
	in
	    shut)
		if $work
		then
		    sudo virsh start $host
		    state=running
		fi
		;;
	    running)
		;;
	    paused)
		if $work
		then
		    sudo virsh resume $host
		    state=running
		fi
		;;
	    '')
		# probably not a VM, like "fuji"
		;;
	    *)
		echo "No clue what do with virsh state \"$state\""
		continue
		;;
	esac
    fi

    if $work
    then
	log=$HOME/Logs/by-date/$today/$host
	touch $log
	mkdir -p $HOME/Logs/by-vm/$host
	if [ ! -L $HOME/Logs/by-vm/$host/$today ]
	then
	    ( cd $HOME/Logs/by-vm/$host; ln -s ../../by-date/$today/$host $today )
	fi

	rm -f $tmp.up
	wait=0
	delta=5
	max_wait=300
	while [ $wait -lt $max_wait ]
	do
	    if ssh $build_user@$host true >/dev/null 2>&1
	    then
		$debug && echo host $host up and ssh for $build_user ok
		if [ "$build_user" != "$qa_user" ]
		then
		    if ssh $qa_user@$host true
		    then
			$debug && echo ssh for $qa_user ok
		    else
			echo Warning cannot ssh for $qa_user to host $host
		    fi
		fi
		touch $tmp.up
		break
	    fi
	    $debug && echo host $host down, waiting $wait
	    sleep $delta
	    wait=`expr $wait + $delta`
	done
	if [ ! -f $tmp.up ]
	then
	    echo "Failed to start VM $host after $max_wait secs ... giving up"
	    continue
	fi

	_script_header daily >$tmp.script
	if [ -n "$efile" ]
	then
	    cat $efile >>$tmp.script
	fi
	_script_git >>$tmp.script
	$cflag && _script_git_check >>$tmp.script
	$pflag && _script_git_pull >>$tmp.script

	if $mflag
	then
	    _script_make "$host" >>$tmp.script
	fi

	$iflag && _script_install >>$tmp.script

	_script_header daily.qa >$tmp.script.qa

	[ -n "$valgrind_opt" ] && echo "$valgrind_opt" >>$tmp.script.qa

	cat >>$tmp.script.qa <<End-of-File

cd $qa_dir
End-of-File

	$qflag && _script_qa >>$tmp.script.qa
	$rflag && _script_qa_rerun >>$tmp.script.qa
	$hflag && _script_harvest >>$tmp.script.qa

	chmod 755 $tmp.script $tmp.script.qa
	# we've sometimes seen the scp fail because we get here a
	# bit too soon after a VM has been rebooted ... be a little
	# more patient
	#
	rm -f $tmp.ok
	for i in 1 2 3
	do
	    if scp $tmp.script $build_user@$host:daily.script >$tmp.out 2>&1
	    then
		touch $tmp.ok
		break
	    else
		cat $tmp.out
		echo "Warning: scp for daily.script failed!"
		sleep 2
	    fi
	done
	if [ ! -f $tmp.ok ]
	then
	    echo "Error: cannot copy daily.script to $host ... giving up"
	    continue
	fi

	if $qflag || $rflag || $hflag
	then
	    rm -f $tmp.ok
	    for i in 1 2 3
	    do
		if scp $tmp.script.qa $qa_user@$host:daily.qa.script >$tmp.out 2>&1
		then
		    touch $tmp.ok
		    break
		else
		    cat $tmp.out
		    echo "Warning: scp for daily.qa.script failed!"
		    sleep 2
		fi
	    done
	    if [ ! -f $tmp.ok ]
	    then
		echo "Error: cannot copy daily.qa.script to $host ... giving up"
		continue
	    fi
	fi

	# actually do the remote work here
	#
	start=`date +%s`
	ssh -t $build_user@$host sh -c ./daily.script
	scp -q $build_user@$host:daily.log $tmp.out
	echo "=== build and install: `date` ===" >>$log
	cat $tmp.out >>$log
	echo "=== END ===" >>$log

	if $qflag || $rflag || $hflag
	then
	    # -o LogLevel=FATAL to get rid of
	    #     connect /tmp/.X11-unix/X0: No such file or directory
	    # from local ssh client, as per suggestion at
	    # https://stackoverflow.com/questions/56825547
	    #
	    ssh -o LogLevel=FATAL -t $qa_user@$host sh -c ./daily.qa.script
	    scp -q $qa_user@$host:daily.qa.log $tmp.out
	    echo "=== qa: `date` ===" >>$log
	    cat $tmp.out >>$log
	    echo "=== END ===" >>$log
	fi

	if $hflag
	then
	    # harvest failures by pulling from QA host
	    echo "Harvest:"
	    if [ -d $HOME/Logs/by-vm/$host/pcpqa ]
	    then
		# old dir name
		mv $HOME/Logs/by-vm/$host/pcpqa $HOME/Logs/by-vm/$host/qa
	    fi
	    if [ ! -d $HOME/Logs/by-vm/$host/qa ]
	    then
		rm -f $HOME/Logs/by-vm/$host/qa
		mkdir -p $HOME/Logs/by-vm/$host/qa
		( cd $HOME/Logs/by-vm/$host/qa;
		    for file in common common.rc common.check common.setup group show-me localconfig
		    do
		    	ln -s $HOME/src/pcp/qa/$file .
		    done
		)
	    fi
	    rm -f $HOME/Logs/by-vm/$host/qa/[0-9]*
	    rm -f $tmp.harvest
	    scp $qa_user@$host:daily.harvest $tmp.harvest >$tmp.out 2>&1
	    if [ -s $tmp.harvest ]
	    then
		for file in `cat $tmp.harvest`
		do
		    echo -n " $file"
		    if scp $qa_user@$host:$qa_dir/$file $HOME/Logs/by-vm/$host/qa >$tmp.err 2>&1
		    then
			:
		    else
			echo
			cat $tmp.err
		    fi
		done
		echo
	    elif [ -s $tmp.out ]
	    then
		echo "scp error ..."
		cat $tmp.out
	    else
		echo "No failures."
	    fi
	fi

	if $iflag || $qflag || $rflag || $hflag
	then
	    # run whatami on QA host
	    ssh -t $qa_user@$host $qa_dir/admin/whatami >$HOME/Logs/by-vm/$host/whatami
	    echo "Info: `sed -e 's/[ 	][ 	]*/ /g' <$HOME/Logs/by-vm/$host/whatami`" | tee -a $HOME/daily.log
	fi

	end=`date +%s`
	echo "$start $end" | awk >>$log '
    { delta = $2 - $1
      hr = int(delta/3600)
      min = int((delta-hr*3600)/60)
      sec = delta % 60
      printf "Elapsed time: %d:%02d:%02d (%d secs)\n", hr, min, sec, delta
    }'

    fi
    
    if $sflag
    then
	doit=false
	case $host
	in
	    vm01|vm03|vm16|vm35|vm36)
		# try to keep these VMs running, unless -f
		$fflag && doit=true
		;;
	    vm*)
		doit=true
		;;
	esac
	if $doit
	then
	    if [ $state = shut ]
	    then
		if $work
		then
		    echo "Warning: already shutdown"  | tee -a $HOME/daily.log
		else
		    echo "Warning: already shutdown"
		fi
	    else
		# see note above re. sudo virsh
		#
		if sudo virsh list | grep " $host  *running" >/dev/null
		then
		    # not all systems have a poweroff(8) command, and
		    # BSD and Linux cannot even agree on the -p or -P
		    # option for shutdown(8) nor how to emit a usage
		    # message, and for OpenBSD "which shutdown" fails
		    # unless run as root (the executable in /sbin is
		    # not only unexecutable but us unreadable for mere
		    # mortals) ... sigh
		    # 
		    ssh -f -t -q $build_user@$host "sync; sleep 2; sync; export PATH=\$PATH:/sbin:/usr/sbin; if sudo which poweroff >/dev/null 2>&1; then sudo poweroff; elif sudo which shutdown >/dev/null 2>&1; then if sudo shutdown -\? 2>&1 | grep 'usage.*-[a-z]*p' >/dev/null; then sudo shutdown -p now; else sudo shutdown -P now; fi; fi" </dev/null &
		    i=0
		    rm -f $tmp.done
		    while [ $i -lt 60 ]
		    do
			if sudo virsh list | grep " $host *running" >/dev/null
			then
			    sleep 1
			    i=`expr $i + 1`
			else
			    touch $tmp.done
			    break
			fi
		    done
		    if [ ! -f $tmp.done ]
		    then
			echo "shutdown failed for $host ... forcing virsh shutdown"
			sudo virsh shutdown $host
			i=0
			rm -f $tmp.done
			while [ $i -lt 10 ]
			do
			    if sudo virsh list | grep " $host *running" >/dev/null
			    then
				sleep 1
				i=`expr $i + 1`
			    else
				touch $tmp.done
				break
			    fi
			done
			if [ ! -f $tmp.done ]
			then
			    echo "virsh shutdown failed for $host ... forcing virsh destroy"
			    # seems a bit harsh, but we've given you
			    # two chances to shutdown
			    #
			    sudo virsh destroy $host
			fi
		    fi
		else
		    sudo virsh list
		    echo "$host already stopped?"
		fi
		if $work
		then
		    echo "Info: shutdown"  | tee -a $HOME/daily.log
		else
		    echo "Info: shutdown"
		fi
	    fi
	else
	    if $work
	    then
		echo "Warning: shutdown skipped"  | tee -a $HOME/daily.log
	    else
		echo "Warning: shutdown skipped"
	    fi
	fi
    fi

    $work && echo "Logs in $log"

done
