#!/usr/bin/python
# -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4
#
# mach - make a chroot
#
# script to set up a chroot using packages and optionally perform builds in it

import sys, os, stat
import getopt, string, commands, urllib, shutil, time, fcntl, re
import rpm # FIXME: find out what rpm version we need !
import random # spinner theming madness
import tempfile # for tempdir for spec extraction
import glob # for globbing paths
import grp # for finding out the mach gid
import sets # to work with python 2.3 as well
from copy import deepcopy # for stuff in dist.d files

# print out debug info if wanted
# uses DEBUG global
def debug (string):
    if DEBUG:
        print "DEBUG: %s" % string

# make sure True and False constants exist

if not hasattr (__builtins__, 'True'):
    __builtins__.True = (1 == 1)
    __builtins__.False = (1 != 1)

# usage and help
usage = 'usage: mach [mach-options] command [command-options-and-arguments]'
help = '''Make a chroot and do stuff in it.

Mach options:
  -v, --version         print mach version
  -r, --root=ROOT       specify a root to work in
  -d, --debug           turn on debugging
  -f, --force           override root lock
  -k, --keep            do not revert buildroot to initial package list
  -s, --sign            sign packages after build
  -m, --md5sum          create md5sum of results after build
  -c, --collect         collect packages and relevant files to .
  -q, --quiet           no spinners, minimal output
  --release=(tag)       add given tag to release tag
  --no-lint             don\'t run rpmlint at the end
  --no-scripts          don\'t run post build scripts
  --canonify            just output the canonical root name and exit

Commands:
  apt-get           run host\'s apt-get on target root
  apt-cache         run host\'s apt-cache on target root
  build             build from (set of) .spec file(s)
                                (passes options given to rpmbuild)
  chroot            chroot into root
  clean             totally cleans the root
  rebuild           rebuild a (set of) .src.rpm(s)
                                (passes options given to rpmbuild)
  rpm               run host\'s rpm on target root
  setup             sets up the root for a particular purpose
        prep                         preparation before package install
        minimal                      minimal set of rpms (bash, glibc)
        base                         base set of rpms for minimal functionality
        build                        everything to build rpms
  status            show status of mach roots
  unlock            override root lock
  yum               run host\'s yum on target root
'''

# list of allowed commands
allowed_commands = ('apt-get', 'apt-cache', 'build',
                    'chroot', 'clean', 'rebuild', 'rpm',
                    'setup', 'status', 'unlock', 'yum')

# read default configuration
config = {
  'force': 0,
  'check': 0,
  'sign': 0,
  'md5sum': 0,
  'collect': 0,
  'release': 0,
  'quiet': 0,
  'lint': 1,
  'scripts': 1,
  'macros': {},
  'defaultroot': 'fedora-6-i386',

  'chroot': '/usr/sbin/mach-helper chroot',
  'mount': '/usr/sbin/mach-helper mount',
  'rpm': '/usr/sbin/mach-helper rpm',
  'umount': '/usr/sbin/mach-helper umount',
  'rm': '/usr/sbin/mach-helper rm',
  'apt-get': '/usr/sbin/mach-helper apt-get',
  'apt-cache': '/usr/bin/apt-cache',
  'yum': '/usr/sbin/mach-helper yum',
  'mknod': '/usr/sbin/mach-helper mknod',
  'runuser': '/bin/su',
  'installer': 'yum',

  # root_*: commands run inside the build root
  'root_rpmbuild': 'rpmbuild',
}
DEBUG = 0

# read system-wide configuration
file = '/etc/mach/conf'
try:
    execfile (file)
except:
    sys.stderr.write ("Could not read config file %s: %s\n" %
                      (file, sys.exc_info()[1]))
    sys.exit (1)

# read dist information

# clear stuff
aptsources = {}
yumsources = {}
packages = {}
sourceslist = {}
aliases = {}

# first, get locations
file = '/etc/mach/location'
try:
    execfile (file)
except:
    sys.stderr.write ("Could not read location file %s: %s\n" %
                      (file, sys.exc_info()[1]))
    sys.exit (1)


# next, get all of the dist.d files
dists = os.listdir ('/etc/mach/dist.d')
dists.sort()
for file in dists:
    # skip various files, backups etc
    if re.search ('^[\.#]|(\.rpm(orig|new|save)|~)$', file):
        continue
    file = os.path.join ('/etc', 'mach', 'dist.d', file)
    try:
        execfile (file)
    except:
        sys.stderr.write ("Could not read dist file %s: %s\n" %
                          (file, sys.exc_info()[1]))
        sys.exit (1)

# read user configuration
file = os.path.join (os.environ['HOME'], '.machrc')
try:
    execfile (file)
except IOError:
    pass

# rpm -q format, users may have customized %_query_all_fmt in ~/.rpmmacros...
qfmt = '%{name}-%{version}-%{release}\n'

# build user/group inside chroot
builduser = 'machbuild'
buildgroup = 'machbuild'

# uid/gid information
import grp
try:
    mach_gid = grp.getgrnam ('mach')[2]
except KeyError:
    sys.stderr.write ("No group 'mach' found")
    sys.exit (1)

debug ('Using mach gid %r' % mach_gid)

### exceptions
class ReturnValue(Exception):
    """
    Raised when a process exits with a non-zero return value.
    Has the return value and output as arguments.
    """
    pass
class StatusValue(Exception):
    """
    Raised when a process does not exit normally.
    Has the status value and the output as arguments.
    """
    pass
     
### objects
class Spec:
    class Error(Exception): pass
    def __init__ (self, path, root, options = ""):
        """
        Parse the given spec file, using additional options for rpmbuild.
        The spec file is evaluated using a chroot into the given root, after
        copying it there.

        @type root: L{Root}
        """
        self.path = path
        self.content = []
        self.vars = {}
        self.options = options
        self.root = root

        # where the copy lives, relative to the root
        self.rootpath = os.path.join (os.path.sep, 'tmp', os.path.basename (
            self.path))

        debug ("Spec.__init__: opening spec %s" % path)
        # check if the spec file has a %prep section
        command = "grep '%%prep' %s" % path
        (status, output) = commands.getstatusoutput (command)
        if os.WIFEXITED (status):
            retval = os.WEXITSTATUS (status)
        if (retval != 0):
            raise self.Error, "spec file %s does not have a %%prep section" % self.path
        # copy the spec file inside the root if not already there
        # it is already there, for example, after mangling and rebuilding from
        # it
        if self.path != self.root.rootdir + self.rootpath:
            try:
                self.root.do_chroot ('rm -f %s' % self.rootpath)
            except ReturnValue, (retval, output):
                raise self.Error, "Failed removing spec file '%s' (return value %d).\nOutput follows:\n%s" % (self.rootpath, retval, output)
 
            copy_grouped (self.path, os.path.join (self.root.rootdir, 'tmp'))

        # check for syntax errors in the spec file first
        command = "rpmbuild -bp --nodeps --force %s --define '__spec_prep_pre exit 0' --define 'setup :' %s" % (options, self.rootpath)
        debug ("Spec:__init__: command: %s" % command)
        try:
            (retval, output) = self.root.do_chroot (command)
        except ReturnValue, (retval, output):
            raise self.Error, "Failed parsing spec file '%s' (return value %d).\nOutput follows:\n%s" % (self.path, retval, output)
 
        self.content = open (self.path).readlines ()
        # expand tags using rpmbuild because results can depend on
        # build options
        for which in ['name', 'version', 'release']:
            try:
                output = self.rpmbuild_prep_run (
                    "echo %%%s; exit 0" % which)
                last = output.split("\n")[-1]
                debug ("result for %s: %s" % (which, last))
                self.vars[which] = last
            except ReturnValue, (value, output):
                raise self.Error ("Could not expand %s" % which)

        # scan for defines in the spec file
        debug ("scanning spec file for %define's")
        matchstr = re.compile ("^\s*%define\s+(\S*)\s+(\S*)$")
        for line in self.content:
            match = matchstr.search (line)
            if match:
                tag = match.expand ("\\1")
                value = match.expand ("\\2")
                debug ("Found define of %s to %s" % (tag, value))
                self.vars[tag] = value
        # now loop over all of the tag values and expand them until we're done
        # FIXME: an anti-loop here would be nice, wouldn't it ?
        # FIXME: for now, we do a single pass loop
        # FIXME: also realize this will bum us out with multiple %{...}, so
        # FIXME: in the future make this non-greedy or something
        matchstr = re.compile ("^(.*)%{(.*)}(.*)$")
        for tag in self.vars.keys ():
            try:
                match = matchstr.search (self.vars[tag])
            except TypeError:
                sys.stderr.write ("Error on searching for tag %s with value %s" % (tag, self.vars[tag]))
            if match:
                expandee = match.expand ("\\2")
                debug ("Need to expand tag %s inside tag %s (%s)" % (expandee, tag, self.vars[tag]))
                if expandee in self.vars.keys ():
                    debug ("Expanding it to %s" % self.vars[expandee])
                    front = match.expand ("\\1")
                    back = match.expand ("\\3")
                    self.vars[tag] = front + self.vars[expandee] + back
                    debug ("Expanded complete line to %s" % self.vars[tag])

    def rpmbuild_prep_run (self, script):
        """
        Use rpmbuild to evaluate the spec file and run the given 'prepare'
        script.
        Executes inside the root.

        Raises ReturnValue (value, output) if a non-zero exit value, or
        returns the output.
        """
        options = self.options
        path = self.rootpath
        command = "rpmbuild -bp --nodeps --force %(options)s " \
            "--define '__spec_prep_pre %(script)s' " \
            "--define 'setup :' " \
            "%(path)s" % locals()
        debug ("Spec:__init__: command: %s" % command)
        (retval, output) = self.root.do_chroot (command)
        if (retval != 0):
            raise ReturnValue, (retval, output)

        return output
 
    def tag (self, which):
        "Get the given (unique) tag from the spec file as a string"
        # if we already have it, return it
        if self.vars.has_key (which):
            return self.vars[which]

        # if we don't, look for it in the file
        matchstr = re.compile ("^" + which + r"\d*\s*:\s*(.*)", re.I)
        for line in self.content:
            match = matchstr.search (line)
            if match:
                value = match.expand ("\\1")
                self.vars[which] = value
                return value
        sys.stderr.write ("WARNING: didn't find tag %s in spec file\n" % which)
        return None

    def tags (self, which):
        "Get all given tags from the spec file as a list"
        result = []
        matchstr = re.compile ("^" + which + r"\d*:\s*(.*)", re.I)
        for line in self.content:
            match = matchstr.search (line)
            if match:
                value = match.expand ("\\1")
                result.append (value)
        return result

    def nvr (self):
        "return name, version, release triplet"
        return (self.vars['name'], self.vars['version'], self.vars['release'])

    def _sourcespatches (self, which):
        # get a dict of Source and Patch lines, given quoted options to build
        # as args
        result = {}
        matchstr = re.compile ("^" + which + r"(\d*):\s*(.*)")
        macro = ""
        if which == "Source": macro = "SOURCEURL"
        if which == "Patch": macro = "PATCHURL"
        for line in self.content:
            match = matchstr.search (line)
            if match:
                number = match.expand ("\\1")
                value = match.expand ("\\2")
                tag = which + number
                if number == "": number = "0"

                try:
                    output = self.rpmbuild_prep_run (
                        "echo %%%s%s; exit 0" % (macro, number))
                    last = output.split("\n")[-1]
                    debug ("result for %s: %s" % (which, last))
                    result[tag] = last
                except ReturnValue, (value, output):
                    error ("Could not expand %s" % which)

        return result

    def sources (self):
           # parse the spec file for all Source: files
           return self._sourcespatches ("Source")

    def patches (self):
           # parse the spec file for all Source: files
           return self._sourcespatches ("Patch")

### abstract object wrapping a src rpm
class SRPM:
    class Error(Exception): pass
    def __init__ (self, path, root, options = ""):
        """
        @type root: L{Root}
        """
        self.path = path
        self.root = root
        self.options = options

        self.header = self._header ()
        self.specname = self._specname ()

        # ignore signatures
    def _header (self):
        "get the rpm header from the src.rpm"
        debug ("Getting RPM header from %s" % self.path)
        ts = rpm.TransactionSet ("", (rpm._RPMVSF_NOSIGNATURES))
        fd = os.open (self.path, os.O_RDONLY)
        try:
            header = ts.hdrFromFdno (fd)
        except:
            print "a whoopsie occurred."
        if not header:
            raise self.Error, "%s doesn't look like a .src.rpm" % self.path
        return header

    def _specname (self):
        "return the name of the specfile inside this src.rpm"
        # FIXME: we found an rhas spec that had something.spec.in as the spec
        # file, so find a better way to get the spec file name
        name = commands.getoutput ('rpm -qlp %s 2> /dev/null | grep \.spec$' % self.path)
        if not name:
            name = commands.getoutput ('rpm -qlp %s 2> /dev/null | grep \.spec.*$' % self.path)
            if not name:
                raise self.Error, "Didn't find spec file in %s" % self.path
        debug ("SRPM.specname: %s" % name)
        return name

    def BuildRequires (self):
        "return a list of BuildRequires for this src.rpm"
        buildreqs = self.header[rpm.RPMTAG_REQUIRENAME]
        debug ("BuildRequires from header: %s" % buildreqs)
        # remove non-packagey things from buildreqs, like autodeps
        # currently removes everything containing  / and all rpmlib() deps
        if buildreqs:
            return filter (lambda x: not (re.search("[/]", x) or re.search("rpmlib", x)), buildreqs)
        else:
            return []

    def results (self):
        """
        Returns a list of package names that can be built from this src.rpm,
        without version and release.
        """
        # FIXME: this extracts the spec file out of the src.rpm and
        # puts it in a temp dir; is this the cleanest option ?
        debug ("Getting possible result packages from src.rpm %s" % self.path)
        path = self.path
        specname = self.specname
        tmpdir = tempfile.mktemp ()
        ensure_dir (tmpdir)
        tmp_remove_cmd = 'rm -rf %s' % tmpdir
        cmd = 'cd %s && rpm2cpio %s \
               | cpio -id --no-absolute-filenames %s \
               > /dev/null 2>&1' % (tmpdir, path, specname)
        os.system (cmd)
        spectmp = os.path.join (tmpdir, self.specname)
        if not os.path.exists (spectmp):
            raise self.Error, "didn't find spec file in %s" % self.path
        # we need to get the results with our options, otherwise the
        # release tag may be wrong; e.g. for FE it would not include dist
        cmd = "rpm -q --qf '%s' %s --specfile %s" % (
            qfmt, self.options, spectmp)
        output = commands.getoutput (cmd)
        debug ("output of rpm -q --specfile: \n%s" % output)
        try:
            spec = Spec (spectmp, self.root, self.options)
        except Spec.Error, message:
            os.system (tmp_remove_cmd)
            raise self.Error, message
            
        (n, v, r) = spec.nvr ()
        # strip v and r off of output
        result = []
        for line in output.split ("\n"):
            result.append (line.replace ('-%s-%s' % (v, r), ''))
        debug ("results of srpm: %s" % result)
        os.system (tmp_remove_cmd)
        return result

### abstract object acting on the root
# FIXME: we can subclass this based on actual method to install stuff
class Root:
    # exceptions that can be raised
    class Locked(Exception): pass
    class Error(Exception): pass

    # read in configuration from disk when creating a root
    def __init__ (self, config):
        self.config = config
        self.hooks  = self.config.get ('hooks', {})
        self.root = self.config['root']
        self.rootdir = get_config_dir (config, 'root')
        self.resultdir = get_config_dir (config, 'result')
        self.statedir = get_config_dir (self.config, 'state')
        self.tmpdir = get_config_dir (self.config, 'tmp')

    # small helper functions
    def host_path (self, target_path):
        "Get a full host path for the given target path."
        # os.path.join drops everything before an arg starting with os.sep
        if target_path.startswith (os.sep):
            target_path = target_path[1:]
        return os.path.join (self.rootdir, target_path)

    def lock (self):
        "Lock this root for operations"
        ensure_dir (self.statedir)
        debug ("locking root")
        lockpath = os.path.join (self.statedir, 'lock')
        if os.path.exists (lockpath):
            if not self.config['force']:
                raise self.Locked
            else:
                print 'warning: overriding lock on root %s' % self.root
        try:
            lockfile = open (lockpath, 'w')
        except:
            raise self.Error, "ERROR: can't create lock file for %s !\n" % self.root
        lockfile.close ()

    def unlock (self, args = []):
        "Unlock this root for operations"
        debug ("unlocking root")
        lockpath = os.path.join (self.statedir, 'lock')
        if os.path.exists (lockpath):
            os.remove (lockpath)

    def get_state (self, which):
        "Returns None if the state file doesn't exist, and contents if it does"
        path = os.path.join (self.statedir, which)
        if os.path.exists (path):
            return open (path, 'r').readlines ()
        else:
            return None

    # default here assures that state files are never empty
    def set_state (self, which, content = ""):
        "Creates the state file and fill it with given contents."
        #FIXME: if we do want Error, what about apt.conf ?
        #"raises Error if it already existed."
        if not content: content = which
        path = os.path.join (self.statedir, which)
        #if os.path.exists (path):
        #    raise self.Error, "State file %s exists already" % path
        debug ('outputting state')
        debug (content)
        debug ('outputted state')
        open (path, 'w').write (content)

    def progress (self):
        "returns True if progress information should be shown"
        if self.config['quiet']: return False
        return True

    def stdout (self, message):
        #FIXME: check verbosity here
        "outputs this message to stdout"
        sys.stdout.write (message)
        sys.stdout.flush ()

    def mount (self):
        "mount proc into chroot"
        # FIXME: the next three lines are workaround code for when not having
        # proc
        # FIXME: rh 72's mount needs /proc/mounts file, let's give it one
        #open (os.path.join (self.rootdir, 'proc', 'mounts'), 'w').close ()
        #return
        ensure_dir (os.path.join (self.rootdir, 'proc'))
        # first umount for completeness; you never know
        self.umount ()
        debug ("mounting proc")
        file = open (os.path.join (self.statedir, 'mount'), "a+")
        command = '%s -t proc proc %s/proc' % (self.config['mount'], self.rootdir)
        debug (command)
        os.system (command)
        file.write ('%s/proc\n' % self.rootdir)
        file.close ()

    # umount all mounted paths
    def umount (self):
        if not self.get_state: return
        mountfile = os.path.join (self.statedir, 'mount')
        if not os.path.exists (mountfile): return
        debug ('cat %s | xargs %s' % (mountfile, self.config['umount']))
        os.system ('cat %s | xargs %s' % (mountfile, self.config['umount']))
        os.remove (mountfile)

    # recreate a given file listed in config['files']
    def config_recreate (self, filename):
        debug ("Recreating %s from config" % filename)
        if self.config['files'].has_key (filename):
            new = self.rootdir + filename
            if os.path.exists (new):
                return
            ensure_dir (os.path.dirname (self.rootdir + filename))
            tmpname = os.path.join ('tmp', os.path.basename (filename))
            file = open (os.path.join (self.rootdir, tmpname), 'w')
            file.write (config['files'][filename])
            file.close ()
            try:
              os.rename (os.path.join (self.rootdir, tmpname), new)
            except OSError:
              raise self.Error, "Could not create %s" % new

    def _splitarg (self, matcher, args):
        "only used by splitargspecs and splitargsrcrpms"
        "put everything from args matching matcher on result"
        "put everything from args not matching matcher on other"
        "return (result, other)"
        result = []
        other = []
        match = re.compile (matcher, re.I)
        for arg in args:
            if match.search (arg):
                result.append (arg)
            else:
                other.append (arg)
        return (other, result)

    # seek through the argument list for specfiles
    def splitargspecs (self, args):
        return self._splitarg ('\.spec$', args)

    # seek through the argument list for srpms
    def splitargsrpms (self, args):
        return self._splitarg ('\.src\.rpm$', args)

    # mach apt-get
    def aptget (self, args):
        if self.config['installer'] != 'apt-get':
            raise self.Error, "Installer is not configured to be apt-get"

        self._setup_prep ()

        self.installer (args, interactive = True)

    # mach yum
    def yum (self, args):
        if self.config['installer'] != 'yum':
            raise self.Error, "Installer is not configured to be yum"

        self._setup_prep ()

        self.installer (args, interactive = True)

    # run the installer outside the root on the root
    # FIXME: virtualize this
    # if message is specified, it will be printed, and newline-terminated
    # properly
    # if no message specified, then it's up to caller
    def installer (self, args, message = None,
                progress = False, interactive = False):
        "run apt-get (arg) or yum (arg) from outside the root on the root"
        if self.config['installer'] == "yum":
            conf = 'yum.conf'
            extras = '--installroot=' + self.rootdir + ' '
        else:
            conf = 'apt.conf'
            extras = ''

        conf = os.path.join (self.statedir, conf)
        debug ("installer: args: %r" % args)
        command = "%s -c %s %s -y %s" % (self.config[self.config['installer']], conf, extras, string.join (args))
        debug ("installer: command %s" % command)
        if DEBUG or interactive:
            retval = os.system (command)
            if retval != 0:
                raise self.Error, "Could not %s %s" % (self.config['installer'], string.join (args))
            return
            
        if message: self.stdout (message)
        if progress: self.stdout (' ...')
        (status, output) = do_with_output (command, progress)
        if message and not progress: self.stdout ('\n')
        if status != 0:
            raise self.Error, "Could not %s %s" % (self.config['installer'], string.join (args))

    def intaptget (self, args):
        if not self.get_state ('base'):
            self.setup (['base', ])
        self.mount ()
        self.installer (args, interactive = True)
        self.umount ()

    def aptcache (self, args, progress = False):
        "run apt-cache (arg) from outside the root on the root"
        conf = os.path.join (self.statedir, 'apt.conf')
        command = "%s -c %s %s" % (self.config['apt-cache'], conf, string.join (args))
        (status, output) = do_with_output (command, progress)
        if status != 0:
            raise self.Error, "Could not apt-cache %s" % string.join (args)
        print output

    def rpm (self, args, progress = False):
        "run rpm (arg) from outside the root on the root, returns output"
        command = "%s --root %s %s" % (self.config['rpm'], self.rootdir, join_single_quoted (args))
        (status, output) = do_with_output (command, progress)
        if status != 0:
            raise self.Error, "Could not rpm %s" % join_single_quoted (args)
        return output

    # main command executer:
    # - executes on host
    # - direct execution with std's connected or output captured (FIXME)
    # - show or don't show progress meter
    # FIXME: user ?
    def do (self, command, progress = True, user = ""):
        "execute given command"
        debug ("Executing %s" % command)
        if progress:
            return do_progress (command)
        else:
            (status, output) = commands.getstatusoutput (command)

        if os.WIFEXITED (status):
            retval = os.WEXITSTATUS (status)
        if (retval != 0):
            raise ReturnValue, (retval, output)
        # FIXME: we send back retvals with a raise now, do the same
        # in do_progress
        return (retval, output)

    def do_chroot (self, command, message = None, progress = False, user = "", fatal = False):
        # if you call do_chroot, then do_chroot is responsible for printing
        # \n
        "execute given command in root"
        # HACK: FIXME:
        # if the cmd already contains -c " as a sequence, then don't wrap
        # it in /bin/bash
        cmd = ""
        if string.find (command, '-c "') > -1:
            cmd = "%s %s %s" % (config['chroot'], self.rootdir, command)
        else:
            # we use double quotes to protect the commandline since
            # we use single quotes to protect the args in command
            cmd = "%s %s %s - root -c \"%s\"" % (config['chroot'],
                                                 self.rootdir,
                                                 self.config['runuser'],
                                                 command)
        # be quiet if we're asked to be so
        if (self.progress () == False): progress = False
        if message: self.stdout (message)
        if progress: self.stdout (' ...')
        (ret, output) = self.do (cmd, progress)
        if message and not progress: self.stdout ('\n')
        if (ret != 0):
            if fatal:
                # FIXME. what about unmounting ?
                sys.stderr.write ("Non-zero return value %d on executing %s\n" % (ret, cmd))
                exit (ret)
        return (ret, output)

    # check the given package list file
    # remove all packages that aren't in this list file
    def check_package_list (self, listfile):
        "check the given package list file and remove packages not listed"
        debug ("checking package list against snapshot")
        if self.config.has_key ('keep'):
            print "Not removing packages from root."
            return

        # as long as we succeed in removing packages, reducing the removal
        # count, we can loop.

        count = -1

        def get_diff (listfile):
            """
            Return tuple of two lists:
            - list of packages installed but not in the given listfile
            - list of packages in the given listfile but not installed
            """
            # get a list of installed packages
            root = self.rootdir
            cmd = "%s --root %s -qa --qf '%s'" % (self.config['rpm'], root, qfmt)
            debug ("running " + cmd)
            output = commands.getoutput (cmd)
            installed = sets.Set (output.split ("\n"))

            # get the list of packages that should be installed
            lines = open (listfile).read ()
            target = sets.Set (lines.split ("\n"))

            diff1 = installed.difference (target)
            diff2 = target.difference (installed)
            return (list (diff1), list (diff2))

        while True:
            (remove, install) = get_diff (listfile)
            if count > -1 and count == len (remove):
                # nothing changed since last loop, so bail
                raise self.Error, "Could not remove packages %s" % " ".join (remove)
            count = len (remove)
            if count == 0:
                break

            debug ("%d packages to remove" % len (remove))

            # try removing them
            try:
                self.stdout ("Reverting to build state")
                if self.progress (): self.stdout (' ...')
                self.rpm (['-ev', '--allmatches', '--nodeps'] + remove, self.progress ())
                if not self.progress (): self.stdout ('\n')
            except self.Error:
                # try harder
                (remove, install) = get_diff (listfile)
                print "WARNING: could not remove all packages, trying harder"
                self.rpm (['-ev', '--allmatches', '--nodeps', '--noscripts'] + remove,
                    self.progress ())

        # now install what's missing
        if install:
            debug("%d packages to reinstall" % len (install))
            self.installer (['-y install %s' % " ".join (install)])

        # since we may have removed stuff needed to be consistent, upgrade
        self._upgrade_packages()

        # since we may have upgraded packages from the build set, make a
        # new snapshot
        if install:
            self._make_package_snapshot('build')

        return True
        
        debug ("retval %d for command %s" % (retval, cmd))
        if retval == 0:
            # no differences
            return True

        cmd = "%s --root %s -qa --qf '%s' \
               | grep -v -f %s - \
               | tr \"\n\" \" \"" % (self.config['rpm'], root, qfmt, listfile)
        debug ("running " + cmd)
        packages = commands.getoutput (cmd)
        debug ("packages to remove: %r" % packages)
        # FIXME: how many ? print the number
        if not packages:
           return True
        sys.stdout.write ("Removing packages ...")
        try:
            self.rpm (['-ev', '--allmatches'] + string.split (packages), self.progress ())
        except self.Error:
            # try harder
            print "WARNING: could not remove all packages, trying harder"
            self.rpm (['-ev', '--allmatches', '--noscripts'] + string.split (packages), self.progress ())
        return True


    # implementation of actual externalized commands
    # actual externalized commands

    def build (self, args):
        "build from a set of spec files by packaging them up as src.rpm and "
        "passing them to rebuild"

        if not args:
            raise self.Error, 'Please supply .spec files to build'

        # check if we can build here yet
        if not self.get_state ('build'):
            self.setup (['build', ])

        self.lock ()

        srpms = [] # resulting set of srpms
        # separate options to rpmbuild from specfiles
        (options, specs) = self.splitargspecs (args)
        debug ("Build options to %s: %s" %
               (self.config['root_rpmbuild'], options))
        # check if the spec files exist and if we can parse the files necessary
        for specfile in specs:
            if not os.path.exists (specfile):
                raise self.Error, "spec %s does not exist !" % specfile
            self.stdout ("Building .src.rpm from %s\n" % os.path.basename (specfile))
            options_string = join_single_quoted (options)
            if self.config.has_key('buildopts'):
                options_string += ' %s' % self.config['buildopts']

            debug ("build: quoted options_string: %s" % options_string)
            try:
                spec = Spec (specfile, self, options_string)
            except Spec.Error, message:
                raise self.Error, message
            (n, v, r) = spec.nvr ()
            # download all referenced files (Sources, Patches)
            downloads = spec.sources ().values () + spec.patches ().values ()
            # which paths will we check for already existing files ?
            # FIXME: add a temp stuff path somewhere instead of using current
            tmppath = "%s/%s-%s-%s" % (self.tmpdir, n, v, r)
            ensure_dir (tmppath)
            # check temporary dir, current dir, and dir where specfile lives
            paths = [tmppath, '.', os.path.dirname (specfile)]
            # check if the files mentioned aren't already on-disk
            files = []
            debug ("build: files to download: %s" % downloads)
            debug ("build: paths to check: %s" % paths)
            if downloads:
                for download in downloads:
                    found = 0
                    filename = os.path.basename (download)
                    for path in paths:
                        filepath = os.path.join (path, filename)
                        if os.path.exists (filepath):
                            self.stdout ("Using %s\n" % filepath)
                            files.append (filepath)
                            found = 1
                    if not found:
                        filepath = os.path.join (tmppath, filename)
                        try:
                            urlgrab (download, filepath)
                        except:
                            raise self.Error, "Could not download %s !" % download
                        files.append (filepath)
                        self.stdout ("Using %s\n" % filepath)
    
            # copy all necessary files to /tmp in root, then chroot to mv them
            for file in files:
                debug ("Getting file %s into SOURCES" % file)
                # sanity check the given file using "file"
                # crap, this completely hangs on fedora core 0.94 :(
#                command = "file -b -i -z %s" % file
#                if file[-7:] == ".tar.gz":
#                    line = os.popen (command).readline ().rstrip ()
#                    if not line[:17] == 'application/x-tar':
#                        raise self.Error, "file doesn't think %s is a tar.gz !" % file
#       # didn't seem to work on MPlayer 0.91 tar.bz2 :(
        #if file[-8:] == ".tar.bz2":
                #    line = os.popen (command).readline ()
                #    if not line == 'application/x-tar       gnu (application/octet-stream)':
                #        raise self.Error, "file doesn't think %s is a tar.bz2 !" % file
                copy_grouped (file, os.path.join (self.rootdir, 'tmp'))
                self.do_chroot ("cd / && mv %s %s" % (
                                os.path.join ('tmp', os.path.basename (file)),
                                os.path.join ('usr', 'src', 'rpm', 'SOURCES')))

            # remove the spec if it existed, then copy
            self.do_chroot ("cd / && rm -f %s" % (
                                os.path.join ('tmp', os.path.basename (specfile))))
            copy_grouped (specfile, os.path.join (self.rootdir, 'tmp'))
            self.do_chroot ("cd / && mv %s %s" % (
                            os.path.join ('tmp', os.path.basename (specfile)),
                            os.path.join ('usr', 'src', 'rpm', 'SPECS')))
            # fix ownership on all of these
            self.do_chroot ("cd /usr/src/rpm && chown -R %s:%s *" %
                            (builduser, buildgroup))
            try:
                self.do_chroot ("cd / && %s %s -bs --nodeps %s" \
                                % (self.config['root_rpmbuild'],
                                   options_string,
                                   os.path.join ('usr', 'src', 'rpm', 'SPECS',
                                                os.path.basename (specfile))),
                                "Creating .src.rpm",
                                  True)
            except ReturnValue:
                raise self.Error, 'could not build .src.rpm'
            (n, v, r) = spec.nvr ()
            srpmname = "%s-%s-%s.src.rpm" % (n, v, r)
            debug ("resulting srpm: %s" % srpmname)
            srpmfile = os.path.join (self.rootdir, 'usr', 'src', 'rpm', 'SRPMS', srpmname)
            if not os.path.exists (srpmfile):
                raise self.Error, '%s not built' % srpmfile
            dest = os.path.join (tmppath, srpmname)
            if os.path.exists (dest):
                os.unlink (dest)
            copy_grouped (self.rootdir + '/usr/src/rpm/SRPMS/' + srpmname, dest)
            srpms.append (dest)

            # remove the built src.rpm in the target's /usr/src/rpm
            os.unlink (srpmfile)

        # ready to build them all
        print "Rebuilding generated .src.rpm's: \n- %s" % string.join (srpms, "\n- ")
        self.unlock ()
        self.rebuild (options + srpms)

    def _bruteclean (self):
        "brutishly clean out the root by using mach-helper rm -rfv"
        command = "%s -rfv %s" % (self.config['rm'], self.rootdir)
        self.umount ()
        sys.stdout.write ("Brutishly removing %s ..." % self.rootdir)
        (status, output) = do_with_output (command, True)
        if status != 0:
            raise self.Error, "Could not remove %s" % self.rootdir

    def clean (self, args = []):
        "clean out the root"
        # does the root still exist ?
        if not os.path.exists (self.rootdir) and \
            not os.path.exists (self.statedir):
            debug ("already clean, returning")
            return

        if os.path.exists (self.rootdir):
            self.umount ()

        # remove all state info
        if os.path.exists (self.statedir):
            debug ("Removing statedir %s" % self.statedir)
            os.system ("rm -rf %s" % self.statedir)

        # remove yum cache for the local packages
        packagesdirname = config['packages']['dir']
        fullpath = os.path.join ('/var', 'cache',
            'mach', packagesdirname, 'yum', "local.%s" % self.root)
        debug ('checking for removal of %s' % fullpath)
        if os.path.exists (fullpath):
            cmd = '%s -rfv %s' % (config['rm'], fullpath)
            (status, output) = do_with_output (cmd, False)

        # remove the root
        if os.path.exists (self.rootdir):

            # do a brute clean if rm is not in the root
            if not os.path.exists (self.rootdir + '/bin/rm'):
                self._bruteclean ()
                return
            # get a list of files and dirs in the root
            files = string.join (os.listdir (self.rootdir))
            self.do_chroot ('cd / && rm -rfv %s' % files,
                "Cleaning out root", True)
            try:
                os.rmdir (self.rootdir)
            except OSError:
                # assume we couldn't delete because not empty and bring out the
                # big guns
                self._bruteclean ()


    # this ensures that locally built RPMS will be used to
    # install from
    # we append the root name because config for this repo
    # will end up in a dir common to all flavors of a base dist
    def update_local_repo (self):
        debug ('update_local_repo()')
        srcs = get_sources_dict (self.config)
        root = self.config['root']
        rootdir = get_config_dir (self.config, 'root')
        if self.config['installer'] == 'yum':
            line = 'file://%s/usr/src/rpm/RPMS.mach-local' % rootdir
        else:
            line = 'rpm-dir file://%s/usr/src rpm mach-local' % rootdir
        srcs['local.%s' % root] = line
        debug ('creating sources list with dict %r' % srcs)
        create_sources_list (self.config, srcs)

        try:
            if self.config['installer'] != 'yum':
                self.installer (["update", ])
        except Root.Error:
            raise self.Error, 'could not update installer'
        debug ('update_local_repo() done')

    # chroot into root
    def chroot (self, args):
        self.mount ()
        # a minimal system might not have su/runuser, yikes
        if not os.path.exists (os.path.join (self.rootdir, 'bin',
                                             self.config['runuser'])):
            if args:
                print "No %s in root, can't execute %s" % (
                    self.config['runuser'], join_single_quoted (args))
                return
            else:
                cmd = "%s %s %s" % (self.config['chroot'], self.rootdir, join_single_quoted (args))
                print "Entering %s, type exit to leave." % self.rootdir
        elif args:
            cmd = "%s %s %s - root -c '%s'" % (self.config['chroot'],
                self.rootdir, self.config['runuser'], join_single_quoted (args))
        else:
            cmd = "%s %s %s - root" % (self.config['chroot'], self.rootdir,
                self.config['runuser'])
            print "Entering %s, type exit to leave." % self.rootdir
        debug ("running %s" % cmd)
        os.system (cmd)
        self.umount ()

    def setup (self, args):
        "Set up the root to a given target"
        target = ''
        if args:
            target = args[0]
        else:
            target = 'base' # default to build
        targets = ('prep', 'minimal', 'base', 'build')

        # see if something real given to setup
        if target not in targets:
            raise self.Error, "don't know how to set up %s" % target
        self.lock ()

        # try each of the targets in turn
        for which in targets:
            debug ("setting up target %s" % which)
            method = '_setup_' + which
            if method not in Root.__dict__.keys ():
                raise self.Error, "no _setup_%s method" % which
            Root.__dict__[method] (self)
            if target == which:
                break
        self.unlock ()

    def rebuild (self, args):
        if not args:
            raise self.Error, 'Please supply .src.rpm files to build'

        (options, paths) = self.splitargsrpms (args)
        # turn the options into a correctly quoted string, and add
        # buildopts from config, since they also affect parsing
        options_string = join_single_quoted (options)
        if self.config.has_key('buildopts'):
            options_string += ' %s' % self.config['buildopts']

        self.setup (['build', ])
        self.lock ()
        # FIXME: we don't need this anymore, do we ?
        #self.config_recreate ('/usr/bin/apt-sigchecker')
        # get pkgs and collect info
        pkgs = {}
        # we use a tmppath including login name because /tmp has the
        # sticky bit set, thus we can't remove someone else's files
        whoami = None
        try:
            whoami = os.getlogin()
        except OSError:
            import pwd
            whoami = pwd.getpwuid(os.geteuid())[0]
        targettmppath = os.path.join (os.sep, 'tmp', whoami)
        hosttmppath = self.host_path (targettmppath)
        ensure_dir (hosttmppath)
        for path in paths:
            # resolve path to basename (without dirs)
            srpmname = os.path.basename (path)
            targetnewpath = os.path.join (targettmppath, srpmname)
            hostnewpath = os.path.join (hosttmppath, srpmname)
            if os.path.exists (hostnewpath):
                os.unlink (hostnewpath)
            try:
                urlgrab (path, hostnewpath)
            except:
                raise self.Error, "Can't find %s" % path
            os.chown (hostnewpath, -1, mach_gid)
            srpm = SRPM (hostnewpath, self, options_string)

            spec = srpm.specname
            buildreqs = srpm.BuildRequires ()
            debug ("BuildRequires: %r" % (buildreqs, ))
            name = srpm.header[rpm.RPMTAG_NAME]
            pkgs[name] = {}
            pkgs[name]['path'] = targetnewpath
            pkgs[name]['buildreqs'] = buildreqs
            pkgs[name]['srpm'] = srpm
            pkgs[name]['srpmname'] = srpmname
            pkgs[name]['header'] = srpm.header
            pkgs[name]['spec'] = spec

        # now figure out build order based on buildrequires of each pkg
        if len(pkgs) > 1:
            deps = [] # list of pkg, dep pairs
            for pkg in pkgs.keys ():
                debug ("Processing %s for build order" % pkgs[pkg]['srpmname'])
                # get results from this package
                results = pkgs[pkg]['srpm'].results ()
                # figure out the build requirements and add
                # pkg -> (results) and buildreq -> (results) to chain
                for result in results:
                    if result != pkg:
                        debug ("adding %s depends on pkg %s" % (result, pkg))
                        deps.append ((result, pkg))
                    if not pkgs[pkg]['buildreqs']: continue
                    for buildreq in pkgs[pkg]['buildreqs']:
                        if (result == buildreq):
                            sys.stderr.write ("WARNING: package %s BuildRequires: itself, packaging error !" % result)
                        else:
                            debug ("adding %s depends on buildreq %s" % (result, buildreq))
                            deps.append ((result, buildreq))

            debug ("topological input: ")
            debug (deps)
            debug ("topological output: ")
            try:
                sorted = topological_sort (deps)
            except CycleError:
                sys.stderr.write ('''
    ERROR: You've hit a dependency loop in the packages you are trying to build.
           This means that the BuildRequires: of the various packages form a loop.
           You can work around this by breaking the loop in two pieces, and first
           building the one chain, then the other, then the one again based on
           the other's new packages.''')

                exit (config)
            sorted.reverse ()
            debug ("order %s" % sorted)
            # now scrub packages not up for build from order
            order = []
            for pkg in sorted:
                if pkg in pkgs.keys ():
                    order.append (pkg)
            debug ("(without packages up for build) order %s" % order)

            # now make sure all packages given are up for build
            for pkg in pkgs.keys ():
                if pkg not in order:
                    order.append (pkg)
            debug ("(with all packages up for build) order %s" % order)
        else:
            order = pkgs

        # now build
        self.mount ()

        resultdirs = [] # will contain each of the resultdir's for this build
        # loop over all package names
        for name in order:
            srpmname = pkgs[name]['srpmname']
            # remove packages if not asked to keep
            self._check_revert_build ()
            # install buildrequires if there are
            debug ("Building %s with package stuff %s" % (name, pkgs[name].keys ()))
            print "Building source rpm %s" % srpmname
            if pkgs[name].has_key ('buildreqs') and pkgs[name]['buildreqs']:
                #FIXME: we filter out packages here that can have multiple
                #versions, and expect the user to install them manually
                #this works around kernel and kernel-source buildreqs
                # command to use to find out if a buildreq is already satisfied
                debug ("Package buildrequires before filtering: %s" % pkgs[name]['buildreqs'])
                chkcmd = "rpm --root=%s -q --whatprovides '%%s'" % self.rootdir
                candidates = pkgs[name]['buildreqs']
                buildreqs = []
                for cand in candidates:
                    if cand == "kernel":
                        print "Warning: make sure you have manually installed the right kernel rpm"
                        print "         and that you are building with --keep"
                        continue
                    elif cand == "kernel-source":
                        print "Warning: make sure you have manually installed the right kernel-source rpm"
                        print "         and that you are building with --keep"
                        continue
                    #FIXME: if an older version of a buildrequire is already
                    #installed, it will get filtered even though we need the
                    #newer version
                    #elif commands.getstatusoutput(chkcmd % cand)[0]:
                    buildreqs.append (cand)
   
                debug ("BuildRequires after filtering: %s" % buildreqs)
                # only try to install if filtering left buildreqs
                if (buildreqs):
                    self.update_local_repo ()
                    try:
                        # wrap all buildreqs in quotes
                        debug ('about to install buildreqs %r' % buildreqs)
                        self.installer ([ '-y install %s' % "'" + "' '".join (buildreqs) + "'", ],
                                     "Installing BuildRequires", self.progress ())
                        # FIXME: build-dep would be nice but only works on a pkg
                        # in the rpm-src tree apparently
                        #self.installer ('build-dep -y %s' % pkgs[name]['path'], True)
                    except Root.Error:
                        raise self.Error, 'could not install buildreqs %s' % buildreqs
                        #continue
            else:
                debug ("No BuildRequires for %s." % srpmname)

            # get the name and create the place where to store results
            h = pkgs[name]['header']
            name = h[rpm.RPMTAG_NAME]
            version = h[rpm.RPMTAG_VERSION]
            release = h[rpm.RPMTAG_RELEASE]
            # if the release tag needs mangling, do it here as well
            if self.config['release']:
                debug ("Mangling release tag with %s" % self.config['release'])
                fullname = '%s-%s-%s.%s' % (name, version, release,
                                            self.config['release'])
            else:
                debug ("Not mangling release tag")
                fullname = '%s-%s-%s' % (name, version, release)
            pkgresultdir = os.path.join (self.resultdir, fullname)
            debug ("Storing result in %s" % pkgresultdir)
            ensure_dir (pkgresultdir)

            # rebuild binary rpm from the src rpm
            # install the src.rpm
            self.do_chroot ("%s -c 'rpm -Uhv %s' %s" %
                (self.config['runuser'], pkgs[name]['path'], builduser))
            specfile = pkgs[name]['spec']

            command = 'cp /usr/src/rpm/SPECS/%s /tmp/%s' % (specfile, specfile)
            self.do_chroot (command)
            # copy specfile to resultdir
            copy_grouped (os.path.join (self.rootdir, 'tmp', specfile), pkgresultdir)
            # mangle spec file if requested
            if self.config['release']:
                # first check if it doesn't already comply !
                # FIXME: since fedora.us now uses .1 as a disttag,
                # this checking breaks, so we disable it for now
                #debug ("self.rootdir: %s" % self.rootdir)
                try:
                    spec = Spec (os.path.join (self.rootdir, 'tmp', specfile), self, options_string)
                except Spec.Error, message:
                    raise self.Error, message
                release = spec.tag ("release")
                debug ("Release of spec file to mangle is %s" % release)
                #if not release.endswith ('.' + self.config['release']):
                if 1:
                    # we quote sed with ' because we quote shell -c with ""
                    # we use [:space:] and ^[:space:], most portable
                    # don't use \s, doesn't work correctly on RH9
                    command = "sed -e 's,^\\\\(Release[[:space:]]*:[[:space:]]*[^[:space:]]*\\\\)[[:space:]]*$,\\\\1.%s,i' /tmp/%s > /usr/src/rpm/SPECS/%s" % (self.config['release'], specfile, specfile)
                    debug ("mangling with %s" % command)
                    self.do_chroot (command)
                    command = "grep Release: /usr/src/rpm/SPECS/%s" % specfile
                    (ret, output) = self.do_chroot (command)
                    debug ("Mangled to: %s" % output)
                else:
                    debug ("Release already contains mangle trailer %s" % self.config['release'])

            # delete the original temp spec file
            command = 'rm /tmp/%s' % specfile
            self.do_chroot (command)

            # detect amount of CPU's on system so _smp_flags gets set
            # correctly on chroot
            if not os.environ.has_key('RPM_BUILD_NCPUS'):
               nrcpus = os.popen('/usr/bin/getconf _NPROCESSORS_ONLN').read().rstrip()
            else:
               nrcpus = os.environ['RPM_BUILD_NCPUS']

            # FIXME: before we used to check for met buildrequires using
            # the host's rpmbuild accessing the target's rpm database,
            # because the target's rpm database may be of a different version.
            # However, this means that any defines that use shell commands
            # get executed on the host, not the target, so end up giving the
            # wrong results, causing conditional BuildRequires to fail.

            # We need to revisit the topic of target rpm databases

            debug ("Checking buildreqs: %s" % command)
            command = "%s -c \"RPM_BUILD_NCPUS=%s %s " \
                      "--define '__spec_prep_pre exit 0' " \
                      "--define 'setup :' " \
                      "-bp %s /usr/src/rpm/SPECS/%s\" - %s" % (
                self.config['runuser'], nrcpus, 
                self.config['root_rpmbuild'],
                options_string.replace('"', '\\"'),
                specfile, builduser)
            try:
                (status, output) = self.do_chroot (command,
                    "Checking BuildRequires for %s" % srpmname,
                    False)
            except ReturnValue, (retval, output):
                raise self.Error, "BuildRequires not met:\n%s" % output

            # rebuild from spec inside chroot, note: using a login shell
            # in order to get a vanilla default environment
            # we use --nodeps here so that the build doesn't try to read
            # the target's database, which might be written in a different
            # version than the host's rpmbuild understands.
            # this is fine now since we check for met buildrequires before.

            command = "%s -c \"RPM_BUILD_NCPUS=%s %s --nodeps -ba %s /usr/src/rpm/SPECS/%s 2>&1\" - %s" % (
                self.config['runuser'], nrcpus, 
                self.config['root_rpmbuild'],
                options_string.replace('"', '\\"'),
                specfile, builduser)
            (status, output) = self.do_chroot (command,
                                               "Rebuilding source rpm %s" % srpmname,
                                               True)
            if (output):
                log = os.path.join (pkgresultdir, 'rpm.log')
                if os.path.exists (log):
                    os.unlink (log)
                handle = open (log, 'w')
                handle.write (output)
                handle.close ()
                os.chown (log, -1, mach_gid)
            if status != 0:
                sys.stderr.write ("ERROR: something went wrong rebuilding the .src.rpm\n")
                sys.stderr.write ("ERROR: inspect rpm build log %s/rpm.log\n" % pkgresultdir)
                self.umount ()
                self.unlock ()
                raise self.Error, "failed to rebuild SRPMs"
            # reinstall and repackage src rpm
            #sys.stdout.write ("Repackaging %s ..." % srpm)
            #do_chroot (config, '%s -Uhv /tmp/%s > /dev/null 2>&1' % (chroot_rpm, srpm))
            #do_chroot (config, "%s -c 'rpm -Uhv /tmp/%s > /dev/null 2>&1' %s" % (self.config['runuser'], srpm, builduser))
            #self.rpm ('-Uhv /tmp/%s > /dev/null 2>&1' % srpm)
            #(status, output) = do_chroot_with_output (config,
            # "%s -c '%s -bs --nodeps /usr/src/rpm/SPECS/%s 2>&1' %s" % (self.config['runuser'], self.config['root_rpmbuild'], spec, builduser),
            #    True)

            # analyze log file and move all of the rpms listed as Wrote:
            (srpm, rpms) = get_rpms_from_log (os.path.join (pkgresultdir, 'rpm.log'))
            # FIXME: install these RPMS based on a boolean ?
            # FIXME: error checks
            hostrpms = map (lambda x: self.rootdir + x, rpms)

            output = self.rpm (['-qp',  '--qf', "%%{name}=%%{epoch}:%%{version}-%%{release} "] + hostrpms)
            #(status, output) = do_chroot_with_output (config, '%s -qp --qf "%%{name}=%%{epoch}:%%{version}-%%{release} " %s' % (chroot_rpm, string.join (rpms, " ")))
            # FIXME: don't install built packages automatically, they can
            # cause conflicts and break sequence builds
            # Need to handle packages without Epochs :/
            # output = string.replace (output, '=(none):', '=');
            #sys.stdout.write ("Installing built RPMS and dependencies ...")
            #self.installer ("-y install %s" % output, True)
            print "Collecting results of build %s" % srpm
            rpms.append (srpm)

            for file in rpms:
                path = self.rootdir + file
                # file already contains starting /
                copy_grouped (path, pkgresultdir)
                # for now, we do, for multiple builds
                # self.do_chroot ('rm ' + file)

            # update internal apt/yum repo
            self.update_local_repo ()
            print "Build of %s succeeded, results in\n%s" % (fullname, pkgresultdir)
            resultdirs.append (pkgresultdir)

        # if we have rpmlint, and it was not turned off, rpmlint the packages
        if self.config['lint']:
            (status, output) = commands.getstatusoutput(
                'rpmlint %s' % os.path.join (pkgresultdir, "*.rpm"))
            if os.WEXITSTATUS(status) != 127:
                # looks like rpmlint existed, so show the output
                print output

        # now sign packages if requested
        if self.config['sign']:
            print "Signing built packages ..."
            cmd = 'rpm --addsign %s' % string.join (resultdirs, '/*.rpm ') + '/*.rpm'
            debug ("running %s" % cmd)
            os.system (cmd)

        # md5sum packages and spec file if requested
        if self.config['md5sum']:
            for pkgresultdir in resultdirs:
                md5sumpath = os.path.join (pkgresultdir, 'md5sum')
                if os.path.exists (md5sumpath):
                    os.unlink (md5sumpath)
                cmd = 'cd %s && md5sum *.rpm *.spec > md5sum' % pkgresultdir
                debug ("running %s" % cmd)
                os.system (cmd)
                os.chown (md5sumpath, -1, mach_gid)

        # clearsign md5sums if enabled and signing requested
        if self.config['sign'] and self.config['md5sum']:
            # FIXME: sadly, we cannot clearsign multiple files ?
            print "clearsigning md5sums ..."
            for resultdir in resultdirs:
                if os.path.exists (os.path.join (pkgresultdir, 'md5sum.asc')):
                    os.unlink (os.path.join (pkgresultdir, 'md5sum.asc'))
                cmd = 'cd %s && md5sum *.rpm > md5sum' % pkgresultdir
                debug ("running %s" % cmd)
                os.system (cmd)
                # FIXME: figure out the proper way to use an agent here
                cmd = 'gpg --use-agent --clearsign %s/md5sum' % pkgresultdir
                debug ("running %s" % cmd)
                os.system (cmd)

        # run success-script if requested
        if self.config['script-success'] and self.config['scripts']:
            for pkgresultdir in resultdirs:
                cmd = '%s %s' % (self.config['script-success'], pkgresultdir)
                debug ("running %s" % cmd)
                os.system (cmd)

        # now collect stuff to . if requested
        if self.config['collect']:
            for pkgresultdir in resultdirs:
                for x in glob.glob (os.path.join (pkgresultdir, '*.rpm')):
                    os.system ('mv %s %s' % (x, os.path.basename (x)))
                for x in glob.glob (os.path.join (pkgresultdir, '*')):
                    os.system ('mv %s %s.%s' % (x, os.path.basename (
                        pkgresultdir), os.path.basename (x)))

        print "Build done."
        self.unlock ()
        return True

        # sign all packages at once so we only need the passphrase once
        # FIXME: repeat if fail

        # collect all packages if requested

        self.umount ()
        self.unlock ()
        return

    def status (self, args):
        if not os.path.exists (get_config_dir (self.config, 'state')):
            print "+ %s is cleared" % self.root
            return

        lockedness = "unlocked"
        if self.get_state ('lock'): lockedness = "unlocked"
        print "+ %s: %s" % (self.root, lockedness)
        # output of du -B M is inconsistent
        command = "%s %s du / --max-depth=0 | tail -n 1 | cut -f 1" % (config['chroot'], self.rootdir)

        try:
            diskusage = "%d MB" % (string.atoi (string.strip (commands.getoutput (command))) / 1024.0)
        except ValueError: # atoi failed
            diskusage = 'Unknown'
        print "  - %s: %s" % (self.rootdir, diskusage)
        command = "du %s --max-depth=0 | tail -n 1 | cut -f 1" % (self.resultdir)
        try:
            diskusage = "%d KB" % string.atoi (string.strip (commands.getoutput (command)))
        except ValueError: # atoi failed
            diskusage = 'Unknown'
        print "  - %s: %s" % (self.resultdir, diskusage)
        if not os.path.exists (self.resultdir):
            print "    - no packages built"
            return
        for dir in os.listdir (self.resultdir):
            # check if there's a .src.rpm
            built = False
            for file in os.listdir (os.path.join (self.resultdir, dir)):
                if file[-8:] == '.src.rpm': built = True
            if not built:
                print "    - %s: build failed" % dir
                continue

            # get disk usage
            command = "du --max-depth=0 -c %s | tail -n 1 | cut -f 1" % (os.path.join (self.resultdir, dir, '*.rpm'))
            diskusage = string.strip (commands.getoutput (command))
            # get signature
            command = "rpm -qip %s | grep Signature" % os.path.join (self.resultdir, dir, '*.src.rpm')
            output = string.strip (commands.getoutput (command))
            # get last word of output
            signature = output.split ()[-1]
            signedness = None
            if signature == "(none)":
                signedness = "unsigned"
            else:
                signedness = "signed with %s" % signature

            print "    - %s: %s K, %s " % (dir, diskusage, signedness)
        print

    # private methods
    def _call_setup_hook (self, state):
        key = 'setup-%s' % state
        if self.hooks.has_key (key):
            status = os.system ("%s '%s' '%s' '%s' '%s'" %
                                (self.hooks[key], state, self.root,
                                self.rootdir, self.statedir))
            if not os.WIFEXITED (status) or os.WEXITSTATUS (status) != 0:
                raise Root.Error, "setup-%s-hook failed" % (state)

    def _setup_prep (self):
        "prepares a given root if not done yet; this ensures that a number"
        "of necessary files exist in the root"
        if self.get_state ('prep'):
            debug ("target prep already set up")
            return

        print "Preparing root"
        # rpm transaction lock directories - can be either of these
        ensure_dir (os.path.join (self.rootdir, 'var', 'lock', 'run'))
        ensure_dir (os.path.join (self.rootdir, 'var', 'lock', 'rpm'))

        # apt bits
        ensure_dir (os.path.join (self.rootdir, 'etc', 'apt'))
        ensure_dir (os.path.join (self.rootdir, 'var', 'lib', 'rpm'))
        ensure_dir (os.path.join (self.statedir, 'apt/etc/apt'))
        ensure_dir (os.path.join (self.statedir,
            'apt', 'var', 'cache', 'apt', 'archives'))
        ensure_dir (os.path.join (self.statedir,
            'apt', 'var', 'cache', 'apt', 'archives', 'partial'))
        ensure_dir (os.path.join (self.statedir,
            'apt', 'var', 'state', 'apt', 'lists', 'partial'))

        # this bit is really dirty.  But it allows me to set up an FC3 target
        # on an FC4 host, allowing runuser to be installed instead of having
        # post scripts failing
        # ideally we should also check for what the host version is, but too
        # lazy to add that now
        if self.root.startswith ('fedora-3'):
            debug ("Setting up a Fedora Core 3 root")
            if os.path.exists (os.path.join ('/usr/lib', 'libselinux-mach.so')):
                debug ('copying in custom libselinux-mach.so')
                # FIXME: what about 64 bit ?
                targetlibdir = os.path.join (self.rootdir, 'usr', 'lib')
                ensure_dir (targetlibdir)
                command = "cp -a /usr/lib/libselinux-mach* %s" % targetlibdir
                debug (command)
                os.system (command)
                
        prifile = os.path.join ('/etc', 'apt', 'rpmpriorities')
        if os.path.exists (prifile):
            command = ('install -m 664 %s %s/apt/etc/apt/rpmpriorities' % (prifile, self.statedir))
            debug (command)
            os.system (command)

        # yum bits
        ensure_dir (os.path.join (self.rootdir, 'var', 'log'))
        open (os.path.join (self.rootdir, 'var', 'log', 'yum.log'), 'w')
        ensure_dir (os.path.join (self.statedir, 'yum/yum.repos.d'))
        ensure_dir (os.path.join (self.rootdir, 'var/cache/mach/yum'))

        # generic bits
        #FIXME: os.chmod (root + '/var/lib/rpm', g+s)
        ensure_dir (os.path.join (self.rootdir, 'dev'))
        devnull = os.path.join (self.rootdir, 'dev', 'null')
        if not os.path.exists (devnull):
            cmd = '%s %s -m 666 c 1 3' % (config['mknod'], devnull)
            try:
                self.do(cmd, progress=False)
            except ReturnValue, (retval, output):
                raise Root.Error, "mach-helper could not mknod /dev/null"

        devzero = os.path.join (self.rootdir, 'dev', 'zero')
        if not os.path.exists (devzero):
            cmd = '%s %s -m 666 c 1 5' % (config['mknod'], devzero)
            try:
                self.do(cmd, progress=False)
            except ReturnValue, (retval, output):
                raise Root.Error, "mach-helper could not mknod /dev/zero"

        devrandom = os.path.join (self.rootdir, 'dev', 'random')
        if not os.path.exists (devrandom):
            cmd = '%s %s -m 666 c 1 8' % (config['mknod'], devrandom)
            try:
                self.do(cmd, progress=False)
            except ReturnValue, (retval, output):
                raise Root.Error, "mach-helper could not mknod /dev/random"

        devurandom = os.path.join (self.rootdir, 'dev', 'urandom')
        if not os.path.exists (devurandom):
            cmd = '%s %s -m 666 c 1 9' % (config['mknod'], devurandom)
            try:
                self.do(cmd, progress=False)
            except ReturnValue, (retval, output):
                raise Root.Error, "mach-helper could not mknod /dev/urandom"


        ensure_dir (os.path.join (self.rootdir, 'etc', 'rpm'))
        open (os.path.join (self.rootdir, 'etc', 'mtab'), 'w')
        open (os.path.join (self.rootdir, 'etc', 'fstab'), 'w')
        ensure_dir (os.path.join (self.rootdir, 'tmp'))
        ensure_dir (os.path.join (self.rootdir, 'var', 'tmp'))
        # check if our rpm understands --promoteepoch
        command = ('rpm --promoteepoch')
        status = os.system (command)
        epochopt = ''
        if os.WIFEXITED (status) and os.WEXITSTATUS (status) == 0:
            epochopt = 'Options { "--promoteepoch"; }'

        # create a custom apt.conf/yum.conf file

        root = self.rootdir
        state = self.statedir
        packagesdirname = self.config['packages']['dir']
        conf = '''
// apt.conf generated by mach

// clear RPM::Pre-Install-Pkgs; // this makes sure packages don\'t get checked
// Allow-Duplicated { "^kernel$"; "^kernel-"; "^alsa-kernel"; "^gpg-pubkey$" };

APT {
    Clean-Installed "false";
    Get {
        Assume-Yes "false";
        Download-Only "false";
        Show-Upgraded "true";
        Fix-Broken "false";
        Ignore-Missing "false";
        Compile "false";
        Archive-Cleanup "false";
    };
};

Acquire {
    Retries "0";
    Http {
        Pipeline-Depth "0";
    }
};

RPM {
    Ignore { };
    Hold { };
    Allow-Duplicated { "^gpg-pubkey$" };
    Source {
        Build-Command "''' + self.config['root_rpmbuild'] + ''' --rebuild";
    };
    GPG-Check "false";
    RootDir "''' + root + '''";
    ''' + epochopt + '''
    Install-Options "--root ''' + root + '''";
    Erase-Options "--root ''' + root + '''";
}
Dir {
     Etc "''' + os.path.join (state, 'apt', 'etc', 'apt') + '''";
     Cache "''' + os.path.join ('/', 'var', 'cache', 'mach', packagesdirname, 'apt') + '''";
     State "''' + os.path.join (state, 'apt', 'var', 'state', 'apt') + '''";
     Bin { scripts "/dev/null"; }; // do not execute lua scripts
}

'''
        self.set_state ("apt.conf", conf)

        # create a custom yum.conf file
        # FIXME: the list of double dots going up should be gotten
        # by calculating the depth of the file tree
        conf = '''
[main]
cachedir = /../../../../..''' + os.path.join ('/var', 'cache', 'mach', packagesdirname, 'yum') + '''
reposdir = /../../../../..''' + os.path.join (state, 'yum', 'yum.repos.d')

        # if the distro's config dict contains "excludearch", add an exclude
        # line for each arch
        if self.config.has_key('excludearch'):
            words = []
            for arch in self.config['excludearch'].split():
                words.append("*.%s" % arch)
            conf = conf + "\nexclude = %s\n" % " ".join(words)

        self.set_state ("yum.conf", conf)

        # create sources.list
        srcs = get_sources_dict (config)
        create_sources_list (config, srcs)

        # run installer update
        if self.config['installer'] != 'yum':
            self.installer (["update", ], "Updating installer",
                self.progress ())

        self._call_setup_hook ("prep")
        # write state file
        self.set_state ("prep")

    def _setup_minimal (self):
        # FIXME: right now we only create these once; shouldn't we redo this
        # every time or something ?
        debug ("Creating config files")
        for filename in self.config['files'].keys ():
            self.config_recreate (filename)

        self.mount ()
        self._install ('minimal')
        self.umount ()

    def _setup_base (self):
        self.mount ()
        # we do kernel and initscripts first, because lots of %pre
        # scripts fail otherwise
        #if not self.get_state ('base'):
        #    debug('Installing kernel and initscripts')
        #    self.installer (['-y install kernel initscripts', ],
        #        "Installing kernel initscripts", True)
        self._install ('base')
        self.umount ()

        # ensure su/runuser is installed
        # don't use os.path.join, since 'runuser' starts with /
        path = os.path.join (self.rootdir, self.config['runuser'])
        path = self.rootdir + self.config['runuser']
        debug('Checking if %s exists' % path)
        if not os.path.exists (path):
            raise self.Error ("Could not find %s in %s" % (
                self.config['runuser'], self.rootdir))

    def _check_revert_build (self):
        # see if we need to revert to the build package list
        debug('Checking if we need to revert to the build package list')
        statefile = os.path.join (self.statedir, 'build')
        self.check_package_list (statefile)

    def _write_macros(self, file):
        # write macros from configuration to given file
        for mac in self.config.get ('macros', {}).items ():
            file.write ('%%%-19s %s\n' % (mac))

    def _upgrade_packages (self):
        # make the installer upgrade everything to the latest version
        if self.config['installer'] == 'yum':
            self.installer (["-y update", ])
        else:
            self.installer (["-y update", ])
            self.installer (["-y upgrade", ])

    def _setup_build (self):
        "set up a root for rpm building"
        # first do the stuff that we want to ensure gets done every time,
        # for example when config file has changed

        # create rpm macros
        debug ("writing macros file")
        path = self.rootdir + '/tmp/macros'
        macros = open (path, 'w')
        macros.write ("%_topdir /usr/src/rpm\n")
        macros.write ("%_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm\n")
        macros.write ("%_unpackaged_files_terminate_build 1\n")
        macros.write ("%_missing_doc_files_terminate_build 1\n")
        self._write_macros (macros)
        macros.close ()
        os.chown(path, -1, mach_gid)
        self.do_chroot ('mv /tmp/macros /etc/rpm', fatal = True)

        # create sources.list; we redo this so people can reconfigure
        # their location file and have it take on the next build
        srcs = get_sources_dict (config)
        create_sources_list (config, srcs)

        # now check if we're already good
        if self.get_state ('build'):
            self._check_revert_build ()
            return
        self.mount ()
        # install packages
        self._install ('build')
        #FIXME: add mach user and group
        self.do_chroot ("echo %s:x:500:%d::/usr/src/rpm:/bin/bash >> /etc/passwd" % (builduser, mach_gid), fatal = True)
        self.do_chroot ("echo %s::%d:%s >> /etc/group" % (buildgroup, mach_gid, builduser), fatal = True)
        self.do_chroot ("mkdir -p /usr/src/rpm", fatal = True)
        self.do_chroot ("chown -R %s:%s /usr/src/rpm" % (builduser, buildgroup), fatal = True)
        self.do_chroot ("%s -c 'cp -p /etc/skel/.bashrc /usr/src/rpm/.bashrc || cp -p /etc/bashrc /usr/src/rpm/.bashrc || :' %s" % (
            self.config['runuser'], builduser), fatal = True)

        # create /boot/kernel.h with a warning
        self.do_chroot ("mkdir -p /boot", fatal = True)
        self.do_chroot ("echo '#ifndef __BOOT_KERNEL_H_' > /boot/kernel.h", fatal = True)
        self.do_chroot ("echo '#define __BOOT_KERNEL_H_' >> /boot/kernel.h", fatal = True)
        self.do_chroot ("echo '#error This is a kernel.h generated by mach, including this indicates a build error !' >> /boot/kernel.h", fatal = True)
        self.do_chroot ("echo '#endif /* __BOOT_KERNEL_H_ */' >> /boot/kernel.h", fatal = True)

        # create dir structure
        for dir in ('RPMS', 'SRPMS', 'SOURCES', 'SPECS', 'BUILD'):
            self.do_chroot ("%s -c 'mkdir -p /usr/src/rpm/%s' %s" %
                            (self.config['runuser'], dir, builduser), fatal = True)

        # this ensures that locally built RPMS are already used for apt-get
        for dir in ('RPMS', 'SRPMS'):
            self.do_chroot ('ln -sf %s /usr/src/rpm/%s.mach-local' % (dir, dir), fatal = True)
        # upgrade packages; that way our snapshot contains the latest versions
        # available and a revert to snapshot will not try to remove packages
        # that have been upgraded from the snapshot
        self._upgrade_packages ()

        self._call_setup_hook ("build")
        self._make_package_snapshot('build')
        self.umount ()

    def _make_package_snapshot (self, statename):
        self.stdout ("Making snapshot ...\n")
        # FIXME: use _with_output for this
        output = self.rpm (['-qa', '--qf', qfmt, ])
        self.set_state (statename, output)


    def _install (self, set):
        "installs the given set of packages into the root."
        "Packages in this set are defined in dist."
        debug ("_install (%s)" % set)
        if self.get_state (set): return
        debug ("Installing set of packages in %s" % set)
        try:
            rpms = self.config['packages'][set]
        except KeyError:
            raise self.Error, "Package set '%s' not defined for this distribution !" % set

        if rpms:
            # do a two stage install if necessary and gloss over errors
            # from first try
            try:
                self.installer (['-y install %s' % rpms, ],
                             "Installing package set '%s'" % set, True)
            except self.Error:
                self.installer (['-y install %s' % rpms, ],
                             "Retrying installing package set '%s'" % set, True)

        else:
            print "No packages in %s set" % rpms
        self.set_state (set)

### helper functions

# return an argument string containing all of the arguments of the given list,
# but each of them surrounded by quotes
# this is done so that ['define', 'kernel 2.4.22'] returns a string like
# 'define' 'kernel 2.4.22'
# so that using mach build --define "kernel 2.4.22" passes correct arguments
# to rpmbuild
def join_single_quoted (args):
    if not args:
        return ""
    else:
        return "'" + string.join (args, "' '") + "'"

# grab given url and store it as filename, preserving timestamps
def urlgrab (url, filename):
    print "Getting %s ..." % url
    opener = urllib.FancyURLopener ()
    try:
        (t, h) = opener.retrieve (url, filename)
    except IOError, e:
        sys.stderr.write ("Couldn't retrieve %s:\n%s\n" % (url, e))
        raise e

    d = None
    try:
        if h:
            d = time.mktime (h.getdate ("Last-Modified") or h.getdate ("Date"))
        if d:
            os.utime (filename, (d, d))
    except:
        sys.stderr.write ("Warning: time stamp not preserved for %s\n" % filename)

    if config.get ('hooks', {}).has_key ('download'):
        status = os.system ("%s '%s' '%s'" % (config['hooks']['download'], url, filename))
        if not os.WIFEXITED (status) or os.WEXITSTATUS (status) != 0:
            raise Root.Error, "Download-hook for %s failed" % url


# unlock and exit
def exit (config):
    unlock (config)
    sys.exit (1)

# run a command, opening a pty and returning fd and pid
def pty_popen (cmd):
    pid, fd = os.forkpty ()
    if pid == 0:
        os.execl ('/bin/sh', 'sh', '-c', cmd)

    else:
        return os.fdopen (fd), pid

# run a command and give a progress spinner that regularly advances
# delta gives the number of secs between each spin iteration
# jump is the number of iterations before jumping one dot further
# returns a tuple:
# - the return status of the app
# - all of the output as one big string
def do_progress (cmd, delta = 0.1, jump = 20):
    # choose a random spinner by jumping through lots of hoops
    spinkey = random.randint (0, len (config['spinner'].keys ()) - 1)
    spinner = config['spinner'][config['spinner'].keys ()[spinkey]]
    i = 0 # jump counter
    j = 0 # spinner state
    timestamp = time.time ()
    running = 1
    size = 0 # size of total output since last spinner update
    output = []
    (fdin, pid) = pty_popen (cmd)
    fcntl.fcntl (fdin, fcntl.F_SETFL, os.O_NONBLOCK) # don't block input
    sys.stdout.write (spinner[0])
    sys.stdout.flush () # otherwise we don't see nothing until first update
    while running:
        block = ""
        try:
            block = fdin.read ()
            # trash all \r\n
            block = string.replace (block, '\r\n', '\n')
            output.append (block)
        except IOError:
            pass
        size = size + len (block)
        if size > 0 and time.time () > timestamp + delta:
            timestamp = time.time ()
            size = 0
            i = i + 1
            j = (j + 1) % len (spinner)
            sys.stdout.write ('\b' + spinner[j % len (spinner)])
            if (i > jump):
                i = 0
                sys.stdout.write ('\b.%s' % spinner[j])
            sys.stdout.flush ()
        time.sleep (0.1)
        try:
            (dip, status) = os.waitpid (pid, os.WNOHANG)
        except OSError:
            running = 0

    # done
    output = string.join (output, '').split ("\n")
    retval = 0
    if os.WIFEXITED (status):
        retval = os.WEXITSTATUS (status)
    if (retval != 0):
        sys.stdout.write ('\b!\n')
        sys.stderr.write ('ERROR: %s failed.\n %s\n' % (cmd, string.join (output, "\n")))
    else:
        sys.stdout.write ('\b.\n')
    return (retval, string.join (output, '\n'))

# execute the given command; possibly with progress bar
# return (return value, output) of command
def do_with_output (command, progress = False):
    debug ("Executing %s" % command)
    if progress:
        return do_progress (command)
    else:
        (status, output) = commands.getstatusoutput (command)

    if os.WIFEXITED (status):
        retval = os.WEXITSTATUS (status)
    #if (retval != 0):
    #    sys.stdout.write ('\b!\n')
        #sys.stderr.write ('ERROR:\n %s' % string.join (output))
    #else:
    #    sys.stdout.write ('\b.\n')
    #FIXME: hack to return no output, we should change this
    if (retval != 0):
        sys.stderr.write ("ERROR: %s failed\n" % command)
    return (retval, output)

# do a topological sort on the given list of pairs
# taken from a mail on the internet and adapted
# given the list of pairs of X needed for Y, return a suggested build list

class CycleError(Exception): pass
def topological_sort (pairlist):
    numpreds = {}   # elt ->; # of predecessors
    successors = {} # elt -> list of successors
    for first, second in pairlist:
        # make sure every elt is a key in numpreds
        if not numpreds.has_key (first):
            numpreds[first] = 0
        if not numpreds.has_key (second):
            numpreds[second] = 0

        # since first < second, second gains a pred ...
        numpreds[second] = numpreds[second] + 1

        # ... and first gains a succ
        if successors.has_key (first):
            successors[first].append (second)
        else:
            successors[first] = [second]

    # suck up everything without a predecessor
    answer = filter (lambda x, numpreds=numpreds:
                         numpreds[x] == 0, numpreds.keys())

    # for everything in answer, knock down the pred count on
    # its successors; note that answer grows *in* the loop
    for x in answer:
        del numpreds[x]
        if successors.has_key (x):
            for y in successors[x]:
                numpreds[y] = numpreds[y] - 1
                if numpreds[y] == 0:
                    answer.append (y)
            # following "del" isn't needed; just makes
            # CycleError details easier to grasp
            del successors[x]

    if numpreds:
        # everything in numpreds has at least one successor ->
        # there's a cycle
        raise CycleError, (answer, numpreds, successors)
    return answer

# get all of the written rpms from the log file and return a tuple
# srpm, list of rpms
def get_rpms_from_log (log):
    rpms = []
    srpm = ''
    matchstr = re.compile ("src\.rpm$")
    output = commands.getoutput ('grep "Wrote: " %s | sort | uniq' % log)
    for line in output.split ("\n"):
        curr = string.replace (line, 'Wrote: ', '')
        if matchstr.search (curr):
            srpm = curr
        else:
            rpms.append (curr)
    return (srpm, rpms)


# return full path to directory based on the root (in config) and which sort
# of directory to get

# FIXME: doing + s is silly
def get_config_dir (config, which):
#FIXME: do error checking on key
    return config['dirs'][which + 's'] + '/' + config['root']

# make sure a dir exists
# we have to recursively create ourselves so we can change group ownership
def ensure_dir (dir):
    debug ("ensuring dir %s" % dir)
    if os.path.exists (dir):
        return

    pieces = dir.split (os.sep)
    path = os.sep
    for piece in pieces:
        path = os.path.join (path, piece)
        if not os.path.exists (path):
            debug ("creating dir %s" % path)
            try:
                os.makedirs (path)
            except OSError:
                sys.stderr.write ("Could not create %s, make sure you have permissions to do so\n" % path)
                sys.exit (1)
            debug ("changing group owner to %r" % mach_gid)
            os.chown (path, -1, mach_gid)

def copy_grouped (source, dest):
    # copy source to destination, making sure dest is owned by the mach group 
    debug ("copying '%s' to '%s'" % (source, dest))
    destpath = dest
    if os.path.isdir (dest):
        destpath = os.path.join (dest, os.path.basename (source))
    if os.path.exists (destpath):
        os.unlink (destpath)
    shutil.copy2 (source, dest)
    os.chown (destpath, -1, mach_gid)

# get the dict of sources.list lines based on the config
# based on sourceslist dict
# returns: a dict of name -> location line
def get_sources_dict (config):
    sourcesdict = {}
    root = config['root']
    for platform in config['sourceslist'][root].keys ():
        for source in config['sourceslist'][root][platform]:
            if config['installer'] == 'yum':
                sourcestype = 'yumsources'
            else:
                sourcestype = 'aptsources'
            try:
                sourcesdict[source] = config[sourcestype][platform][source]
            except KeyError:
                sys.stderr.write ("no %s key found in config['%s']['%s']\n" % (source, sourcestype, platform))
                sys.exit (1)


    debug ("sources.list: " + string.join (sourcesdict.values(), "\n"))
    return sourcesdict

# create config files for the installer giving the locations of
# repositories
def create_sources_list (config, list):
    root = get_config_dir (config, 'root')
    statedir = get_config_dir (config, 'state')
    try:
        cachedir = config['dirs']['cache']
    except KeyError:
        sys.stderr.write (
            "ERROR: mach 0.4.9 added the 'cache' key to config['dirs'],\n"
            "but it is missing from your config file.  Please add.\n")
        sys.exit (1)
        

    if config['installer'] == 'yum':
        debug ("Creating files in yum.repos.d")
        for source in list.keys():
            line = list[source]
            debug ('writing file for repo %s' % source)
            debug ('repo has URI %s' % line)
            sources = open (statedir + '/yum/yum.repos.d/%s.repo' % source, 'w')
            sources.write("[%s]\nname=%s\nbaseurl=%s\nenabled=1\n" % (source, source, line))
            # we want to set metadata_expire to 0 for local files, so that we
            # can immediately use results of local builds
            if line.startswith('file://'):
                v = yum_version ()
                debug ('local repository, and yum version %r' % (v, ))
                if v >= (2, 6, 0):
                    debug ('adding line to expire_metadata for local repo')
                    sources.write("metadata_expire=0\n")
                else:
                    debug ('yum < 2.6.0, no need to add expire_metadata')

            ensure_dir (os.path.join (root, 'var/cache/mach/yum/%s/packages' % source))
            ensure_dir (os.path.join (root, 'var/cache/mach/yum/%s/headers' % source))
            if source.startswith('local.'):
                debug ('updating local repository %s' % source)
                cmd = 'createrepo %s/usr/src/rpm/RPMS.mach-local' % root
                (status, output) = do_with_output (cmd, False)
                if status != 0:
                    raise Exception, "Could not update local yum repository"
                # at some point yum seems to not properly realize the repo is
                # updated; do the equivalent of yum clean metadata only for
                # the local repo
                packagesdirname = config['packages']['dir']
                for file in ['cachecookie', 'primary.xml.gz', 'repomd.xml']:
                    fullpath = os.path.join ('/var', 'cache',
                        'mach', packagesdirname, 'yum', source, file)
                    debug ('checking for removal of %s' % fullpath)
                    if os.path.exists (fullpath):
                        cmd = '%s %s' % (config['rm'], fullpath)
                        (status, output) = do_with_output (cmd, False)

            ensure_dir (os.path.join (root, 'var/cache/mach/yum/%s/packages' % source))
            sources.close ()
    else:
        debug ("Creating sources.list")
        sources = open (statedir + '/apt/etc/apt/sources.list', 'w')
        sources.write (string.join (list.values(), "\n"))
        sources.close ()

# lock a given root; creates a lock file in the statedir under root
def lock (config):
    statedir = get_config_dir (config, 'state')
    ensure_dir (statedir)
    lockpath = statedir + '/lock'
    if os.path.exists (lockpath):
        if not config['force']:
            sys.stderr.write ('ERROR: %s already locked !\n' % config['root'])
            return False
        else:
            print 'warning: overriding lock on root %s' % config['root']
    try:
        lockfile = open (lockpath, 'w')
    except:
        sys.stderr.write ('ERROR: can''t create lock file for %s !\n' % config['root'])
        return False
    lockfile.close ()
    return True

# unlock a given root; removes lock file in the statedir under root
def unlock (config):
    statedir = get_config_dir (config, 'state')
    lockpath = statedir + '/lock'
    if os.path.exists (lockpath):
        os.remove (lockpath)
    return True

# return the state, or nothing if state isn't there
def get_state (config, state):
    statedir = get_config_dir (config, 'state')
    if os.path.exists (statedir + '/' + state):
        return statedir + '/' + state
    return

# check if the current user is in the right group
def user_check ():
    if os.getuid () != 0:
        machgid = grp.getgrnam ('mach')[2]
        if not machgid in os.getgroups():
            sys.stderr.write ("ERROR: user is not in group mach, please add !\n")
            sys.exit (1)

# return the yum version as a tuple
def yum_version ():
    # my FC5 yum prints "Loading installonlyn" plugin as the first line.  ugh.
    output = commands.getoutput ('yum --version | tail -n 1')
    version = output.rstrip ()
    l = map (int, version.split ("."))
    return tuple (l)

# check if everything is in order to actually do stuff
def sanity_check (config):
    user_check ()

    # check if partial dir exists
    try:
        aptarchivesdir = config['dirs']['aptarchives']
    except KeyError:
        #  before 0.4.9, the key was 'archives'
        aptarchivesdir = config['dirs']['archives']

    partial = os.path.join(aptarchivesdir, 'partial')
    ensure_dir (partial)

    # check if mach-helper is suid
    if not (os.stat ('/usr/sbin/mach-helper')[0] & stat.S_ISUID):
        sys.stderr.write ("ERROR: /usr/sbin/mach-helper is not setuid !\n")
        sys.exit (1)

    # we're fine
    return True

# print out status for each of the roots set up
# FIXME: maybe also add result dirs, since we could have results for cleaned
# roots, or no results for roots that are set up
def status (config):
    statedir = config['dirs']['states']
    dirs = os.listdir (statedir)
    for dir in dirs:
        # only give status if it's a known root dir
        config['root'] = dir
        root = Root (config)
        root.status (None)

# main function
def main (config, args):
    global DEBUG # we might change it

    canonify   = 0
    cli_config = {}

    try:
        opts, args = getopt.getopt (args, 'r:hdfksmcqv',
                                    ['root=', 'help', 'debug', 'force', 'keep',
                                     'sign', 'md5sum', 'collect', 'quiet',
                                     'release=', 'no-lint',
                                     'no-scripts', 'canonify',
                                     'version'])
    except getopt.error, exc:
        sys.stderr.write ('ERROR: %s\n' % str (exc))
        sys.exit (1)

    # parse environment
    try:
        root = os.environ['MACH_ROOT']
    except:
        root = config['defaultroot']

    # parse config options
    for opt, arg in opts:
        if opt in ('-h', '--help'):
            print usage
            print help
            sys.exit (0)
        if opt in ('-v', '--version'):
            print "mach (make a chroot) 0.9.1"
            print "Written by Thomas Vander Stichele <thomas at apestaart dot org>."
            sys.exit (0)
        elif opt in ('-r', '--root'):
            root = arg
        elif opt in ('-d', '--debug'):
            DEBUG = 1
        elif opt in ('-f', '--force'):
            cli_config['force'] = 1
        elif opt in ('-k', '--keep'):
            cli_config['keep'] = 1
        elif opt in ('-s', '--sign'):
            cli_config['sign'] = 1
        elif opt in ('-m', '--md5sum'):
            cli_config['md5sum'] = 1
        elif opt in ('-c', '--collect'):
            cli_config['collect'] = 1
        elif opt in ('-q', '--quiet'):
            cli_config['quiet'] = 1
        elif opt == '--release':
            cli_config['release'] = arg
        elif opt == '--no-lint':
            cli_config['lint'] = 0
        elif opt == '--no-scripts':
            cli_config['scripts'] = 0
        elif opt == '--canonify':
            canonify = 1

    # resolve root aliases to real root names, check for duplicates
    seen_aliases = []
    for name in aliases.keys ():
        if name in seen_aliases:
            sys.stderr.write ('Root "%s" is ambiguous\n' % al)
            sys.exit (1)
        seen_aliases.append (name)
        for al in aliases[name]:
            if al in seen_aliases:
                sys.stderr.write ('Root "%s" is ambiguous\n' % al)
                sys.exit (1)
            if al == root: root = name
            seen_aliases.append (al)
    del seen_aliases

    if canonify:
        print root
        sys.exit(0)

    debug ("This is mach (make a chroot) 0.9.1")
    debug ("real root name is %s" % root)
    # pull in root-specific configuration
    # FIXME: even nicer would be if we could just do config[rootname] = ...
    # and copy that over
    try:
        config['packages'] = packages[root]
    except KeyError:
        sys.stderr.write ('No definition for packages found for %s\n' % root)
        sys.exit (1)
    try:
        config['aptsources'] = aptsources
        config['yumsources'] = yumsources
        config['sourceslist'] = sourceslist
    except KeyError:
        sys.stderr.write ('No sources information found for %s\n' % root)
        sys.exit (1)
    #try:
    #    config['aliases'] = aliases
    #except:
    #    debug ("no aliases for this root")

    if config.has_key (root):
        for key in config[root].keys ():
            # Merge dicts, assign others
            if config.has_key (key) and hasattr (config[key], 'update'):
                debug ("updating config['%s'] with %s" % (key, config[root][key]))
                config[key].update(config[root][key])
            else:
                debug ("setting config['%s'] to %s" % (key, config[root][key]))
                config[key] = config[root][key]

    for key in cli_config.keys():
        debug ("setting config['%s'] to %s from CLI" % (key, cli_config[key]))
        config[key] = cli_config[key]

    # debug output options
    debug ("root: %s" % root)

    # process options
    config['root'] = root

    # check if everything is ready to go
    sanity_check (config)

    # run command
    if not args:
        print usage
        print help
        sys.exit (1)
    debug ("main: args: %s" % args)
    command = args[0]
    debug ("main: running %s" % command)

    root = Root (config)
    output = ""
    if command in allowed_commands:
        # no '-' allowed is silly but what can we do about it ?
        # this is also a good place to intercept commands that need to
        # be interactive
        if command == "apt-get": command = "intaptget"
        if command == "apt-cache": command = "aptcache"
        if command == "yum": command = "yum"

        if command == "status":
            status (config)
            return

        if not command in Root.__dict__.keys():
            sys.stderr.write ("No %s method defined\n" % command)
            sys.exit (1)

        try:
            # we changed behaviour to pass the list of args instead of a string
            output = Root.__dict__[command] (root, args[1:])
        except Root.Locked:
            sys.stderr.write ("Root is locked.  Use -f to override lock.\n")
        except Root.Error, message:
            sys.stderr.write ("ERROR: %s\n" % message)
            root.unlock ()
            sys.exit (1)
        except ReturnValue, (retval, output):
            sys.stderr.write ("Return value: %s\n" % retval)
            sys.stderr.write (output)
            root.unlock ()
            sys.exit (retval)
        except KeyboardInterrupt:
            sys.stderr.write ("\nAborted.\n")
            root.unlock ()
            sys.exit (1)
        if output and output != True:
            print output
    else:
        sys.stderr.write ("No such command '%s'\n" % command)
    #try:
    #globals()[command](config, args)
    #except:
    #    sys.stderr.write ('mach: unsupported command %s\n' % command)
    #    sys.exit (1)

# run main program

if __name__ == '__main__':
    # run command specified or go into interpreter mode
    if len (sys.argv) > 1:
        main (config, sys.argv[1:])
    else:
        user_check ()
        print "starting mach interpreter ..."
        running = True;
        while running:
            sys.stdout.write ("> ")
            command = sys.stdin.readline ()
            main (config, string.split (command))


