diff options
Diffstat (limited to 'buildscripts/makealldists.py')
| -rw-r--r-- | buildscripts/makealldists.py | 289 |
1 files changed, 289 insertions, 0 deletions
diff --git a/buildscripts/makealldists.py b/buildscripts/makealldists.py new file mode 100644 index 0000000..762700e --- /dev/null +++ b/buildscripts/makealldists.py @@ -0,0 +1,289 @@ +#!/usr/bin/python + +from __future__ import with_statement +import subprocess +import sys +import os +import time +import tempfile +import errno +import glob +import shutil +import settings +import simples3 + +def s3bucket(): + return simples3.S3Bucket(settings.bucket, settings.id, settings.key) + +def s3cp (bucket, filename, s3name): + defaultacl="public-read" + print "putting %s to %s" % (filename, s3name) + bucket.put(s3name, open(filename, "rb").read(), acl=defaultacl) + +def pushrepo(repodir): + files=subprocess.Popen(['find', repodir, '-type', 'f'], stdout=subprocess.PIPE).communicate()[0][:-1].split('\n') + bucket=s3bucket() + olddebs=[t[0] for t in bucket.listdir(prefix='distros/') if t[0].endswith('.deb')] + newdebs=[] + for fn in files: + if len(fn) == 0: + continue + tail = fn[len(repodir):] + # Note: be very careful not to produce s3names containing + # sequences of repeated slashes: s3 doesn't treat a////b as + # equivalent to a/b. + s3name1='distros-archive/'+time.strftime('%Y%m%d')+tail + s3name2='distros'+tail + s3cp(bucket, fn, s3name1) + s3cp(bucket, fn, s3name2) + if s3name1.endswith('.deb'): + newdebs.append(s3name1) + # FIXME: we ought to clean out old debs eventually, but this will + # blow away too much if we're trying to push a subset of what's + # supposed to be available. + #[bucket.delete(deb) for deb in set(olddebs).difference(set(newdebs))] + +def cat (inh, outh): + inh.seek(0) + for line in inh: + outh.write(line) + inh.close() + +# This generates all tuples from mixed-radix counting system, essentially. +def gen(listlist): + dim=len(listlist) + a=[0 for ignore in listlist] + while True: + yield [listlist[i][a[i]] for i in range(dim)] + a[0]+=1 + for j in range(dim): + if a[j] == len(listlist[j]): + if j<dim-1: + a[j+1]+=1 + else: + return + a[j]=0 + +def dirify(string): + return (string if string[-1:] in '\/' else string+'/') +def fileify(string): + return (string if string[-1:] not in '\/' else string.rstrip('\/')) + +# WTF: os.makedirs errors if the leaf exists? +def makedirs(f): + try: + os.makedirs(f) + except OSError: # as exc: # Python >2.5 + exc=sys.exc_value + if exc.errno == errno.EEXIST: + pass + else: + raise exc + + + +# This is a fairly peculiar thing to want to do, but our build process +# creates several apt repositories for each mongo version we build on +# any given Debian/Ubutnu release. To merge repositories together, we +# must concatenate the Packages.gz files. +def merge_directories_concatenating_conflicts (target, sources): + print sources + target = dirify(target) + for source in sources: + source = dirify(source) + files = subprocess.Popen(["find", source, "-type", "f"], stdout=subprocess.PIPE).communicate()[0].split('\n') + for f in files: + if f == '': + continue + rel = f[len(source):] + o=target+rel + makedirs(os.path.dirname(o)) + with open(f) as inh: + with open(target+rel, "a") as outh: + outh.write(inh.read()) + + +def parse_mongo_version_spec(spec): + l = spec.split(':') + if len(l) == 1: + l+=['',''] + elif len(l) == 2: + l+=[''] + return l + +def logfh(distro, distro_version, arch): + prefix = "%s-%s-%s.log." % (distro, distro_version, arch) + # This is a NamedTemporaryFile mostly so that I can tail(1) them + # as we go. + return tempfile.NamedTemporaryFile("w+b", -1, prefix=prefix) + +def spawn(distro, distro_version, arch, spec, directory, opts): + argv = ["python", "makedist.py"] + opts + [ directory, distro, distro_version, arch ] + [ spec ] +# cmd = "mkdir -p %s; cd %s; touch foo.deb; echo %s %s %s %s %s | tee Packages " % ( directory, directory, directory, distro, distro_version, arch, mongo_version ) +# print cmd +# argv = ["sh", "-c", cmd] + fh = logfh(distro, distro_version, arch) + print >> fh, "Running %s" % argv + # it's often handy to be able to run these things at the shell + # manually. FIXME: this ought to be slightly less than thoroughly + # ignorant of quoting issues (as is is now). + print >> fh, " ".join(argv) + fh.flush() + proc = subprocess.Popen(argv, stdin=None, stdout=fh, stderr=fh) + return (proc, fh, distro, distro_version, arch, spec) + +def win(name, logfh, winfh): + logfh.seek(0) + print >> winfh, "=== Winner %s ===" % name + cat(logfh, winfh) + print >> winfh, "=== End winner %s ===" % name + +def lose(name, logfh, losefh): + logfh.seek(0) + print >> losefh, "=== Loser %s ===" % name + cat(logfh, losefh) + print >> losefh, "=== End loser %s ===" % name + +def wait(procs, winfh, losefh, winners, losers): + print "." + sys.stdout.flush() + try: + (pid, stat) = os.wait() + except OSError, err: + print >> sys.stderr, "This shouldn't happen." + print >> sys.stderr, err + next + if pid: + [tup] = [tup for tup in procs if tup[0].pid == pid] + (proc, logfh, distro, distro_version, arch, spec) = tup + procs.remove(tup) + name = "%s %s %s" % (distro, distro_version, arch) + if os.WIFEXITED(stat): + if os.WEXITSTATUS(stat) == 0: + win(name, logfh, winfh) + winners.append(name) + else: + lose(name, logfh, losefh) + losers.append(name) + if os.WIFSIGNALED(stat): + lose(name, logfh, losefh) + losers.append(name) + + + +def __main__(): + # FIXME: getopt & --help. + print " ".join(sys.argv) + branches = sys.argv[-1] + makedistopts = sys.argv[1:-1] + + # Output from makedist.py goes here. + outputroot=tempfile.mkdtemp() + repodir=tempfile.mkdtemp() + + print "makedist output under: %s\ncombined repo: %s\n" % (outputroot, repodir) + sys.stdout.flush() + # Add more dist/version/architecture tuples as they're supported. + dists = (("ubuntu", "10.4"), + ("ubuntu", "9.10"), + ("ubuntu", "9.4"), + ("ubuntu", "8.10"), + ("debian", "5.0"), + ("centos", "5.4"), + ("fedora", "11"), + ("fedora", "12")) + arches = ("x86", "x86_64") +# mongos = branches.split(',') + # Run a makedist for each distro/version/architecture tuple above. + winners = [] + losers = [] + winfh=tempfile.TemporaryFile() + losefh=tempfile.TemporaryFile() + procs = [] + count = 0 + for ((distro, distro_version), arch, spec) in gen([dists, arches, [branches]]): + # FIXME: now x86 fedoras on RackSpace circa 04/10. + if distro == "fedora" and arch == "x86": + continue + count+=1 + opts = makedistopts + if distro in ["debian", "ubuntu"]: + outputdir = "%s/deb/%s" % (outputroot, distro) + elif distro in ["centos", "fedora", "redhat"]: + outputdir = "%s/rpm/%s/%s/os" % (outputroot, distro, distro_version) + else: + raise Exception("unsupported distro %s" % distro) + #opts += ["--subdirs"] + + procs.append(spawn(distro, distro_version, arch, spec, outputdir, opts)) + + if len(procs) == 8: + wait(procs, winfh, losefh, winners, losers) + + while procs: + wait(procs, winfh, losefh, winners, losers) + + winfh.seek(0) + losefh.seek(0) + nwinners=len(winners) + nlosers=len(losers) + print "%d winners; %d losers" % (nwinners, nlosers) + cat(winfh, sys.stdout) + cat(losefh, sys.stdout) + print "%d winners; %d losers" % (nwinners, nlosers) + if count == nwinners + nlosers: + print "All jobs accounted for" +# return 0 + else: + print "Lost some jobs...?" + return 1 + + sys.stdout.flush() + sys.stderr.flush() + + # this is sort of ridiculous, but the outputs from rpmbuild look + # like RPM/<arch>, but the repo wants to look like + # <arch>/RPM. + for dist in os.listdir(outputroot+'/rpm'): + if dist in ["centos", "fedora", "redhat"]: + distdir="%s/rpm/%s" % (outputroot, dist) + rpmdirs = subprocess.Popen(["find", distdir, "-type", "d", "-a", "-name", "RPMS"], stdout=subprocess.PIPE).communicate()[0].split('\n')[:-1] + for rpmdir in rpmdirs: + for arch in os.listdir(rpmdir): + archdir="%s/../%s" % (rpmdir, arch) + os.mkdir(archdir) + os.rename("%s/%s" % (rpmdir, arch), "%s/RPMS" % (archdir,)) + os.rmdir(rpmdir) + + + for flavor in os.listdir(outputroot): + argv=["python", "mergerepositories.py", flavor, "%s/%s" % (outputroot, flavor), repodir] + print "running %s" % argv + print " ".join(argv) + r = subprocess.Popen(argv).wait() + if r != 0: + raise Exception("mergerepositories.py exited %d" % r) + print repodir + pushrepo(repodir) + shutil.rmtree(outputroot) + shutil.rmtree(repodir) + + return 0 + + +if __name__ == '__main__': + __main__() + + +# FIXME: this ought to be someplace else. + +# FIXME: remove this comment when the buildbot does this. After this +# program, run something that amounts to +# +# find /tmp/distros -name *.deb -or -name Packages.gz | while read f; do echo "./s3cp.py $f ${f#/tmp/}"; done +# +# where ./s3cp.py is a trivial s3 put executable in this directory. + +# merge_directories_concatenating_conflicts('/tmp/distros/debian', '/tmp/distros-20100222/debian/HEAD', '/tmp/distros-20100222/debian/r1.3.2','/tmp/distros-20100222/debian/v1.2') + +# merge_directories_concatenating_conflicts('/tmp/distros/ubuntu', '/tmp/distros-20100222/ubuntu/HEAD', '/tmp/distros-20100222/ubuntu/r1.3.2', '/tmp/distros-20100222/ubuntu/v1.2') |