summaryrefslogtreecommitdiff
path: root/buildscripts
diff options
context:
space:
mode:
authorAntonin Kral <a.kral@bobek.cz>2010-08-11 12:38:57 +0200
committerAntonin Kral <a.kral@bobek.cz>2010-08-11 12:38:57 +0200
commit7645618fd3914cb8a20561625913c20d49504a49 (patch)
tree8370f846f58f6d71165b7a0e2eda04648584ec76 /buildscripts
parent68c73c3c7608b4c87f07440dc3232801720b1168 (diff)
downloadmongodb-7645618fd3914cb8a20561625913c20d49504a49.tar.gz
Imported Upstream version 1.6.0
Diffstat (limited to 'buildscripts')
-rw-r--r--buildscripts/bb.py2
-rw-r--r--buildscripts/buildboost.bat54
-rw-r--r--buildscripts/buildboost64.bat61
-rw-r--r--buildscripts/cleanbb.py59
-rw-r--r--buildscripts/confluence_export.py29
-rw-r--r--buildscripts/distmirror.py139
-rw-r--r--buildscripts/docs.py120
-rwxr-xr-x[-rw-r--r--]buildscripts/errorcodes.py59
-rw-r--r--buildscripts/makealldists.py289
-rw-r--r--buildscripts/makedist.py645
-rw-r--r--buildscripts/mergerepositories.py194
-rwxr-xr-xbuildscripts/smoke.py522
-rw-r--r--buildscripts/utils.py80
13 files changed, 1971 insertions, 282 deletions
diff --git a/buildscripts/bb.py b/buildscripts/bb.py
index e1e36f6..1e87828 100644
--- a/buildscripts/bb.py
+++ b/buildscripts/bb.py
@@ -16,7 +16,7 @@ def checkOk():
print( "excpted version [" + m + "]" )
from subprocess import Popen, PIPE
- diff = Popen( [ "git", "diff", "origin/v1.4" ], stdout=PIPE ).communicate()[ 0 ]
+ diff = Popen( [ "git", "diff", "origin/v1.2" ], stdout=PIPE ).communicate()[ 0 ]
if len(diff) > 0:
print( diff )
raise Exception( "build bot broken?" )
diff --git a/buildscripts/buildboost.bat b/buildscripts/buildboost.bat
new file mode 100644
index 0000000..b1b8ad0
--- /dev/null
+++ b/buildscripts/buildboost.bat
@@ -0,0 +1,54 @@
+@echo off
+
+cls
+echo This script builds the boost libs that MongoDB requires on Windows.
+echo We assume boost source is in machine's \boost directory.
+echo You can get boost at www.boost.org.
+echo .
+echo Note: you will want boost v1.42 or higher with VS2010.
+echo .
+echo We assume you have bjam. To build bjam:
+echo cd tools\jam\src
+echo build.bat
+echo .
+
+cd \boost
+echo bin\bjam --version
+bin\bjam --version
+
+echo .
+echo .
+echo .
+echo About to build release libraries
+pause
+cls
+bin\bjam variant=release runtime-link=static link=static --with-filesystem --with-thread --with-date_time --with-program_options --layout=versioned threading=multi toolset=msvc
+echo .
+echo .
+echo .
+echo About to try to move libs from /boost/stage/lib to /boost/lib/
+pause
+cls
+rem bjam makes extra copies without the ver #; we kill those:
+del stage\lib\*s.lib
+move stage\lib\* lib\
+
+echo .
+echo .
+echo .
+echo About to build debug libraries
+pause
+cls
+bin\bjam variant=debug --with-filesystem --with-thread --with-date_time --with-program_options --layout=versioned threading=multi toolset=msvc
+
+echo .
+echo .
+echo .
+echo About to try to move libs from /boost/stage/lib to /boost/lib/
+pause
+cls
+rem bjam makes extra copies without the ver #; we kill those:
+del stage\lib\*-gd.lib
+move stage\lib\* lib\
+
+echo Done - try running "dir \boost\lib\"
diff --git a/buildscripts/buildboost64.bat b/buildscripts/buildboost64.bat
new file mode 100644
index 0000000..86f3e11
--- /dev/null
+++ b/buildscripts/buildboost64.bat
@@ -0,0 +1,61 @@
+@echo off
+
+rem 64 bit version
+rem address-model=64
+
+rem run
+rem bin\bjam --clean
+rem if you switch compilers etc.
+
+cls
+echo This script builds the (64 bit) boost libs that MongoDB requires on Windows.
+echo We assume boost source is in machine's \boost directory.
+echo You can get boost at www.boost.org.
+echo .
+echo Note: you will want boost v1.42 or higher with VS2010.
+echo .
+echo We assume you have bjam. To build bjam:
+echo cd tools\jam\src
+echo build.bat
+echo .
+
+cd \boost
+echo bin\bjam --version
+bin\bjam --version
+
+echo .
+echo .
+echo .
+echo About to build release libraries
+pause
+cls
+bin\bjam --build-dir=c:\temp\boost64 address-model=64 variant=release runtime-link=static link=static --with-filesystem --with-thread --with-date_time --with-program_options --layout=versioned threading=multi toolset=msvc
+echo .
+echo .
+echo .
+echo About to try to move libs from /boost/stage/lib to /boost/lib/
+pause
+cls
+rem bjam makes extra copies without the ver #; we kill those:
+del stage\lib\*s.lib
+move stage\lib\* lib\
+
+echo .
+echo .
+echo .
+echo About to build debug libraries
+pause
+cls
+bin\bjam --build-dir=c:\temp\boost64 address-model=64 variant=debug --with-filesystem --with-thread --with-date_time --with-program_options --layout=versioned threading=multi toolset=msvc
+
+echo .
+echo .
+echo .
+echo About to try to move libs from /boost/stage/lib to /boost/lib/
+pause
+cls
+rem bjam makes extra copies without the ver #; we kill those:
+del stage\lib\*-gd.lib
+move stage\lib\* lib\
+
+echo Done - try running "dir \boost\lib\"
diff --git a/buildscripts/cleanbb.py b/buildscripts/cleanbb.py
index 68a8012..261519a 100644
--- a/buildscripts/cleanbb.py
+++ b/buildscripts/cleanbb.py
@@ -3,17 +3,39 @@ import sys
import os
import utils
import time
+from optparse import OptionParser
+
+cwd = os.getcwd();
+if cwd.find("buildscripts" ) > 0 :
+ cwd = cwd.partition( "buildscripts" )[0]
+
+print( "cwd [" + cwd + "]" )
+
+def shouldKill( c ):
+ if c.find( cwd ) >= 0:
+ return True
+
+ if ( c.find( "buildbot" ) >= 0 or c.find( "slave" ) ) and c.find( "/mongo/" ) >= 0:
+ return True
+
+ return False
def killprocs( signal="" ):
- cwd = os.getcwd();
- if cwd.find("buildscripts" ) > 0 :
- cwd = cwd.partition( "buildscripts" )[0]
killed = 0
- for x in utils.getprocesslist():
+ l = utils.getprocesslist()
+ print( "num procs:" + str( len( l ) ) )
+ if len(l) == 0:
+ print( "no procs" )
+ try:
+ print( execsys( "/sbin/ifconfig -a" ) )
+ except Exception,e:
+ print( "can't get interfaces" + str( e ) )
+
+ for x in l:
x = x.lstrip()
- if x.find( cwd ) < 0:
+ if not shouldKill( x ):
continue
pid = x.partition( " " )[0]
@@ -24,20 +46,31 @@ def killprocs( signal="" ):
return killed
-def cleanup( root ):
+def cleanup( root , nokill ):
+ if nokill:
+ print "nokill requested, not killing anybody"
+ else:
+ if killprocs() > 0:
+ time.sleep(3)
+ killprocs("-9")
+
# delete all regular files, directories can stay
# NOTE: if we delete directories later, we can't delete diskfulltest
for ( dirpath , dirnames , filenames ) in os.walk( root , topdown=False ):
for x in filenames:
- os.remove( dirpath + "/" + x )
+ foo = dirpath + "/" + x
+ print( "removing: " + foo )
+ os.remove( foo )
- if killprocs() > 0:
- time.sleep(3)
- killprocs("-9")
if __name__ == "__main__":
+ parser = OptionParser(usage="read the script")
+ parser.add_option("--nokill", dest='nokill', default=False, action='store_true')
+ (options, args) = parser.parse_args()
+
root = "/data/db/"
- if len( sys.argv ) > 1:
- root = sys.argv[1]
- cleanup( root )
+ if len(args) > 0:
+ root = args[0]
+
+ cleanup( root , options.nokill )
diff --git a/buildscripts/confluence_export.py b/buildscripts/confluence_export.py
index 956605b..29cdde6 100644
--- a/buildscripts/confluence_export.py
+++ b/buildscripts/confluence_export.py
@@ -14,10 +14,15 @@ import shutil
import subprocess
import sys
import urllib2
+sys.path[0:0] = [""]
+import simples3
from suds.client import Client
-SOAP_URI = "http://mongodb.onconfluence.com/rpc/soap-axis/confluenceservice-v1?wsdl"
+import settings
+
+HTML_URI = "http://mongodb.onconfluence.com/rpc/soap-axis/confluenceservice-v1?wsdl"
+PDF_URI = "http://www.mongodb.org/rpc/soap-axis/pdfexport?wsdl"
USERNAME = "soap"
PASSWORD = "soap"
AUTH_URI = "http://www.mongodb.org/login.action?os_authType=basic"
@@ -25,12 +30,18 @@ TMP_DIR = "confluence-tmp"
TMP_FILE = "confluence-tmp.zip"
-def export_and_get_uri():
- client = Client(SOAP_URI)
+def export_html_and_get_uri():
+ client = Client(HTML_URI)
auth = client.service.login(USERNAME, PASSWORD)
return client.service.exportSpace(auth, "DOCS", "TYPE_HTML")
+def export_pdf_and_get_uri():
+ client = Client(PDF_URI)
+ auth = client.service.login(USERNAME, PASSWORD)
+ return client.service.exportSpace(auth, "DOCS")
+
+
def login_and_download(docs):
cookie_jar = cookielib.CookieJar()
cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar)
@@ -69,11 +80,21 @@ def overwrite(src, dest):
os.symlink(os.path.abspath(target), os.path.abspath(current))
+def write_to_s3(pdf):
+ s3 = simples3.S3Bucket(settings.bucket, settings.id, settings.key)
+ name = "docs/mongodb-docs-%s.pdf" % datetime.date.today()
+ s3.put(name, pdf, acl="public-read")
+
+
def main(dir):
+ # HTML
rmdir(TMP_DIR)
- extract_to_dir(login_and_download(export_and_get_uri()), TMP_DIR)
+ extract_to_dir(login_and_download(export_html_and_get_uri()), TMP_DIR)
overwrite("%s/DOCS/" % TMP_DIR, dir)
+ # PDF
+ write_to_s3(login_and_download(export_pdf_and_get_uri()).read())
+
if __name__ == "__main__":
try:
diff --git a/buildscripts/distmirror.py b/buildscripts/distmirror.py
new file mode 100644
index 0000000..1902e2a
--- /dev/null
+++ b/buildscripts/distmirror.py
@@ -0,0 +1,139 @@
+#!/usr/bin/python
+
+# Download mongodb stuff (at present builds, sources, docs, but not
+# drivers).
+
+# Usage: <progname> [directory] # directory defaults to cwd.
+
+# FIXME: this script is fairly sloppy.
+import sys
+import os
+import urllib2
+import time
+import hashlib
+import warnings
+
+written_files = []
+def get(url, filename):
+ # A little safety check.
+ if filename in written_files:
+ raise Exception('not overwriting file %s (already written in this session)' % filename)
+ else:
+ written_files.append(filename)
+ print "downloading %s to %s" % (url, filename)
+ open(filename, 'w').write(urllib2.urlopen(url).read())
+
+
+def checkmd5(md5str, filename):
+ m = hashlib.md5()
+ m.update(open(filename, 'rb').read())
+ d = m.hexdigest()
+ if d != md5str:
+ warnings.warn("md5sum mismatch for file %s: wanted %s; got %s" % (filename, md5str, d))
+
+osarches=(("osx", ("i386", "i386-tiger", "x86_64"), ("tgz", )),
+ ("linux", ("i686", "x86_64"), ("tgz", )),
+ ("win32", ("i386", "x86_64"), ("zip", )),
+ ("sunos5", ("i86pc", "x86_64"), ("tgz", )),
+ ("src", ("src", ), ("tar.gz", "zip")), )
+
+# KLUDGE: this will need constant editing.
+versions = ("1.4.2", "1.5.1", "latest")
+
+url_format = "http://downloads.mongodb.org/%s/mongodb-%s-%s.%s"
+filename_format = "mongodb-%s-%s.%s"
+
+def core_server():
+ for version in versions:
+ for (os, architectures, archives) in osarches:
+ for architecture in architectures:
+ for archive in archives:
+ osarch = os + '-' + architecture if architecture != 'src' else 'src'
+ # ugh.
+ if architecture == 'src' and version == 'latest':
+ if archive == 'tar.gz':
+ archive2 = 'tarball'
+ elif archive == 'zip':
+ archive2 == 'zipball'
+ url = "http://github.com/mongodb/mongo/"+archive2+"/master"
+ version2 = "master"
+ else:
+ version2 = version if architecture != 'src' else 'r'+version
+ url = url_format % (os, osarch, version2, archive)
+ # ugh ugh
+ md5url = url+'.md5' if architecture != 'src' else None
+ filename = filename_format % (osarch, version2, archive)
+ get(url, filename)
+ if md5url:
+ print "fetching md5 url " + md5url
+ md5str = urllib2.urlopen(md5url).read()
+ checkmd5(md5str, filename)
+
+def drivers():
+ # Drivers... FIXME: drivers.
+ driver_url_format = "http://github.com/mongodb/mongo-%s-driver/%s/%s"
+ driver_filename_format = "mongo-%s-driver-%s.%s"
+ drivers=(("python", ("1.6", "master"), ("zipball", "tarball"), None),
+ ("ruby", ("0.20", "master"), ("zipball", "tarball"), None),
+ ("c", ("v0.1", "master"), ("zipball", "tarball"), None),
+ # FIXME: PHP, Java, and Csharp also have zips and jars of
+ # precompiled relesaes.
+ ("php", ("1.0.6", "master"), ("zipball", "tarball"), None),
+ ("java", ("r1.4", "r2.0rc1", "master"), ("zipball", "tarball"), None),
+ # And Csharp is in a different github place, too.
+ ("csharp", ("0.82.2", "master"), ("zipball", "tarball"),
+ "http://github.com/samus/mongodb-%s/%s/%s"),
+ )
+
+ for (lang, releases, archives, url_format) in drivers:
+ for release in releases:
+ for archive in archives:
+ url = (url_format if url_format else driver_url_format) % (lang, archive, release)
+ if archive == 'zipball':
+ extension = 'zip'
+ elif archive == 'tarball':
+ extension = 'tgz'
+ else:
+ raise Exception('unknown archive format %s' % archive)
+ filename = driver_filename_format % (lang, release, extension)
+ get(url, filename)
+ # ugh ugh ugh
+ if lang == 'csharp' and release != 'master':
+ url = 'http://github.com/downloads/samus/mongodb-csharp/MongoDBDriver-Release-%.zip' % (release)
+ filename = 'MongoDBDriver-Release-%.zip' % (release)
+ get(url, filename)
+ if lang == 'java' and release != 'master':
+ get('http://github.com/downloads/mongodb/mongo-java-driver/mongo-%s.jar' % (release), 'mongo-%s.jar' % (release))
+ # I have no idea what's going on with the PHP zipfiles.
+ if lang == 'php' and release == '1.0.6':
+ get('http://github.com/downloads/mongodb/mongo-php-driver/mongo-1.0.6-php5.2-osx.zip', 'mongo-1.0.6-php5.2-osx.zip')
+ get('http://github.com/downloads/mongodb/mongo-php-driver/mongo-1.0.6-php5.3-osx.zip', 'mongo-1.0.6-php5.3-osx.zip')
+
+def docs():
+ # FIXME: in principle, the doc PDFs could be out of date.
+ docs_url = time.strftime("http://downloads.mongodb.org/docs/mongodb-docs-%Y-%m-%d.pdf")
+ docs_filename = time.strftime("mongodb-docs-%Y-%m-%d.pdf")
+ get(docs_url, docs_filename)
+
+def extras():
+ # Extras
+ extras = ("http://media.mongodb.org/zips.json", )
+ for extra in extras:
+ if extra.rfind('/') > -1:
+ filename = extra[extra.rfind('/')+1:]
+ else:
+ raise Exception('URL %s lacks a slash?' % extra)
+ get(extra, filename)
+
+if len(sys.argv) > 1:
+ dir=sys.argv[1]
+ os.makedirs(dir)
+ os.chdir(dir)
+
+print """NOTE: the md5sums for all the -latest tarballs are out of
+date. You will probably see warnings as this script runs. (If you
+don't, feel free to delete this note.)"""
+core_server()
+drivers()
+docs()
+extras()
diff --git a/buildscripts/docs.py b/buildscripts/docs.py
new file mode 100644
index 0000000..719b5af
--- /dev/null
+++ b/buildscripts/docs.py
@@ -0,0 +1,120 @@
+"""Build the C++ client docs and the MongoDB server docs.
+"""
+
+from __future__ import with_statement
+import os
+import shutil
+import socket
+import subprocess
+import time
+import urllib2
+
+import markdown
+
+
+def clean_dir(dir):
+ try:
+ shutil.rmtree(dir)
+ except:
+ pass
+ os.makedirs(dir)
+
+
+def convert_dir(source, dest):
+ clean_dir(dest)
+
+ for x in os.listdir(source + "/"):
+ if not x.endswith(".md"):
+ continue
+
+ with open("%s/%s" % (source, x)) as f:
+ raw = f.read()
+
+ html = markdown.markdown(raw)
+ print(x)
+
+ with open("%s/%s" % (dest, x.replace(".md", ".html")), 'w') as o:
+ o.write(html)
+
+
+def check_mongo():
+ sock = socket.socket()
+ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+ sock.settimeout(1)
+ sock.connect(("localhost", 31999))
+ sock.close()
+
+def did_mongod_start(timeout=20):
+ while timeout > 0:
+ time.sleep(1)
+ try:
+ check_mongo()
+ return True
+ except Exception,e:
+ print e
+ timeout = timeout - 1
+ return False
+
+def stop(proc):
+ try:
+ proc.terminate()
+ except AttributeError:
+ os.kill(proc.pid, 15)
+
+def commands_list(out):
+ clean_dir("dummy_data_dir")
+ with open("/dev/null") as null:
+ try:
+ p = subprocess.Popen(["./mongod", "--dbpath", "dummy_data_dir",
+ "--port", "31999", "--rest"], stdout=null, stderr=null)
+ except:
+ print "No mongod? Skipping..."
+ return
+ if not did_mongod_start():
+ print "Slow mongod? Skipping..."
+ stop(p)
+ return
+ print "Started mongod"
+
+ with open(out, "w") as f:
+ f.write("<base href='http://localhost:28017'/>")
+ f.write(urllib2.urlopen("http://localhost:32999/_commands").read())
+
+ print "Stopping mongod"
+ stop(p)
+
+def gen_cplusplus(dir):
+ clean_dir(dir)
+ clean_dir("docs/doxygen")
+
+ # Too noisy...
+ with open("/dev/null") as null:
+ subprocess.call(["doxygen", "doxygenConfig"], stdout=null, stderr=null)
+
+ os.rename("docs/doxygen/html", dir)
+
+
+def version():
+ """Get the server version from doxygenConfig.
+ """
+ with open("doxygenConfig") as f:
+ for line in f.readlines():
+ if line.startswith("PROJECT_NUMBER"):
+ return line.split("=")[1].strip()
+
+
+def main():
+ v = version()
+ print("Generating server docs in docs/html/internal/%s" % v)
+ convert_dir("docs", "docs/html/internal/%s" % v)
+ print("Generating commands list")
+ commands_list("docs/html/internal/%s/commands.html" % v)
+ shutil.rmtree("dummy_data_dir")
+ print("Generating C++ docs in docs/html/cplusplus/%s" % v)
+ gen_cplusplus("docs/html/cplusplus/%s" % v)
+
+
+if __name__ == "__main__":
+ main()
+
+
diff --git a/buildscripts/errorcodes.py b/buildscripts/errorcodes.py
index 7a7e017..d87b7ad 100644..100755
--- a/buildscripts/errorcodes.py
+++ b/buildscripts/errorcodes.py
@@ -3,23 +3,24 @@
import os
import sys
import re
+import utils
def getAllSourceFiles( arr=None , prefix="." ):
if arr is None:
arr = []
for x in os.listdir( prefix ):
- if x.startswith( "." ) or x.startswith( "pcre-" ) or x.startswith( "32bit" ) or x.startswith( "mongodb-" ):
+ if x.startswith( "." ) or x.startswith( "pcre-" ) or x.startswith( "32bit" ) or x.startswith( "mongodb-" ) or x.startswith("debian") or x.startswith( "mongo-cxx-driver" ):
continue
full = prefix + "/" + x
- if os.path.isdir( full ):
+ if os.path.isdir( full ) and not os.path.islink( full ):
getAllSourceFiles( arr , full )
else:
if full.endswith( ".cpp" ) or full.endswith( ".h" ) or full.endswith( ".c" ):
arr.append( full )
return arr
-
+
assertNames = [ "uassert" , "massert" ]
def assignErrorCodes():
@@ -43,6 +44,8 @@ def assignErrorCodes():
out.close()
+codes = []
+
def readErrorCodes( callback ):
ps = [ re.compile( "([um]asser(t|ted)) *\( *(\d+)" ) ,
re.compile( "(User|Msg)Exceptio(n)\( *(\d+)" )
@@ -52,6 +55,7 @@ def readErrorCodes( callback ):
for line in open( x ):
for p in ps:
for m in p.findall( line ):
+ codes.append( ( x , lineNum , line , m[2] ) )
callback( x , lineNum , line , m[2] )
lineNum = lineNum + 1
@@ -78,8 +82,57 @@ def checkErrorCodes():
readErrorCodes( checkDups )
return len( errors ) == 0
+def getBestMessage( err , start ):
+ err = err.partition( start )[2]
+ if not err:
+ return ""
+ err = err.partition( "\"" )[2]
+ if not err:
+ return ""
+ err = err.rpartition( "\"" )[0]
+ if not err:
+ return ""
+ return err
+
+def genErrorOutput():
+
+ g = utils.getGitVersion()
+
+ if os.path.exists( "docs/errors.md" ):
+ i = open( "docs/errors.md" , "r" )
+
+
+ out = open( "docs/errors.md" , 'w' )
+ out.write( "MongoDB Error Codes\n==========\n\n\n" )
+
+ prev = ""
+ seen = {}
+
+ codes.sort( key=lambda x: x[0]+"-"+x[3] )
+ for f,l,line,num in codes:
+ if num in seen:
+ continue
+ seen[num] = True
+
+ if f.startswith( "./" ):
+ f = f[2:]
+
+ if f != prev:
+ out.write( "\n\n" )
+ out.write( f + "\n----\n" )
+ prev = f
+
+ url = "http://github.com/mongodb/mongo/blob/" + g + "/" + f + "#L" + str(l)
+
+ out.write( "* " + str(num) + " [code](" + url + ") " + getBestMessage( line , str(num) ) + "\n" )
+
+ out.write( "\n" )
+ out.close()
+
if __name__ == "__main__":
ok = checkErrorCodes()
print( "ok:" + str( ok ) )
print( "next: " + str( getNextCode() ) )
+ if ok:
+ genErrorOutput()
diff --git a/buildscripts/makealldists.py b/buildscripts/makealldists.py
new file mode 100644
index 0000000..762700e
--- /dev/null
+++ b/buildscripts/makealldists.py
@@ -0,0 +1,289 @@
+#!/usr/bin/python
+
+from __future__ import with_statement
+import subprocess
+import sys
+import os
+import time
+import tempfile
+import errno
+import glob
+import shutil
+import settings
+import simples3
+
+def s3bucket():
+ return simples3.S3Bucket(settings.bucket, settings.id, settings.key)
+
+def s3cp (bucket, filename, s3name):
+ defaultacl="public-read"
+ print "putting %s to %s" % (filename, s3name)
+ bucket.put(s3name, open(filename, "rb").read(), acl=defaultacl)
+
+def pushrepo(repodir):
+ files=subprocess.Popen(['find', repodir, '-type', 'f'], stdout=subprocess.PIPE).communicate()[0][:-1].split('\n')
+ bucket=s3bucket()
+ olddebs=[t[0] for t in bucket.listdir(prefix='distros/') if t[0].endswith('.deb')]
+ newdebs=[]
+ for fn in files:
+ if len(fn) == 0:
+ continue
+ tail = fn[len(repodir):]
+ # Note: be very careful not to produce s3names containing
+ # sequences of repeated slashes: s3 doesn't treat a////b as
+ # equivalent to a/b.
+ s3name1='distros-archive/'+time.strftime('%Y%m%d')+tail
+ s3name2='distros'+tail
+ s3cp(bucket, fn, s3name1)
+ s3cp(bucket, fn, s3name2)
+ if s3name1.endswith('.deb'):
+ newdebs.append(s3name1)
+ # FIXME: we ought to clean out old debs eventually, but this will
+ # blow away too much if we're trying to push a subset of what's
+ # supposed to be available.
+ #[bucket.delete(deb) for deb in set(olddebs).difference(set(newdebs))]
+
+def cat (inh, outh):
+ inh.seek(0)
+ for line in inh:
+ outh.write(line)
+ inh.close()
+
+# This generates all tuples from mixed-radix counting system, essentially.
+def gen(listlist):
+ dim=len(listlist)
+ a=[0 for ignore in listlist]
+ while True:
+ yield [listlist[i][a[i]] for i in range(dim)]
+ a[0]+=1
+ for j in range(dim):
+ if a[j] == len(listlist[j]):
+ if j<dim-1:
+ a[j+1]+=1
+ else:
+ return
+ a[j]=0
+
+def dirify(string):
+ return (string if string[-1:] in '\/' else string+'/')
+def fileify(string):
+ return (string if string[-1:] not in '\/' else string.rstrip('\/'))
+
+# WTF: os.makedirs errors if the leaf exists?
+def makedirs(f):
+ try:
+ os.makedirs(f)
+ except OSError: # as exc: # Python >2.5
+ exc=sys.exc_value
+ if exc.errno == errno.EEXIST:
+ pass
+ else:
+ raise exc
+
+
+
+# This is a fairly peculiar thing to want to do, but our build process
+# creates several apt repositories for each mongo version we build on
+# any given Debian/Ubutnu release. To merge repositories together, we
+# must concatenate the Packages.gz files.
+def merge_directories_concatenating_conflicts (target, sources):
+ print sources
+ target = dirify(target)
+ for source in sources:
+ source = dirify(source)
+ files = subprocess.Popen(["find", source, "-type", "f"], stdout=subprocess.PIPE).communicate()[0].split('\n')
+ for f in files:
+ if f == '':
+ continue
+ rel = f[len(source):]
+ o=target+rel
+ makedirs(os.path.dirname(o))
+ with open(f) as inh:
+ with open(target+rel, "a") as outh:
+ outh.write(inh.read())
+
+
+def parse_mongo_version_spec(spec):
+ l = spec.split(':')
+ if len(l) == 1:
+ l+=['','']
+ elif len(l) == 2:
+ l+=['']
+ return l
+
+def logfh(distro, distro_version, arch):
+ prefix = "%s-%s-%s.log." % (distro, distro_version, arch)
+ # This is a NamedTemporaryFile mostly so that I can tail(1) them
+ # as we go.
+ return tempfile.NamedTemporaryFile("w+b", -1, prefix=prefix)
+
+def spawn(distro, distro_version, arch, spec, directory, opts):
+ argv = ["python", "makedist.py"] + opts + [ directory, distro, distro_version, arch ] + [ spec ]
+# cmd = "mkdir -p %s; cd %s; touch foo.deb; echo %s %s %s %s %s | tee Packages " % ( directory, directory, directory, distro, distro_version, arch, mongo_version )
+# print cmd
+# argv = ["sh", "-c", cmd]
+ fh = logfh(distro, distro_version, arch)
+ print >> fh, "Running %s" % argv
+ # it's often handy to be able to run these things at the shell
+ # manually. FIXME: this ought to be slightly less than thoroughly
+ # ignorant of quoting issues (as is is now).
+ print >> fh, " ".join(argv)
+ fh.flush()
+ proc = subprocess.Popen(argv, stdin=None, stdout=fh, stderr=fh)
+ return (proc, fh, distro, distro_version, arch, spec)
+
+def win(name, logfh, winfh):
+ logfh.seek(0)
+ print >> winfh, "=== Winner %s ===" % name
+ cat(logfh, winfh)
+ print >> winfh, "=== End winner %s ===" % name
+
+def lose(name, logfh, losefh):
+ logfh.seek(0)
+ print >> losefh, "=== Loser %s ===" % name
+ cat(logfh, losefh)
+ print >> losefh, "=== End loser %s ===" % name
+
+def wait(procs, winfh, losefh, winners, losers):
+ print "."
+ sys.stdout.flush()
+ try:
+ (pid, stat) = os.wait()
+ except OSError, err:
+ print >> sys.stderr, "This shouldn't happen."
+ print >> sys.stderr, err
+ next
+ if pid:
+ [tup] = [tup for tup in procs if tup[0].pid == pid]
+ (proc, logfh, distro, distro_version, arch, spec) = tup
+ procs.remove(tup)
+ name = "%s %s %s" % (distro, distro_version, arch)
+ if os.WIFEXITED(stat):
+ if os.WEXITSTATUS(stat) == 0:
+ win(name, logfh, winfh)
+ winners.append(name)
+ else:
+ lose(name, logfh, losefh)
+ losers.append(name)
+ if os.WIFSIGNALED(stat):
+ lose(name, logfh, losefh)
+ losers.append(name)
+
+
+
+def __main__():
+ # FIXME: getopt & --help.
+ print " ".join(sys.argv)
+ branches = sys.argv[-1]
+ makedistopts = sys.argv[1:-1]
+
+ # Output from makedist.py goes here.
+ outputroot=tempfile.mkdtemp()
+ repodir=tempfile.mkdtemp()
+
+ print "makedist output under: %s\ncombined repo: %s\n" % (outputroot, repodir)
+ sys.stdout.flush()
+ # Add more dist/version/architecture tuples as they're supported.
+ dists = (("ubuntu", "10.4"),
+ ("ubuntu", "9.10"),
+ ("ubuntu", "9.4"),
+ ("ubuntu", "8.10"),
+ ("debian", "5.0"),
+ ("centos", "5.4"),
+ ("fedora", "11"),
+ ("fedora", "12"))
+ arches = ("x86", "x86_64")
+# mongos = branches.split(',')
+ # Run a makedist for each distro/version/architecture tuple above.
+ winners = []
+ losers = []
+ winfh=tempfile.TemporaryFile()
+ losefh=tempfile.TemporaryFile()
+ procs = []
+ count = 0
+ for ((distro, distro_version), arch, spec) in gen([dists, arches, [branches]]):
+ # FIXME: now x86 fedoras on RackSpace circa 04/10.
+ if distro == "fedora" and arch == "x86":
+ continue
+ count+=1
+ opts = makedistopts
+ if distro in ["debian", "ubuntu"]:
+ outputdir = "%s/deb/%s" % (outputroot, distro)
+ elif distro in ["centos", "fedora", "redhat"]:
+ outputdir = "%s/rpm/%s/%s/os" % (outputroot, distro, distro_version)
+ else:
+ raise Exception("unsupported distro %s" % distro)
+ #opts += ["--subdirs"]
+
+ procs.append(spawn(distro, distro_version, arch, spec, outputdir, opts))
+
+ if len(procs) == 8:
+ wait(procs, winfh, losefh, winners, losers)
+
+ while procs:
+ wait(procs, winfh, losefh, winners, losers)
+
+ winfh.seek(0)
+ losefh.seek(0)
+ nwinners=len(winners)
+ nlosers=len(losers)
+ print "%d winners; %d losers" % (nwinners, nlosers)
+ cat(winfh, sys.stdout)
+ cat(losefh, sys.stdout)
+ print "%d winners; %d losers" % (nwinners, nlosers)
+ if count == nwinners + nlosers:
+ print "All jobs accounted for"
+# return 0
+ else:
+ print "Lost some jobs...?"
+ return 1
+
+ sys.stdout.flush()
+ sys.stderr.flush()
+
+ # this is sort of ridiculous, but the outputs from rpmbuild look
+ # like RPM/<arch>, but the repo wants to look like
+ # <arch>/RPM.
+ for dist in os.listdir(outputroot+'/rpm'):
+ if dist in ["centos", "fedora", "redhat"]:
+ distdir="%s/rpm/%s" % (outputroot, dist)
+ rpmdirs = subprocess.Popen(["find", distdir, "-type", "d", "-a", "-name", "RPMS"], stdout=subprocess.PIPE).communicate()[0].split('\n')[:-1]
+ for rpmdir in rpmdirs:
+ for arch in os.listdir(rpmdir):
+ archdir="%s/../%s" % (rpmdir, arch)
+ os.mkdir(archdir)
+ os.rename("%s/%s" % (rpmdir, arch), "%s/RPMS" % (archdir,))
+ os.rmdir(rpmdir)
+
+
+ for flavor in os.listdir(outputroot):
+ argv=["python", "mergerepositories.py", flavor, "%s/%s" % (outputroot, flavor), repodir]
+ print "running %s" % argv
+ print " ".join(argv)
+ r = subprocess.Popen(argv).wait()
+ if r != 0:
+ raise Exception("mergerepositories.py exited %d" % r)
+ print repodir
+ pushrepo(repodir)
+ shutil.rmtree(outputroot)
+ shutil.rmtree(repodir)
+
+ return 0
+
+
+if __name__ == '__main__':
+ __main__()
+
+
+# FIXME: this ought to be someplace else.
+
+# FIXME: remove this comment when the buildbot does this. After this
+# program, run something that amounts to
+#
+# find /tmp/distros -name *.deb -or -name Packages.gz | while read f; do echo "./s3cp.py $f ${f#/tmp/}"; done
+#
+# where ./s3cp.py is a trivial s3 put executable in this directory.
+
+# merge_directories_concatenating_conflicts('/tmp/distros/debian', '/tmp/distros-20100222/debian/HEAD', '/tmp/distros-20100222/debian/r1.3.2','/tmp/distros-20100222/debian/v1.2')
+
+# merge_directories_concatenating_conflicts('/tmp/distros/ubuntu', '/tmp/distros-20100222/ubuntu/HEAD', '/tmp/distros-20100222/ubuntu/r1.3.2', '/tmp/distros-20100222/ubuntu/v1.2')
diff --git a/buildscripts/makedist.py b/buildscripts/makedist.py
index 35383b9..1928b76 100644
--- a/buildscripts/makedist.py
+++ b/buildscripts/makedist.py
@@ -1,23 +1,13 @@
#!/usr/bin/env python
-# makedist.py: make a distro package (on an EC2 instance)
+# makedist.py: make a distro package (on an EC2 (or sometimes
+# RackSpace) instance)
# For ease of use, put a file called settings.py someplace in your
# sys.path, containing something like the following:
# makedist = {
-# # ec2-api-tools needs the following two set in the process
-# # environment.
-# "EC2_HOME": "/path/to/ec2-api-tools",
-# # The EC2 tools won't run at all unless this variable is set to a directory
-# # relative to which a "bin/java" exists.
-# "JAVA_HOME" : "/usr",
-# # All the ec2-api-tools take these two as arguments.
-# # Alternatively, you can set the environment variables EC2_PRIVATE_KEY and EC2_CERT
-# # respectively, leave these two out of settings.py, and let the ec2 tools default.
-# "ec2_pkey": "/path/to/pk-file.pem"
-# "ec2_cert" : "/path/to/cert-file.pem"
-# # This gets supplied to ec2-run-instances to rig up an ssh key for
+# # This gets supplied to EC2 to rig up an ssh key for
# # the remote user.
# "ec2_sshkey" : "key-id",
# # And so we need to tell our ssh processes where to find the
@@ -54,6 +44,14 @@ import socket
import time
import os.path
import tempfile
+import string
+import settings
+
+from libcloud.types import Provider
+from libcloud.providers import get_driver
+from libcloud.drivers.ec2 import EC2NodeDriver, NodeImage
+from libcloud.base import Node, NodeImage, NodeSize, NodeState
+from libcloud.ssh import ParamikoSSHClient
# For the moment, we don't handle any of the errors we raise, so it
# suffices to have a simple subclass of Exception that just
@@ -141,139 +139,125 @@ class EC2InstanceConfigurator(BaseConfigurator):
(("centos", "5.4", "x86_64"), "ami-ccb35ea5"),
(("fedora", "8", "x86_64"), "ami-2547a34c"),
(("fedora", "8", "x86"), "ami-5647a33f"))),
+ ("rackspace_imgname",
+ ((("fedora", "11", "x86_64"), "Fedora 11"),
+ (("fedora", "12", "x86_64"), "Fedora 12"),
+ (("fedora", "13", "x86_64"), "Fedora 13"))),
("ec2_mtype",
((("*", "*", "x86"), "m1.small"),
(("*", "*", "x86_64"), "m1.large"))),
]
+class nodeWrapper(object):
+ def __init__(self, configurator, **kwargs):
+ self.terminate = False if "no_terminate" in kwargs else True
+ self.use_internal_name = False
+
+ def getHostname(self):
+ internal_name=self.node.private_ip[0]
+ public_name=self.node.public_ip[0]
+ if not (internal_name or external_name):
+ raise Exception('host has no name?')
+ if self.use_internal_name:
+ # FIXME: by inspection, it seems this is sometimes the
+ # empty string. Dunno if that's EC2 or libcloud being
+ # stupid, but it's not good.
+ if internal_name:
+ return internal_name
+ else:
+ return public_name
+ else:
+ return public_name
+
+ def initwait(self):
+ print "waiting for node to spin up"
+ # Wait for EC2 to tell us the node is running.
+ while 1:
+ n=None
+ # EC2 sometimes takes a while to report a node.
+ for i in range(6):
+ nodes = [n for n in self.list_nodes() if (n.id==self.node.id)]
+ if len(nodes)>0:
+ n=nodes[0]
+ break
+ else:
+ time.sleep(10)
+ if not n:
+ raise Exception("couldn't find node with id %s" % self.node.id)
+ if n.state == NodeState.PENDING:
+ time.sleep(10)
+ else:
+ self.node = n
+ break
+ print "ok"
+ # Now wait for the node's sshd to be accepting connections.
+ print "waiting for ssh"
+ sshwait = True
+ if sshwait == False:
+ return
+ while sshwait:
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ try:
+ try:
+ s.connect((self.node.public_ip[0], 22))
+ sshwait = False
+ print "connected on port 22 (ssh)"
+ time.sleep(15) # arbitrary timeout, in case the
+ # remote sshd is slow.
+ except socket.error, err:
+ pass
+ finally:
+ s.close()
+ time.sleep(3) # arbitrary timeout
+ print "ok"
+
+ def __enter__(self):
+ self.start()
+ # Note: we don't do an initwait() in __enter__ because if an
+ # exception is raised during __enter__, __exit__ doesn't get
+ # run (and by inspection RackSpace doesn't let you kill a node
+ # that hasn't finished booting yet).
+ return self
+
+ def __exit__(self, type, value, traceback):
+ self.stop()
-class EC2Instance (object):
+ def stop(self):
+ if self.terminate:
+ print "Destroying node %s" % self.node.id
+ self.node.destroy()
+ else:
+ print "Not terminating EC2 instance %s." % self.node.id
+
+ def setup(self):
+ pass
+
+class EC2Instance (nodeWrapper):
def __init__(self, configurator, **kwargs):
+ super(EC2Instance, self).__init__(configurator, **kwargs)
# Stuff we need to start an instance: AMI name, key and cert
# files. AMI and mtype default to configuration in this file,
# but can be overridden.
self.ec2_ami = configurator.findOrDefault(kwargs, "ec2_ami")
self.ec2_mtype = configurator.findOrDefault(kwargs, "ec2_mtype")
-
self.use_internal_name = True if "use_internal_name" in kwargs else False
-
- # Authentication stuff defaults according to the conventions
- # of the ec2-api-tools.
- self.ec2_cert=kwargs["ec2_cert"]
- self.ec2_pkey=kwargs["ec2_pkey"]
self.ec2_sshkey=kwargs["ec2_sshkey"]
# FIXME: this needs to be a commandline option
self.ec2_groups = ["default", "buildbot-slave", "dist-slave"]
- self.terminate = False if "no_terminate" in kwargs else True
- def parsedesc (self, hdl):
- line1=hdl.readline()
- splitline1=line1.split()
- (_, reservation, unknown1, groupstr) = splitline1[:4]
- groups = groupstr.split(',')
- self.ec2_reservation = reservation
- self.ec2_unknown1 = unknown1
- self.ec2_groups = groups
- # I haven't seen more than 4 data fields in one of these
- # descriptions, but what do I know?
- if len(splitline1)>4:
- print >> sys.stderr, "more than 4 fields in description line 1\n%s\n" % line1
- self.ec2_extras1 = splitline1[4:]
- line2=hdl.readline()
- splitline2=line2.split()
- # The jerks make it tricky to parse line 2: the fields are
- # dependent on the instance's state.
- (_, instance, ami, status_or_hostname) = splitline2[:4]
- self.ec2_instance = instance
- if ami != self.ec2_ami:
- print >> sys.stderr, "warning: AMI in description isn't AMI we invoked\nwe started %s, but got\n%s", (self.ec2_ami, line2)
- # FIXME: are there other non-running statuses?
- if status_or_hostname in ["pending", "terminated"]:
- self.ec2_status = status_or_hostname
- self.ec2_running = False
- index = 4
- self.ec2_storage = splitline2[index+8]
- else:
- self.ec2_running = True
- index = 6
- self.ec2_status = splitline2[5]
- self.ec2_external_hostname = splitline2[3]
- self.ec2_internal_hostname = splitline2[4]
- self.ec2_external_ipaddr = splitline2[index+8]
- self.ec2_internal_ipaddr = splitline2[index+9]
- self.ec2_storage = splitline2[index+10]
- (sshkey, unknown2, mtype, starttime, zone, unknown3, unknown4, monitoring) = splitline2[index:index+8]
- # FIXME: potential disagreement with the supplied sshkey?
- self.ec2_sshkey = sshkey
- self.ec2_unknown2 = unknown2
- # FIXME: potential disagreement with the supplied mtype?
- self.ec2_mtype = mtype
- self.ec2_starttime = starttime
- self.ec2_zone = zone
- self.ec2_unknown3 = unknown3
- self.ec2_unknown4 = unknown4
- self.ec2_monitoring = monitoring
def start(self):
"Fire up a fresh EC2 instance."
- groups = reduce(lambda x, y : x+y, [["-g", i] for i in self.ec2_groups], [])
- argv = ["ec2-run-instances",
- self.ec2_ami, "-K", self.ec2_pkey, "-C", self.ec2_cert,
- "-k", self.ec2_sshkey, "-t", self.ec2_mtype] + groups
- self.ec2_running = False
- print "running %s" % argv
- proc = subprocess.Popen(argv, stdout=subprocess.PIPE)
- try:
- self.parsedesc(proc.stdout)
- if self.ec2_instance == "":
- raise SimpleError("instance id is empty")
- else:
- print "Instance id: %s" % self.ec2_instance
- finally:
- r = proc.wait()
- if r != 0:
- raise SimpleError("ec2-run-instances exited %d", r)
-
- def initwait(self):
- # poll the instance description until we get a hostname.
- # Note: it seems there can be a time interval after
- # ec2-run-instance finishes during which EC2 will tell us that
- # the instance ID doesn't exist. This is sort of bad.
- state = "pending"
- numtries = 0
- giveup = 5
-
- while not self.ec2_running:
- time.sleep(15) # arbitrary
- argv = ["ec2-describe-instances", "-K", self.ec2_pkey, "-C", self.ec2_cert, self.ec2_instance]
- proc = subprocess.Popen(argv, stdout=subprocess.PIPE)
- try:
- self.parsedesc(proc.stdout)
- except Exception, e:
- r = proc.wait()
- if r < giveup:
- print sys.stderr, str(e)
- continue
- else:
- raise SimpleError("ec2-describe-instances exited %d", r)
- numtries+=1
-
- def stop(self):
- if self.terminate:
- LocalHost.runLocally(["ec2-terminate-instances", "-K", self.ec2_pkey, "-C", self.ec2_cert, self.ec2_instance])
- else:
- print "Not terminating EC2 instance %s." % self.ec2_instance
-
- def __enter__(self):
- self.start()
- return self
-
- def __exit__(self, type, value, traceback):
- self.stop()
-
- def getHostname(self):
- return self.ec2_internal_hostname if self.use_internal_name else self.ec2_external_hostname
+ EC2 = get_driver(Provider.EC2)
+ self.driver = EC2NodeDriver(settings.id, settings.key)
+ image = NodeImage(self.ec2_ami, self.ec2_ami, EC2)
+ size = NodeSize(self.ec2_mtype, self.ec2_mtype, None, None, None, None, EC2)
+ self.node = self.driver.create_node(image=image, name=self.ec2_ami, size=size, keyname=self.ec2_sshkey, securitygroup=self.ec2_groups)
+ print "Created node %s" % self.node.id
+
+ def list_nodes(self):
+ return self.driver.list_nodes()
class SshConnectionConfigurator (BaseConfigurator):
def __init__(self, **kwargs):
@@ -287,6 +271,7 @@ class SshConnectionConfigurator (BaseConfigurator):
(("ubuntu", "9.4", "*"), "root"),
(("ubuntu", "8.10", "*"), "root"),
(("ubuntu", "8.4", "*"), "ubuntu"),
+ (("fedora", "*", "*"), "root"),
(("centos", "*", "*"), "root"))),
]
@@ -300,28 +285,7 @@ class SshConnection (object):
# Gets set to False when we think we can ssh in.
self.sshwait = True
- def sshWait(self):
- "Poll until somebody's listening on port 22"
-
- if self.sshwait == False:
- return
- while self.sshwait:
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- try:
- try:
- s.connect((self.ssh_host, 22))
- self.sshwait = False
- print "connected on port 22 (ssh)"
- time.sleep(15) # arbitrary timeout, in case the
- # remote sshd is slow.
- except socket.error, err:
- pass
- finally:
- s.close()
- time.sleep(3) # arbitrary timeout
-
def initSsh(self):
- self.sshWait()
ctlpath="/tmp/ec2-ssh-%s-%s-%s" % (self.ssh_host, self.ssh_login, os.getpid())
argv = ["ssh", "-o", "StrictHostKeyChecking no",
"-M", "-o", "ControlPath %s" % ctlpath,
@@ -349,7 +313,6 @@ class SshConnection (object):
self.ssh_host] + argv)
def sendFiles(self, files):
- self.sshWait()
for (localfile, remotefile) in files:
LocalHost.runLocally(["scp", "-o", "StrictHostKeyChecking no",
"-o", "ControlMaster auto",
@@ -360,8 +323,6 @@ class SshConnection (object):
("" if remotefile is None else remotefile) ])
def recvFiles(self, files):
- self.sshWait()
- print files
for (remotefile, localfile) in files:
LocalHost.runLocally(["scp", "-o", "StrictHostKeyChecking no",
"-o", "ControlMaster auto",
@@ -402,7 +363,8 @@ s/^Package:.*mongodb/Package: {pkg_name}{pkg_name_suffix}\\
Conflicts: {pkg_name_conflicts}/' debian/control; ) || exit 1
( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i 's|$(CURDIR)/debian/mongodb/|$(CURDIR)/debian/{pkg_name}{pkg_name_suffix}/|g' debian/rules) || exit 1
( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i 's|debian/mongodb.manpages|debian/{pkg_name}{pkg_name_suffix}.manpages|g' debian/rules) || exit 1
-( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i '/^Name:/s/.*/Name: {pkg_name}{pkg_name_suffix}/; /^Version:/s/.*/Version: {pkg_version}/;' rpm/mongo.spec )
+( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i '/^Name:/s/.*/Name: {pkg_name}{pkg_name_suffix}\\
+Conflicts: {pkg_name_conflicts}/; /^Version:/s/.*/Version: {pkg_version}/; /Requires.*mongo/s/mongo/{pkg_name}{pkg_name_suffix}/;' rpm/mongo.spec )
# Debian systems require some ridiculous workarounds to get an init
# script at /etc/init.d/mongodb when the packge name isn't the init
# script name. Note: dh_installinit --name won't work, because that
@@ -412,6 +374,22 @@ Conflicts: {pkg_name_conflicts}/' debian/control; ) || exit 1
ln debian/init.d debian/{pkg_name}{pkg_name_suffix}.mongodb.init &&
ln debian/mongodb.upstart debian/{pkg_name}{pkg_name_suffix}.mongodb.upstart &&
sed -i 's/dh_installinit/dh_installinit --name=mongodb/' debian/rules) || exit 1
+( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && cat debian/rules)
+( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && cat rpm/mongo.spec)
+"""
+
+ # If we're just packaging up nightlies, do this:
+ nightly_build_mangle_files="""
+( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i '/scons[[:space:]]*$/d; s^scons.*install^mkdir -p debian/{pkg_name}{pkg_name_suffix} \&\& wget http://downloads.mongodb.org/linux/mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& tar xzvf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& find `tar tzf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz | sed "s|/.*||" | sort -u | head -n1` -mindepth 1 -maxdepth 1 -type d | xargs -n1 -IARG mv -v ARG debian/{pkg_name}{pkg_name_suffix}/usr \&\& (rm debian/{pkg_name}{pkg_name_suffix}/usr/bin/mongosniff || true)^' debian/rules)
+( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i 's/^BuildRequires:.*//; s/scons.*\ -c//; s/scons.*\ all//; s^scons.*install^(mkdir -p $RPM_BUILD_ROOT/usr ; cd /tmp \&\& curl http://downloads.mongodb.org/linux/mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz > mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& tar xzvf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz \&\& find `tar tzf mongodb-linux-{mongo_arch}-{mongo_pub_version}.tgz | sed "s|/.*||" | sort -u | head -n1` -mindepth 1 -maxdepth 1 -type d | xargs -n1 -IARG cp -pRv ARG $RPM_BUILD_ROOT/usr \&\& (rm -r $RPM_BUILD_ROOT/usr/bin/mongosniff $RPM_BUILD_ROOT/usr/lib64/libmongoclient.a $RPM_BUILD_ROOT/usr/lib/libmongoclient.a $RPM_BUILD_ROOT/usr/include/mongo || true))^' rpm/mongo.spec)
+# Upstream nightlies no longer contain libmongoclient.
+( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i '/%package devel/{{N;N;d;}}; /%description devel/{{N;N;N;N;N;d;}}; /%files devel/{{N;N;N;d;}};' rpm/mongo.spec )
+( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && cat debian/rules)
+( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && cat rpm/mongo.spec)
+"""
+#$RPM_BUILD_ROOT/usr/lib/libmongoclient.a $RPM_BUILD_ROOT/usr/lib64/libmongoclient.a
+ mangle_files_for_new_deb_xulrunner_commands = """
+( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}" && sed -i 's/xulrunner-dev/xulrunner-1.9.2-dev/g' debian/control )
"""
mangle_files_for_ancient_redhat_commands = """
@@ -432,8 +410,10 @@ mkdir -p "{pkg_product_dir}/{distro_version}/10gen/binary-{distro_arch}"
mkdir -p "{pkg_product_dir}/{distro_version}/10gen/source"
( cd "{pkg_name}{pkg_name_suffix}-{pkg_version}"; debuild ) || exit 1
# Try installing it
-dpkg -i *.deb
+dpkg -i {pkg_name}{pkg_name_suffix}*.deb
ps ax | grep mongo || {{ echo "no running mongo" >/dev/stderr; exit 1; }}
+dpkg --remove $(for f in {pkg_name}{pkg_name_suffix}*.deb ; do echo ${{f%%_*}}; done)
+dpkg --purge $(for f in {pkg_name}{pkg_name_suffix}*.deb ; do echo ${{f%%_*}}; done)
cp {pkg_name}{pkg_name_suffix}*.deb "{pkg_product_dir}/{distro_version}/10gen/binary-{distro_arch}"
cp {pkg_name}{pkg_name_suffix}*.dsc "{pkg_product_dir}/{distro_version}/10gen/source"
cp {pkg_name}{pkg_name_suffix}*.tar.gz "{pkg_product_dir}/{distro_version}/10gen/source"
@@ -445,10 +425,11 @@ rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/{distro_arch}/epel-release
yum -y install {pkg_prereq_str}
"""
rpm_build_commands="""
-for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS; do mkdir -p /usr/src/redhat/$d; done
-cp -v "{pkg_name}{pkg_name_suffix}-{pkg_version}/rpm/mongo.spec" /usr/src/redhat/SPECS
-tar -cpzf /usr/src/redhat/SOURCES/"{pkg_name}{pkg_name_suffix}-{pkg_version}".tar.gz "{pkg_name}{pkg_name_suffix}-{pkg_version}"
-rpmbuild -ba /usr/src/redhat/SPECS/mongo.spec
+for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS; do mkdir -p {rpmbuild_dir}/$d; done
+cp -v "{pkg_name}{pkg_name_suffix}-{pkg_version}/rpm/mongo.spec" {rpmbuild_dir}/SPECS/{pkg_name}{pkg_name_suffix}.spec
+tar -cpzf {rpmbuild_dir}/SOURCES/"{pkg_name}{pkg_name_suffix}-{pkg_version}".tar.gz "{pkg_name}{pkg_name_suffix}-{pkg_version}"
+rpmbuild -ba --target={distro_arch} {rpmbuild_dir}/SPECS/{pkg_name}{pkg_name_suffix}.spec
+# FIXME: should install the rpms, check if mongod is running.
"""
# FIXME: this is clean, but adds 40 minutes or so to the build process.
old_rpm_precommands = """
@@ -474,25 +455,28 @@ rpm -ivh /usr/src/redhat/RPMS/{distro_arch}/boost-devel-1.38.0-1.{distro_arch}.r
# On very old Debianoids, libboost-<foo>-dev will be some old
# boost that's not as thready as we want, but which Eliot says
- # will work.
- very_old_deb_prereqs = ["libboost-thread-dev", "libboost-filesystem-dev", "libboost-program-options-dev", "libboost-date-time-dev", "libboost-dev", "xulrunner1.9-dev"]
+ # will work; on very new Debianoids, libbost-<foo>-dev is what we
+ # want.
+ unversioned_deb_boost_prereqs = ["libboost-thread-dev", "libboost-filesystem-dev", "libboost-program-options-dev", "libboost-date-time-dev", "libboost-dev"]
+ # On some in-between Debianoids, libboost-<foo>-dev is still a
+ # 1.34, but 1.35 packages are available, so we want those.
+ versioned_deb_boost_prereqs = ["libboost-thread1.35-dev", "libboost-filesystem1.35-dev", "libboost-program-options1.35-dev", "libboost-date-time1.35-dev", "libboost1.35-dev"]
- # On less old (but still old!) Debianoids, libboost-<foo>-dev is
- # still a 1.34, but 1.35 packages are available, so we want those.
- old_deb_prereqs = ["libboost-thread1.35-dev", "libboost-filesystem1.35-dev", "libboost-program-options1.35-dev", "libboost-date-time1.35-dev", "libboost1.35-dev", "xulrunner-dev"]
+ unversioned_deb_xulrunner_prereqs = ["xulrunner-dev"]
- # On newer Debianoids, libbost-<foo>-dev is some sufficiently new
- # thing.
- new_deb_prereqs = [ "libboost-thread-dev", "libboost-filesystem-dev", "libboost-program-options-dev", "libboost-date-time-dev", "libboost-dev", "xulrunner-dev" ]
+ old_versioned_deb_xulrunner_prereqs = ["xulrunner-1.9-dev"]
+ new_versioned_deb_xulrunner_prereqs = ["xulrunner-1.9.2-dev"]
common_deb_prereqs = [ "build-essential", "dpkg-dev", "libreadline-dev", "libpcap-dev", "libpcre3-dev", "git-core", "scons", "debhelper", "devscripts", "git-core" ]
centos_preqres = ["js-devel", "readline-devel", "pcre-devel", "gcc-c++", "scons", "rpm-build", "git" ]
- fedora_prereqs = ["js-devel", "readline-devel", "pcre-devel", "gcc-c++", "scons", "rpm-build", "git" ]
+ fedora_prereqs = ["js-devel", "readline-devel", "pcre-devel", "gcc-c++", "scons", "rpm-build", "git", "curl" ]
def __init__(self, **kwargs):
super(ScriptFileConfigurator, self).__init__(**kwargs)
- if kwargs["mongo_version"][0] == 'r':
+ # FIXME: this method is disabled until we get back around to
+ # actually building from source.
+ if None: # kwargs["mongo_version"][0] == 'r':
self.get_mongo_commands = """
wget -Otarball.tgz "http://github.com/mongodb/mongo/tarball/{mongo_version}";
tar xzf tarball.tgz
@@ -502,7 +486,9 @@ mv "`tar tzf tarball.tgz | sed 's|/.*||' | sort -u | head -n1`" "{pkg_name}{pkg_
self.get_mongo_commands = """
git clone git://github.com/mongodb/mongo.git
"""
- if kwargs['mongo_version'][0] == 'v':
+ # This is disabled for the moment. it's for building the
+ # tip of some versioned branch.
+ if None: #kwargs['mongo_version'][0] == 'v':
self.get_mongo_commands +="""
( cd mongo && git archive --prefix="{pkg_name}{pkg_name_suffix}-{pkg_version}/" "`git log origin/{mongo_version} | sed -n '1s/^commit //p;q'`" ) | tar xf -
"""
@@ -518,81 +504,188 @@ git clone git://github.com/mongodb/mongo.git
self.configuration += [("pkg_product_dir",
((("ubuntu", "*", "*"), self.deb_productdir),
(("debian", "*", "*"), self.deb_productdir),
- (("fedora", "*", "*"), self.rpm_productdir),
- (("centos", "*", "*"), self.rpm_productdir))),
+ (("fedora", "*", "*"), "~/rpmbuild/RPMS"),
+ (("centos", "*", "*"), "/usr/src/redhat/RPMS"))),
("pkg_prereqs",
((("ubuntu", "9.4", "*"),
- self.old_deb_prereqs + self.common_deb_prereqs),
+ self.versioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
(("ubuntu", "9.10", "*"),
- self.new_deb_prereqs + self.common_deb_prereqs),
+ self.unversioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
(("ubuntu", "10.4", "*"),
- self.new_deb_prereqs + self.common_deb_prereqs),
+ self.unversioned_deb_boost_prereqs + self.new_versioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
(("ubuntu", "8.10", "*"),
- self.old_deb_prereqs + self.common_deb_prereqs),
+ self.versioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
(("ubuntu", "8.4", "*"),
- self.very_old_deb_prereqs + self.common_deb_prereqs),
+ self.unversioned_deb_boost_prereqs + self.old_versioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
(("debian", "5.0", "*"),
- self.old_deb_prereqs + self.common_deb_prereqs),
- (("fedora", "8", "*"),
+ self.versioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
+ (("fedora", "*", "*"),
self.fedora_prereqs),
(("centos", "5.4", "*"),
self.centos_preqres))),
+ # FIXME: this is deprecated
("commands",
((("debian", "*", "*"),
- self.preamble_commands + self.deb_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.deb_build_commands),
- (("ubuntu", "*", "*"),
+ self.deb_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.deb_build_commands),
+ (("ubuntu", "10.4", "*"),
+ self.preamble_commands + self.deb_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.mangle_files_for_new_deb_xulrunner_commands + self.deb_build_commands),
+ (("ubuntu", "*", "*"),
self.preamble_commands + self.deb_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.deb_build_commands),
(("centos", "*", "*"),
self.preamble_commands + self.old_rpm_precommands + self.rpm_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.mangle_files_for_ancient_redhat_commands + self.rpm_build_commands),
(("fedora", "*", "*"),
self.preamble_commands + self.old_rpm_precommands + self.rpm_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.rpm_build_commands))),
+ ("preamble_commands",
+ ((("*", "*", "*"), self.preamble_commands),
+ )),
+ ("install_prereqs",
+ ((("debian", "*", "*"), self.deb_prereq_commands),
+ (("ubuntu", "*", "*"), self.deb_prereq_commands),
+ (("centos", "*", "*"), self.rpm_prereq_commands),
+ (("fedora", "*", "*"), self.rpm_prereq_commands))),
+ ("get_mongo",
+ ((("*", "*", "*"), self.get_mongo_commands),
+ )),
+ ("mangle_mongo",
+ ((("debian", "*", "*"), self.mangle_files_commands),
+ (("ubuntu", "10.4", "*"),
+ self.mangle_files_commands + self.mangle_files_for_new_deb_xulrunner_commands),
+ (("ubuntu", "*", "*"), self.mangle_files_commands),
+ (("centos", "*", "*"),
+ self.mangle_files_commands + self.mangle_files_for_ancient_redhat_commands),
+ (("fedora", "*", "*"),
+ self.mangle_files_commands))),
+ ("build_prerequisites",
+ ((("fedora", "*", "*"), self.old_rpm_precommands),
+ (("centos", "*", "*"), self.old_rpm_precommands),
+ (("*", "*", "*"), ''))),
+ ("install_for_packaging",
+ ((("debian", "*", "*"),""),
+ (("ubuntu", "*", "*"),""),
+ (("fedora", "*", "*"), ""),
+ (("centos", "*", "*"),""))),
+ ("build_package",
+ ((("debian", "*", "*"),
+ self.deb_build_commands),
+ (("ubuntu", "*", "*"),
+ self.deb_build_commands),
+ (("fedora", "*", "*"),
+ self.rpm_build_commands),
+ (("centos", "*", "*"),
+ self.rpm_build_commands))),
("pkg_name",
((("debian", "*", "*"), "mongodb"),
(("ubuntu", "*", "*"), "mongodb"),
(("centos", "*", "*"), "mongo"),
-
- (("fedora", "*", "*"), "mongo")
- )),
+ (("fedora", "*", "*"), "mongo"))),
+ # FIXME: there should be a command-line argument for this.
("pkg_name_conflicts",
- ((("*", "*", "*"), ["", "-stable", "-unstable", "-snapshot"]),
- ))
- ]
+ ((("*", "*", "*"), ["", "-stable", "-unstable", "-snapshot", "-oldstable"]),
+ )),
+ ("rpmbuild_dir",
+ ((("fedora", "*", "*"), "~/rpmbuild"),
+ (("centos", "*", "*"), "/usr/src/redhat"),
+ (("*", "*","*"), ''),
+ )),
+ ]
class ScriptFile(object):
def __init__(self, configurator, **kwargs):
- self.mongo_version = kwargs["mongo_version"]
- self.pkg_version = kwargs["pkg_version"]
- self.pkg_name_suffix = kwargs["pkg_name_suffix"] if "pkg_name_suffix" in kwargs else ""
+ self.configurator = configurator
+ self.mongo_version_spec = kwargs['mongo_version_spec']
+ self.mongo_arch = kwargs["arch"] if kwargs["arch"] == "x86_64" else "i686"
self.pkg_prereqs = configurator.default("pkg_prereqs")
self.pkg_name = configurator.default("pkg_name")
self.pkg_product_dir = configurator.default("pkg_product_dir")
- self.pkg_name_conflicts = configurator.default("pkg_name_conflicts") if self.pkg_name_suffix else []
- self.pkg_name_conflicts.remove(self.pkg_name_suffix) if self.pkg_name_suffix and self.pkg_name_suffix in self.pkg_name_conflicts else []
- self.formatter = configurator.default("commands")
+ #self.formatter = configurator.default("commands")
self.distro_name = configurator.default("distro_name")
self.distro_version = configurator.default("distro_version")
self.distro_arch = configurator.default("distro_arch")
+ def bogoformat(self, fmt, **kwargs):
+ r = ''
+ i = 0
+ while True:
+ c = fmt[i]
+ if c in '{}':
+ i+=1
+ c2=fmt[i]
+ if c2 == c:
+ r+=c
+ else:
+ j=i
+ while True:
+ p=fmt[j:].find('}')
+ if p == -1:
+ raise Exception("malformed format string starting at %d: no closing brace" % i)
+ else:
+ j+=p
+ if len(fmt) > (j+1) and fmt[j+1]=='}':
+ j+=2
+ else:
+ break
+ key = fmt[i:j]
+ r+=kwargs[key]
+ i=j
+ else:
+ r+=c
+ i+=1
+ if i==len(fmt):
+ return r
+
+ def fmt(self, formatter, **kwargs):
+ try:
+ return string.Formatter.format(formatter, kwargs)
+ finally:
+ return self.bogoformat(formatter, **kwargs)
+
def genscript(self):
- return self.formatter.format(mongo_version=self.mongo_version,
- distro_name=self.distro_name,
- distro_version=self.distro_version,
- distro_arch=self.distro_arch,
- pkg_prereq_str=" ".join(self.pkg_prereqs),
- pkg_name=self.pkg_name,
- pkg_name_suffix=self.pkg_name_suffix,
- pkg_version=self.pkg_version,
- pkg_product_dir=self.pkg_product_dir,
- # KLUDGE: rpm specs and deb
- # control files use
- # comma-separated conflicts,
- # but there's no reason to
- # suppose this works elsewhere
- pkg_name_conflicts = ", ".join([self.pkg_name+conflict for conflict in self.pkg_name_conflicts])
- )
+ script=''
+ formatter = self.configurator.default("preamble_commands") + self.configurator.default("install_prereqs")
+ script+=self.fmt(formatter,
+ distro_name=self.distro_name,
+ distro_version=self.distro_version,
+ distro_arch=self.distro_arch,
+ pkg_name=self.pkg_name,
+ pkg_product_dir=self.pkg_product_dir,
+ mongo_arch=self.mongo_arch,
+ pkg_prereq_str=" ".join(self.pkg_prereqs),
+ )
+
+ specs=self.mongo_version_spec.split(',')
+ for spec in specs:
+ (version, pkg_name_suffix, pkg_version) = parse_mongo_version_spec(spec)
+ mongo_version = version if version[0] != 'n' else ('HEAD' if version == 'nlatest' else 'r'+version[1:]) #'HEAD'
+ mongo_pub_version = version.lstrip('n') if version[0] in 'n' else 'latest'
+ pkg_name_suffix = pkg_name_suffix if pkg_name_suffix else ''
+ pkg_version = pkg_version
+ pkg_name_conflicts = list(self.configurator.default("pkg_name_conflicts") if pkg_name_suffix else [])
+ pkg_name_conflicts.remove(pkg_name_suffix) if pkg_name_suffix and pkg_name_suffix in pkg_name_conflicts else []
+ formatter = self.configurator.default("get_mongo") + self.configurator.default("mangle_mongo") + (self.configurator.nightly_build_mangle_files if version[0] == 'n' else '') +(self.configurator.default("build_prerequisites") if version[0] != 'n' else '') + self.configurator.default("install_for_packaging") + self.configurator.default("build_package")
+ script+=self.fmt(formatter,
+ mongo_version=mongo_version,
+ distro_name=self.distro_name,
+ distro_version=self.distro_version,
+ distro_arch=self.distro_arch,
+ pkg_prereq_str=" ".join(self.pkg_prereqs),
+ pkg_name=self.pkg_name,
+ pkg_name_suffix=pkg_name_suffix,
+ pkg_version=pkg_version,
+ pkg_product_dir=self.pkg_product_dir,
+ # KLUDGE: rpm specs and deb
+ # control files use
+ # comma-separated conflicts,
+ # but there's no reason to
+ # suppose this works elsewhere
+ pkg_name_conflicts = ", ".join([self.pkg_name+conflict for conflict in pkg_name_conflicts]),
+ mongo_arch=self.mongo_arch,
+ mongo_pub_version=mongo_pub_version,
+ rpmbuild_dir=self.configurator.default('rpmbuild_dir'))
+ script+='rm -rf mongo'
+ return script
def __enter__(self):
self.localscript=None
@@ -614,6 +707,69 @@ class ScriptFile(object):
class Configurator(SshConnectionConfigurator, EC2InstanceConfigurator, ScriptFileConfigurator, BaseHostConfigurator):
def __init__(self, **kwargs):
super(Configurator, self).__init__(**kwargs)
+
+class rackspaceInstance(nodeWrapper):
+ def __init__(self, configurator, **kwargs):
+ super(rackspaceInstance, self).__init__(configurator, **kwargs)
+ self.imgname=configurator.default('rackspace_imgname')
+
+ def start(self):
+ driver = get_driver(Provider.RACKSPACE)
+ self.conn = driver(settings.rackspace_account, settings.rackspace_api_key)
+ name=self.imgname+'-'+str(os.getpid())
+ images=filter(lambda x: (x.name.find(self.imgname) > -1), self.conn.list_images())
+ sizes=self.conn.list_sizes()
+ sizes.sort(cmp=lambda x,y: int(x.ram)<int(y.ram))
+ node = None
+ if len(images) > 1:
+ raise Exception("too many images with \"%s\" in the name" % self.imgname)
+ if len(images) < 1:
+ raise Exception("too few images with \"%s\" in the name" % self.imgname)
+ image = images[0]
+ self.node = self.conn.create_node(image=image, name=name, size=sizes[0])
+ # Note: the password is available only in the response to the
+ # create_node request, not in subsequent list_nodes()
+ # requests; so although the node objects we get back from
+ # list_nodes() are usuable for most things, we must hold onto
+ # the initial password.
+ self.password = self.node.extra['password']
+ print self.node
+
+ def list_nodes(self):
+ return self.conn.list_nodes()
+
+ def setup(self):
+ self.putSshKey()
+
+ def putSshKey(self):
+ keyfile=settings.makedist['ssh_keyfile']
+ ssh = ParamikoSSHClient(hostname = self.node.public_ip[0], password = self.password)
+ ssh.connect()
+ print "putting ssh public key"
+ ssh.put(".ssh/authorized_keys", contents=open(keyfile+'.pub').read(), chmod=0600)
+ print "ok"
+
+def parse_mongo_version_spec (spec):
+ foo = spec.split(":")
+ mongo_version = foo[0] # this can be a commit id, a
+ # release id "r1.2.2", or a branch name
+ # starting with v.
+ if len(foo) > 1:
+ pkg_name_suffix = foo[1]
+ if len(foo) > 2 and foo[2]:
+ pkg_version = foo[2]
+ else:
+ pkg_version = time.strftime("%Y%m%d")
+ if not pkg_name_suffix:
+ if mongo_version[0] in ["r", "v"]:
+ nums = mongo_version.split(".")
+ if int(nums[1]) % 2 == 0:
+ pkg_name_suffix = "-stable"
+ else:
+ pkg_name_suffix = "-unstable"
+ else:
+ pkg_name_suffix = ""
+ return (mongo_version, pkg_name_suffix, pkg_version)
def main():
# checkEnvironment()
@@ -629,59 +785,28 @@ def main():
try:
import settings
if "makedist" in dir ( settings ):
- for key in ["EC2_HOME", "JAVA_HOME"]:
- if key in settings.makedist:
- os.environ[key] = settings.makedist[key]
- for key in ["ec2_pkey", "ec2_cert", "ec2_sshkey", "ssh_keyfile" ]:
+ for key in ["ec2_sshkey", "ssh_keyfile", "gpg_homedir" ]:
if key not in kwargs and key in settings.makedist:
kwargs[key] = settings.makedist[key]
except Exception, err:
print "No settings: %s. Continuing anyway..." % err
pass
- # Ensure that PATH contains $EC2_HOME/bin
- vars = ["EC2_HOME", "JAVA_HOME"]
- for var in vars:
- if os.getenv(var) == None:
- raise SimpleError("Environment variable %s is unset; did you create a settings.py?", var)
-
- if len([True for x in os.environ["PATH"].split(":") if x.find(os.environ["EC2_HOME"]) > -1]) == 0:
- os.environ["PATH"]=os.environ["EC2_HOME"]+"/bin:"+os.environ["PATH"]
-
-
kwargs["distro_name"] = distro_name
kwargs["distro_version"] = distro_version
kwargs["arch"] = arch
-
- foo = mongo_version_spec.split(":")
- kwargs["mongo_version"] = foo[0] # this can be a commit id, a
- # release id "r1.2.2", or a
- # branch name starting with v.
- if len(foo) > 1:
- kwargs["pkg_name_suffix"] = foo[1]
- if len(foo) > 2 and foo[2]:
- kwargs["pkg_version"] = foo[2]
- else:
- kwargs["pkg_version"] = time.strftime("%Y%m%d")
-
+ kwargs['mongo_version_spec'] = mongo_version_spec
+
+ kwargs["localdir"] = rootdir
# FIXME: this should also include the mongo version or something.
- if "subdirs" in kwargs:
- kwargs["localdir"] = "%s/%s/%s/%s" % (rootdir, distro_name, distro_version, arch, kwargs["mongo_version"])
- else:
- kwargs["localdir"] = rootdir
+# if "subdirs" in kwargs:
+# kwargs["localdir"] = "%s/%s/%s/%s/%s" % (rootdir, distro_name, distro_version, arch, kwargs["mongo_version"])
+# else:
+
- if "pkg_name_suffix" not in kwargs:
- if kwargs["mongo_version"][0] in ["r", "v"]:
- nums = kwargs["mongo_version"].split(".")
- if int(nums[1]) % 2 == 0:
- kwargs["pkg_name_suffix"] = "-stable"
- else:
- kwargs["pkg_name_suffix"] = "-unstable"
- else:
- kwargs["pkg_name_suffix"] = ""
- kwargs['local_gpg_dir'] = kwargs["local_gpg_dir"] if "local_gpg_dir" in kwargs else os.path.expanduser("~/.gnupg")
+ kwargs['gpg_homedir'] = kwargs["gpg_homedir"] if "gpg_homedir" in kwargs else os.path.expanduser("~/.gnupg")
configurator = Configurator(**kwargs)
LocalHost.runLocally(["mkdir", "-p", kwargs["localdir"]])
with ScriptFile(configurator, **kwargs) as script:
@@ -689,15 +814,18 @@ def main():
print """# Going to run the following on a fresh AMI:"""
print f.read()
time.sleep(10)
- with EC2Instance(configurator, **kwargs) as ec2:
- ec2.initwait()
- kwargs["ssh_host"] = ec2.getHostname()
+ # FIXME: it's not the best to have two different pathways for
+ # the different hosting services, but...
+ with EC2Instance(configurator, **kwargs) if kwargs['distro_name'] != 'fedora' else rackspaceInstance(configurator, **kwargs) as host:
+ host.initwait()
+ host.setup()
+ kwargs["ssh_host"] = host.getHostname()
with SshConnection(configurator, **kwargs) as ssh:
ssh.runRemotely(["uname -a; ls /"])
ssh.runRemotely(["mkdir", "pkg"])
if "local_mongo_dir" in kwargs:
ssh.sendFiles([(kwargs["local_mongo_dir"]+'/'+d, "pkg") for d in ["rpm", "debian"]])
- ssh.sendFiles([(kwargs['local_gpg_dir'], ".gnupg")])
+ ssh.sendFiles([(kwargs['gpg_homedir'], ".gnupg")])
ssh.sendFiles([(script.localscript, "makedist.sh")])
ssh.runRemotely((["sudo"] if ssh.ssh_login != "root" else [])+ ["sh", "makedist.sh"])
ssh.recvFiles([(script.pkg_product_dir, kwargs['localdir'])])
@@ -709,7 +837,7 @@ def processArguments():
("N", "no-terminate", False, "Leave the EC2 instance running at the end of the job", None),
("S", "subdirs", False, "Create subdirectories of the output directory based on distro name, version, and architecture", None),
("I", "use-internal-name", False, "Use the EC2 internal hostname for sshing", None),
- (None, "local-gpg-dir", True, "Local directory of gpg junk", "STRING"),
+ (None, "gpg-homedir", True, "Local directory of gpg junk", "STRING"),
(None, "local-mongo-dir", True, "Copy packaging files from local mongo checkout", "DIRECTORY"),
]
shortopts = "".join([t[0] + (":" if t[2] else "") for t in flagspec if t[0] is not None])
@@ -746,11 +874,12 @@ def processArguments():
MONGO-VERSION-SPEC has the syntax
Commit(:Pkg-Name-Suffix(:Pkg-Version)). If Commit starts with an 'r',
-build from a tagged release; if Commit starts with a 'v', build from
-the HEAD of a version branch; otherwise, build whatever git commit is
-identified by Commit. Pkg-Name-Suffix gets appended to the package
-name, and defaults to "-stable" and "-unstable" if Commit looks like
-it designates a stable or unstable release/branch, respectively.
+build from a tagged release; if Commit starts with an 'n', package up
+a nightly build; if Commit starts with a 'v', build from the HEAD of a
+version branch; otherwise, build whatever git commit is identified by
+Commit. Pkg-Name-Suffix gets appended to the package name, and
+defaults to "-stable" and "-unstable" if Commit looks like it
+designates a stable or unstable release/branch, respectively.
Pkg-Version is used as the package version, and defaults to YYYYMMDD.
Examples:
@@ -779,8 +908,7 @@ Options:"""
print "%-20s\t%s." % ("%4s--%s%s:" % ("-%s, " % t[0] if t[0] else "", t[1], ("="+t[4]) if t[4] else ""), t[3])
print """
Mandatory arguments to long options are also mandatory for short
-options. Some EC2 arguments default to (and override) environment
-variables; see the ec2-api-tools documentation."""
+options."""
sys.exit(0)
if "usage" in kwargs:
@@ -796,4 +924,5 @@ if __name__ == "__main__":
# Examples:
-# ./makedist.py --local-gpg-dir=$HOME/10gen/dst/dist-gnupg /tmp/ubuntu ubuntu 8.10 x86_64 HEAD:-snapshot
+# ./makedist.py /tmp/ubuntu ubuntu 8.10 x86_64 HEAD:-snapshot,v1.4:-stable,v1.5:-unstable
+# ./makedist.py /tmp/ubuntu ubuntu 8.10 x86_64 nlatest:-snapshot,n1.4.2:-stable,n1.5.0:-unstable
diff --git a/buildscripts/mergerepositories.py b/buildscripts/mergerepositories.py
new file mode 100644
index 0000000..bc50d08
--- /dev/null
+++ b/buildscripts/mergerepositories.py
@@ -0,0 +1,194 @@
+#!/usr/bin/python
+
+from __future__ import with_statement
+from libcloud.types import Provider
+from libcloud.providers import get_driver
+from libcloud.drivers.ec2 import EC2NodeDriver, NodeImage
+from libcloud.base import Node, NodeImage, NodeSize, NodeState
+
+# libcloud's SSH client seems to be one of those pointless wrappers
+# that (at the moment) both doesn't add anything to the thing it wraps
+# (Paramiko) and also fails to expose the underlying thing's features.
+# What's wrong with people?
+#from libcloud.ssh import SSHClient
+
+import time
+import sys
+import settings
+import subprocess
+import os
+import socket
+
+EC2 = get_driver(Provider.EC2)
+EC2Driver=EC2NodeDriver(settings.id, settings.key)
+
+def tryEC2():
+
+ image=NodeImage('ami-bf07ead6', 'ubuntu 10.4', EC2)
+ size=NodeSize('m1.large', 'large', None, None, None, None, EC2)
+
+ node = None
+ try:
+ node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, keyname="kp1", securitygroup=['default', 'dist-slave', 'buildbot-slave'])
+ print node
+ print node.id
+ while node.state == NodeState.PENDING:
+ time.sleep(3)
+ finally:
+ if node:
+ node.destroy()
+
+
+class node(object):
+ def initWait(self):
+ while 1:
+ n=None
+ # EC2 sometimes takes a while to report a node.
+ for i in range(6):
+ nodes = [n for n in self.list_nodes() if (n.id==self.node.id)]
+ if len(nodes)>0:
+ n=nodes[0]
+ break
+ else:
+ time.sleep(10)
+ if not n:
+ raise Exception("couldn't find node with id %s" % self.node.id)
+ if n.state == NodeState.PENDING:
+ time.sleep(10)
+ else:
+ self.node = n
+ break
+ print "ok"
+ # Now wait for the node's sshd to be accepting connections.
+ print "waiting for ssh"
+ sshwait = True
+ if sshwait == False:
+ return
+ while sshwait:
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ try:
+ try:
+ s.connect((self.node.public_ip[0], 22))
+ sshwait = False
+ print "connected on port 22 (ssh)"
+ time.sleep(15) # arbitrary timeout, in case the
+ # remote sshd is slow.
+ except socket.error, err:
+ pass
+ finally:
+ s.close()
+ time.sleep(3) # arbitrary timeout
+ print "ok"
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, arg0, arg1, arg2):
+ print "shutting down node %s" % self.node
+ self.node.destroy()
+
+# I don't think libcloud's Nodes implement __enter__ and __exit__, and
+# I like the with statement for ensuring that we don't leak nodes when
+# we don't have to.
+class ec2node(node):
+ def list_nodes(self):
+ return EC2Driver.list_nodes()
+
+class ubuntuNode(ec2node):
+ def __init__(self):
+ image=NodeImage('ami-bf07ead6', 'ubuntu 10.4', EC2)
+ size=NodeSize('m1.large', 'large', None, None, None, None, EC2)
+
+ self.node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, securitygroup=['default', 'dist-slave', 'buildbot-slave'], keyname='kp1')
+
+class centosNode(ec2node):
+ def __init__(self):
+ image=NodeImage('ami-ccb35ea5', 'ubuntu 10.4', EC2)
+ size=NodeSize('m1.large', 'large', None, None, None, None, EC2)
+
+ self.node = EC2Driver.create_node(image=image, name="ubuntu-test", size=size, securitygroup=['default', 'dist-slave', 'buildbot-slave'], keyname='kp1')
+
+class rackspaceNode(node):
+ def list_nodes(self):
+ self.conn.list_nodes()
+
+class fedora11Node(rackspaceNode):
+ def __init__(self):
+ driver = get_driver(Provider.RACKSPACE)
+ self.conn = driver(settings.rackspace_account, settings.rackspace_api_key)
+ string='Fedora 11'
+ images=filter(lambda x: (x.name.find(string) > -1), self.conn.list_images())
+ sizes=self.conn.list_sizes()
+ sizes.sort(cmp=lambda x,y: int(x.ram)<int(y.ram))
+ node = None
+ if len(images) != 1:
+ raise "too many images with \"%s\" in the name" % string
+ image = images[0]
+ self.node = self.conn.create_node(image=image, name=string, size=sizes[0])
+ print self.node
+ self.password = self.node.extra['password']
+
+class Err(Exception):
+ pass
+
+def merge_yum_repo(dir, outdir):
+ dirtail=dir.rstrip('\/').split('/')[-1]
+ keyfile=settings.makedist['ssh_keyfile']
+ makeyumrepo="""find . -name RPMS | while read dir; do (cd $dir/.. && createrepo .); done"""
+ with centosNode() as centos:
+ centos.initWait()
+ print centos.node
+ run_for_effect(["scp", "-o", "StrictHostKeyChecking no","-i", keyfile, "-r", dir, "root@"+centos.node.public_ip[0]+":"])
+ run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "root@"+centos.node.public_ip[0], "cd ./" + dirtail + " && " + makeyumrepo])
+ run_for_effect(["scp", "-o", "StrictHostKeyChecking no", "-i", keyfile, "-r", "root@"+centos.node.public_ip[0]+":./"+dirtail +'/*', outdir])
+
+
+
+def merge_apt_repo(dir, outdir):
+ dirtail=dir.rstrip('\/').split('/')[-1]
+
+ gpgdir=settings.makedist['gpg_homedir']
+ keyfile=settings.makedist['ssh_keyfile']
+
+ makeaptrepo="""for x in debian ubuntu; do (cd $x; for d in `find . -name *.deb | sed 's|^./||; s|/[^/]*$||' | sort -u`; do dpkg-scanpackages $d > $d/Packages; gzip -9c $d/Packages > $d/Packages.gz; done) ; done"""
+ makereleaseprologue="""Origin: 10gen
+Label: 10gen
+Suite: 10gen
+Codename: VVVVVV
+Version: VVVVVV
+Architectures: i386 amd64
+Components: 10gen
+Description: 10gen packages"""
+ makeaptrelease="""find . -maxdepth 3 -mindepth 3 | while read d; do ( cd $d && (echo '%s' | sed s/VVVVVV/$(basename $(pwd))/; apt-ftparchive release .) > /tmp/Release && mv /tmp/Release . && gpg -r `gpg --list-keys | grep uid | awk '{print $(NF)}'` --no-secmem-warning --no-tty -abs --output Release.gpg Release ); done""" % makereleaseprologue
+ with ubuntuNode() as ubuntu:
+ ubuntu.initWait()
+ print ubuntu.node
+ run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "sudo", "sh", "-c", "\"export DEBIAN_FRONTEND=noninteractive; apt-get update; apt-get -y install debhelper\""])
+ run_for_effect(["scp", "-o", "StrictHostKeyChecking no","-i", keyfile, "-r", dir, "ubuntu@"+ubuntu.node.public_ip[0]+":"])
+ run_for_effect(["scp", "-o", "StrictHostKeyChecking no","-i", keyfile, "-r", gpgdir, "ubuntu@"+ubuntu.node.public_ip[0]+":.gnupg"])
+ run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "sh", "-c", "\"ls -lR ./" + dirtail + "\""])
+ run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "cd ./"+dirtail + " && " + makeaptrepo])
+ run_for_effect(["ssh", "-o", "StrictHostKeyChecking no","-i", keyfile, "ubuntu@"+ubuntu.node.public_ip[0], "cd ./"+dirtail + " && " + makeaptrelease])
+ run_for_effect(["scp", "-o", "StrictHostKeyChecking no", "-i", keyfile, "-r", "ubuntu@"+ubuntu.node.public_ip[0]+":./"+dirtail +'/*', outdir])
+
+
+def run_for_effect(argv):
+ print " ".join(argv)
+ r=subprocess.Popen(argv).wait()
+ if r!=0:
+ raise Err("subprocess %s exited %d" % (argv, r))
+
+if __name__ == "__main__":
+ (flavor, dir, outdir) = sys.argv[-3:]
+
+ if flavor == "deb":
+ merge_apt_repo(dir, outdir)
+ elif flavor == "rpm":
+ merge_yum_repo(dir, outdir)
+ else:
+ Err("unknown pkg flavor %s" % flavor)
+ # TODO: yum repositories
+
+
+ #main()
+ #tryRackSpace()
diff --git a/buildscripts/smoke.py b/buildscripts/smoke.py
new file mode 100755
index 0000000..0023226
--- /dev/null
+++ b/buildscripts/smoke.py
@@ -0,0 +1,522 @@
+#!/usr/bin/python
+
+# smoke.py: run some mongo tests.
+
+# Bugs, TODOs:
+
+# 0 Some tests hard-code pathnames relative to the mongo repository,
+# so the smoke.py process and all its children must be run with the
+# mongo repo as current working directory. That's kinda icky.
+
+# 1 The tests that are implemented as standalone executables ("test",
+# "perftest"), don't take arguments for the dbpath, but
+# unconditionally use "/tmp/unittest".
+
+# 2 mongod output gets intermingled with mongo output, and it's often
+# hard to find error messages in the slop. Maybe have smoke.py do
+# some fancier wrangling of child process output?
+
+# 3 Some test suites run their own mongods, and so don't need us to
+# run any mongods around their execution. (It's harmless to do so,
+# but adds noise in the output.)
+
+# 4 Running a separate mongo shell for each js file is slower than
+# loading js files into one mongo shell process. Maybe have runTest
+# queue up all filenames ending in ".js" and run them in one mongo
+# shell at the "end" of testing?
+
+# 5 Right now small-oplog implies master/slave replication. Maybe
+# running with replication should be an orthogonal concern. (And
+# maybe test replica set replication, too.)
+
+# 6 We use cleanbb.py to clear out the dbpath, but cleanbb.py kills
+# off all mongods on a box, which means you can't run two smoke.py
+# jobs on the same host at once. So something's gotta change.
+
+from __future__ import with_statement
+from subprocess import Popen, PIPE, call
+import os
+import sys
+import utils
+import time
+import socket
+from optparse import OptionParser
+import atexit
+import glob
+import shutil
+import re
+import parser
+
+mongoRepo = os.getcwd() #'./'
+testPath = None
+
+mongodExecutable = "./mongod"
+mongodPort = "32000"
+shellExecutable = "./mongo"
+continueOnFailure = False
+oneMongodPerTest = False
+
+tests = []
+winners = []
+losers = {}
+
+# Finally, atexit functions seem to be a little oblivious to whether
+# Python is exiting because of an error, so we'll use this to
+# communicate with the report() function.
+exit_bad = True
+
+# For replication hash checking
+replicated_dbs = []
+lost_in_slave = []
+lost_in_master = []
+screwy_in_slave = {}
+
+smokeDbPrefix = ''
+smallOplog = False
+
+# This class just implements the with statement API, for a sneaky
+# purpose below.
+class nothing(object):
+ def __enter__(self):
+ return self
+ def __exit__(self, type, value, traceback):
+ return not isinstance(value, Exception)
+
+class mongod(object):
+ def __init__(self, **kwargs):
+ self.kwargs = kwargs
+ self.proc = None
+
+ def __enter__(self):
+ self.start()
+ return self
+
+ def __exit__(self, type, value, traceback):
+ try:
+ self.stop()
+ except Exception, e:
+ print >> sys.stderr, "error shutting down mongod"
+ print >> sys.stderr, e
+ return not isinstance(value, Exception)
+
+ def ensureTestDirs(self):
+ utils.ensureDir( smokeDbPrefix + "/tmp/unittest/" )
+ utils.ensureDir( smokeDbPrefix + "/data/" )
+ utils.ensureDir( smokeDbPrefix + "/data/db/" )
+
+ def checkMongoPort( self, port=27017 ):
+ sock = socket.socket()
+ sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+ sock.settimeout(1)
+ sock.connect(("localhost", int(port)))
+ sock.close()
+
+ def didMongodStart( self, port=mongodPort, timeout=20 ):
+ while timeout > 0:
+ time.sleep( 1 )
+ try:
+ self.checkMongoPort( int(port) )
+ return True
+ except Exception,e:
+ print >> sys.stderr, e
+ timeout = timeout - 1
+ return False
+
+ def start(self):
+ global mongodPort
+ global mongod
+ if self.proc:
+ print >> sys.stderr, "probable bug: self.proc already set in start()"
+ return
+ self.ensureTestDirs()
+ dirName = smokeDbPrefix + "/data/db/sconsTests/"
+ self.port = int(mongodPort)
+ self.slave = False
+ if 'slave' in self.kwargs:
+ dirName = smokeDbPrefix + '/data/db/sconsTestsSlave/'
+ srcport = mongodPort
+ self.port += 1
+ self.slave = True
+ if os.path.exists ( dirName ):
+ if 'slave' in self.kwargs:
+ argv = ["python", "buildscripts/cleanbb.py", '--nokill', dirName]
+
+ else:
+ argv = ["python", "buildscripts/cleanbb.py", dirName]
+ call( argv )
+ utils.ensureDir( dirName )
+ argv = [mongodExecutable, "--port", str(self.port), "--dbpath", dirName]
+ if self.kwargs.get('smallOplog'):
+ argv += ["--master", "--oplogSize", "10"]
+ if self.slave:
+ argv += ['--slave', '--source', 'localhost:'+str(srcport)]
+ print "running " + " ".join(argv)
+ self.proc = Popen(argv)
+ if not self.didMongodStart( self.port ):
+ raise Exception( "Failed to start mongod" )
+
+ if self.slave:
+ while True:
+ argv = [shellExecutable, "--port", str(self.port), "--quiet", "--eval", 'db.printSlaveReplicationInfo()']
+ res = Popen(argv, stdout=PIPE).communicate()[0]
+ if res.find('initial sync') < 0:
+ break
+
+
+
+ def stop(self):
+ if not self.proc:
+ print >> sys.stderr, "probable bug: self.proc unset in stop()"
+ return
+ try:
+ # This function not available in Python 2.5
+ self.proc.terminate()
+ except AttributeError:
+ if os.sys.platform == "win32":
+ import win32process
+ win32process.TerminateProcess(self.proc._handle, -1)
+ else:
+ from os import kill
+ kill( self.proc.pid, 15 )
+ self.proc.wait()
+ sys.stderr.flush()
+ sys.stdout.flush()
+
+class Bug(Exception):
+ def __str__(self):
+ return 'bug in smoke.py: ' + super(Bug, self).__str__()
+
+class TestFailure(Exception):
+ pass
+
+class TestExitFailure(TestFailure):
+ def __init__(self, *args):
+ self.path = args[0]
+ self.status=args[1]
+ def __str__(self):
+ return "test %s exited with status %d" % (self.path, self.status)
+
+class TestServerFailure(TestFailure):
+ def __init__(self, *args):
+ self.path = args[0]
+ self.status = -1 # this is meaningless as an exit code, but
+ # that's the point.
+ def __str__(self):
+ return 'mongod not running after executing test %s' % self.path
+
+def checkDbHashes(master, slave):
+ # Need to pause a bit so a slave might catch up...
+ if not slave.slave:
+ raise(Bug("slave instance doesn't have slave attribute set"))
+
+ print "waiting for slave to catch up..."
+ ARB=10 # ARBITRARY
+ time.sleep(ARB)
+ while True:
+ # FIXME: it's probably better to do an empty insert and a
+ # getLastError() to force a sync.
+ argv = [shellExecutable, "--port", str(slave.port), "--quiet", "--eval", 'db.printSlaveReplicationInfo()']
+ res = Popen(argv, stdout=PIPE).communicate()[0]
+ m = re.search('(\d+)secs ', res)
+ if int(m.group(1)) > ARB: #res.find('initial sync') < 0:
+ break
+ time.sleep(3)
+
+ # FIXME: maybe make this run dbhash on all databases?
+ for mongod in [master, slave]:
+ argv = [shellExecutable, "--port", str(mongod.port), "--quiet", "--eval", "x=db.runCommand('dbhash'); printjson(x.collections)"]
+ hashstr = Popen(argv, stdout=PIPE).communicate()[0]
+ # WARNING FIXME KLUDGE et al.: this is sleazy and unsafe.
+ mongod.dict = eval(hashstr)
+
+ global lost_in_slave, lost_in_master, screwy_in_slave, replicated_dbs
+
+ for db in replicated_dbs:
+ if db not in slave.dict:
+ lost_in_slave.append(db)
+ mhash = master.dict[db]
+ shash = slave.dict[db]
+ if mhash != shash:
+ screwy_in_slave[db] = mhash + "/" + shash
+ for db in slave.dict.keys():
+ if db not in master.dict:
+ lost_in_master.append(db)
+ replicated_dbs += master.dict.keys()
+
+# Blech.
+def skipTest(path):
+ if smallOplog:
+ if os.path.basename(path) in ["cursor8.js", "indexh.js"]:
+ return True
+ return False
+
+def runTest(test):
+ (path, usedb) = test
+ (ignore, ext) = os.path.splitext(path)
+ if skipTest(path):
+ print "skippping " + path
+ return
+ if ext == ".js":
+ argv=[shellExecutable, "--port", mongodPort]
+ if not usedb:
+ argv += ["--nodb"]
+ if smallOplog:
+ argv += ["--eval", 'testingReplication = true;']
+ argv += [path]
+ elif ext in ["", ".exe"]:
+ # Blech.
+ if os.path.basename(path) in ["test", "test.exe", "perftest", "perftest.exe"]:
+ argv=[path]
+ # more blech
+ elif os.path.basename(path) == 'mongos':
+ argv=[path, "--test"]
+ else:
+ argv=[testPath and os.path.abspath(os.path.join(testPath, path)) or path,
+ "--port", mongodPort]
+ else:
+ raise Bug("fell off in extenstion case: %s" % path)
+ print " *******************************************"
+ print " Test : " + os.path.basename(path) + " ..."
+ t1=time.time()
+ # FIXME: we don't handle the case where the subprocess
+ # hangs... that's bad.
+ r = call(argv, cwd=testPath)
+ t2=time.time()
+ print " " + str((t2-t1)*1000) + "ms"
+ if r != 0:
+ raise TestExitFailure(path, r)
+ if Popen( [ mongodExecutable, "msg", "ping", mongodPort ], stdout=PIPE ).communicate()[0].count( "****ok" ) == 0:
+ raise TestServerFailure(path)
+ if call( [ mongodExecutable, "msg", "ping", mongodPort ] ) != 0:
+ raise TestServerFailure(path)
+ print ""
+
+def runTests(tests):
+ # If we're in one-mongo-per-test mode, we instantiate a nothing
+ # around the loop, and a mongod inside the loop.
+
+ # FIXME: some suites of tests start their own mongod, so don't
+ # need this. (So long as there are no conflicts with port,
+ # dbpath, etc., and so long as we shut ours down properly,
+ # starting this mongod shouldn't break anything, though.)
+ with nothing() if oneMongodPerTest else mongod(smallOplog=smallOplog) as master1:
+ with nothing() if oneMongodPerTest else (mongod(slave=True) if smallOplog else nothing()) as slave1:
+ for test in tests:
+ try:
+ with mongod(smallOplog=smallOplog) if oneMongodPerTest else nothing() as master2:
+ with mongod(slave=True) if oneMongodPerTest and smallOplog else nothing() as slave2:
+ runTest(test)
+ winners.append(test)
+ if isinstance(slave2, mongod):
+ checkDbHashes(master2, slave2)
+ except TestFailure, f:
+ try:
+ print f
+ # Record the failing test and re-raise.
+ losers[f.path] = f.status
+ raise f
+ except TestServerFailure, f:
+ if not oneMongodPerTest:
+ return 2
+ except TestFailure, f:
+ if not continueOnFailure:
+ return 1
+ if isinstance(slave1, mongod):
+ checkDbHashes(master1, slave1)
+
+ return 0
+
+def report():
+ print "%d test%s succeeded" % (len(winners), '' if len(winners) == 1 else 's')
+ num_missed = len(tests) - (len(winners) + len(losers.keys()))
+ if num_missed:
+ print "%d tests didn't get run" % num_missed
+ if losers:
+ print "The following tests failed (with exit code):"
+ for loser in losers:
+ print "%s\t%d" % (loser, losers[loser])
+
+ def missing(lst, src, dst):
+ if lst:
+ print """The following collections were present in the %s but not the %s
+at the end of testing:""" % (src, dst)
+ for db in lst:
+ print db
+ missing(lost_in_slave, "master", "slave")
+ missing(lost_in_master, "slave", "master")
+ if screwy_in_slave:
+ print """The following collections has different hashes in master and slave
+at the end of testing:"""
+ for db in screwy_in_slave.keys():
+ print "%s\t %s" % (db, screwy_in_slave[db])
+ if smallOplog and not (lost_in_master or lost_in_slave or screwy_in_slave):
+ print "replication ok for %d collections" % (len(replicated_dbs))
+ if (exit_bad or losers or lost_in_slave or lost_in_master or screwy_in_slave):
+ status = 1
+ else:
+ status = 0
+ exit (status)
+
+def expandSuites(suites):
+ globstr = None
+ global mongoRepo, tests
+ for suite in suites:
+ if suite == 'smokeAll':
+ tests = []
+ expandSuites(['smoke', 'smokePerf', 'smokeClient', 'smokeJs', 'smokeJsPerf', 'smokeJsSlowNightly', 'smokeJsSlowWeekly', 'smokeParallel', 'smokeClone', 'smokeParallel', 'smokeRepl', 'smokeAuth', 'smokeSharding', 'smokeTool'])
+ break
+ if suite == 'smoke':
+ if os.sys.platform == "win32":
+ program = 'test.exe'
+ else:
+ program = 'test'
+ (globstr, usedb) = (program, False)
+ elif suite == 'smokePerf':
+ if os.sys.platform == "win32":
+ program = 'perftest.exe'
+ else:
+ program = 'perftest'
+ (globstr, usedb) = (program, False)
+ elif suite == 'smokeJs':
+ # FIXME: _runner.js seems equivalent to "[!_]*.js".
+ #(globstr, usedb) = ('_runner.js', True)
+ (globstr, usedb) = ('[!_]*.js', True)
+ elif suite == 'smokeQuota':
+ (globstr, usedb) = ('quota/*.js', True)
+ elif suite == 'smokeJsPerf':
+ (globstr, usedb) = ('perf/*.js', True)
+ elif suite == 'smokeDisk':
+ (globstr, usedb) = ('disk/*.js', True)
+ elif suite == 'smokeJsSlowNightly':
+ (globstr, usedb) = ('slowNightly/*.js', True)
+ elif suite == 'smokeJsSlowWeekly':
+ (globstr, usedb) = ('slowWeekly/*.js', True)
+ elif suite == 'smokeParallel':
+ (globstr, usedb) = ('parallel/*.js', True)
+ elif suite == 'smokeClone':
+ (globstr, usedb) = ('clone/*.js', False)
+ elif suite == 'smokeRepl':
+ (globstr, usedb) = ('repl/*.js', False)
+ elif suite == 'smokeReplSets':
+ (globstr, usedb) = ('replsets/*.js', False)
+ elif suite == 'smokeAuth':
+ (globstr, usedb) = ('auth/*.js', False)
+ elif suite == 'smokeSharding':
+ (globstr, usedb) = ('sharding/*.js', False)
+ elif suite == 'smokeTool':
+ (globstr, usedb) = ('tool/*.js', False)
+ # well, the above almost works for everything...
+ elif suite == 'smokeClient':
+ paths = ["firstExample", "secondExample", "whereExample", "authTest", "clientTest", "httpClientTest"]
+ if os.sys.platform == "win32":
+ paths = [path+'.exe' for path in paths]
+ # hack
+ tests += [(testPath and path or os.path.join(mongoRepo, path), False) for path in paths]
+ elif suite == 'mongosTest':
+ if os.sys.platform == "win32":
+ program = 'mongos.exe'
+ else:
+ program = 'mongos'
+ tests += [(os.path.join(mongoRepo, program), False)]
+ else:
+ raise Exception('unknown test suite %s' % suite)
+
+ if globstr:
+ globstr = os.path.join(mongoRepo, (os.path.join(('jstests/' if globstr.endswith('.js') else ''), globstr)))
+ paths = glob.glob(globstr)
+ paths.sort()
+ tests += [(path, usedb) for path in paths]
+ if not tests:
+ raise Exception( "no tests found" )
+ return tests
+
+def main():
+ parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*")
+ parser.add_option('--mode', dest='mode', default='suite',
+ help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests. (default "suite")')
+ # Some of our tests hard-code pathnames e.g., to execute, so until
+ # th we don't have the freedom to run from anyplace.
+# parser.add_option('--mongo-repo', dest='mongoRepo', default=None,
+# help='Top-level directory of mongo checkout to use. (default: script will make a guess)')
+ parser.add_option('--test-path', dest='testPath', default=None,
+ help="Path to the test executables to run "
+ "(currently only used for smokeClient)")
+ parser.add_option('--mongod', dest='mongodExecutable', #default='./mongod',
+ help='Path to mongod to run (default "./mongod")')
+ parser.add_option('--port', dest='mongodPort', default="32000",
+ help='Port the mongod will bind to (default 32000)')
+ parser.add_option('--mongo', dest='shellExecutable', #default="./mongo",
+ help='Path to mongo, for .js test files (default "./mongo")')
+ parser.add_option('--continue-on-failure', dest='continueOnFailure',
+ action="store_true", default=False,
+ help='If supplied, continue testing even after a test fails')
+ parser.add_option('--one-mongod-per-test', dest='oneMongodPerTest',
+ action="store_true", default=False,
+ help='If supplied, run each test in a fresh mongod')
+ parser.add_option('--from-file', dest='File',
+ help="Run tests/suites named in FILE, one test per line, '-' means stdin")
+ parser.add_option('--smoke-db-prefix', dest='smokeDbPrefix', default='',
+ help="Prefix to use for the mongods' dbpaths.")
+ parser.add_option('--small-oplog', dest='smallOplog', default=False,
+ action="store_true",
+ help='Run tests with master/slave replication & use a small oplog')
+ global tests
+ (options, tests) = parser.parse_args()
+
+# global mongoRepo
+# if options.mongoRepo:
+# pass
+# mongoRepo = options.mongoRepo
+# else:
+# prefix = ''
+# while True:
+# if os.path.exists(prefix+'buildscripts'):
+# mongoRepo = os.path.normpath(prefix)
+# break
+# else:
+# prefix += '../'
+# # FIXME: will this be a device's root directory on
+# # Windows?
+# if os.path.samefile('/', prefix):
+# raise Exception("couldn't guess the mongo repository path")
+
+ print tests
+
+ global mongoRepo, mongodExecutable, mongodPort, shellExecutable, continueOnFailure, oneMongodPerTest, smallOplog, smokeDbPrefix, testPath
+ testPath = options.testPath
+ mongodExecutable = options.mongodExecutable if options.mongodExecutable else os.path.join(mongoRepo, 'mongod')
+ mongodPort = options.mongodPort if options.mongodPort else mongodPort
+ shellExecutable = options.shellExecutable if options.shellExecutable else os.path.join(mongoRepo, 'mongo')
+ continueOnFailure = options.continueOnFailure if options.continueOnFailure else continueOnFailure
+ oneMongodPerTest = options.oneMongodPerTest if options.oneMongodPerTest else oneMongodPerTest
+ smokeDbPrefix = options.smokeDbPrefix
+ smallOplog = options.smallOplog
+
+ if options.File:
+ if options.File == '-':
+ tests = sys.stdin.readlines()
+ else:
+ with open(options.File) as f:
+ tests = f.readlines()
+ tests = [t.rstrip('\n') for t in tests]
+
+ if not tests:
+ raise Exception( "no tests specified" )
+ # If we're in suite mode, tests is a list of names of sets of tests.
+ if options.mode == 'suite':
+ # Suites: smoke, smokePerf, smokeJs, smokeQuota, smokeJsPerf,
+ # smokeJsSlow, smokeParalell, smokeClone, smokeRepl, smokeDisk
+ suites = tests
+ tests = []
+ expandSuites(suites)
+ elif options.mode == 'files':
+ tests = [(os.path.abspath(test), True) for test in tests]
+
+ runTests(tests)
+ global exit_bad
+ exit_bad = False
+
+atexit.register(report)
+
+if __name__ == "__main__":
+ main()
diff --git a/buildscripts/utils.py b/buildscripts/utils.py
index 41d6767..1ca2fdd 100644
--- a/buildscripts/utils.py
+++ b/buildscripts/utils.py
@@ -2,9 +2,50 @@
import re
import socket
import time
-
+import os
# various utilities that are handy
+def getGitBranch():
+ if not os.path.exists( ".git" ):
+ return None
+
+ version = open( ".git/HEAD" ,'r' ).read().strip()
+ if not version.startswith( "ref: " ):
+ return version
+ version = version.split( "/" )
+ version = version[len(version)-1]
+ return version
+
+def getGitBranchString( prefix="" , postfix="" ):
+ t = re.compile( '[/\\\]' ).split( os.getcwd() )
+ if len(t) > 2 and t[len(t)-1] == "mongo":
+ par = t[len(t)-2]
+ m = re.compile( ".*_([vV]\d+\.\d+)$" ).match( par )
+ if m is not None:
+ return prefix + m.group(1).lower() + postfix
+ if par.find("Nightly") > 0:
+ return ""
+
+
+ b = getGitBranch()
+ if b == None or b == "master":
+ return ""
+ return prefix + b + postfix
+
+def getGitVersion():
+ if not os.path.exists( ".git" ):
+ return "nogitversion"
+
+ version = open( ".git/HEAD" ,'r' ).read().strip()
+ if not version.startswith( "ref: " ):
+ return version
+ version = version[5:]
+ f = ".git/" + version
+ if not os.path.exists( f ):
+ return version
+ return open( f , 'r' ).read().strip()
+
+
def execsys( args ):
import subprocess
if isinstance( args , str ):
@@ -24,6 +65,40 @@ def getprocesslist():
r = re.compile( "[\r\n]+" )
return r.split( raw )
+
+def removeIfInList( lst , thing ):
+ if thing in lst:
+ lst.remove( thing )
+
+def findVersion( root , choices ):
+ for c in choices:
+ if ( os.path.exists( root + c ) ):
+ return root + c
+ raise "can't find a version of [" + root + "] choices: " + choices
+
+def choosePathExist( choices , default=None):
+ for c in choices:
+ if c != None and os.path.exists( c ):
+ return c
+ return default
+
+def filterExists(paths):
+ return filter(os.path.exists, paths)
+
+def ensureDir( name ):
+ d = os.path.dirname( name )
+ if not os.path.exists( d ):
+ print( "Creating dir: " + name );
+ os.makedirs( d )
+ if not os.path.exists( d ):
+ raise "Failed to create dir: " + name
+
+
+def distinctAsString( arr ):
+ s = set()
+ for x in arr:
+ s.add( str(x) )
+ return list(s)
def checkMongoPort( port=27017 ):
sock = socket.socket()
@@ -32,6 +107,7 @@ def checkMongoPort( port=27017 ):
sock.connect(("localhost", port))
sock.close()
+
def didMongodStart( port=27017 , timeout=20 ):
while timeout > 0:
time.sleep( 1 )
@@ -41,7 +117,5 @@ def didMongodStart( port=27017 , timeout=20 ):
except Exception,e:
print( e )
timeout = timeout - 1
-
return False
-
generated by cgit v1.2.3 (git 2.46.0) at 2025-12-08 10:15:05 +0000