summaryrefslogtreecommitdiff
path: root/buildscripts
diff options
context:
space:
mode:
Diffstat (limited to 'buildscripts')
-rw-r--r--buildscripts/distmirror.py2
-rwxr-xr-xbuildscripts/errorcodes.py21
-rw-r--r--buildscripts/frob_version.py2
-rw-r--r--buildscripts/hacks_ubuntu.py2
-rw-r--r--buildscripts/makealldists.py20
-rw-r--r--buildscripts/makedist.py32
-rw-r--r--buildscripts/mergerepositories.py2
-rw-r--r--buildscripts/s3del.py36
-rwxr-xr-xbuildscripts/smoke.py425
-rw-r--r--buildscripts/utils.py21
10 files changed, 290 insertions, 273 deletions
diff --git a/buildscripts/distmirror.py b/buildscripts/distmirror.py
index 1902e2a..7af1a89 100644
--- a/buildscripts/distmirror.py
+++ b/buildscripts/distmirror.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
# Download mongodb stuff (at present builds, sources, docs, but not
# drivers).
diff --git a/buildscripts/errorcodes.py b/buildscripts/errorcodes.py
index d87b7ad..a105647 100755
--- a/buildscripts/errorcodes.py
+++ b/buildscripts/errorcodes.py
@@ -1,32 +1,17 @@
-#!/usr/bin/python
+#!/usr/bin/env python
import os
import sys
import re
import utils
-def getAllSourceFiles( arr=None , prefix="." ):
- if arr is None:
- arr = []
-
- for x in os.listdir( prefix ):
- if x.startswith( "." ) or x.startswith( "pcre-" ) or x.startswith( "32bit" ) or x.startswith( "mongodb-" ) or x.startswith("debian") or x.startswith( "mongo-cxx-driver" ):
- continue
- full = prefix + "/" + x
- if os.path.isdir( full ) and not os.path.islink( full ):
- getAllSourceFiles( arr , full )
- else:
- if full.endswith( ".cpp" ) or full.endswith( ".h" ) or full.endswith( ".c" ):
- arr.append( full )
-
- return arr
assertNames = [ "uassert" , "massert" ]
def assignErrorCodes():
cur = 10000
for root in assertNames:
- for x in getAllSourceFiles():
+ for x in utils.getAllSourceFiles():
print( x )
didAnything = False
fixed = ""
@@ -50,7 +35,7 @@ def readErrorCodes( callback ):
ps = [ re.compile( "([um]asser(t|ted)) *\( *(\d+)" ) ,
re.compile( "(User|Msg)Exceptio(n)\( *(\d+)" )
]
- for x in getAllSourceFiles():
+ for x in utils.getAllSourceFiles():
lineNum = 1
for line in open( x ):
for p in ps:
diff --git a/buildscripts/frob_version.py b/buildscripts/frob_version.py
index 7b89e0b..560a8ed 100644
--- a/buildscripts/frob_version.py
+++ b/buildscripts/frob_version.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
from __future__ import with_statement
import tempfile
diff --git a/buildscripts/hacks_ubuntu.py b/buildscripts/hacks_ubuntu.py
index 81deddd..977d2df 100644
--- a/buildscripts/hacks_ubuntu.py
+++ b/buildscripts/hacks_ubuntu.py
@@ -21,7 +21,7 @@ def foundxulrunner( env , options ):
if best is None:
- print( "warning: using ubuntu without xulrunner-dev. we reccomend installing it" )
+ print( "warning: using ubuntu without xulrunner-dev. we recommend installing it" )
return False
incroot = "/usr/include/" + best + "/"
diff --git a/buildscripts/makealldists.py b/buildscripts/makealldists.py
index 762700e..6b6f365 100644
--- a/buildscripts/makealldists.py
+++ b/buildscripts/makealldists.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
from __future__ import with_statement
import subprocess
@@ -184,14 +184,16 @@ def __main__():
print "makedist output under: %s\ncombined repo: %s\n" % (outputroot, repodir)
sys.stdout.flush()
# Add more dist/version/architecture tuples as they're supported.
- dists = (("ubuntu", "10.4"),
+ dists = (("ubuntu", "10.10"),
+ ("ubuntu", "10.4"),
("ubuntu", "9.10"),
("ubuntu", "9.4"),
- ("ubuntu", "8.10"),
+ #("ubuntu", "8.10"),
("debian", "5.0"),
("centos", "5.4"),
- ("fedora", "11"),
- ("fedora", "12"))
+ #("fedora", "12"),
+ ("fedora", "13"),
+ ("fedora", "14"))
arches = ("x86", "x86_64")
# mongos = branches.split(',')
# Run a makedist for each distro/version/architecture tuple above.
@@ -202,7 +204,7 @@ def __main__():
procs = []
count = 0
for ((distro, distro_version), arch, spec) in gen([dists, arches, [branches]]):
- # FIXME: now x86 fedoras on RackSpace circa 04/10.
+ # FIXME: no x86 fedoras on RackSpace circa 04/10.
if distro == "fedora" and arch == "x86":
continue
count+=1
@@ -264,9 +266,9 @@ def __main__():
if r != 0:
raise Exception("mergerepositories.py exited %d" % r)
print repodir
- pushrepo(repodir)
- shutil.rmtree(outputroot)
- shutil.rmtree(repodir)
+ #pushrepo(repodir)
+ #shutil.rmtree(outputroot)
+ #shutil.rmtree(repodir)
return 0
diff --git a/buildscripts/makedist.py b/buildscripts/makedist.py
index 1928b76..b5387c2 100644
--- a/buildscripts/makedist.py
+++ b/buildscripts/makedist.py
@@ -123,7 +123,9 @@ class EC2InstanceConfigurator(BaseConfigurator):
def __init__(self, **kwargs):
super(EC2InstanceConfigurator, self).__init__(**kwargs)
self.configuration += [("ec2_ami",
- ((("ubuntu", "10.4", "x86_64"), "ami-bf07ead6"),
+ ((("ubuntu", "10.10", "x86_64"), "ami-688c7801"),
+ (("ubuntu", "10.10", "x86"), "ami-1a837773"),
+ (("ubuntu", "10.4", "x86_64"), "ami-bf07ead6"),
(("ubuntu", "10.4", "x86"), "ami-f707ea9e"),
(("ubuntu", "9.10", "x86_64"), "ami-55739e3c"),
(("ubuntu", "9.10", "x86"), "ami-bb709dd2"),
@@ -140,9 +142,9 @@ class EC2InstanceConfigurator(BaseConfigurator):
(("fedora", "8", "x86_64"), "ami-2547a34c"),
(("fedora", "8", "x86"), "ami-5647a33f"))),
("rackspace_imgname",
- ((("fedora", "11", "x86_64"), "Fedora 11"),
- (("fedora", "12", "x86_64"), "Fedora 12"),
- (("fedora", "13", "x86_64"), "Fedora 13"))),
+ ((("fedora", "12", "x86_64"), "Fedora 12"),
+ (("fedora", "13", "x86_64"), "Fedora 13"),
+ (("fedora", "14", "x86_64"), "Fedora 14"))),
("ec2_mtype",
((("*", "*", "x86"), "m1.small"),
(("*", "*", "x86_64"), "m1.large"))),
@@ -266,6 +268,7 @@ class SshConnectionConfigurator (BaseConfigurator):
# FLAW: this actually depends more on the AMI
# than the triple.
((("debian", "*", "*"), "root"),
+ (("ubuntu", "10.10", "*"), "ubuntu"),
(("ubuntu", "10.4", "*"), "ubuntu"),
(("ubuntu", "9.10", "*"), "ubuntu"),
(("ubuntu", "9.4", "*"), "root"),
@@ -420,8 +423,12 @@ cp {pkg_name}{pkg_name_suffix}*.tar.gz "{pkg_product_dir}/{distro_version}/10gen
dpkg-scanpackages "{pkg_product_dir}/{distro_version}/10gen/binary-{distro_arch}" /dev/null | gzip -9c > "{pkg_product_dir}/{distro_version}/10gen/binary-{distro_arch}/Packages.gz"
dpkg-scansources "{pkg_product_dir}/{distro_version}/10gen/source" /dev/null | gzip -9c > "{pkg_product_dir}/{distro_version}/10gen/source/Sources.gz"
"""
- rpm_prereq_commands = """
-rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/{distro_arch}/epel-release-5-3.noarch.rpm
+ centos_prereq_commands = """
+rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/{distro_arch}/epel-release-5-4.noarch.rpm
+yum -y install {pkg_prereq_str}
+"""
+ fedora_prereq_commands = """
+#rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/{distro_arch}/epel-release-5-4.noarch.rpm
yum -y install {pkg_prereq_str}
"""
rpm_build_commands="""
@@ -462,6 +469,7 @@ rpm -ivh /usr/src/redhat/RPMS/{distro_arch}/boost-devel-1.38.0-1.{distro_arch}.r
# 1.34, but 1.35 packages are available, so we want those.
versioned_deb_boost_prereqs = ["libboost-thread1.35-dev", "libboost-filesystem1.35-dev", "libboost-program-options1.35-dev", "libboost-date-time1.35-dev", "libboost1.35-dev"]
+ new_versioned_deb_boost_prereqs = ["libboost-thread1.42-dev", "libboost-filesystem1.42-dev", "libboost-program-options1.42-dev", "libboost-date-time1.42-dev", "libboost1.42-dev"]
unversioned_deb_xulrunner_prereqs = ["xulrunner-dev"]
old_versioned_deb_xulrunner_prereqs = ["xulrunner-1.9-dev"]
@@ -511,6 +519,8 @@ git clone git://github.com/mongodb/mongo.git
self.versioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
(("ubuntu", "9.10", "*"),
self.unversioned_deb_boost_prereqs + self.unversioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
+ (("ubuntu", "10.10", "*"),
+ self.new_versioned_deb_boost_prereqs + self.new_versioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
(("ubuntu", "10.4", "*"),
self.unversioned_deb_boost_prereqs + self.new_versioned_deb_xulrunner_prereqs + self.common_deb_prereqs),
(("ubuntu", "8.10", "*"),
@@ -532,22 +542,24 @@ git clone git://github.com/mongodb/mongo.git
(("ubuntu", "*", "*"),
self.preamble_commands + self.deb_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.deb_build_commands),
(("centos", "*", "*"),
- self.preamble_commands + self.old_rpm_precommands + self.rpm_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.mangle_files_for_ancient_redhat_commands + self.rpm_build_commands),
+ self.preamble_commands + self.old_rpm_precommands + self.centos_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.mangle_files_for_ancient_redhat_commands + self.rpm_build_commands),
(("fedora", "*", "*"),
- self.preamble_commands + self.old_rpm_precommands + self.rpm_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.rpm_build_commands))),
+ self.preamble_commands + self.old_rpm_precommands + self.fedora_prereq_commands + self.get_mongo_commands + self.mangle_files_commands + self.rpm_build_commands))),
("preamble_commands",
((("*", "*", "*"), self.preamble_commands),
)),
("install_prereqs",
((("debian", "*", "*"), self.deb_prereq_commands),
(("ubuntu", "*", "*"), self.deb_prereq_commands),
- (("centos", "*", "*"), self.rpm_prereq_commands),
- (("fedora", "*", "*"), self.rpm_prereq_commands))),
+ (("centos", "*", "*"), self.centos_prereq_commands),
+ (("fedora", "*", "*"), self.fedora_prereq_commands))),
("get_mongo",
((("*", "*", "*"), self.get_mongo_commands),
)),
("mangle_mongo",
((("debian", "*", "*"), self.mangle_files_commands),
+ (("ubuntu", "10.10", "*"),
+ self.mangle_files_commands + self.mangle_files_for_new_deb_xulrunner_commands),
(("ubuntu", "10.4", "*"),
self.mangle_files_commands + self.mangle_files_for_new_deb_xulrunner_commands),
(("ubuntu", "*", "*"), self.mangle_files_commands),
diff --git a/buildscripts/mergerepositories.py b/buildscripts/mergerepositories.py
index bc50d08..028b6e2 100644
--- a/buildscripts/mergerepositories.py
+++ b/buildscripts/mergerepositories.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
from __future__ import with_statement
from libcloud.types import Provider
diff --git a/buildscripts/s3del.py b/buildscripts/s3del.py
new file mode 100644
index 0000000..7967de6
--- /dev/null
+++ b/buildscripts/s3del.py
@@ -0,0 +1,36 @@
+
+import os
+import sys
+import time
+
+sys.path.append( "." )
+sys.path.append( ".." )
+sys.path.append( "../../" )
+sys.path.append( "../../../" )
+
+import simples3
+import settings
+import subprocess
+
+# check s3 for md5 hashes
+
+def check_dir( bucket , prefix , todel ):
+
+ for ( key , modify , etag , size ) in bucket.listdir( prefix=prefix ):
+ if key.find( todel ) < 0:
+ continue
+ print( key )
+ time.sleep( 2 )
+ bucket.delete( key )
+
+def clean( todel ):
+
+
+ bucket = simples3.S3Bucket( settings.bucket , settings.id , settings.key )
+
+ for x in [ "osx" , "linux" , "win32" , "sunos5" , "src" ]:
+ check_dir( bucket , x , todel )
+
+
+if __name__ == "__main__":
+ clean( sys.argv[1] )
diff --git a/buildscripts/smoke.py b/buildscripts/smoke.py
index 0023226..5fdd26f 100755
--- a/buildscripts/smoke.py
+++ b/buildscripts/smoke.py
@@ -1,8 +1,8 @@
-#!/usr/bin/python
+#!/usr/bin/env python
# smoke.py: run some mongo tests.
-# Bugs, TODOs:
+# Bugs, TODOs:
# 0 Some tests hard-code pathnames relative to the mongo repository,
# so the smoke.py process and all its children must be run with the
@@ -34,49 +34,48 @@
# jobs on the same host at once. So something's gotta change.
from __future__ import with_statement
-from subprocess import Popen, PIPE, call
+
+import glob
+from optparse import OptionParser
import os
+import parser
+import re
+import shutil
+import socket
+from subprocess import (Popen,
+ PIPE,
+ call)
import sys
-import utils
import time
-import socket
-from optparse import OptionParser
-import atexit
-import glob
-import shutil
-import re
-import parser
-mongoRepo = os.getcwd() #'./'
-testPath = None
+from pymongo import Connection
+
+import utils
-mongodExecutable = "./mongod"
-mongodPort = "32000"
-shellExecutable = "./mongo"
-continueOnFailure = False
-oneMongodPerTest = False
+# TODO clean this up so we don't need globals...
+mongo_repo = os.getcwd() #'./'
+test_path = None
+mongod_executable = None
+mongod_port = None
+shell_executable = None
+continue_on_failure = None
tests = []
winners = []
losers = {}
-# Finally, atexit functions seem to be a little oblivious to whether
-# Python is exiting because of an error, so we'll use this to
-# communicate with the report() function.
-exit_bad = True
-
# For replication hash checking
-replicated_dbs = []
+replicated_collections = []
lost_in_slave = []
lost_in_master = []
screwy_in_slave = {}
-smokeDbPrefix = ''
-smallOplog = False
+smoke_db_prefix = ''
+small_oplog = False
# This class just implements the with statement API, for a sneaky
# purpose below.
-class nothing(object):
+class Nothing(object):
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
@@ -99,23 +98,23 @@ class mongod(object):
print >> sys.stderr, e
return not isinstance(value, Exception)
- def ensureTestDirs(self):
- utils.ensureDir( smokeDbPrefix + "/tmp/unittest/" )
- utils.ensureDir( smokeDbPrefix + "/data/" )
- utils.ensureDir( smokeDbPrefix + "/data/db/" )
+ def ensure_test_dirs(self):
+ utils.ensureDir(smoke_db_prefix + "/tmp/unittest/")
+ utils.ensureDir(smoke_db_prefix + "/data/")
+ utils.ensureDir(smoke_db_prefix + "/data/db/")
- def checkMongoPort( self, port=27017 ):
+ def check_mongo_port(self, port=27017):
sock = socket.socket()
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
sock.settimeout(1)
sock.connect(("localhost", int(port)))
sock.close()
-
- def didMongodStart( self, port=mongodPort, timeout=20 ):
+
+ def did_mongod_start(self, port=mongod_port, timeout=20):
while timeout > 0:
- time.sleep( 1 )
+ time.sleep(1)
try:
- self.checkMongoPort( int(port) )
+ self.check_mongo_port(int(port))
return True
except Exception,e:
print >> sys.stderr, e
@@ -123,47 +122,45 @@ class mongod(object):
return False
def start(self):
- global mongodPort
+ global mongod_port
global mongod
if self.proc:
print >> sys.stderr, "probable bug: self.proc already set in start()"
return
- self.ensureTestDirs()
- dirName = smokeDbPrefix + "/data/db/sconsTests/"
- self.port = int(mongodPort)
+ self.ensure_test_dirs()
+ dir_name = smoke_db_prefix + "/data/db/sconsTests/"
+ self.port = int(mongod_port)
self.slave = False
if 'slave' in self.kwargs:
- dirName = smokeDbPrefix + '/data/db/sconsTestsSlave/'
- srcport = mongodPort
+ dir_name = smoke_db_prefix + '/data/db/sconsTestsSlave/'
+ srcport = mongod_port
self.port += 1
self.slave = True
- if os.path.exists ( dirName ):
+ if os.path.exists(dir_name):
if 'slave' in self.kwargs:
- argv = ["python", "buildscripts/cleanbb.py", '--nokill', dirName]
-
+ argv = ["python", "buildscripts/cleanbb.py", '--nokill', dir_name]
else:
- argv = ["python", "buildscripts/cleanbb.py", dirName]
- call( argv )
- utils.ensureDir( dirName )
- argv = [mongodExecutable, "--port", str(self.port), "--dbpath", dirName]
- if self.kwargs.get('smallOplog'):
- argv += ["--master", "--oplogSize", "10"]
+ argv = ["python", "buildscripts/cleanbb.py", dir_name]
+ call(argv)
+ utils.ensureDir(dir_name)
+ argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
+ if self.kwargs.get('small_oplog'):
+ argv += ["--master", "--oplogSize", "128"]
if self.slave:
- argv += ['--slave', '--source', 'localhost:'+str(srcport)]
+ argv += ['--slave', '--source', 'localhost:' + str(srcport)]
print "running " + " ".join(argv)
self.proc = Popen(argv)
- if not self.didMongodStart( self.port ):
- raise Exception( "Failed to start mongod" )
-
+ if not self.did_mongod_start(self.port):
+ raise Exception("Failed to start mongod")
+
if self.slave:
- while True:
- argv = [shellExecutable, "--port", str(self.port), "--quiet", "--eval", 'db.printSlaveReplicationInfo()']
- res = Popen(argv, stdout=PIPE).communicate()[0]
- if res.find('initial sync') < 0:
- break
-
-
-
+ local = Connection(port=self.port, slave_okay=True).local
+ synced = False
+ while not synced:
+ synced = True
+ for source in local.sources.find(fields=["syncedTo"]):
+ synced = synced and "syncedTo" in source and source["syncedTo"]
+
def stop(self):
if not self.proc:
print >> sys.stderr, "probable bug: self.proc unset in stop()"
@@ -177,11 +174,14 @@ class mongod(object):
win32process.TerminateProcess(self.proc._handle, -1)
else:
from os import kill
- kill( self.proc.pid, 15 )
+ kill(self.proc.pid, 15)
self.proc.wait()
sys.stderr.flush()
sys.stdout.flush()
-
+
+ def wait_for_repl(self):
+ Connection(port=self.port).test.smokeWait.insert({}, w=2, wtimeout=5*60*1000)
+
class Bug(Exception):
def __str__(self):
return 'bug in smoke.py: ' + super(Bug, self).__str__()
@@ -192,6 +192,7 @@ class TestFailure(Exception):
class TestExitFailure(TestFailure):
def __init__(self, *args):
self.path = args[0]
+
self.status=args[1]
def __str__(self):
return "test %s exited with status %d" % (self.path, self.status)
@@ -204,48 +205,41 @@ class TestServerFailure(TestFailure):
def __str__(self):
return 'mongod not running after executing test %s' % self.path
-def checkDbHashes(master, slave):
+def check_db_hashes(master, slave):
# Need to pause a bit so a slave might catch up...
if not slave.slave:
raise(Bug("slave instance doesn't have slave attribute set"))
- print "waiting for slave to catch up..."
- ARB=10 # ARBITRARY
- time.sleep(ARB)
- while True:
- # FIXME: it's probably better to do an empty insert and a
- # getLastError() to force a sync.
- argv = [shellExecutable, "--port", str(slave.port), "--quiet", "--eval", 'db.printSlaveReplicationInfo()']
- res = Popen(argv, stdout=PIPE).communicate()[0]
- m = re.search('(\d+)secs ', res)
- if int(m.group(1)) > ARB: #res.find('initial sync') < 0:
- break
- time.sleep(3)
+ print "waiting for slave to catch up"
+ master.wait_for_repl()
+ print "caught up!"
# FIXME: maybe make this run dbhash on all databases?
for mongod in [master, slave]:
- argv = [shellExecutable, "--port", str(mongod.port), "--quiet", "--eval", "x=db.runCommand('dbhash'); printjson(x.collections)"]
- hashstr = Popen(argv, stdout=PIPE).communicate()[0]
- # WARNING FIXME KLUDGE et al.: this is sleazy and unsafe.
- mongod.dict = eval(hashstr)
+ mongod.dbhash = Connection(port=mongod.port, slave_okay=True).test.command("dbhash")
+ mongod.dict = mongod.dbhash["collections"]
+
+ global lost_in_slave, lost_in_master, screwy_in_slave, replicated_collections
- global lost_in_slave, lost_in_master, screwy_in_slave, replicated_dbs
+ replicated_collections += master.dict.keys()
- for db in replicated_dbs:
+ for db in replicated_collections:
if db not in slave.dict:
lost_in_slave.append(db)
mhash = master.dict[db]
shash = slave.dict[db]
if mhash != shash:
screwy_in_slave[db] = mhash + "/" + shash
+
for db in slave.dict.keys():
if db not in master.dict:
lost_in_master.append(db)
- replicated_dbs += master.dict.keys()
+
+
# Blech.
def skipTest(path):
- if smallOplog:
+ if small_oplog:
if os.path.basename(path) in ["cursor8.js", "indexh.js"]:
return True
return False
@@ -254,78 +248,79 @@ def runTest(test):
(path, usedb) = test
(ignore, ext) = os.path.splitext(path)
if skipTest(path):
- print "skippping " + path
+ print "skipping " + path
return
if ext == ".js":
- argv=[shellExecutable, "--port", mongodPort]
+ argv = [shell_executable, "--port", mongod_port]
if not usedb:
- argv += ["--nodb"]
- if smallOplog:
+ argv += ["--nodb"]
+ if small_oplog:
argv += ["--eval", 'testingReplication = true;']
argv += [path]
elif ext in ["", ".exe"]:
# Blech.
if os.path.basename(path) in ["test", "test.exe", "perftest", "perftest.exe"]:
- argv=[path]
+ argv = [path]
# more blech
elif os.path.basename(path) == 'mongos':
- argv=[path, "--test"]
+ argv = [path, "--test"]
else:
- argv=[testPath and os.path.abspath(os.path.join(testPath, path)) or path,
- "--port", mongodPort]
+ argv = [test_path and os.path.abspath(os.path.join(test_path, path)) or path,
+ "--port", mongod_port]
else:
raise Bug("fell off in extenstion case: %s" % path)
print " *******************************************"
print " Test : " + os.path.basename(path) + " ..."
- t1=time.time()
+ t1 = time.time()
# FIXME: we don't handle the case where the subprocess
# hangs... that's bad.
- r = call(argv, cwd=testPath)
- t2=time.time()
- print " " + str((t2-t1)*1000) + "ms"
+ r = call(argv, cwd=test_path)
+ t2 = time.time()
+ print " " + str((t2 - t1) * 1000) + "ms"
if r != 0:
raise TestExitFailure(path, r)
- if Popen( [ mongodExecutable, "msg", "ping", mongodPort ], stdout=PIPE ).communicate()[0].count( "****ok" ) == 0:
- raise TestServerFailure(path)
- if call( [ mongodExecutable, "msg", "ping", mongodPort ] ) != 0:
+
+ try:
+ c = Connection( "127.0.0.1" , int(mongod_port) )
+ except Exception,e:
raise TestServerFailure(path)
- print ""
-def runTests(tests):
- # If we're in one-mongo-per-test mode, we instantiate a nothing
- # around the loop, and a mongod inside the loop.
+ print ""
+def run_tests(tests):
# FIXME: some suites of tests start their own mongod, so don't
# need this. (So long as there are no conflicts with port,
# dbpath, etc., and so long as we shut ours down properly,
# starting this mongod shouldn't break anything, though.)
- with nothing() if oneMongodPerTest else mongod(smallOplog=smallOplog) as master1:
- with nothing() if oneMongodPerTest else (mongod(slave=True) if smallOplog else nothing()) as slave1:
+
+ # The reason we use with is so that we get __exit__ semantics
+
+ with mongod(small_oplog=small_oplog) as master:
+ with mongod(slave=True) if small_oplog else Nothing() as slave:
+ if small_oplog:
+ master.wait_for_repl()
+
for test in tests:
try:
- with mongod(smallOplog=smallOplog) if oneMongodPerTest else nothing() as master2:
- with mongod(slave=True) if oneMongodPerTest and smallOplog else nothing() as slave2:
- runTest(test)
+ runTest(test)
winners.append(test)
- if isinstance(slave2, mongod):
- checkDbHashes(master2, slave2)
except TestFailure, f:
try:
print f
# Record the failing test and re-raise.
losers[f.path] = f.status
raise f
- except TestServerFailure, f:
- if not oneMongodPerTest:
- return 2
+ except TestServerFailure, f:
+ return 2
except TestFailure, f:
- if not continueOnFailure:
+ if not continue_on_failure:
return 1
- if isinstance(slave1, mongod):
- checkDbHashes(master1, slave1)
+ if isinstance(slave, mongod):
+ check_db_hashes(master, slave)
return 0
+
def report():
print "%d test%s succeeded" % (len(winners), '' if len(winners) == 1 else 's')
num_missed = len(tests) - (len(winners) + len(losers.keys()))
@@ -335,7 +330,7 @@ def report():
print "The following tests failed (with exit code):"
for loser in losers:
print "%s\t%d" % (loser, losers[loser])
-
+
def missing(lst, src, dst):
if lst:
print """The following collections were present in the %s but not the %s
@@ -349,149 +344,124 @@ at the end of testing:""" % (src, dst)
at the end of testing:"""
for db in screwy_in_slave.keys():
print "%s\t %s" % (db, screwy_in_slave[db])
- if smallOplog and not (lost_in_master or lost_in_slave or screwy_in_slave):
- print "replication ok for %d collections" % (len(replicated_dbs))
- if (exit_bad or losers or lost_in_slave or lost_in_master or screwy_in_slave):
- status = 1
- else:
- status = 0
- exit (status)
+ if small_oplog and not (lost_in_master or lost_in_slave or screwy_in_slave):
+ print "replication ok for %d collections" % (len(replicated_collections))
+ if losers or lost_in_slave or lost_in_master or screwy_in_slave:
+ raise Exception("Test failures")
+
-def expandSuites(suites):
+def expand_suites(suites):
globstr = None
- global mongoRepo, tests
+ tests = []
for suite in suites:
- if suite == 'smokeAll':
- tests = []
- expandSuites(['smoke', 'smokePerf', 'smokeClient', 'smokeJs', 'smokeJsPerf', 'smokeJsSlowNightly', 'smokeJsSlowWeekly', 'smokeParallel', 'smokeClone', 'smokeParallel', 'smokeRepl', 'smokeAuth', 'smokeSharding', 'smokeTool'])
- break
- if suite == 'smoke':
+ if suite == 'all':
+ return expand_suites(['test', 'perf', 'client', 'js', 'jsPerf', 'jsSlowNightly', 'jsSlowWeekly', 'parallel', 'clone', 'parallel', 'repl', 'auth', 'sharding', 'tool'])
+ if suite == 'test':
if os.sys.platform == "win32":
program = 'test.exe'
else:
program = 'test'
(globstr, usedb) = (program, False)
- elif suite == 'smokePerf':
+ elif suite == 'perf':
if os.sys.platform == "win32":
program = 'perftest.exe'
else:
program = 'perftest'
(globstr, usedb) = (program, False)
- elif suite == 'smokeJs':
- # FIXME: _runner.js seems equivalent to "[!_]*.js".
- #(globstr, usedb) = ('_runner.js', True)
- (globstr, usedb) = ('[!_]*.js', True)
- elif suite == 'smokeQuota':
- (globstr, usedb) = ('quota/*.js', True)
- elif suite == 'smokeJsPerf':
- (globstr, usedb) = ('perf/*.js', True)
- elif suite == 'smokeDisk':
- (globstr, usedb) = ('disk/*.js', True)
- elif suite == 'smokeJsSlowNightly':
- (globstr, usedb) = ('slowNightly/*.js', True)
- elif suite == 'smokeJsSlowWeekly':
- (globstr, usedb) = ('slowWeekly/*.js', True)
- elif suite == 'smokeParallel':
- (globstr, usedb) = ('parallel/*.js', True)
- elif suite == 'smokeClone':
- (globstr, usedb) = ('clone/*.js', False)
- elif suite == 'smokeRepl':
- (globstr, usedb) = ('repl/*.js', False)
- elif suite == 'smokeReplSets':
- (globstr, usedb) = ('replsets/*.js', False)
- elif suite == 'smokeAuth':
- (globstr, usedb) = ('auth/*.js', False)
- elif suite == 'smokeSharding':
- (globstr, usedb) = ('sharding/*.js', False)
- elif suite == 'smokeTool':
- (globstr, usedb) = ('tool/*.js', False)
- # well, the above almost works for everything...
- elif suite == 'smokeClient':
+ elif suite == 'client':
paths = ["firstExample", "secondExample", "whereExample", "authTest", "clientTest", "httpClientTest"]
if os.sys.platform == "win32":
- paths = [path+'.exe' for path in paths]
+ paths = [path + '.exe' for path in paths]
# hack
- tests += [(testPath and path or os.path.join(mongoRepo, path), False) for path in paths]
+ tests += [(test_path and path or os.path.join(mongo_repo, path), False) for path in paths]
elif suite == 'mongosTest':
if os.sys.platform == "win32":
program = 'mongos.exe'
else:
program = 'mongos'
- tests += [(os.path.join(mongoRepo, program), False)]
+ tests += [(os.path.join(mongo_repo, program), False)]
+ elif os.path.exists( suite ):
+ tests += [ ( os.path.join( mongo_repo , suite ) , True ) ]
else:
- raise Exception('unknown test suite %s' % suite)
+ try:
+ globstr, usedb = {"js": ("[!_]*.js", True),
+ "quota": ("quota/*.js", True),
+ "jsPerf": ("perf/*.js", True),
+ "disk": ("disk/*.js", True),
+ "jsSlowNightly": ("slowNightly/*.js", True),
+ "jsSlowWeekly": ("slowWeekly/*.js", True),
+ "parallel": ("parallel/*.js", True),
+ "clone": ("clone/*.js", False),
+ "repl": ("repl/*.js", False),
+ "replSets": ("replsets/*.js", False),
+ "dur": ("dur/*.js", False),
+ "auth": ("auth/*.js", False),
+ "sharding": ("sharding/*.js", False),
+ "tool": ("tool/*.js", False)}[suite]
+ except KeyError:
+ raise Exception('unknown test suite %s' % suite)
if globstr:
- globstr = os.path.join(mongoRepo, (os.path.join(('jstests/' if globstr.endswith('.js') else ''), globstr)))
+ globstr = os.path.join(mongo_repo, (os.path.join(('jstests/' if globstr.endswith('.js') else ''), globstr)))
paths = glob.glob(globstr)
paths.sort()
tests += [(path, usedb) for path in paths]
- if not tests:
- raise Exception( "no tests found" )
+
return tests
+def add_exe(e):
+ if os.sys.platform.startswith( "win" ) and not e.endswith( ".exe" ):
+ e += ".exe"
+ return e
+
def main():
+ global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, smoke_db_prefix, test_path
parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*")
parser.add_option('--mode', dest='mode', default='suite',
- help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests. (default "suite")')
+ help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests (%default)')
# Some of our tests hard-code pathnames e.g., to execute, so until
- # th we don't have the freedom to run from anyplace.
-# parser.add_option('--mongo-repo', dest='mongoRepo', default=None,
-# help='Top-level directory of mongo checkout to use. (default: script will make a guess)')
- parser.add_option('--test-path', dest='testPath', default=None,
- help="Path to the test executables to run "
- "(currently only used for smokeClient)")
- parser.add_option('--mongod', dest='mongodExecutable', #default='./mongod',
- help='Path to mongod to run (default "./mongod")')
- parser.add_option('--port', dest='mongodPort', default="32000",
- help='Port the mongod will bind to (default 32000)')
- parser.add_option('--mongo', dest='shellExecutable', #default="./mongo",
- help='Path to mongo, for .js test files (default "./mongo")')
- parser.add_option('--continue-on-failure', dest='continueOnFailure',
+ # that changes we don't have the freedom to run from anyplace.
+ # parser.add_option('--mongo-repo', dest='mongo_repo', default=None,
+ parser.add_option('--test-path', dest='test_path', default=None,
+ help="Path to the test executables to run, "
+ "currently only used for 'client' (%default)")
+ parser.add_option('--mongod', dest='mongod_executable', default=os.path.join(mongo_repo, 'mongod'),
+ help='Path to mongod to run (%default)')
+ parser.add_option('--port', dest='mongod_port', default="32000",
+ help='Port the mongod will bind to (%default)')
+ parser.add_option('--mongo', dest='shell_executable', default=os.path.join(mongo_repo, 'mongo'),
+ help='Path to mongo, for .js test files (%default)')
+ parser.add_option('--continue-on-failure', dest='continue_on_failure',
action="store_true", default=False,
help='If supplied, continue testing even after a test fails')
- parser.add_option('--one-mongod-per-test', dest='oneMongodPerTest',
- action="store_true", default=False,
- help='If supplied, run each test in a fresh mongod')
parser.add_option('--from-file', dest='File',
help="Run tests/suites named in FILE, one test per line, '-' means stdin")
- parser.add_option('--smoke-db-prefix', dest='smokeDbPrefix', default='',
- help="Prefix to use for the mongods' dbpaths.")
- parser.add_option('--small-oplog', dest='smallOplog', default=False,
+ parser.add_option('--smoke-db-prefix', dest='smoke_db_prefix', default=smoke_db_prefix,
+ help="Prefix to use for the mongods' dbpaths ('%default')")
+ parser.add_option('--small-oplog', dest='small_oplog', default=False,
action="store_true",
help='Run tests with master/slave replication & use a small oplog')
global tests
(options, tests) = parser.parse_args()
-# global mongoRepo
-# if options.mongoRepo:
-# pass
-# mongoRepo = options.mongoRepo
-# else:
-# prefix = ''
-# while True:
-# if os.path.exists(prefix+'buildscripts'):
-# mongoRepo = os.path.normpath(prefix)
-# break
-# else:
-# prefix += '../'
-# # FIXME: will this be a device's root directory on
-# # Windows?
-# if os.path.samefile('/', prefix):
-# raise Exception("couldn't guess the mongo repository path")
-
print tests
- global mongoRepo, mongodExecutable, mongodPort, shellExecutable, continueOnFailure, oneMongodPerTest, smallOplog, smokeDbPrefix, testPath
- testPath = options.testPath
- mongodExecutable = options.mongodExecutable if options.mongodExecutable else os.path.join(mongoRepo, 'mongod')
- mongodPort = options.mongodPort if options.mongodPort else mongodPort
- shellExecutable = options.shellExecutable if options.shellExecutable else os.path.join(mongoRepo, 'mongo')
- continueOnFailure = options.continueOnFailure if options.continueOnFailure else continueOnFailure
- oneMongodPerTest = options.oneMongodPerTest if options.oneMongodPerTest else oneMongodPerTest
- smokeDbPrefix = options.smokeDbPrefix
- smallOplog = options.smallOplog
-
+ test_path = options.test_path
+
+ mongod_executable = add_exe(options.mongod_executable)
+ if not os.path.exists(mongod_executable):
+ raise Exception("no mongod found in this directory.")
+
+ mongod_port = options.mongod_port
+
+ shell_executable = add_exe( options.shell_executable )
+ if not os.path.exists(shell_executable):
+ raise Exception("no mongo shell found in this directory.")
+
+ continue_on_failure = options.continue_on_failure
+ smoke_db_prefix = options.smoke_db_prefix
+ small_oplog = options.small_oplog
+
if options.File:
if options.File == '-':
tests = sys.stdin.readlines()
@@ -500,23 +470,20 @@ def main():
tests = f.readlines()
tests = [t.rstrip('\n') for t in tests]
- if not tests:
- raise Exception( "no tests specified" )
# If we're in suite mode, tests is a list of names of sets of tests.
if options.mode == 'suite':
- # Suites: smoke, smokePerf, smokeJs, smokeQuota, smokeJsPerf,
- # smokeJsSlow, smokeParalell, smokeClone, smokeRepl, smokeDisk
- suites = tests
- tests = []
- expandSuites(suites)
+ tests = expand_suites(tests)
elif options.mode == 'files':
tests = [(os.path.abspath(test), True) for test in tests]
- runTests(tests)
- global exit_bad
- exit_bad = False
+ if not tests:
+ raise Exception( "no tests specified" )
+
+ try:
+ run_tests(tests)
+ finally:
+ report()
-atexit.register(report)
if __name__ == "__main__":
main()
diff --git a/buildscripts/utils.py b/buildscripts/utils.py
index 1ca2fdd..8021d87 100644
--- a/buildscripts/utils.py
+++ b/buildscripts/utils.py
@@ -5,10 +5,27 @@ import time
import os
# various utilities that are handy
+def getAllSourceFiles( arr=None , prefix="." ):
+ if arr is None:
+ arr = []
+
+ for x in os.listdir( prefix ):
+ if x.startswith( "." ) or x.startswith( "pcre-" ) or x.startswith( "32bit" ) or x.startswith( "mongodb-" ) or x.startswith("debian") or x.startswith( "mongo-cxx-driver" ):
+ continue
+ full = prefix + "/" + x
+ if os.path.isdir( full ) and not os.path.islink( full ):
+ getAllSourceFiles( arr , full )
+ else:
+ if full.endswith( ".cpp" ) or full.endswith( ".h" ) or full.endswith( ".c" ):
+ arr.append( full )
+
+ return arr
+
+
def getGitBranch():
if not os.path.exists( ".git" ):
return None
-
+
version = open( ".git/HEAD" ,'r' ).read().strip()
if not version.startswith( "ref: " ):
return version
@@ -45,7 +62,6 @@ def getGitVersion():
return version
return open( f , 'r' ).read().strip()
-
def execsys( args ):
import subprocess
if isinstance( args , str ):
@@ -65,7 +81,6 @@ def getprocesslist():
r = re.compile( "[\r\n]+" )
return r.split( raw )
-
def removeIfInList( lst , thing ):
if thing in lst:
lst.remove( thing )
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-16 04:41:54 +0000