From 582fc32574a3b158c81e49cb00e6ae59205e66ba Mon Sep 17 00:00:00 2001 From: Antonin Kral Date: Thu, 17 Mar 2011 00:05:43 +0100 Subject: Imported Upstream version 1.8.0 --- jstests/replsets/auth1.js | 184 ++++++++++++++ jstests/replsets/buildindexes.js | 86 +++++++ jstests/replsets/cloneDb.js | 52 ++++ jstests/replsets/config1.js | 21 ++ jstests/replsets/fastsync.js | 117 +++++++++ jstests/replsets/getlasterror_w2.js | 36 +++ jstests/replsets/groupAndMapReduce.js | 105 ++++++++ jstests/replsets/initial_sync1.js | 129 ++++++++++ jstests/replsets/initial_sync2.js | 179 ++++++++++++++ jstests/replsets/initial_sync3.js | 87 +++++++ jstests/replsets/ismaster1.js | 36 +++ jstests/replsets/key1 | 1 + jstests/replsets/key2 | 1 + jstests/replsets/remove1.js | 132 ++++++++++ jstests/replsets/replset2.js | 252 +++++++++---------- jstests/replsets/replset3.js | 130 ++++++---- jstests/replsets/replset5.js | 42 +--- jstests/replsets/replset_remove_node.js | 9 +- jstests/replsets/replsetarb2.js | 2 + jstests/replsets/replsetarb3.js | 144 +++++++++++ jstests/replsets/replsetfreeze.js | 105 ++++++++ jstests/replsets/rollback.js | 333 +++++++++++++------------ jstests/replsets/rollback2.js | 423 +++++++++++++++++--------------- jstests/replsets/rollback3.js | 39 ++- jstests/replsets/rslib.js | 63 +++++ jstests/replsets/slaveDelay2.js | 104 ++++++++ jstests/replsets/slavedelay1.js | 127 ++++++++++ jstests/replsets/sync1.js | 30 ++- jstests/replsets/sync_passive.js | 89 +++++++ jstests/replsets/sync_passive2.js | 120 +++++++++ jstests/replsets/toostale.js | 121 +++++++++ jstests/replsets/two_initsync.js | 1 + jstests/replsets/twosets.js | 19 +- 33 files changed, 2739 insertions(+), 580 deletions(-) create mode 100644 jstests/replsets/auth1.js create mode 100644 jstests/replsets/buildindexes.js create mode 100644 jstests/replsets/cloneDb.js create mode 100644 jstests/replsets/config1.js create mode 100644 jstests/replsets/fastsync.js create mode 100644 jstests/replsets/getlasterror_w2.js create mode 100644 jstests/replsets/groupAndMapReduce.js create mode 100644 jstests/replsets/initial_sync1.js create mode 100644 jstests/replsets/initial_sync2.js create mode 100644 jstests/replsets/initial_sync3.js create mode 100644 jstests/replsets/ismaster1.js create mode 100644 jstests/replsets/key1 create mode 100644 jstests/replsets/key2 create mode 100644 jstests/replsets/remove1.js create mode 100644 jstests/replsets/replsetarb3.js create mode 100644 jstests/replsets/replsetfreeze.js create mode 100644 jstests/replsets/rslib.js create mode 100644 jstests/replsets/slaveDelay2.js create mode 100644 jstests/replsets/slavedelay1.js create mode 100644 jstests/replsets/sync_passive.js create mode 100644 jstests/replsets/sync_passive2.js create mode 100644 jstests/replsets/toostale.js (limited to 'jstests/replsets') diff --git a/jstests/replsets/auth1.js b/jstests/replsets/auth1.js new file mode 100644 index 0000000..4945869 --- /dev/null +++ b/jstests/replsets/auth1.js @@ -0,0 +1,184 @@ +// check replica set authentication + +load("jstests/replsets/rslib.js"); + +var name = "rs_auth1"; +var port = allocatePorts(4); +var path = "jstests/replsets/"; + + +print("reset permissions"); +run("chmod", "644", path+"key1"); +run("chmod", "644", path+"key2"); + + +print("try starting mongod"); +var m = runMongoProgram( "mongod", "--keyFile", path+"key1", "--port", port[0], "--dbpath", "/data/db/" + name); + + +print("should fail with wrong permissions"); +assert.eq(m, 2, "mongod should exit w/ 2: permissions too open"); +stopMongod(port[0]); + + +print("change permissions on #1 & #2"); +run("chmod", "600", path+"key1"); +run("chmod", "600", path+"key2"); + + +print("add a user to server0: foo"); +m = startMongodTest( port[0], name+"-0", 0 ); +m.getDB("admin").addUser("foo", "bar"); +m.getDB("test").addUser("bar", "baz"); +print("make sure user is written before shutting down"); +m.getDB("test").getLastError(); +stopMongod(port[0]); + + +print("start up rs"); +var rs = new ReplSetTest({"name" : name, "nodes" : 3, "startPort" : port[0]}); +m = rs.restart(0, {"keyFile" : path+"key1"}); +var s = rs.start(1, {"keyFile" : path+"key1"}); +var s2 = rs.start(2, {"keyFile" : path+"key1"}); + +var result = m.getDB("admin").auth("foo", "bar"); +assert.eq(result, 1, "login failed"); +result = m.getDB("admin").runCommand({replSetInitiate : rs.getReplSetConfig()}); +assert.eq(result.ok, 1, "couldn't initiate: "+tojson(result)); + +var master = rs.getMaster().getDB("test"); +wait(function() { + var status = master.adminCommand({replSetGetStatus:1}); + return status.members && status.members[1].state == 2 && status.members[2].state == 2; + }); + +master.foo.insert({x:1}); +master.runCommand({getlasterror:1, w:3, wtimeout:60000}); + + +print("try some legal and illegal reads"); +var r = master.foo.findOne(); +assert.eq(r.x, 1); + +s.setSlaveOk(); +slave = s.getDB("test"); + +function doQueryOn(p) { + var err = {}; + try { + r = p.foo.findOne(); + } + catch(e) { + if (typeof(JSON) != "undefined") { + err = JSON.parse(e.substring(6)); + } + else if (e.indexOf("10057") > 0) { + err.code = 10057; + } + } + assert.eq(err.code, 10057); +}; + +doQueryOn(slave); +master.adminCommand({logout:1}); +doQueryOn(master); + + +result = slave.auth("bar", "baz"); +assert.eq(result, 1); + +r = slave.foo.findOne(); +assert.eq(r.x, 1); + + +print("add some data"); +master.auth("bar", "baz"); +for (var i=0; i<1000; i++) { + master.foo.insert({x:i, foo : "bar"}); +} +master.runCommand({getlasterror:1, w:3, wtimeout:60000}); + + +print("fail over"); +rs.stop(0); + +wait(function() { + function getMaster(s) { + var result = s.getDB("admin").runCommand({isMaster: 1}); + printjson(result); + if (result.ismaster) { + master = s.getDB("test"); + return true; + } + return false; + } + + if (getMaster(s) || getMaster(s2)) { + return true; + } + return false; + }); + + +print("add some more data 1"); +master.auth("bar", "baz"); +for (var i=0; i<1000; i++) { + master.foo.insert({x:i, foo : "bar"}); +} +master.runCommand({getlasterror:1, w:3, wtimeout:60000}); + + +print("resync"); +rs.restart(0); + + +print("add some more data 2"); +for (var i=0; i<1000; i++) { + master.foo.insert({x:i, foo : "bar"}); +} +master.runCommand({getlasterror:1, w:3, wtimeout:60000}); + + +print("add member with wrong key"); +var conn = new MongodRunner(port[3], "/data/db/"+name+"-3", null, null, ["--replSet","rs_auth1","--rest","--oplogSize","2", "--keyFile", path+"key2"], {no_bind : true}); +conn.start(); + + +master.getSisterDB("admin").auth("foo", "bar"); +var config = master.getSisterDB("local").system.replset.findOne(); +config.members.push({_id : 3, host : getHostName()+":"+port[3]}); +config.version++; +try { + master.adminCommand({replSetReconfig:config}); +} +catch (e) { + print("error: "+e); +} +reconnect(master); +master.getSisterDB("admin").auth("foo", "bar"); + + +print("shouldn't ever sync"); +for (var i = 0; i<30; i++) { + print("iteration: " +i); + var results = master.adminCommand({replSetGetStatus:1}); + printjson(results); + assert(results.members[3].state != 2); + sleep(1000); +} + + +print("stop member"); +stopMongod(port[3]); + + +print("start back up with correct key"); +conn = new MongodRunner(port[3], "/data/db/"+name+"-3", null, null, ["--replSet","rs_auth1","--rest","--oplogSize","2", "--keyFile", path+"key1"], {no_bind : true}); +conn.start(); + +wait(function() { + var results = master.adminCommand({replSetGetStatus:1}); + printjson(results); + return results.members[3].state == 2; + }); + diff --git a/jstests/replsets/buildindexes.js b/jstests/replsets/buildindexes.js new file mode 100644 index 0000000..76de797 --- /dev/null +++ b/jstests/replsets/buildindexes.js @@ -0,0 +1,86 @@ +doTest = function( signal ) { + + var name = "buildIndexes"; + var host = getHostName(); + + var replTest = new ReplSetTest( {name: name, nodes: 3} ); + + var nodes = replTest.startSet(); + + var config = replTest.getReplSetConfig(); + config.members[2].priority = 0; + config.members[2].buildIndexes = false; + + replTest.initiate(config); + + var master = replTest.getMaster().getDB(name); + var slaveConns = replTest.liveNodes.slaves; + var slave = []; + for (var i in slaveConns) { + slaveConns[i].setSlaveOk(); + slave.push(slaveConns[i].getDB(name)); + } + replTest.awaitReplication(); + + print("creating an index on x"); + master.x.ensureIndex({y : 1}); + printjson(master.x.stats()); + + for (var i=0; i<100; i++) { + master.x.insert({x:1,y:"abc",c:1}); + } + + replTest.awaitReplication(); + + printjson(slave[0].runCommand({count: "x"})); + var ns = master.x+""; + print("namespace: "+ns); + + // can't query system.indexes from slave, so we'll look at coll.stats() + printjson(slave[0].adminCommand({replSetGetStatus:1})); + printjson(slave[0].getSisterDB("local").system.replset.findOne()); + printjson(master.stats()); + printjson(slave[0].stats()); + printjson(slave[1].stats()); + printjson(master.x.stats()); + printjson(slave[0].x.stats()); + printjson(slave[1].x.stats()); + print("sleeping"); + sleep(20000); + var indexes = slave[0].stats().indexes; + assert.eq(indexes, 2, 'number of indexes'); + + indexes = slave[1].stats().indexes; + assert.eq(indexes, 1); + + + indexes = slave[0].x.stats().indexSizes; + printjson(indexes); + + var count = 0; + for (var i in indexes) { + count++; + if (i == "_id_") { + continue; + } + print(i); + print(i.match(/y_/)); + assert(i.match(/y_/)); + } + + assert.eq(count, 2); + + indexes = slave[1].x.stats().indexSizes; + printjson(indexes); + + count = 0; + for (var i in indexes) { + count++; + } + + assert.eq(count, 1); + + replTest.stopSet(15); +} + +doTest(15); diff --git a/jstests/replsets/cloneDb.js b/jstests/replsets/cloneDb.js new file mode 100644 index 0000000..6d2d0f3 --- /dev/null +++ b/jstests/replsets/cloneDb.js @@ -0,0 +1,52 @@ +// Test for cloning a db from a replica set [SERVER-1643] -Tony + +load('jstests/libs/grid.js') + +doTest = function( signal ) { + + var N = 2000 + + // ~1KB string + var Text = '' + for (var i = 0; i < 40; i++) + Text += 'abcdefghijklmnopqrstuvwxyz' + + // Create replica set + var repset = new ReplicaSet ('testSet', 3) .begin() + var master = repset.getMaster() + var db1 = master.getDB('test') + + // Insert data + for (var i = 0; i < N; i++) { + db1['foo'].insert({x: i, text: Text}) + db1.getLastError(2) // wait to be copied to at least one secondary + } + + // Create single server + var solo = new Server ('singleTarget') + var soloConn = solo.begin() + var db2 = soloConn.getDB('test') + + // Clone db from replica set to single server + db2.cloneDatabase (repset.getURL()) + + // Confirm clone worked + assert.eq (Text, db2['foo'] .findOne({x: N-1}) ['text'], 'cloneDatabase failed (test1)') + + // Now test the reverse direction + db1 = master.getDB('test2') + db2 = soloConn.getDB('test2') + for (var i = 0; i < N; i++) { + db2['foo'].insert({x: i, text: Text}) + db2.getLastError() + } + db1.cloneDatabase (solo.host()) + assert.eq (Text, db2['foo'] .findOne({x: N-1}) ['text'], 'cloneDatabase failed (test2)') + + // Shut down replica set and single server + solo.end() + repset.stopSet( signal ) +} + +doTest( 15 ); +print("replsets/cloneDb.js SUCCESS"); diff --git a/jstests/replsets/config1.js b/jstests/replsets/config1.js new file mode 100644 index 0000000..748ce8f --- /dev/null +++ b/jstests/replsets/config1.js @@ -0,0 +1,21 @@ +doTest = function( signal ) { + var name = 'config1'; + + var replTest = new ReplSetTest( {name: name, nodes: 3} ); + var nodes = replTest.startSet(); + + var config = replTest.getReplSetConfig(); + config.settings = {"heartbeatSleep" : .5, heartbeatTimeout : .8}; + + replTest.initiate(config); + + // Call getMaster to return a reference to the node that's been + // elected master. + var master = replTest.getMaster(); + + config = master.getDB("local").system.replset.findOne(); + assert.eq(config.settings.heartbeatSleep, .5); + assert.eq(config.settings.heartbeatTimeout, .8); +}; + +doTest(15); diff --git a/jstests/replsets/fastsync.js b/jstests/replsets/fastsync.js new file mode 100644 index 0000000..d7c3905 --- /dev/null +++ b/jstests/replsets/fastsync.js @@ -0,0 +1,117 @@ +/* + * 1. insert 100000 objects + * 2. export to two dbpaths + * 3. add one node w/fastsync + * 4. check that we never get "errmsg" : "initial sync cloning db: whatever" + * 5. check writes are replicated + */ + +var w = 0; +var wait = function(f) { + w++; + var n = 0; + while (!f()) { + if( n % 4 == 0 ) + print("toostale.js waiting " + w); + if (++n == 4) { + print("" + f); + } + assert(n < 200, 'tried 200 times, giving up'); + sleep(1000); + } +} + +var reconnect = function(a) { + wait(function() { + try { + a.getDB("foo").bar.stats(); + return true; + } catch(e) { + print(e); + return false; + } + }); +}; + +ports = allocatePorts( 3 ); + +var basename = "jstests_fastsync"; +var basePath = "/data/db/" + basename; +var hostname = getHostName(); + +var pargs = new MongodRunner( ports[ 0 ], basePath + "-p", false, false, + ["--replSet", basename, "--oplogSize", 2], + {no_bind : true} ); +p = pargs.start(); + +var admin = p.getDB("admin"); +var foo = p.getDB("foo"); +var local = p.getDB("local"); + +var config = {_id : basename, members : [{_id : 0, host : hostname+":"+ports[0]}]}; +printjson(config); +var result = admin.runCommand({replSetInitiate : config}); +print("result:"); +printjson(result); + +var count = 0; +while (count < 10 && result.ok != 1) { + count++; + sleep(2000); + result = admin.runCommand({replSetInitiate : config}); +} + +assert(result.ok, tojson(result)); +assert.soon(function() { return admin.runCommand({isMaster:1}).ismaster; }); + +print("1"); +for (var i=0; i<100000; i++) { + foo.bar.insert({date : new Date(), x : i, str : "all the talk on the market"}); +} +print("total in foo: "+foo.bar.count()); + + +print("2"); +admin.runCommand( {fsync:1,lock:1} ); +copyDbpath( basePath + "-p", basePath + "-s" ); +admin.$cmd.sys.unlock.findOne(); + + +print("3"); +var sargs = new MongodRunner( ports[ 1 ], basePath + "-s", false, false, + ["--replSet", basename, "--fastsync", + "--oplogSize", 2], {no_bind : true} ); +var reuseData = true; +sargs.start(reuseData); + +config = local.system.replset.findOne(); +config.version++; +config.members.push({_id:1, host:hostname+":"+ports[1]}); + +result = admin.runCommand({replSetReconfig : config}); +assert(result.ok, "reconfig worked"); +reconnect(p); + +print("4"); +var status = admin.runCommand({replSetGetStatus : 1}); +var count = 0; +while (status.members[1].state != 2 && count < 200) { + print("not a secondary yet"); + if (count % 10 == 0) { + printjson(status); + } + assert(!status.members[1].errmsg || !status.members[1].errmsg.match("^initial sync cloning db")); + + sleep(1000); + + // disconnection could happen here + try { + status = admin.runCommand({replSetGetStatus : 1}); + } + catch (e) { + print(e); + } + count++; +} + +assert.eq(status.members[1].state, 2); diff --git a/jstests/replsets/getlasterror_w2.js b/jstests/replsets/getlasterror_w2.js new file mode 100644 index 0000000..795e667 --- /dev/null +++ b/jstests/replsets/getlasterror_w2.js @@ -0,0 +1,36 @@ +// BUG: [SERVER-1768] replica set getlasterror {w: 2} after 2000 +// inserts hangs while secondary servers log "replSet error RS102 too stale to catch up" every once in a while + +function newReplicaSet (name, numServers) { + var rs = new ReplSetTest({name: name, nodes: numServers}) + rs.startSet() + rs.initiate() + rs.awaitReplication() + return rs +} + +function go() { +var N = 2000 + +// ~1KB string +var Text = '' +for (var i = 0; i < 40; i++) + Text += 'abcdefghijklmnopqrstuvwxyz' + +// Create replica set of 3 servers +var repset = newReplicaSet('repset', 3) +var conn = repset.getMaster() +var db = conn.getDB('test') + +// Add data to it +for (var i = 0; i < N; i++) + db['foo'].insert({x: i, text: Text}) + +// wait to be copied to at least one secondary (BUG hangs here) +db.getLastError(2) + +print('getlasterror_w2.js SUCCESS') +} + +// turn off until fixed +//go(); diff --git a/jstests/replsets/groupAndMapReduce.js b/jstests/replsets/groupAndMapReduce.js new file mode 100644 index 0000000..539fe44 --- /dev/null +++ b/jstests/replsets/groupAndMapReduce.js @@ -0,0 +1,105 @@ +doTest = function( signal ) { + + // Test basic replica set functionality. + // -- Replication + // -- Failover + + // Replica set testing API + // Create a new replica set test. Specify set name and the number of nodes you want. + var replTest = new ReplSetTest( {name: 'testSet', nodes: 3} ); + + // call startSet() to start each mongod in the replica set + // this returns a list of nodes + var nodes = replTest.startSet(); + + // Call initiate() to send the replSetInitiate command + // This will wait for initiation + replTest.initiate(); + + // Call getMaster to return a reference to the node that's been + // elected master. + var master = replTest.getMaster(); + + // save some records + var len = 100 + for (var i = 0; i < len; ++i) { + master.getDB("foo").foo.save({a: i}); + } + + // This method will check the oplogs of the master + // and slaves in the set and wait until the change has replicated. + replTest.awaitReplication(); + print("Sleeping 10s for slaves to go to secondary state"); + sleep(10000); + + slaves = replTest.liveNodes.slaves; + assert( slaves.length == 2, "Expected 2 slaves but length was " + slaves.length ); + slaves.forEach(function(slave) { + // try to read from slave + slave.slaveOk = true; + var count = slave.getDB("foo").foo.count(); + printjson( count ); + assert.eq( len , count , "slave count wrong: " + slave ); + + print("Doing a findOne to verify we can get a row"); + var one = slave.getDB("foo").foo.findOne(); + printjson(one); + +// stats = slave.getDB("foo").adminCommand({replSetGetStatus:1}); +// printjson(stats); + + print("Calling group() with slaveOk=true, must succeed"); + slave.slaveOk = true; + count = slave.getDB("foo").foo.group({initial: {n:0}, reduce: function(obj,out){out.n++;}}); + printjson( count ); + assert.eq( len , count[0].n , "slave group count wrong: " + slave ); + + print("Calling group() with slaveOk=false, must fail"); + slave.slaveOk = false; + try { + count = slave.getDB("foo").foo.group({initial: {n:0}, reduce: function(obj,out){out.n++;}}); + assert(false, "group() succeeded with slaveOk=false"); + } catch (e) { + print("Received exception: " + e); + } + + print("Calling inline mr() with slaveOk=true, must succeed"); + slave.slaveOk = true; + map = function() { emit(this.a, 1); }; + reduce = function(key, vals) { var sum = 0; for (var i = 0; i < vals.length; ++i) { sum += vals[i]; } return sum; }; + slave.getDB("foo").foo.mapReduce(map, reduce, {out: { "inline" : 1}}); + + print("Calling mr() to collection with slaveOk=true, must fail"); + try { + slave.getDB("foo").foo.mapReduce(map, reduce, "output"); + assert(false, "mapReduce() to collection succeeded on slave"); + } catch (e) { + print("Received exception: " + e); + } + + print("Calling inline mr() with slaveOk=false, must fail"); + slave.slaveOk = false; + try { + slave.getDB("foo").foo.mapReduce(map, reduce, {out: { "inline" : 1}}); + assert(false, "mapReduce() succeeded on slave with slaveOk=false"); + } catch (e) { + print("Received exception: " + e); + } + print("Calling mr() to collection with slaveOk=false, must fail"); + try { + slave.getDB("foo").foo.mapReduce(map, reduce, "output"); + assert(false, "mapReduce() to collection succeeded on slave with slaveOk=false"); + } catch (e) { + print("Received exception: " + e); + } + + }); + + + + // Shut down the set and finish the test. + replTest.stopSet( signal ); +} + +doTest( 15 ); +print("SUCCESS"); diff --git a/jstests/replsets/initial_sync1.js b/jstests/replsets/initial_sync1.js new file mode 100644 index 0000000..ee30b4e --- /dev/null +++ b/jstests/replsets/initial_sync1.js @@ -0,0 +1,129 @@ +/** + * Test killing the secondary during initially sync + * + * 1. Bring up set + * 2. Insert some data + * 4. Make sure synced + * 5. Freeze #2 + * 6. Bring up #3 + * 7. Kill #2 in the middle of syncing + * 8. Eventually it should become a secondary + * 9. Bring #2 back up + * 10. Insert some stuff + * 11. Everyone happy eventually + */ + +load("jstests/replsets/rslib.js"); +var basename = "jstests_initsync1"; + + +print("1. Bring up set"); +var replTest = new ReplSetTest( {name: basename, nodes: 2} ); +var conns = replTest.startSet(); +replTest.initiate(); + +var master = replTest.getMaster(); +var foo = master.getDB("foo"); +var admin = master.getDB("admin"); + +var slave1 = replTest.liveNodes.slaves[0]; +var admin_s1 = slave1.getDB("admin"); +var local_s1 = slave1.getDB("local"); + +print("2. Insert some data"); +for (var i=0; i<10000; i++) { + foo.bar.insert({date : new Date(), x : i, str : "all the talk on the market"}); +} +print("total in foo: "+foo.bar.count()); + + +print("4. Make sure synced"); +replTest.awaitReplication(); + + +print("5. Freeze #2"); +admin_s1.runCommand({replSetFreeze:999999}); + + +print("6. Bring up #3"); +var ports = allocatePorts( 3 ); +var basePath = "/data/db/" + basename; +var hostname = getHostName(); + +var sargs = new MongodRunner( ports[ 2 ], basePath, false, false, + ["--replSet", basename, "--oplogSize", 2], + {no_bind : true} ); +var slave2 = sargs.start(); +var local_s2 = slave2.getDB("local"); +var admin_s2 = slave2.getDB("admin"); + +var config = replTest.getReplSetConfig(); +config.version = 2; +config.members.push({_id:2, host:hostname+":"+ports[2]}); + +try { + admin.runCommand({replSetReconfig:config}); +} +catch(e) { + print(e); +} +reconnect(slave1); +reconnect(slave2); + +wait(function() { + var config2 = local_s1.system.replset.findOne(); + var config3 = local_s2.system.replset.findOne(); + + printjson(config2); + printjson(config3); + + return config2.version == config.version && + (config3 && config3.version == config.version); + }); + +wait(function() { + var status = admin_s2.runCommand({replSetGetStatus:1}); + printjson(status); + return status.members && + (status.members[2].state == 3 || status.members[2].state == 2); + }); + + +print("7. Kill #2 in the middle of syncing"); +replTest.stop(1); + + +print("8. Eventually it should become a secondary"); +print("if initial sync has started, this will cause it to fail and sleep for 5 minutes"); +sleep(5*60*1000); +wait(function() { + var status = admin_s2.runCommand({replSetGetStatus:1}); + occasionally(function() { printjson(status); }); + return status.members[2].state == 2; + }); + + +print("9. Bring #2 back up"); +replTest.start(1, {}, true); +reconnect(slave1); +wait(function() { + var status = admin_s1.runCommand({replSetGetStatus:1}); + printjson(status); + return status.ok == 1 && status.members && + status.members[1].state == 2 || status.members[1].state == 1; + }); + + +/** + * TODO: this fails on buildbot + * see SERVER-2550 +print("10. Insert some stuff"); +master = replTest.getMaster(); +for (var i=0; i<10000; i++) { + foo.bar.insert({date : new Date(), x : i, str : "all the talk on the market"}); +} + + +print("11. Everyone happy eventually"); +replTest.awaitReplication(); +*/ diff --git a/jstests/replsets/initial_sync2.js b/jstests/replsets/initial_sync2.js new file mode 100644 index 0000000..3ad3972 --- /dev/null +++ b/jstests/replsets/initial_sync2.js @@ -0,0 +1,179 @@ +/** + * Test killing the primary during initial sync + * and don't allow the other secondary to become primary + * + * 1. Bring up set + * 2. Insert some data + * 4. Make sure synced + * 5. Freeze #2 + * 6. Bring up #3 + * 7. Kill #1 in the middle of syncing + * 8. Check that #3 makes it into secondary state + * 9. Bring #1 back up + * 10. Initial sync should succeed + * 11. Insert some stuff + * 12. Everyone happy eventually + */ + +load("jstests/replsets/rslib.js"); +var basename = "jstests_initsync2"; + +var doTest = function() { + +print("1. Bring up set"); +var replTest = new ReplSetTest( {name: basename, nodes: 2} ); +var conns = replTest.startSet(); +replTest.initiate(); + +var master = replTest.getMaster(); +var origMaster = master; +var foo = master.getDB("foo"); +var admin = master.getDB("admin"); + +var slave1 = replTest.liveNodes.slaves[0]; +var admin_s1 = slave1.getDB("admin"); +var local_s1 = slave1.getDB("local"); + +print("2. Insert some data"); +for (var i=0; i<10000; i++) { + foo.bar.insert({date : new Date(), x : i, str : "all the talk on the market"}); +} +print("total in foo: "+foo.bar.count()); + + +print("4. Make sure synced"); +replTest.awaitReplication(); + + +print("5. Freeze #2"); +admin_s1.runCommand({replSetFreeze:999999}); + + +print("6. Bring up #3"); +var ports = allocatePorts( 3 ); +var basePath = "/data/db/" + basename; +var hostname = getHostName(); + +var sargs = new MongodRunner( ports[ 2 ], basePath, false, false, + ["--replSet", basename, "--oplogSize", 2], + {no_bind : true} ); +var slave2 = sargs.start(); +var local_s2 = slave2.getDB("local"); +var admin_s2 = slave2.getDB("admin"); + +var config = replTest.getReplSetConfig(); +config.version = 2; +config.members.push({_id:2, host:hostname+":"+ports[2]}); + +try { + admin.runCommand({replSetReconfig:config}); +} +catch(e) { + print(e); +} +reconnect(slave1); +reconnect(slave2); + +wait(function() { + var config2 = local_s1.system.replset.findOne(); + var config3 = local_s2.system.replset.findOne(); + + printjson(config2); + printjson(config3); + + return config2.version == config.version && + (config3 && config3.version == config.version); + }); +admin_s2.runCommand({replSetFreeze:999999}); + + +wait(function() { + var status = admin_s2.runCommand({replSetGetStatus:1}); + printjson(status); + return status.members && + (status.members[2].state == 3 || status.members[2].state == 2); + }); + + +print("7. Kill #1 in the middle of syncing"); +replTest.stop(0); + + +print("8. Check that #3 makes it into secondary state"); +wait(function() { + var status = admin_s2.runCommand({replSetGetStatus:1}); + occasionally(function() { printjson(status);}, 10); + if (status.members[2].state == 2 || status.members[2].state == 1) { + return true; + } + return false; + }); + + +print("9. Bring #1 back up"); +replTest.start(0, {}, true); +reconnect(master); +wait(function() { + var status = admin.runCommand({replSetGetStatus:1}); + printjson(status); + return status.members && + (status.members[0].state == 1 || status.members[0].state == 2); + }); + + +print("10. Initial sync should succeed"); +wait(function() { + var status = admin_s2.runCommand({replSetGetStatus:1}); + printjson(status); + return status.members && + status.members[2].state == 2 || status.members[2].state == 1; + }); + + +print("11. Insert some stuff"); +// ReplSetTest doesn't find master correctly unless all nodes are defined by +// ReplSetTest +for (var i = 0; i<30; i++) { + var result = admin.runCommand({isMaster : 1}); + if (result.ismaster) { + break; + } + else if (result.primary) { + master = connect(result.primary+"/admin").getMongo(); + break; + } + sleep(1000); +} + +for (var i=0; i<10000; i++) { + foo.bar.insert({date : new Date(), x : i, str : "all the talk on the market"}); +} + + +print("12. Everyone happy eventually"); +// if 3 is master... +if (master+"" != origMaster+"") { + print("3 is master"); + slave2 = origMaster; +} + +wait(function() { + var op1 = getLatestOp(master); + var op2 = getLatestOp(slave1); + var op3 = getLatestOp(slave2); + + occasionally(function() { + print("latest ops:"); + printjson(op1); + printjson(op2); + printjson(op3); + }); + + return friendlyEqual(getLatestOp(master), getLatestOp(slave1)) && + friendlyEqual(getLatestOp(master), getLatestOp(slave2)); + }); + +replTest.stopSet(); +}; + +doTest(); diff --git a/jstests/replsets/initial_sync3.js b/jstests/replsets/initial_sync3.js new file mode 100644 index 0000000..471aa16 --- /dev/null +++ b/jstests/replsets/initial_sync3.js @@ -0,0 +1,87 @@ +/* test initial sync options + * + * {state : 1} + * {state : 2} + * {name : host+":"+port} + * {_id : 2} + * {optime : now} + * {optime : 1970} + */ + +load("jstests/replsets/rslib.js"); +var name = "initialsync3"; +var host = getHostName(); +var port = allocatePorts(7); + +print("Start set with three nodes"); +var replTest = new ReplSetTest( {name: name, nodes: 7} ); +var nodes = replTest.startSet(); +replTest.initiate({ + _id : name, + members : [ + {_id:0, host : host+":"+port[0]}, + {_id:1, host : host+":"+port[1], initialSync : {state : 1}}, + {_id:2, host : host+":"+port[2], initialSync : {state : 2}}, + {_id:3, host : host+":"+port[3], initialSync : {name : host+":"+port[2]}}, + {_id:4, host : host+":"+port[4], initialSync : {_id : 2}}, + {_id:5, host : host+":"+port[5], initialSync : {optime : new Date()}}, + {_id:6, host : host+":"+port[6], initialSync : {optime : new Date(0)}} + ]}); + +var master = replTest.getMaster(); + +print("Initial sync"); +master.getDB("foo").bar.baz.insert({x:1}); + +print("Make sure everyone's secondary"); +wait(function() { + var status = master.getDB("admin").runCommand({replSetGetStatus:1}); + occasionally(function() { + printjson(status); + }); + + if (!status.members) { + return false; + } + + for (i=0; i<7; i++) { + if (status.members[i].state != 1 && status.members[i].state != 2) { + return false; + } + } + return true; + + }); + +replTest.awaitReplication(); + +replTest.stopSet(); + +print("reconfig"); + +var rs2 = new ReplSetTest( {name: 'reconfig-isync3', nodes: 3} ); +rs2.startSet(); +rs2.initiate(); + +master = rs2.getMaster(); +var config = master.getDB("local").system.replset.findOne(); +config.version++; +config.members[0].initialSync = {state : 2}; +config.members[1].initialSync = {state : 1}; +try { + master.getDB("admin").runCommand({replSetReconfig : config}); +} +catch(e) { + print("trying to reconfigure: "+e); +} + +master = rs2.getMaster(); +config = master.getDB("local").system.replset.findOne(); + +assert(typeof(config.members[0].initialSync) == "object"); +assert.eq(config.members[0].initialSync.state, 2); +assert.eq(config.members[1].initialSync.state, 1); + +rs2.stopSet(); + +print("initialSync3 success!"); diff --git a/jstests/replsets/ismaster1.js b/jstests/replsets/ismaster1.js new file mode 100644 index 0000000..22865e5 --- /dev/null +++ b/jstests/replsets/ismaster1.js @@ -0,0 +1,36 @@ +/** + * 1. Check passive field in isMaster + */ + +load("jstests/replsets/rslib.js"); + +var name = "ismaster"; +var host = getHostName(); + +var replTest = new ReplSetTest( {name: name, nodes: 3} ); + +var nodes = replTest.startSet(); + +var config = replTest.getReplSetConfig(); +config.members[1].priority = 0; +config.members[2].priority = 0; + +replTest.initiate(config); + +var master = replTest.getMaster(); +wait(function() { + var result = master.getDB("admin").runCommand({replSetGetStatus:1}); + return result.members && result.members[0].state == 1 && + result.members[1].state == 2 && result.members[2].state == 2; + }); + +var result = master.getDB("admin").runCommand({isMaster:1}); +assert(!('passive' in result), tojson(result)); + +result = replTest.liveNodes.slaves[0].getDB("admin").runCommand({isMaster:1}); +assert('passive' in result, tojson(result)); + +result = replTest.liveNodes.slaves[1].getDB("admin").runCommand({isMaster:1}); +assert('passive' in result, tojson(result)); + +replTest.stopSet(); diff --git a/jstests/replsets/key1 b/jstests/replsets/key1 new file mode 100644 index 0000000..b5c19e4 --- /dev/null +++ b/jstests/replsets/key1 @@ -0,0 +1 @@ +foop de doop diff --git a/jstests/replsets/key2 b/jstests/replsets/key2 new file mode 100644 index 0000000..cbde821 --- /dev/null +++ b/jstests/replsets/key2 @@ -0,0 +1 @@ +other key diff --git a/jstests/replsets/remove1.js b/jstests/replsets/remove1.js new file mode 100644 index 0000000..ebd17d6 --- /dev/null +++ b/jstests/replsets/remove1.js @@ -0,0 +1,132 @@ +/* test removing a node from a replica set + * + * Start set with three nodes + * Initial sync + * Remove slave1 + * Remove slave2 + * Bring slave1 back up + * Bring slave2 back up + * Add them back as slave + * Make sure everyone's secondary + */ + +load("jstests/replsets/rslib.js"); +var name = "removeNodes"; +var host = getHostName(); + + +print("Start set with three nodes"); +var replTest = new ReplSetTest( {name: name, nodes: 3} ); +var nodes = replTest.startSet(); +replTest.initiate(); +var master = replTest.getMaster(); + + +print("Initial sync"); +master.getDB("foo").bar.baz.insert({x:1}); + +replTest.awaitReplication(); + + +print("Remove slave2"); +var config = replTest.getReplSetConfig(); + +config.members.pop(); +config.version = 2; +try { + master.getDB("admin").runCommand({replSetReconfig:config}); +} +catch(e) { + print(e); +} +reconnect(master); + + +print("Remove slave1"); +config.members.pop(); +config.version = 3; +try { + master.getDB("admin").runCommand({replSetReconfig:config}); +} +catch(e) { + print(e); +} +reconnect(master); + +print("sleeping 1"); +sleep(10000); +// these are already down, but this clears their ports from memory so that they +// can be restarted later +stopMongod(replTest.getPort(1)); +stopMongod(replTest.getPort(2)); + + +print("Bring slave1 back up"); +var paths = [ replTest.getPath(1), replTest.getPath(2) ]; +var ports = allocatePorts(2, replTest.getPort(2)+1); +var args = ["mongod", "--port", ports[0], "--dbpath", paths[0], "--noprealloc", "--smallfiles", "--rest"]; +var conn = startMongoProgram.apply( null, args ); +conn.getDB("local").system.replset.remove(); +printjson(conn.getDB("local").runCommand({getlasterror:1})); +print(conn); +print("sleeping 2"); +sleep(10000); +stopMongod(ports[0]); + +replTest.restart(1); + + +print("Bring slave2 back up"); +args[2] = ports[1]; +args[4] = paths[1]; +conn = startMongoProgram.apply( null, args ); +conn.getDB("local").system.replset.remove(); +print("path: "+paths[1]); +print("sleeping 3"); +sleep(10000); +stopMongod(ports[1]); + +replTest.restart(2); +sleep(10000); + + +print("Add them back as slaves"); +config.members.push({_id:1, host : host+":"+replTest.getPort(1)}); +config.members.push({_id:2, host : host+":"+replTest.getPort(2)}); +config.version = 4; +wait(function() { + try { + master.getDB("admin").runCommand({replSetReconfig:config}); + } + catch(e) { + print(e); + } + reconnect(master); + + master.setSlaveOk(); + var newConfig = master.getDB("local").system.replset.findOne(); + return newConfig.version == 4; + }); + + +print("Make sure everyone's secondary"); +wait(function() { + var status = master.getDB("admin").runCommand({replSetGetStatus:1}); + occasionally(function() { + printjson(status); + }); + + if (!status.members || status.members.length != 3) { + return false; + } + + for (var i = 0; i<3; i++) { + if (status.members[i].state != 1 && status.members[i].state != 2) { + return false; + } + } + return true; + }); + +replTest.stopSet(); + diff --git a/jstests/replsets/replset2.js b/jstests/replsets/replset2.js index f18b467..4849620 100644 --- a/jstests/replsets/replset2.js +++ b/jstests/replsets/replset2.js @@ -1,126 +1,126 @@ -print("\n\nreplset2.js BEGIN"); - -doTest = function (signal) { - - // FAILING TEST - // See below: - - // Test replication with getLastError - - // Replica set testing API - // Create a new replica set test. Specify set name and the number of nodes you want. - var replTest = new ReplSetTest({ name: 'testSet', nodes: 3, oplogSize: 5 }); - - // call startSet() to start each mongod in the replica set - // this returns a list of nodes - var nodes = replTest.startSet(); - - // Call initiate() to send the replSetInitiate command - // This will wait for initiation - replTest.initiate(); - - var testDB = "repl-test"; - - // Call getMaster to return a reference to the node that's been - // elected master. - var master = replTest.getMaster(); - - // Wait for replication to a single node - master.getDB(testDB).bar.insert({ n: 1 }); - - // Wait for initial sync - replTest.awaitReplication(); - - var slaves = replTest.liveNodes.slaves; - slaves.forEach(function (slave) { slave.setSlaveOk(); }); - - var failed = false; - var callGetLastError = function (w, timeout, db) { - try { - var result = master.getDB(db).getLastErrorObj(w, timeout); - print("replset2.js getLastError result: " + tojson(result)); - if (result['ok'] != 1) { - print("replset2.js FAILURE getlasterror not ok"); - failed = true; - } - } - catch (e) { - print("\nreplset2.js exception in getLastError: " + e + '\n'); - throw e; - } - } - - // Test getlasterror with multiple inserts - // TEST FAILS HEREg - print("\n\nreplset2.js **** Try inserting a multiple records -- first insert ****") - - printjson(master.getDB("admin").runCommand("replSetGetStatus")); - - master.getDB(testDB).foo.insert({ n: 1 }); - master.getDB(testDB).foo.insert({ n: 2 }); - master.getDB(testDB).foo.insert({ n: 3 }); - - print("\nreplset2.js **** TEMP 1 ****") - - printjson(master.getDB("admin").runCommand("replSetGetStatus")); - - callGetLastError(3, 25000, testDB); - - print("replset2.js **** TEMP 1a ****") - - m1 = master.getDB(testDB).foo.findOne({ n: 1 }); - printjson(m1); - assert(m1['n'] == 1, "replset2.js Failed to save to master on multiple inserts"); - - print("replset2.js **** TEMP 1b ****") - - var s0 = slaves[0].getDB(testDB).foo.findOne({ n: 1 }); - assert(s0['n'] == 1, "replset2.js Failed to replicate to slave 0 on multiple inserts"); - - var s1 = slaves[1].getDB(testDB).foo.findOne({ n: 1 }); - assert(s1['n'] == 1, "replset2.js Failed to replicate to slave 1 on multiple inserts"); - - // Test getlasterror with a simple insert - print("replset2.js **** Try inserting a single record ****") - master.getDB(testDB).dropDatabase(); - master.getDB(testDB).foo.insert({ n: 1 }); - callGetLastError(3, 10000, testDB); - - m1 = master.getDB(testDB).foo.findOne({ n: 1 }); - printjson(m1); - assert(m1['n'] == 1, "replset2.js Failed to save to master"); - - s0 = slaves[0].getDB(testDB).foo.findOne({ n: 1 }); - assert(s0['n'] == 1, "replset2.js Failed to replicate to slave 0"); - - s1 = slaves[1].getDB(testDB).foo.findOne({ n: 1 }); - assert(s1['n'] == 1, "replset2.js Failed to replicate to slave 1"); - - // Test getlasterror with large insert - print("replset2.js **** Try inserting many records ****") +print("\n\nreplset2.js BEGIN"); + +doTest = function (signal) { + + // FAILING TEST + // See below: + + // Test replication with getLastError + + // Replica set testing API + // Create a new replica set test. Specify set name and the number of nodes you want. + var replTest = new ReplSetTest({ name: 'testSet', nodes: 3, oplogSize: 5 }); + + // call startSet() to start each mongod in the replica set + // this returns a list of nodes + var nodes = replTest.startSet(); + + // Call initiate() to send the replSetInitiate command + // This will wait for initiation + replTest.initiate(); + + var testDB = "repl-test"; + + // Call getMaster to return a reference to the node that's been + // elected master. + var master = replTest.getMaster(); + + // Wait for replication to a single node + master.getDB(testDB).bar.insert({ n: 1 }); + + // Wait for initial sync + replTest.awaitReplication(); + + var slaves = replTest.liveNodes.slaves; + slaves.forEach(function (slave) { slave.setSlaveOk(); }); + + var failed = false; + var callGetLastError = function (w, timeout, db) { + try { + var result = master.getDB(db).getLastErrorObj(w, timeout); + print("replset2.js getLastError result: " + tojson(result)); + if (result['ok'] != 1) { + print("replset2.js FAILURE getlasterror not ok"); + failed = true; + } + } + catch (e) { + print("\nreplset2.js exception in getLastError: " + e + '\n'); + throw e; + } + } + + // Test getlasterror with multiple inserts + // TEST FAILS HEREg + print("\n\nreplset2.js **** Try inserting a multiple records -- first insert ****") + + printjson(master.getDB("admin").runCommand("replSetGetStatus")); + + master.getDB(testDB).foo.insert({ n: 1 }); + master.getDB(testDB).foo.insert({ n: 2 }); + master.getDB(testDB).foo.insert({ n: 3 }); + + print("\nreplset2.js **** TEMP 1 ****") + + printjson(master.getDB("admin").runCommand("replSetGetStatus")); + + callGetLastError(3, 25000, testDB); + + print("replset2.js **** TEMP 1a ****") + + m1 = master.getDB(testDB).foo.findOne({ n: 1 }); + printjson(m1); + assert(m1['n'] == 1, "replset2.js Failed to save to master on multiple inserts"); + + print("replset2.js **** TEMP 1b ****") + + var s0 = slaves[0].getDB(testDB).foo.findOne({ n: 1 }); + assert(s0['n'] == 1, "replset2.js Failed to replicate to slave 0 on multiple inserts"); + + var s1 = slaves[1].getDB(testDB).foo.findOne({ n: 1 }); + assert(s1['n'] == 1, "replset2.js Failed to replicate to slave 1 on multiple inserts"); + + // Test getlasterror with a simple insert + print("replset2.js **** Try inserting a single record ****") + master.getDB(testDB).dropDatabase(); + master.getDB(testDB).foo.insert({ n: 1 }); + callGetLastError(3, 10000, testDB); + + m1 = master.getDB(testDB).foo.findOne({ n: 1 }); + printjson(m1); + assert(m1['n'] == 1, "replset2.js Failed to save to master"); + + s0 = slaves[0].getDB(testDB).foo.findOne({ n: 1 }); + assert(s0['n'] == 1, "replset2.js Failed to replicate to slave 0"); + + s1 = slaves[1].getDB(testDB).foo.findOne({ n: 1 }); + assert(s1['n'] == 1, "replset2.js Failed to replicate to slave 1"); + + // Test getlasterror with large insert + print("replset2.js **** Try inserting many records ****") try { - bigData = new Array(2000).toString() - for (var n = 0; n < 1000; n++) { - master.getDB(testDB).baz.insert({ n: n, data: bigData }); - } - callGetLastError(3, 60000, testDB); - - print("replset2.js **** V1 ") - - var verifyReplication = function (nodeName, collection) { - data = collection.findOne({ n: 1 }); - assert(data['n'] == 1, "replset2.js Failed to save to " + nodeName); - data = collection.findOne({ n: 999 }); - assert(data['n'] == 999, "replset2.js Failed to save to " + nodeName); - } - - print("replset2.js **** V2 ") - - verifyReplication("master", master.getDB(testDB).baz); - verifyReplication("slave 0", slaves[0].getDB(testDB).baz); - verifyReplication("slave 1", slaves[1].getDB(testDB).baz); - - assert(failed == false, "replset2.js Replication with getLastError failed. See errors."); + bigData = new Array(2000).toString() + for (var n = 0; n < 1000; n++) { + master.getDB(testDB).baz.insert({ n: n, data: bigData }); + } + callGetLastError(3, 60000, testDB); + + print("replset2.js **** V1 ") + + var verifyReplication = function (nodeName, collection) { + data = collection.findOne({ n: 1 }); + assert(data['n'] == 1, "replset2.js Failed to save to " + nodeName); + data = collection.findOne({ n: 999 }); + assert(data['n'] == 999, "replset2.js Failed to save to " + nodeName); + } + + print("replset2.js **** V2 ") + + verifyReplication("master", master.getDB(testDB).baz); + verifyReplication("slave 0", slaves[0].getDB(testDB).baz); + verifyReplication("slave 1", slaves[1].getDB(testDB).baz); + + assert(failed == false, "replset2.js Replication with getLastError failed. See errors."); } catch(e) { print("ERROR: " + e); @@ -132,10 +132,10 @@ doTest = function (signal) { printjson(slaves[1].getDB("local").oplog.rs.find().sort({"$natural": -1}).limit(1).next()); } - - replTest.stopSet(signal); + + replTest.stopSet(signal); } -doTest( 15 ); - +doTest( 15 ); + print("\nreplset2.js SUCCESS\n"); diff --git a/jstests/replsets/replset3.js b/jstests/replsets/replset3.js index 8126b9d..faa0627 100644 --- a/jstests/replsets/replset3.js +++ b/jstests/replsets/replset3.js @@ -1,56 +1,80 @@ - -doTest = function( signal ) { - - // Test replica set step down - - // Replica set testing API - // Create a new replica set test. Specify set name and the number of nodes you want. - var replTest = new ReplSetTest( {name: 'testSet', nodes: 3} ); - - // call startSet() to start each mongod in the replica set - // this returns a list of nodes - var nodes = replTest.startSet(); - - // Call initiate() to send the replSetInitiate command - // This will wait for initiation - replTest.initiate(); - - // Get master node - var master = replTest.getMaster(); - - // Write some data to master - // NOTE: this test fails unless we write some data. - master.getDB("foo").foo.save({a: 1}); - master.getDB("foo").runCommand({getlasterror: 1, w:3, wtimeout: 20000}); - - // Step down master - master.getDB("admin").runCommand({replSetStepDown: true}); - - try { - var new_master = replTest.getMaster(); - } - catch( err ) { - throw( "Could not elect new master before timeout." ); - } - - assert( master != new_master, "Old master shouldn't be equal to new master." ); - - // Make sure that slaves are still up - var result = new_master.getDB("admin").runCommand({replSetGetStatus: 1}); - assert( result['ok'] == 1, "Could not verify that slaves were still up:" + result ); - - slaves = replTest.liveNodes.slaves; - assert.soon(function() { - res = slaves[0].getDB("admin").runCommand({replSetGetStatus: 1}) - return res.myState == 2; - }, "Slave 0 state not ready."); - - assert.soon(function() { - res = slaves[1].getDB("admin").runCommand({replSetGetStatus: 1}) - return res.myState == 2; - }, "Slave 1 state not ready."); - - replTest.stopSet( 15 ); + +doTest = function (signal) { + + // Test replica set step down + + // Replica set testing API + // Create a new replica set test. Specify set name and the number of nodes you want. + var replTest = new ReplSetTest({ name: 'testSet', nodes: 3 }); + + // call startSet() to start each mongod in the replica set + // this returns a list of nodes + var nodes = replTest.startSet(); + + // Call initiate() to send the replSetInitiate command + // This will wait for initiation + replTest.initiate(); + + // Get master node + var master = replTest.getMaster(); + + // Write some data to master + // NOTE: this test fails unless we write some data. + master.getDB("foo").foo.save({ a: 1 }); + master.getDB("foo").runCommand({ getlasterror: 1, w: 3, wtimeout: 20000 }); + + var phase = 1; + + print(phase++); + + // Step down master. Note: this may close our connection! + try { + master.getDB("admin").runCommand({ replSetStepDown: true }); + } catch (err) { + print("caught: " + err + " on stepdown"); + } + + print(phase++); + + try { + var new_master = replTest.getMaster(); + } + catch (err) { + throw ("Could not elect new master before timeout."); + } + + print(phase++); + + assert(master != new_master, "Old master shouldn't be equal to new master."); + + print(phase++); + + // Make sure that slaves are still up + var result = new_master.getDB("admin").runCommand({ replSetGetStatus: 1 }); + assert(result['ok'] == 1, "Could not verify that slaves were still up:" + result); + + print(phase++); + + slaves = replTest.liveNodes.slaves; + assert.soon(function () { + try { + res = slaves[0].getDB("admin").runCommand({ replSetGetStatus: 1 }) + } catch (err) { } + return res.myState == 2; + }, "Slave 0 state not ready."); + + print(phase++); + + assert.soon(function () { + try { + res = slaves[1].getDB("admin").runCommand({ replSetGetStatus: 1 }) + } catch (err) { } + return res.myState == 2; + }, "Slave 1 state not ready."); + + print("replset3.js SUCCESS"); + + replTest.stopSet(15); } doTest( 15 ); diff --git a/jstests/replsets/replset5.js b/jstests/replsets/replset5.js index fe1761e..13ee5c9 100644 --- a/jstests/replsets/replset5.js +++ b/jstests/replsets/replset5.js @@ -23,15 +23,15 @@ doTest = function (signal) { master.getDB("barDB").bar.save({ a: 1 }); replTest.awaitReplication(); - // These writes should be replicated immediately - master.getDB(testDB).foo.insert({ n: 1 }); - master.getDB(testDB).foo.insert({ n: 2 }); - master.getDB(testDB).foo.insert({ n: 3 }); - - // *** NOTE ***: The default doesn't seem to be propogating. - // When I run getlasterror with no defaults, the slaves don't have the data: - // These getlasterror commands can be run individually to verify this. - //master.getDB("admin").runCommand({ getlasterror: 1, w: 3, wtimeout: 20000 }); + // These writes should be replicated immediately + var docNum = 5000; + for(var n=0; n 0); + + +print("5"); +config.members[2].arbiterOnly = true; +reconfig(); + + +print("6"); +statusSoon(7); +assert.eq(replTest.liveNodes.slaves[1].getDB("local").oplog.rs.count(), 0); + + +print("7"); +delete config.members[2].arbiterOnly; +reconfig(); + + +print("8"); +statusSoon(2); +assert(replTest.liveNodes.slaves[1].getDB("local").oplog.rs.count() > 0); + + +print("9"); +for (var i = 0; i < 10000; i++) { + master.getDB("foo").bar.insert({increment : i, c : 0, foo : "kasdlfjaklsdfalksdfakldfmalksdfmaklmfalkfmkafmdsaklfma", date : new Date(), d : Date()}); +} + + +print("10"); +config.members[2].arbiterOnly = true; +reconfig(); + + +print("11"); +statusSoon(7); +assert.eq(replTest.liveNodes.slaves[1].getDB("local").oplog.rs.count(), 0); +*/ + +replTest.stopSet( 15 ); + diff --git a/jstests/replsets/replsetfreeze.js b/jstests/replsets/replsetfreeze.js new file mode 100644 index 0000000..3721ba5 --- /dev/null +++ b/jstests/replsets/replsetfreeze.js @@ -0,0 +1,105 @@ +/* + * 1: initialize set + * 2: step down m1 + * 3: freeze set for 30 seconds + * 4: check no one is master for 30 seconds + * 5: check for new master + * 6: step down new master + * 7: freeze for 30 seconds + * 8: unfreeze + * 9: check we get a new master within 30 seconds + */ + + +var w = 0; +var wait = function(f) { + w++; + var n = 0; + while (!f()) { + if( n % 4 == 0 ) + print("toostale.js waiting " + w); + if (++n == 4) { + print("" + f); + } + assert(n < 200, 'tried 200 times, giving up'); + sleep(1000); + } +} + +var reconnect = function(a) { + wait(function() { + try { + a.getDB("foo").bar.stats(); + return true; + } catch(e) { + print(e); + return false; + } + }); +}; + + +print("1: initialize set"); +var replTest = new ReplSetTest( {name: 'unicomplex', nodes: 3} ); +var nodes = replTest.nodeList(); +var conns = replTest.startSet(); +var config = {"_id" : "unicomplex", "members" : [ + {"_id" : 0, "host" : nodes[0] }, + {"_id" : 1, "host" : nodes[1] }, + {"_id" : 2, "host" : nodes[2], "arbiterOnly" : true}]}; +var r = replTest.initiate(config); +var master = replTest.getMaster(); + + +print("2: step down m1"); +try { + master.getDB("admin").runCommand({replSetStepDown : 1}); +} +catch(e) { + print(e); +} +reconnect(master); + +print("3: freeze set for 30 seconds"); +master.getDB("admin").runCommand({replSetFreeze : 30}); + + +print("4: check no one is master for 30 seconds"); +var start = (new Date()).getTime(); +while ((new Date()).getTime() - start < 30000) { + var result = master.getDB("admin").runCommand({isMaster:1}); + assert.eq(result.ismaster, false); + assert.eq(result.primary, undefined); + sleep(1000); +} + + +print("5: check for new master"); +master = replTest.getMaster(); + + +print("6: step down new master"); +try { + master.getDB("admin").runCommand({replSetStepDown : 1}); +} +catch(e) { + print(e); +} +reconnect(master); + + +print("7: freeze for 30 seconds"); +master.getDB("admin").runCommand({replSetFreeze : 30}); +sleep(1000); + + +print("8: unfreeze"); +master.getDB("admin").runCommand({replSetFreeze : 0}); + + +print("9: check we get a new master within 30 seconds"); +master = replTest.getMaster(); + + +replTest.stopSet( 15 ); + diff --git a/jstests/replsets/rollback.js b/jstests/replsets/rollback.js index 8840371..6370e41 100644 --- a/jstests/replsets/rollback.js +++ b/jstests/replsets/rollback.js @@ -1,155 +1,186 @@ -// test rollback in replica sets - -// try running as : -// -// mongo --nodb rollback.js | tee out | grep -v ^m31 -// - -var debugging = 0; - -function pause(s) { - print(s); - while (debugging) { - sleep(3000); - print(s); - } -} - -function deb(obj) { - if( debugging ) { - print("\n\n\n" + obj + "\n\n"); - } -} - -w = 0; - -function wait(f) { - w++; - var n = 0; - while (!f()) { - if( n % 4 == 0 ) - print("rollback.js waiting " + w); - if (++n == 4) { - print("" + f); - } - sleep(1000); - } -} - -doTest = function (signal) { - - var replTest = new ReplSetTest({ name: 'unicomplex', nodes: 3 }); - var nodes = replTest.nodeList(); - //print(tojson(nodes)); - - var conns = replTest.startSet(); - var r = replTest.initiate({ "_id": "unicomplex", - "members": [ - { "_id": 0, "host": nodes[0] }, - { "_id": 1, "host": nodes[1] }, - { "_id": 2, "host": nodes[2], arbiterOnly: true}] - }); - - // Make sure we have a master - var master = replTest.getMaster(); - a_conn = conns[0]; - A = a_conn.getDB("admin"); - b_conn = conns[1]; - a_conn.setSlaveOk(); - b_conn.setSlaveOk(); - B = b_conn.getDB("admin"); - assert(master == conns[0], "conns[0] assumed to be master"); - assert(a_conn == master); - - //deb(master); - - // Make sure we have an arbiter - assert.soon(function () { - res = conns[2].getDB("admin").runCommand({ replSetGetStatus: 1 }); - return res.myState == 7; - }, "Arbiter failed to initialize."); - - // Wait for initial replication - var a = a_conn.getDB("foo"); - var b = b_conn.getDB("foo"); - - /* force the oplog to roll */ - if (new Date() % 2 == 0) { - print("ROLLING OPLOG AS PART OF TEST (we only do this sometimes)"); - var pass = 1; - var first = a.getSisterDB("local").oplog.rs.find().sort({ $natural: 1 }).limit(1)[0]; - a.roll.insert({ x: 1 }); - while (1) { - for (var i = 0; i < 10000; i++) - a.roll.update({}, { $inc: { x: 1} }); - var op = a.getSisterDB("local").oplog.rs.find().sort({ $natural: 1 }).limit(1)[0]; - if (tojson(op.h) != tojson(first.h)) { - printjson(op); - printjson(first); - break; - } - pass++; - a.getLastError(2); // unlikely secondary isn't keeping up, but let's avoid possible intermittent issues with that. - } - print("PASSES FOR OPLOG ROLL: " + pass); - } - else { - print("NO ROLL"); - } - - a.bar.insert({ q: 1, a: "foo" }); - a.bar.insert({ q: 2, a: "foo", x: 1 }); - a.bar.insert({ q: 3, bb: 9, a: "foo" }); - - assert(a.bar.count() == 3, "t.count"); - - // wait for secondary to get this data - wait(function () { return b.bar.count() == 3; }); - - A.runCommand({ replSetTest: 1, blind: true }); - wait(function () { return B.isMaster().ismaster; }); - - b.bar.insert({ q: 4 }); - b.bar.insert({ q: 5 }); - b.bar.insert({ q: 6 }); - assert(b.bar.count() == 6, "u.count"); - - // a should not have the new data as it was in blind state. - B.runCommand({ replSetTest: 1, blind: true }); - A.runCommand({ replSetTest: 1, blind: false }); - wait(function () { return !B.isMaster().ismaster; }); - wait(function () { return A.isMaster().ismaster; }); - - assert(a.bar.count() == 3, "t is 3"); - a.bar.insert({ q: 7 }); - a.bar.insert({ q: 8 }); - { - assert(a.bar.count() == 5); - var x = a.bar.find().toArray(); - assert(x[0].q == 1, '1'); - assert(x[1].q == 2, '2'); - assert(x[2].q == 3, '3'); - assert(x[3].q == 7, '7'); - assert(x[4].q == 8, '8'); - } - - // A is 1 2 3 7 8 - // B is 1 2 3 4 5 6 - - // bring B back online - B.runCommand({ replSetTest: 1, blind: false }); - - wait(function () { return B.isMaster().ismaster || B.isMaster().secondary; }); - - // everyone is up here... - assert(A.isMaster().ismaster || A.isMaster().secondary, "A up"); - assert(B.isMaster().ismaster || B.isMaster().secondary, "B up"); - - friendlyEqual(a.bar.find().sort({ _id: 1 }).toArray(), b.bar.find().sort({ _id: 1 }).toArray(), "server data sets do not match"); - - pause("rollback.js SUCCESS"); - replTest.stopSet(signal); +// test rollback in replica sets + +// try running as : +// +// mongo --nodb rollback.js | tee out | grep -v ^m31 +// + +var debugging = 0; + +function pause(s) { + print(s); + while (debugging) { + sleep(3000); + print(s); + } +} + +function deb(obj) { + if( debugging ) { + print("\n\n\n" + obj + "\n\n"); + } +} + +w = 0; + +function wait(f) { + w++; + var n = 0; + while (!f()) { + if( n % 4 == 0 ) + print("rollback.js waiting " + w); + if (++n == 4) { + print("" + f); + } + assert(n < 200, 'tried 200 times, giving up'); + sleep(1000); + } } +doTest = function (signal) { + + var replTest = new ReplSetTest({ name: 'unicomplex', nodes: 3 }); + var nodes = replTest.nodeList(); + //print(tojson(nodes)); + + var conns = replTest.startSet(); + var r = replTest.initiate({ "_id": "unicomplex", + "members": [ + { "_id": 0, "host": nodes[0] }, + { "_id": 1, "host": nodes[1] }, + { "_id": 2, "host": nodes[2], arbiterOnly: true}] + }); + + // Make sure we have a master + var master = replTest.getMaster(); + a_conn = conns[0]; + A = a_conn.getDB("admin"); + b_conn = conns[1]; + a_conn.setSlaveOk(); + b_conn.setSlaveOk(); + B = b_conn.getDB("admin"); + assert(master == conns[0], "conns[0] assumed to be master"); + assert(a_conn == master); + + //deb(master); + + // Make sure we have an arbiter + assert.soon(function () { + res = conns[2].getDB("admin").runCommand({ replSetGetStatus: 1 }); + return res.myState == 7; + }, "Arbiter failed to initialize."); + + // Wait for initial replication + var a = a_conn.getDB("foo"); + var b = b_conn.getDB("foo"); + + /* force the oplog to roll */ + if (new Date() % 2 == 0) { + print("ROLLING OPLOG AS PART OF TEST (we only do this sometimes)"); + var pass = 1; + var first = a.getSisterDB("local").oplog.rs.find().sort({ $natural: 1 }).limit(1)[0]; + a.roll.insert({ x: 1 }); + while (1) { + for (var i = 0; i < 10000; i++) + a.roll.update({}, { $inc: { x: 1} }); + var op = a.getSisterDB("local").oplog.rs.find().sort({ $natural: 1 }).limit(1)[0]; + if (tojson(op.h) != tojson(first.h)) { + printjson(op); + printjson(first); + break; + } + pass++; + a.getLastError(2); // unlikely secondary isn't keeping up, but let's avoid possible intermittent issues with that. + } + print("PASSES FOR OPLOG ROLL: " + pass); + } + else { + print("NO ROLL"); + } + + a.bar.insert({ q: 1, a: "foo" }); + a.bar.insert({ q: 2, a: "foo", x: 1 }); + a.bar.insert({ q: 3, bb: 9, a: "foo" }); + + assert(a.bar.count() == 3, "t.count"); + + // wait for secondary to get this data + wait(function () { return b.bar.count() == 3; }); + + A.runCommand({ replSetTest: 1, blind: true }); + reconnect(a,b); + wait(function () { return B.isMaster().ismaster; }); + + b.bar.insert({ q: 4 }); + b.bar.insert({ q: 5 }); + b.bar.insert({ q: 6 }); + assert(b.bar.count() == 6, "u.count"); + + // a should not have the new data as it was in blind state. + B.runCommand({ replSetTest: 1, blind: true }); + print("*************** wait for server to reconnect ****************"); + reconnect(a,b); + A.runCommand({ replSetTest: 1, blind: false }); + reconnect(a,b); + + print("*************** B ****************"); + wait(function () { try { return !B.isMaster().ismaster; } catch(e) { return false; } }); + print("*************** A ****************"); + reconnect(a,b); + wait(function () { + try { + return A.isMaster().ismaster; + } catch(e) { + return false; + } + }); + + assert(a.bar.count() == 3, "t is 3"); + a.bar.insert({ q: 7 }); + a.bar.insert({ q: 8 }); + { + assert(a.bar.count() == 5); + var x = a.bar.find().toArray(); + assert(x[0].q == 1, '1'); + assert(x[1].q == 2, '2'); + assert(x[2].q == 3, '3'); + assert(x[3].q == 7, '7'); + assert(x[4].q == 8, '8'); + } + + // A is 1 2 3 7 8 + // B is 1 2 3 4 5 6 + + // bring B back online + B.runCommand({ replSetTest: 1, blind: false }); + reconnect(a,b); + + wait(function () { return B.isMaster().ismaster || B.isMaster().secondary; }); + + // everyone is up here... + assert(A.isMaster().ismaster || A.isMaster().secondary, "A up"); + assert(B.isMaster().ismaster || B.isMaster().secondary, "B up"); + replTest.awaitReplication(); + + friendlyEqual(a.bar.find().sort({ _id: 1 }).toArray(), b.bar.find().sort({ _id: 1 }).toArray(), "server data sets do not match"); + + pause("rollback.js SUCCESS"); + replTest.stopSet(signal); +}; + + +var reconnect = function(a,b) { + wait(function() { + try { + a.bar.stats(); + b.bar.stats(); + return true; + } catch(e) { + print(e); + return false; + } + }); +}; + print("rollback.js"); doTest( 15 ); diff --git a/jstests/replsets/rollback2.js b/jstests/replsets/rollback2.js index 483d221..46fb548 100644 --- a/jstests/replsets/rollback2.js +++ b/jstests/replsets/rollback2.js @@ -1,201 +1,232 @@ -// test rollback in replica sets - -// try running as : -// -// mongo --nodb rollback.js | tee out | grep -v ^m31 -// - -var debugging = 0; - -function pause(s) { - print(s); - while (debugging) { - sleep(3000); - print(s); - } -} - -function deb(obj) { - if( debugging ) { - print("\n\n\n" + obj + "\n\n"); - } -} - -w = 0; - -function wait(f) { - w++; - var n = 0; - while (!f()) { - if (n % 4 == 0) - print("rollback2.js waiting " + w); - if (++n == 4) { - print("" + f); - } - sleep(1000); - } -} - -function dbs_match(a, b) { - print("dbs_match"); - - var ac = a.system.namespaces.find().sort({name:1}).toArray(); - var bc = b.system.namespaces.find().sort({name:1}).toArray(); - if (!friendlyEqual(ac, bc)) { - print("dbs_match: namespaces don't match"); - print("\n\n"); - printjson(ac); - print("\n\n"); - printjson(bc); - print("\n\n"); - return false; - } - - var c = a.getCollectionNames(); - for( var i in c ) { - print("checking " + c[i]); - if( !friendlyEqual( a[c[i]].find().sort({_id:1}).toArray(), b[c[i]].find().sort({_id:1}).toArray() ) ) { - print("dbs_match: collections don't match " + c[i]); - return false; - } - } - return true; -} - -/* these writes will be initial data and replicate everywhere. */ -function doInitialWrites(db) { - t = db.bar; - t.insert({ q:0}); - t.insert({ q: 1, a: "foo" }); - t.insert({ q: 2, a: "foo", x: 1 }); - t.insert({ q: 3, bb: 9, a: "foo" }); - t.insert({ q: 40, a: 1 }); - t.insert({ q: 40, a: 2 }); - t.insert({ q: 70, txt: 'willremove' }); - - db.createCollection("kap", { capped: true, size: 5000 }); - db.kap.insert({ foo: 1 }) - - // going back to empty on capped is a special case and must be tested - db.createCollection("kap2", { capped: true, size: 5501 }); -} - -/* these writes on one primary only and will be rolled back. */ -function doItemsToRollBack(db) { - t = db.bar; - t.insert({ q: 4 }); - t.update({ q: 3 }, { q: 3, rb: true }); - - t.remove({ q: 40 }); // multi remove test - - t.update({ q: 2 }, { q: 39, rb: true }); - - // rolling back a delete will involve reinserting the item(s) - t.remove({ q: 1 }); - - t.update({ q: 0 }, { $inc: { y: 1} }); - - db.kap.insert({ foo: 2 }) - db.kap2.insert({ foo: 2 }) - - // create a collection (need to roll back the whole thing) - db.newcoll.insert({ a: true }); - - // create a new empty collection (need to roll back the whole thing) - db.createCollection("abc"); -} - -function doWritesToKeep2(db) { - t = db.bar; - t.insert({ txt: 'foo' }); - t.remove({ q: 70 }); - t.update({ q: 0 }, { $inc: { y: 33} }); -} - -function verify(db) { - print("verify"); - t = db.bar; - assert(t.find({ q: 1 }).count() == 1); - assert(t.find({ txt: 'foo' }).count() == 1); - assert(t.find({ q: 4 }).count() == 0); -} - -doTest = function (signal) { - - var replTest = new ReplSetTest({ name: 'unicomplex', nodes: 3 }); - var nodes = replTest.nodeList(); - //print(tojson(nodes)); - - var conns = replTest.startSet(); - var r = replTest.initiate({ "_id": "unicomplex", - "members": [ - { "_id": 0, "host": nodes[0] }, - { "_id": 1, "host": nodes[1] }, - { "_id": 2, "host": nodes[2], arbiterOnly: true}] - }); - - // Make sure we have a master - var master = replTest.getMaster(); - a_conn = conns[0]; - A = a_conn.getDB("admin"); - b_conn = conns[1]; - a_conn.setSlaveOk(); - b_conn.setSlaveOk(); - B = b_conn.getDB("admin"); - assert(master == conns[0], "conns[0] assumed to be master"); - assert(a_conn == master); - - //deb(master); - - // Make sure we have an arbiter - assert.soon(function () { - res = conns[2].getDB("admin").runCommand({ replSetGetStatus: 1 }); - return res.myState == 7; - }, "Arbiter failed to initialize."); - - // Wait for initial replication - var a = a_conn.getDB("foo"); - var b = b_conn.getDB("foo"); - doInitialWrites(a); - - // wait for secondary to get this data - wait(function () { return b.bar.count() == a.bar.count(); }); - - A.runCommand({ replSetTest: 1, blind: true }); - wait(function () { return B.isMaster().ismaster; }); - - doItemsToRollBack(b); - - // a should not have the new data as it was in blind state. - B.runCommand({ replSetTest: 1, blind: true }); - A.runCommand({ replSetTest: 1, blind: false }); - wait(function () { return !B.isMaster().ismaster; }); - wait(function () { return A.isMaster().ismaster; }); - - assert(a.bar.count() >= 1, "count check"); - doWritesToKeep2(a); - - // A is 1 2 3 7 8 - // B is 1 2 3 4 5 6 - - // bring B back online - // as A is primary, B will roll back and then catch up - B.runCommand({ replSetTest: 1, blind: false }); - - wait(function () { return B.isMaster().ismaster || B.isMaster().secondary; }); - - // everyone is up here... - assert(A.isMaster().ismaster || A.isMaster().secondary, "A up"); - assert(B.isMaster().ismaster || B.isMaster().secondary, "B up"); - - verify(a); - - assert( dbs_match(a,b), "server data sets do not match after rollback, something is wrong"); - - pause("rollback2.js SUCCESS"); - replTest.stopSet(signal); +// a test of rollback in replica sets +// +// try running as : +// +// mongo --nodb rollback2.js | tee out | grep -v ^m31 +// + +var debugging = 0; + +function pause(s) { + print(s); + while (debugging) { + sleep(3000); + print(s); + } +} + +function deb(obj) { + if( debugging ) { + print("\n\n\n" + obj + "\n\n"); + } +} + +w = 0; + +function wait(f) { + w++; + var n = 0; + while (!f()) { + if (n % 4 == 0) + print("rollback2.js waiting " + w); + if (++n == 4) { + print("" + f); + } + assert(n < 200, 'tried 200 times, giving up'); + sleep(1000); + } +} + +function dbs_match(a, b) { + print("dbs_match"); + + var ac = a.system.namespaces.find().sort({name:1}).toArray(); + var bc = b.system.namespaces.find().sort({name:1}).toArray(); + if (!friendlyEqual(ac, bc)) { + print("dbs_match: namespaces don't match"); + print("\n\n"); + printjson(ac); + print("\n\n"); + printjson(bc); + print("\n\n"); + return false; + } + + var c = a.getCollectionNames(); + for( var i in c ) { + print("checking " + c[i]); + if( !friendlyEqual( a[c[i]].find().sort({_id:1}).toArray(), b[c[i]].find().sort({_id:1}).toArray() ) ) { + print("dbs_match: collections don't match " + c[i]); + return false; + } + } + return true; +} + +/* these writes will be initial data and replicate everywhere. */ +function doInitialWrites(db) { + t = db.bar; + t.insert({ q:0}); + t.insert({ q: 1, a: "foo" }); + t.insert({ q: 2, a: "foo", x: 1 }); + t.insert({ q: 3, bb: 9, a: "foo" }); + t.insert({ q: 40, a: 1 }); + t.insert({ q: 40, a: 2 }); + t.insert({ q: 70, txt: 'willremove' }); + + db.createCollection("kap", { capped: true, size: 5000 }); + db.kap.insert({ foo: 1 }) + + // going back to empty on capped is a special case and must be tested + db.createCollection("kap2", { capped: true, size: 5501 }); +} + +/* these writes on one primary only and will be rolled back. */ +function doItemsToRollBack(db) { + t = db.bar; + t.insert({ q: 4 }); + t.update({ q: 3 }, { q: 3, rb: true }); + + t.remove({ q: 40 }); // multi remove test + + t.update({ q: 2 }, { q: 39, rb: true }); + + // rolling back a delete will involve reinserting the item(s) + t.remove({ q: 1 }); + + t.update({ q: 0 }, { $inc: { y: 1} }); + + db.kap.insert({ foo: 2 }) + db.kap2.insert({ foo: 2 }) + + // create a collection (need to roll back the whole thing) + db.newcoll.insert({ a: true }); + + // create a new empty collection (need to roll back the whole thing) + db.createCollection("abc"); } +function doWritesToKeep2(db) { + t = db.bar; + t.insert({ txt: 'foo' }); + t.remove({ q: 70 }); + t.update({ q: 0 }, { $inc: { y: 33} }); +} + +function verify(db) { + print("verify"); + t = db.bar; + assert(t.find({ q: 1 }).count() == 1); + assert(t.find({ txt: 'foo' }).count() == 1); + assert(t.find({ q: 4 }).count() == 0); +} + +doTest = function (signal) { + + var replTest = new ReplSetTest({ name: 'unicomplex', nodes: 3 }); + var nodes = replTest.nodeList(); + //print(tojson(nodes)); + + var conns = replTest.startSet(); + var r = replTest.initiate({ "_id": "unicomplex", + "members": [ + { "_id": 0, "host": nodes[0] }, + { "_id": 1, "host": nodes[1] }, + { "_id": 2, "host": nodes[2], arbiterOnly: true}] + }); + + // Make sure we have a master + var master = replTest.getMaster(); + a_conn = conns[0]; + A = a_conn.getDB("admin"); + b_conn = conns[1]; + a_conn.setSlaveOk(); + b_conn.setSlaveOk(); + B = b_conn.getDB("admin"); + assert(master == conns[0], "conns[0] assumed to be master"); + assert(a_conn == master); + + //deb(master); + + // Make sure we have an arbiter + assert.soon(function () { + res = conns[2].getDB("admin").runCommand({ replSetGetStatus: 1 }); + return res.myState == 7; + }, "Arbiter failed to initialize."); + + // Wait for initial replication + var a = a_conn.getDB("foo"); + var b = b_conn.getDB("foo"); + wait(function () { + var status = A.runCommand({replSetGetStatus : 1}); + return status.members[1].state == 2; + }); + + doInitialWrites(a); + + // wait for secondary to get this data + wait(function () { return b.bar.count() == a.bar.count(); }); + wait(function () { + var status = A.runCommand({replSetGetStatus : 1}); + return status.members[1].state == 2; + }); + + + A.runCommand({ replSetTest: 1, blind: true }); + reconnect(a, b); + + wait(function () { return B.isMaster().ismaster; }); + + doItemsToRollBack(b); + + // a should not have the new data as it was in blind state. + B.runCommand({ replSetTest: 1, blind: true }); + reconnect(a, b); + A.runCommand({ replSetTest: 1, blind: false }); + reconnect(a,b); + + wait(function () { try { return !B.isMaster().ismaster; } catch(e) { return false; } }); + wait(function () { try { return A.isMaster().ismaster; } catch(e) { return false; } }); + + assert(a.bar.count() >= 1, "count check"); + doWritesToKeep2(a); + + // A is 1 2 3 7 8 + // B is 1 2 3 4 5 6 + + // bring B back online + // as A is primary, B will roll back and then catch up + B.runCommand({ replSetTest: 1, blind: false }); + reconnect(a,b); + + wait(function () { return B.isMaster().ismaster || B.isMaster().secondary; }); + + // everyone is up here... + assert(A.isMaster().ismaster || A.isMaster().secondary, "A up"); + assert(B.isMaster().ismaster || B.isMaster().secondary, "B up"); + replTest.awaitReplication(); + + verify(a); + + assert( dbs_match(a,b), "server data sets do not match after rollback, something is wrong"); + + pause("rollback2.js SUCCESS"); + replTest.stopSet(signal); +}; + +var reconnect = function(a,b) { + wait(function() { + try { + a.bar.stats(); + b.bar.stats(); + return true; + } catch(e) { + print(e); + return false; + } + }); +}; + print("rollback2.js"); doTest( 15 ); diff --git a/jstests/replsets/rollback3.js b/jstests/replsets/rollback3.js index 5c2f2f1..fa923d8 100755 --- a/jstests/replsets/rollback3.js +++ b/jstests/replsets/rollback3.js @@ -30,10 +30,10 @@ function wait(f) { if (n % 4 == 0) print("rollback3.js waiting " + w); if (++n == 4) { - print("" + f); - } - if (n == 200) { - print("rollback3.js failing waited too long"); + print("" + f); + } + if (n == 200) { + print("rollback3.js failing waited too long"); throw "wait error"; } sleep(1000); @@ -188,15 +188,20 @@ doTest = function (signal) { wait(function () { return b.bar.count() == a.bar.count(); }); A.runCommand({ replSetTest: 1, blind: true }); - wait(function () { return B.isMaster().ismaster; }); + reconnect(a,b); + wait(function () { try { return B.isMaster().ismaster; } catch(e) { return false; } }); doItemsToRollBack(b); // a should not have the new data as it was in blind state. B.runCommand({ replSetTest: 1, blind: true }); + reconnect(a,b); + A.runCommand({ replSetTest: 1, blind: false }); - wait(function () { return !B.isMaster().ismaster; }); - wait(function () { return A.isMaster().ismaster; }); + reconnect(a,b); + + wait(function () { try { return !B.isMaster().ismaster; } catch(e) { return false; } }); + wait(function () { try { return A.isMaster().ismaster; } catch(e) { return false; } }); assert(a.bar.count() >= 1, "count check"); doWritesToKeep2(a); @@ -207,18 +212,34 @@ doTest = function (signal) { // bring B back online // as A is primary, B will roll back and then catch up B.runCommand({ replSetTest: 1, blind: false }); + reconnect(a,b); wait(function () { return B.isMaster().ismaster || B.isMaster().secondary; }); // everyone is up here... assert(A.isMaster().ismaster || A.isMaster().secondary, "A up"); assert(B.isMaster().ismaster || B.isMaster().secondary, "B up"); - + replTest.awaitReplication(); + assert( dbs_match(a,b), "server data sets do not match after rollback, something is wrong"); pause("rollback3.js SUCCESS"); replTest.stopSet(signal); -} +}; + + +var reconnect = function(a,b) { + wait(function() { + try { + a.bar.stats(); + b.bar.stats(); + return true; + } catch(e) { + print(e); + return false; + } + }); +}; print("rollback3.js"); doTest( 15 ); diff --git a/jstests/replsets/rslib.js b/jstests/replsets/rslib.js new file mode 100644 index 0000000..c072829 --- /dev/null +++ b/jstests/replsets/rslib.js @@ -0,0 +1,63 @@ + +var count = 0; +var w = 0; + +var wait = function(f) { + w++; + var n = 0; + while (!f()) { + if( n % 4 == 0 ) + print("waiting " + w); + if (++n == 4) { + print("" + f); + } + assert(n < 200, 'tried 200 times, giving up'); + sleep(1000); + } +}; + +/** + * Use this to do something once every 4 iterations. + * + * 
+ * for (i=0; i<1000; i++) {
+ *   occasionally(function() { print("4 more iterations"); });
+ * }
+ *
+ */ +var occasionally = function(f, n) { + var interval = n || 4; + if (count % interval == 0) { + f(); + } + count++; +}; + +var reconnect = function(a) { + wait(function() { + try { + // make this work with either dbs or connections + if (typeof(a.getDB) == "function") { + a.getDB("foo").bar.stats(); + } + else { + a.bar.stats(); + } + return true; + } catch(e) { + print(e); + return false; + } + }); +}; + + +var getLatestOp = function(server) { + server.getDB("admin").getMongo().setSlaveOk(); + var log = server.getDB("local")['oplog.rs']; + var cursor = log.find({}).sort({'$natural': -1}).limit(1); + if (cursor.hasNext()) { + return cursor.next(); + } + return null; +}; diff --git a/jstests/replsets/slaveDelay2.js b/jstests/replsets/slaveDelay2.js new file mode 100644 index 0000000..2d9dd1f --- /dev/null +++ b/jstests/replsets/slaveDelay2.js @@ -0,0 +1,104 @@ + +var name = "slaveDelay2"; +var host = getHostName(); + +var waitForAllMembers = function(master) { + var ready = false; + + outer: + while (true) { + var state = master.getSisterDB("admin").runCommand({replSetGetStatus:1}); + + for (var m in state.members) { + if (state.members[m].state != 2 && state.members[m].state != 1) { + sleep(10000); + continue outer; + } + } + + printjson(state); + print("okay, everyone is primary or secondary"); + return; + } +}; + + +var initialize = function() { + var replTest = new ReplSetTest( {name: name, nodes: 1} ); + + var nodes = replTest.startSet(); + + replTest.initiate(); + + var master = replTest.getMaster().getDB(name); + + waitForAllMembers(master); + + return replTest; +}; + +var populate = function(master) { + // insert records + for (var i =0; i<1000; i++) { + master.foo.insert({_id:1}); + } + + master.runCommand({getlasterror:1}); +} + +doTest = function( signal ) { + var replTest = initialize(); + var master = replTest.getMaster().getDB(name); + populate(master); + var admin = master.getSisterDB("admin"); + + /** + * start a slave with a long delay (1 hour) and do some writes while it is + * initializing. Make sure it syncs all of these writes before going into + * syncDelay. + */ + var conn = startMongodTest(31008, name + "-sd", 0, { useHostname: true, replSet: name }); + conn.setSlaveOk(); + + config = master.getSisterDB("local").system.replset.findOne(); + config.version++; + config.members.push({_id : 1, host : host+":31008",priority:0, slaveDelay:3600}); + var ok = admin.runCommand({replSetReconfig : config}); + assert.eq(ok.ok,1); + + // do inserts during initial sync + count = 0; + while (count < 10) { + for (var i = 100*count; i<100*(count+1); i++) { + master.foo.insert({x:i}); + } + + //check if initial sync is done + var state = master.getSisterDB("admin").runCommand({replSetGetStatus:1}); + printjson(state); + if (state.members[1].state == 2) { + break; + } + + count++; + } + + // throw out last 100 inserts, but make sure the others were applied + if (count == 0) { + print("NOTHING TO CHECK"); + replTest.stopSet(); + return; + } + + // wait a bit for the syncs to be applied + waitForAllMembers(master); + + for (var i=0; i<(100*count); i++) { + var obj = conn.getDB(name).foo.findOne({x : i}); + assert(obj); + } + + replTest.stopSet(); +} + +doTest(15); diff --git a/jstests/replsets/slavedelay1.js b/jstests/replsets/slavedelay1.js new file mode 100644 index 0000000..e549822 --- /dev/null +++ b/jstests/replsets/slavedelay1.js @@ -0,0 +1,127 @@ + +var waitForAllMembers = function(master) { + var ready = false; + + outer: + while (true) { + var state = master.getSisterDB("admin").runCommand({replSetGetStatus:1}); + printjson(state); + + for (var m in state.members) { + if (state.members[m].state != 2 && state.members[m].state != 1) { + sleep(10000); + continue outer; + } + } + return; + } +}; + + +doTest = function( signal ) { + + var name = "slaveDelay"; + var host = getHostName(); + + var replTest = new ReplSetTest( {name: name, nodes: 3} ); + + var nodes = replTest.startSet(); + + /* set slaveDelay to 30 seconds */ + var config = replTest.getReplSetConfig(); + config.members[2].priority = 0; + config.members[2].slaveDelay = 30; + + replTest.initiate(config); + + var master = replTest.getMaster().getDB(name); + var slaveConns = replTest.liveNodes.slaves; + var slave = []; + for (var i in slaveConns) { + var d = slaveConns[i].getDB(name); + d.getMongo().setSlaveOk(); + slave.push(d); + } + + waitForAllMembers(master); + + // insert a record + master.foo.insert({x:1}); + master.runCommand({getlasterror:1, w:2}); + + var doc = master.foo.findOne(); + assert.eq(doc.x, 1); + + // make sure slave has it + var doc = slave[0].foo.findOne(); + assert.eq(doc.x, 1); + + // make sure delayed slave doesn't have it + assert.eq(slave[1].foo.findOne(), null); + + // wait 35 seconds + sleep(35000); + + // now delayed slave should have it + assert.eq(slave[1].foo.findOne().x, 1); + + + /************* Part 2 *******************/ + + // how about non-initial sync? + + for (var i=0; i<100; i++) { + master.foo.insert({_id : i, "foo" : "bar"}); + } + master.runCommand({getlasterror:1,w:2}); + + assert.eq(master.foo.findOne({_id : 99}).foo, "bar"); + assert.eq(slave[0].foo.findOne({_id : 99}).foo, "bar"); + assert.eq(slave[1].foo.findOne({_id : 99}), null); + + sleep(35000); + + assert.eq(slave[1].foo.findOne({_id : 99}).foo, "bar"); + + /************* Part 3 *******************/ + + // how about if we add a new server? will it sync correctly? + + var conn = startMongodTest( 31007 , name+"-part3" , 0 , {useHostname : true, replSet : name} ); + + config = master.getSisterDB("local").system.replset.findOne(); + printjson(config); + config.version++; + config.members.push({_id : 3, host : host+":31007",priority:0, slaveDelay:10}); + + var admin = master.getSisterDB("admin"); + try { + var ok = admin.runCommand({replSetReconfig : config}); + assert.eq(ok.ok,1); + } + catch(e) { + print(e); + } + + master = replTest.getMaster().getDB(name); + + waitForAllMembers(master); + + sleep(15000); + + // it should be all caught up now + + master.foo.insert({_id : 123, "x" : "foo"}); + master.runCommand({getlasterror:1,w:2}); + + conn.setSlaveOk(); + assert.eq(conn.getDB(name).foo.findOne({_id:123}), null); + + sleep(15000); + + assert.eq(conn.getDB(name).foo.findOne({_id:123}).x, "foo"); + + replTest.stopSet(); +} + +doTest(15); diff --git a/jstests/replsets/sync1.js b/jstests/replsets/sync1.js index e60d128..af16044 100644 --- a/jstests/replsets/sync1.js +++ b/jstests/replsets/sync1.js @@ -1,5 +1,7 @@ // test rollback of replica sets +load("jstests/replsets/rslib.js"); + var debugging=0; w = 0; @@ -50,7 +52,7 @@ doTest = function (signal) { dbs[0].bar.ensureIndex({ w: 1 }); var ok = false; - var inserts = 100000; + var inserts = 10000; print("\nsync1.js ********************************************************************** part 5"); @@ -62,7 +64,7 @@ doTest = function (signal) { do { sleep(1000); status = dbs[0].getSisterDB("admin").runCommand({ replSetGetStatus: 1 }); - } while (status.members[1].state != 2 && status.members[2].state != 2); + } while (status.members[1].state != 2 || status.members[2].state != 2); print("\nsync1.js ********************************************************************** part 6"); dbs[0].getSisterDB("admin").runCommand({ replSetTest: 1, blind: true }); @@ -125,12 +127,14 @@ doTest = function (signal) { try { printjson(dbs[1].isMaster()); printjson(dbs[1].bar.count()); + printjson(dbs[1].adminCommand({replSetGetStatus : 1})); } catch (e) { print(e); } print("dbs[2]:"); try { printjson(dbs[2].isMaster()); printjson(dbs[2].bar.count()); + printjson(dbs[2].adminCommand({replSetGetStatus : 1})); } catch (e) { print(e); } assert(false, "sync1.js too many exceptions, failing"); @@ -161,10 +165,22 @@ doTest = function (signal) { print("\nsync1.js ********************************************************************** part 10"); // now, let's see if rollback works - var result = dbs[0].getSisterDB("admin").runCommand({ replSetTest: 1, blind: false }); + wait(function() { + try { + dbs[0].adminCommand({ replSetTest: 1, blind: false }); + } + catch(e) { + print(e); + } + reconnect(dbs[0]); + reconnect(dbs[1]); + + var status = dbs[1].adminCommand({replSetGetStatus:1}); + return status.members[0].health == 1; + }); + + dbs[0].getMongo().setSlaveOk(); - - printjson(result); sleep(5000); // now this should resync @@ -192,6 +208,10 @@ doTest = function (signal) { count++; if (count == 100) { + printjson(dbs[0].isMaster()); + printjson(dbs[0].adminCommand({replSetGetStatus:1})); + printjson(dbs[1].isMaster()); + printjson(dbs[1].adminCommand({replSetGetStatus:1})); pause("FAIL part 11"); assert(false, "replsets/\nsync1.js fails timing out"); replTest.stopSet(signal); diff --git a/jstests/replsets/sync_passive.js b/jstests/replsets/sync_passive.js new file mode 100644 index 0000000..d3e8ef4 --- /dev/null +++ b/jstests/replsets/sync_passive.js @@ -0,0 +1,89 @@ +/** + * Test syncing from non-primaries. + * + * Start a set. + * Inital sync. + * Kill member 1. + * Add some data. + * Kill member 0. + * Restart member 1. + * Check that it syncs. + * Add some data. + * Kill member 1. + * Restart member 0. + * Check that it syncs. + */ + +load("jstests/replsets/rslib.js"); + +var name = "sync_passive"; +var host = getHostName(); + +var replTest = new ReplSetTest( {name: name, nodes: 3} ); + +var nodes = replTest.startSet(); + +/* set slaveDelay to 30 seconds */ +var config = replTest.getReplSetConfig(); +config.members[2].priority = 0; + +replTest.initiate(config); + +var master = replTest.getMaster().getDB("test"); +var server0 = master; +var server1 = replTest.liveNodes.slaves[0]; + +print("Initial sync"); +for (var i=0;i<100;i++) { + master.foo.insert({x:i}); +} +replTest.awaitReplication(); + + +print("stop #1"); +replTest.stop(1); + + +print("add some data"); +for (var i=0;i<1000;i++) { + master.bar.insert({x:i}); +} +replTest.awaitReplication(); + + +print("stop #0"); +replTest.stop(0); + + +print("restart #1"); +replTest.restart(1); + + +print("check sync"); +replTest.awaitReplication(); + + +print("add data"); +reconnect(server1); +master = replTest.getMaster().getDB("test"); +for (var i=0;i<1000;i++) { + master.bar.insert({x:i}); +} +replTest.awaitReplication(); + + +print("kill #1"); +replTest.stop(1); + + +print("restart #0"); +replTest.restart(0); +reconnect(server0); + + +print("wait for sync"); +replTest.awaitReplication(); + + +print("bring #1 back up, make sure everything's okay"); +replTest.restart(1); diff --git a/jstests/replsets/sync_passive2.js b/jstests/replsets/sync_passive2.js new file mode 100644 index 0000000..230d71c --- /dev/null +++ b/jstests/replsets/sync_passive2.js @@ -0,0 +1,120 @@ +/** + * Test syncing from non-primaries. + */ + +load("jstests/replsets/rslib.js"); + +var name = "sync_passive2"; +var host = getHostName(); + +var replTest = new ReplSetTest( {name: name, nodes: 5} ); +var nodes = replTest.startSet(); + +// 0: master +// 1: arbiter +// 2: slave a +// 3: slave b +// 4: slave c +var config = replTest.getReplSetConfig(); +config.members[1].arbiterOnly = true; +for (i=2; i