Imported Upstream version 2.0.0

9 files changed, 276 insertions, 79 deletions

diff --git a/jstests/dur/closeall.js b/jstests/dur/closeall.js
index f169f06..9131aed 100644
--- a/jstests/dur/closeall.js
+++ b/jstests/dur/closeall.js
@@ -8,7 +8,9 @@ function f() {
     var ourdb = "closealltest";
 
     print("closeall.js start mongod variant:" + variant);
-    var options = (new Date()-0)%2==0 ? 8 : 0;
+    var R = (new Date()-0)%2;
+    var QuickCommits = (new Date()-0)%3 == 0;
+    var options = R==0 ? 8 : 0; // 8 is DurParanoid
     print("closeall.js --durOptions " + options);
     var N = 1000;
     if (options) 
@@ -23,6 +25,10 @@ function f() {
     // we'll use two connections to make a little parallelism
     var db1 = conn.getDB(ourdb);
     var db2 = new Mongo(db1.getMongo().host).getDB(ourdb);
+    if( QuickCommits ) {
+	print("closeall.js QuickCommits variant (using a small syncdelay)");
+	assert( db2.adminCommand({setParameter:1, syncdelay:5}).ok );
+    }
 
     print("closeall.js run test");
 
@@ -34,9 +40,9 @@ function f() {
 	        db1.foo.update({ x: 99 }, { a: 1, b: 2, c: 3, d: 4 });
 	    if (i % 100 == 0)
 	        db1.foo.find();
-	    if( i == 800 )

+	    if( i == 800 )
 	        db1.foo.ensureIndex({ x: 1 });
-        var res = null;

+        var res = null;
         try {
 	    if( variant == 1 )
 		sleep(0);
@@ -44,37 +50,37 @@ function f() {
 		sleep(1);
 	    else if( variant == 3 && i % 10 == 0 )
 		print(i);
-            res = db2.adminCommand("closeAllDatabases");

-        }

-        catch (e) {

-            sleep(5000); // sleeping a little makes console output order prettier

-            print("\n\n\nFAIL closeall.js closeAllDatabases command invocation threw an exception. i:" + i);

-            try {

-                print("getlasterror:");

-                printjson(db2.getLastErrorObj());

-                print("trying one more closealldatabases:");

-                res = db2.adminCommand("closeAllDatabases");

-                printjson(res);

-            }

-            catch (e) {

-                print("got another exception : " + e);

-            }

-            print("\n\n\n");

-            // sleep a little to capture possible mongod output?

-            sleep(2000);

-            throw e;

-        }

-	    assert( res.ok, "closeAllDatabases res.ok=false");

-	}

-

-	print("closeall.js end test loop.  slave.foo.count:");

-	print(slave.foo.count());

-

-	print("closeall.js shutting down servers");

-	stopMongod(30002);

-	stopMongod(30001);

-}

-

-f();

+            res = db2.adminCommand("closeAllDatabases");
+        }
+        catch (e) {
+            sleep(5000); // sleeping a little makes console output order prettier
+            print("\n\n\nFAIL closeall.js closeAllDatabases command invocation threw an exception. i:" + i);
+            try {
+                print("getlasterror:");
+                printjson(db2.getLastErrorObj());
+                print("trying one more closealldatabases:");
+                res = db2.adminCommand("closeAllDatabases");
+                printjson(res);
+            }
+            catch (e) {
+                print("got another exception : " + e);
+            }
+            print("\n\n\n");
+            // sleep a little to capture possible mongod output?
+            sleep(2000);
+            throw e;
+        }
+	    assert( res.ok, "closeAllDatabases res.ok=false");
+	}
+
+	print("closeall.js end test loop.  slave.foo.count:");
+	print(slave.foo.count());
+
+	print("closeall.js shutting down servers");
+	stopMongod(30002);
+	stopMongod(30001);
+}
+
+f();
 sleep(500);
 print("SUCCESS closeall.js");
diff --git a/jstests/dur/data/empty.bson b/jstests/dur/data/empty.bson
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/jstests/dur/data/empty.bson
diff --git a/jstests/dur/diskfull.js b/jstests/dur/diskfull.js
index da45c20..c123ea1 100644
--- a/jstests/dur/diskfull.js
+++ b/jstests/dur/diskfull.js
@@ -14,23 +14,23 @@ for ( i in files ) {
 if ( !doIt ) {
     print( "path " + startPath + " missing, skipping diskfull test" );
     doIt = false;
-}

-

-function checkNoJournalFiles(path, pass) {

-    var files = listFiles(path);

-    if (files.some(function (f) { return f.name.indexOf("prealloc") < 0; })) {

-        if (pass == null) {

-            // wait a bit longer for mongod to potentially finish if it is still running.

-            sleep(10000);

-            return checkNoJournalFiles(path, 1);

-        }

-        print("\n\n\n");

-        print("FAIL path:" + path);

-        print("unexpected files:");

-        printjson(files);

-        assert(false, "FAIL a journal/lsn file is present which is unexpected");

-    }

-}

+}
+
+function checkNoJournalFiles(path, pass) {
+    var files = listFiles(path);
+    if (files.some(function (f) { return f.name.indexOf("prealloc") < 0; })) {
+        if (pass == null) {
+            // wait a bit longer for mongod to potentially finish if it is still running.
+            sleep(10000);
+            return checkNoJournalFiles(path, 1);
+        }
+        print("\n\n\n");
+        print("FAIL path:" + path);
+        print("unexpected files:");
+        printjson(files);
+        assert(false, "FAIL a journal/lsn file is present which is unexpected");
+    }
+}
 
 /** Clear dbpath without removing and recreating diskfulltest directory, as resetDbpath does */
 function clear() {
@@ -56,7 +56,9 @@ function work() {
             d.foo.insert( { _id:i, b:big } );
         }
         
-        d.getLastError();
+        gle = d.getLastError();
+        if ( gle )
+            throw gle;
     } catch ( e ) {
         print( e );
         raise( e );
@@ -86,9 +88,8 @@ function runFirstMongodAndFillDisk() {
     conn = startMongodNoReset("--port", 30001, "--dbpath", startPath, "--dur", "--smallfiles", "--durOptions", 8, "--noprealloc");
     
     assert.throws( work, null, "no exception thrown when exceeding disk capacity" );
-    waitMongoProgramOnPort( 30001 );
-    
-    // the above wait doesn't work on windows
+    stopMongod( 30001 );
+
     sleep(5000);    
 }
 
@@ -104,9 +105,9 @@ function runSecondMongdAndRecover() {
     // stopMongod seems to be asynchronous (hmmm) so we sleep here.
     sleep(5000);
     
-    // at this point, after clean shutdown, there should be no journal files

-    log("check no journal files");

-    checkNoJournalFiles(startPath + "/journal/");

+    // at this point, after clean shutdown, there should be no journal files
+    log("check no journal files");
+    checkNoJournalFiles(startPath + "/journal/");
     
     log();    
 }
@@ -133,4 +134,4 @@ if ( doIt ) {
     
     print(testname + " SUCCESS");    
 
-}
\ No newline at end of file
+}
diff --git a/jstests/dur/dropdb.js b/jstests/dur/dropdb.js
index 7f82cd7..2aabd4a 100644
--- a/jstests/dur/dropdb.js
+++ b/jstests/dur/dropdb.js
@@ -73,21 +73,28 @@ function verify() {
     var d = conn.getDB("test");
     var count = d.foo.count();
     if (count != 1) {
-	print("going to fail, count mismatch in verify()");
+	    print("going to fail, test.foo.count() != 1 in verify()"); 
         sleep(10000); // easier to read the output this way
         print("\n\n\ndropdb.js FAIL test.foo.count() should be 1 but is : " + count);
-	print(d.foo.count() + "\n\n\n");
+	    print(d.foo.count() + "\n\n\n");
         assert(false);
     }
     assert(d.foo.findOne()._id == 100, "100");
 
     print("dropdb.js teste.foo.findOne:");
-    printjson(conn.getDB("teste").foo.findOne());
-
-    var teste = conn.getDB("teste");
-    print("dropdb count " + teste.foo.count());
+    printjson(conn.getDB("teste").foo.findOne());

+

+    var teste = conn.getDB("teste");

+    var testecount = teste.foo.count();

+    if (testecount != 1) {

+        print("going to fail, teste.foo.count() != 1 in verify()");

+        sleep(10000); // easier to read the output this way

+        print("\n\n\ndropdb.js FAIL teste.foo.count() should be 1 but is : " + testecount);

+        print("\n\n\n");

+        assert(false);

+    } 
+    print("teste.foo.count() = " + teste.foo.count());
     assert(teste.foo.findOne()._id == 99, "teste");
-
 }
 
 if (debugging) {
diff --git a/jstests/dur/dur1.js b/jstests/dur/dur1.js
index 4c8f1bf..299ac30 100755
--- a/jstests/dur/dur1.js
+++ b/jstests/dur/dur1.js
@@ -75,7 +75,7 @@ function work() {
 }
 
 function verify() { 
-    log("verify");

+    log("verify test.foo.count == 2");

     var d = conn.getDB("test");

     var ct = d.foo.count();

     if (ct != 2) {

@@ -99,37 +99,38 @@ var path1 = "/data/db/" + testname+"nodur";
 var path2 = "/data/db/" + testname+"dur";
 
 // non-durable version
-log();
+log("run mongod without journaling");
 conn = startMongodEmpty("--port", 30000, "--dbpath", path1, "--nodur", "--smallfiles");
 work();
 stopMongod(30000);
 
 // durable version
-log();
-conn = startMongodEmpty("--port", 30001, "--dbpath", path2, "--dur", "--smallfiles", "--durOptions", 8);
+log("run mongod with --journal");
+conn = startMongodEmpty("--port", 30001, "--dbpath", path2, "--journal", "--smallfiles", "--journalOptions", 8);
 work();
 
 // wait for group commit.
-printjson(conn.getDB('admin').runCommand({getlasterror:1, fsync:1}));
-
-// kill the process hard
+printjson(conn.getDB('admin').runCommand({getlasterror:1, fsync:1}));

+

+// kill the process hard

+log("kill 9");
 stopMongod(30001, /*signal*/9);
 
 // journal file should be present, and non-empty as we killed hard
 
 // restart and recover
-log();
-conn = startMongodNoReset("--port", 30002, "--dbpath", path2, "--dur", "--smallfiles", "--durOptions", 8);
+log("restart mongod --journal and recover");
+conn = startMongodNoReset("--port", 30002, "--dbpath", path2, "--journal", "--smallfiles", "--journalOptions", 8);
 verify();
 
-log("stop");
+log("stop mongod");
 stopMongod(30002);
 
 // stopMongod seems to be asynchronous (hmmm) so we sleep here.
-sleep(5000);
+// sleep(5000);
 
 // at this point, after clean shutdown, there should be no journal files
-log("check no journal files");
+log("check no journal files (after presumably clean shutdown)");
 checkNoJournalFiles(path2 + "/journal");
 
 log("check data matches ns");
diff --git a/jstests/dur/dur1_tool.js b/jstests/dur/dur1_tool.js
new file mode 100755
index 0000000..5090b5b
--- /dev/null
+++ b/jstests/dur/dur1_tool.js
@@ -0,0 +1,152 @@
+/* 
+   test durability option with tools (same a dur1.js but use mongorestore to do repair)
+*/
+
+var debugging = false;
+var testname = "dur1_tool";
+var step = 1;
+var conn = null;

+

+function checkNoJournalFiles(path, pass) {

+    var files = listFiles(path);

+    if (files.some(function (f) { return f.name.indexOf("prealloc") < 0; })) {

+        if (pass == null) {

+            // wait a bit longer for mongod to potentially finish if it is still running.

+            sleep(10000);

+            return checkNoJournalFiles(path, 1);

+        }

+        print("\n\n\n");

+        print("FAIL path:" + path);

+        print("unexpected files:");

+        printjson(files);

+        assert(false, "FAIL a journal/lsn file is present which is unexpected");

+    }

+}

+
+function runDiff(a, b) {
+    function reSlash(s) {
+        var x = s;
+        if (_isWindows()) {
+            while (1) {
+                var y = x.replace('/', '\\');
+                if (y == x)
+                    break;
+                x = y;
+            }
+        }
+        return x;
+    }
+    a = reSlash(a);
+    b = reSlash(b);
+    print("diff " + a + " " + b);
+    return run("diff", a, b);
+}
+
+function log(str) {
+    print();
+    if(str)
+        print(testname+" step " + step++ + " " + str);
+    else
+        print(testname+" step " + step++);
+}
+
+// if you do inserts here, you will want to set _id.  otherwise they won't match on different 
+// runs so we can't do a binary diff of the resulting files to check they are consistent.
+function work() {
+    log("work");
+    var d = conn.getDB("test");
+    d.foo.insert({ _id: 3, x: 22 });
+    d.foo.insert({ _id: 4, x: 22 });
+    d.a.insert({ _id: 3, x: 22, y: [1, 2, 3] });
+    d.a.insert({ _id: 4, x: 22, y: [1, 2, 3] });
+    d.a.update({ _id: 4 }, { $inc: { x: 1} });
+
+    // try building an index.  however, be careful as object id's in system.indexes would vary, so we do it manually:
+    d.system.indexes.insert({ _id: 99, ns: "test.a", key: { x: 1 }, name: "x_1", v: 0 });
+
+//    d.a.update({ _id: 4 }, { $inc: { x: 1} });
+//    d.a.reIndex();
+
+    // assure writes applied in case we kill -9 on return from this function
+    d.getLastError();

+

+    log("endwork");

+    return d;
+}
+
+function verify() { 
+    log("verify test.foo.count == 2");

+    var d = conn.getDB("test");

+    var ct = d.foo.count();

+    if (ct != 2) {

+        print("\n\n\nFAIL dur1.js count is wrong in verify(): " + ct + "\n\n\n");

+        assert(ct == 2);

+    }
+}
+
+if( debugging ) { 
+    // mongod already running in debugger
+    conn = db.getMongo();
+    work();
+    sleep(30000);
+    quit();
+}
+
+log();
+
+// directories
+var path1 = "/data/db/" + testname+"nodur";
+var path2 = "/data/db/" + testname+"dur";
+
+// non-durable version
+log("run mongod without journaling");
+conn = startMongodEmpty("--port", 30000, "--dbpath", path1, "--nodur", "--smallfiles");
+work();
+stopMongod(30000);
+
+// durable version
+log("run mongod with --journal");
+conn = startMongodEmpty("--port", 30001, "--dbpath", path2, "--journal", "--smallfiles", "--journalOptions", 8);
+work();
+
+// wait for group commit.
+printjson(conn.getDB('admin').runCommand({getlasterror:1, fsync:1}));

+

+// kill the process hard

+log("kill 9");
+stopMongod(30001, /*signal*/9);
+
+// journal file should be present, and non-empty as we killed hard
+
+// mongorestore with --dbpath and --journal options should do a recovery pass
+// empty.bson is an empty file so it won't actually insert anything
+log("use mongorestore to recover");
+runMongoProgram("mongorestore", "--dbpath", path2, "--journal", "-d", "test", "-c", "empty", "jstests/dur/data/empty.bson");
+
+// stopMongod seems to be asynchronous (hmmm) so we sleep here.
+// sleep(5000);
+
+// at this point, after clean shutdown, there should be no journal files
+log("check no journal files (after presumably clean shutdown)");
+checkNoJournalFiles(path2 + "/journal");
+
+log("check data matches ns");
+var diff = runDiff(path1 + "/test.ns", path2 + "/test.ns");
+if (diff != "") {
+    print("\n\n\nDIFFERS\n");
+    print(diff);
+}
+assert(diff == "", "error test.ns files differ");
+
+log("check data matches .0");
+var diff = runDiff(path1 + "/test.0", path2 + "/test.0");
+if (diff != "") {
+    print("\n\n\nDIFFERS\n");
+    print(diff);
+}
+assert(diff == "", "error test.0 files differ");
+
+log("check data matches done");
+
+print(testname + " SUCCESS");
+
diff --git a/jstests/dur/indexbg.js b/jstests/dur/indexbg.js
new file mode 100644
index 0000000..e78ae4a
--- /dev/null
+++ b/jstests/dur/indexbg.js
@@ -0,0 +1,7 @@
+path = '/data/db/indexbg_dur';
+
+m = startMongodEmpty( '--port', 30001, '--dbpath', path, '--journal', '--smallfiles', '--journalOptions', 24 );
+t = m.getDB( 'test' ).test;
+t.save( {x:1} );
+t.createIndex( {x:1}, {background:true} );
+t.count();
diff --git a/jstests/dur/indexbg2.js b/jstests/dur/indexbg2.js
new file mode 100644
index 0000000..6a0af24
--- /dev/null
+++ b/jstests/dur/indexbg2.js
@@ -0,0 +1,19 @@
+path = '/data/db/indexbg2_dur';
+
+m = startMongodEmpty( '--port', 30001, '--dbpath', path, '--journal', '--smallfiles' );
+
+t = m.getDB( 'test' ).test;
+t.createIndex( {a:1} );
+t.createIndex( {b:1} );
+t.createIndex( {x:1}, {background:true} );
+for( var i = 0; i < 1000; ++i ) {
+ 	t.insert( {_id:i,a:'abcd',b:'bcde',x:'four score and seven years ago'} );
+    t.remove( {_id:i} );
+}
+sleep( 1000 );
+for( var i = 1000; i < 2000; ++i ) {
+ 	t.insert( {_id:i,a:'abcd',b:'bcde',x:'four score and seven years ago'} );
+    t.remove( {_id:i} );
+}
+t.insert( {_id:2000,a:'abcd',b:'bcde',x:'four score and seven years ago'} );
+assert( !t.getDB().getLastError() );
diff --git a/jstests/dur/manyRestart.js b/jstests/dur/manyRestart.js
index 04e4318..79f72a4 100755
--- a/jstests/dur/manyRestart.js
+++ b/jstests/dur/manyRestart.js
@@ -116,6 +116,10 @@ conn = startMongodEmpty("--port", 30000, "--dbpath", path1, "--nodur", "--smallf
 work();
 stopMongod(30000);
 
+// hail mary for windows
+// Sat Jun 11 14:07:57 Error: boost::filesystem::create_directory: Access is denied: "\data\db\manyRestartsdur" (anon):1
+sleep(1000);
+
 log("starting 30001");
 conn = startMongodEmpty("--port", 30001, "--dbpath", path2, "--dur", "--smallfiles", "--durOptions", 8);
 work();