Merge commit 'upstream/1.8.2'

23 files changed, 91 insertions, 39 deletions

diff --git a/db/btree.cpp b/db/btree.cpp
index d547a1b..242c534 100644
--- a/db/btree.cpp
+++ b/db/btree.cpp
@@ -1188,8 +1188,6 @@ namespace mongo {
             out() << "   " << thisLoc.toString() << ".insertHere " << key.toString() << '/' << recordLoc.toString() << ' '
                   << lchild.toString() << ' ' << rchild.toString() << " keypos:" << keypos << endl;
 
-        DiskLoc oldLoc = thisLoc;
-
         if ( !basicInsert(thisLoc, keypos, recordLoc, key, order) ) {
             thisLoc.btreemod()->split(thisLoc, keypos, recordLoc, key, order, lchild, rchild, idx);
             return;
diff --git a/db/cmdline.cpp b/db/cmdline.cpp
index 900a782..2a10fb5 100644
--- a/db/cmdline.cpp
+++ b/db/cmdline.cpp
@@ -20,6 +20,7 @@
 #include "cmdline.h"
 #include "commands.h"
 #include "../util/processinfo.h"
+#include "../util/message.h"
 #include "security_key.h"
 
 #ifdef _WIN32
@@ -50,6 +51,7 @@ namespace mongo {
         ("quiet", "quieter output")
         ("port", po::value<int>(&cmdLine.port), "specify port number")
         ("bind_ip", po::value<string>(&cmdLine.bind_ip), "comma separated list of ip addresses to listen on - all local ips by default")
+        ("maxConns",po::value<int>(), "max number of simultaneous connections")
         ("logpath", po::value<string>() , "log file to send write to instead of stdout - has to be a file, not directory" )
         ("logappend" , "append to logpath instead of over-writing" )
         ("pidfilepath", po::value<string>(), "full path to pidfile (if not set, no pidfile is created)")
@@ -163,6 +165,19 @@ namespace mongo {
             cmdLine.quiet = true;
         }
 
+        if ( params.count( "maxConns" ) ) {
+            int newSize = params["maxConns"].as<int>();
+            if ( newSize < 5 ) {
+                out() << "maxConns has to be at least 5" << endl;
+                dbexit( EXIT_BADOPTIONS );
+            }
+            else if ( newSize >= 10000000 ) {
+                out() << "maxConns can't be greater than 10000000" << endl;
+                dbexit( EXIT_BADOPTIONS );
+            }
+            connTicketHolder.resize( newSize );
+        }
+
         string logpath;
 
 #ifndef _WIN32
diff --git a/db/commands.cpp b/db/commands.cpp
index 770d035..30bdc54 100644
--- a/db/commands.cpp
+++ b/db/commands.cpp
@@ -127,7 +127,6 @@ namespace mongo {
         if ( strcmp(p, ".$cmd") != 0 ) return false;
 
         bool ok = false;
-        bool valid = false;
 
         BSONElement e = jsobj.firstElement();
         map<string,Command*>::iterator i;
@@ -138,7 +137,6 @@ namespace mongo {
            migrated over to the command object format.
            */
         else if ( (i = _commands->find(e.fieldName())) != _commands->end() ) {
-            valid = true;
             string errmsg;
             Command *c = i->second;
             if ( c->adminOnly() && !startsWith(ns, "admin.") ) {
diff --git a/db/commands/distinct.cpp b/db/commands/distinct.cpp
index 2e26bcd..7b2f6a8 100644
--- a/db/commands/distinct.cpp
+++ b/db/commands/distinct.cpp
@@ -109,7 +109,7 @@ namespace mongo {
 
                         int now = bb.len();
 
-                        uassert(10044,  "distinct too big, 4mb cap", ( now + e.size() + 1024 ) < bufSize );
+                        uassert(10044,  "distinct too big, 16mb cap", ( now + e.size() + 1024 ) < bufSize );
 
                         arr.append( e );
                         BSONElement x( start + now );
diff --git a/db/commands/mr.cpp b/db/commands/mr.cpp
index 16c604a..b9f5b59 100644
--- a/db/commands/mr.cpp
+++ b/db/commands/mr.cpp
@@ -758,7 +758,18 @@ namespace mongo {
         BSONObj fast_emit( const BSONObj& args ) {
             uassert( 10077 , "fast_emit takes 2 args" , args.nFields() == 2 );
             uassert( 13069 , "an emit can't be more than half max bson size" , args.objsize() < ( BSONObjMaxUserSize / 2 ) );
-            (*_tl)->emit( args );
+            
+            if ( args.firstElement().type() == Undefined ) {
+                BSONObjBuilder b( args.objsize() );
+                b.appendNull( "" );
+                BSONObjIterator i( args );
+                i.next();
+                b.append( i.next() );
+                (*_tl)->emit( b.obj() );
+            }
+            else {
+                (*_tl)->emit( args );
+            }
             return BSONObj();
         }
 
diff --git a/db/cursor.h b/db/cursor.h
index 9797d66..d17b698 100644
--- a/db/cursor.h
+++ b/db/cursor.h
@@ -113,6 +113,8 @@ namespace mongo {
         // The implementation may return different matchers depending on the
         // position of the cursor.  If matcher() is nonzero at the start,
         // matcher() should be checked each time advance() is called.
+        // Implementations which generate their own matcher should return this
+        // to avoid a matcher being set manually.
         virtual CoveredIndexMatcher *matcher() const { return 0; }
 
         // A convenience function for setting the value of matcher() manually
diff --git a/db/db.cpp b/db/db.cpp
index 97869c5..579b4a1 100644
--- a/db/db.cpp
+++ b/db/db.cpp
@@ -639,7 +639,6 @@ int main(int argc, char* argv[]) {
     ("journalOptions", po::value<int>(), "journal diagnostic options")
     ("ipv6", "enable IPv6 support (disabled by default)")
     ("jsonp","allow JSONP access via http (has security implications)")
-    ("maxConns",po::value<int>(), "max number of simultaneous connections")
     ("noauth", "run without security")
     ("nohttpinterface", "disable http interface")
     ("noprealloc", "disable data file preallocation - will often hurt performance")
@@ -996,18 +995,6 @@ int main(int argc, char* argv[]) {
         if ( params.count( "profile" ) ) {
             cmdLine.defaultProfile = params["profile"].as<int>();
         }
-        if ( params.count( "maxConns" ) ) {
-            int newSize = params["maxConns"].as<int>();
-            if ( newSize < 5 ) {
-                out() << "maxConns has to be at least 5" << endl;
-                dbexit( EXIT_BADOPTIONS );
-            }
-            else if ( newSize >= 10000000 ) {
-                out() << "maxConns can't be greater than 10000000" << endl;
-                dbexit( EXIT_BADOPTIONS );
-            }
-            connTicketHolder.resize( newSize );
-        }
         if (params.count("nounixsocket")) {
             noUnixSocket = true;
         }
diff --git a/db/dbcommands.cpp b/db/dbcommands.cpp
index 8974bd3..cf0857a 100644
--- a/db/dbcommands.cpp
+++ b/db/dbcommands.cpp
@@ -94,9 +94,10 @@ namespace mongo {
         virtual void help( stringstream& help ) const {
             help << "return error status of the last operation on this connection\n"
                  << "options:\n"
-                 << "  fsync - fsync before returning, or wait for journal commit if running with --dur\n"
-                 << "  w - await replication to w servers (including self) before returning\n"
-                 << "  wtimeout - timeout for w in milliseconds";
+                 << "  { fsync:true } - fsync before returning, or wait for journal commit if running with --journal\n"
+                 << "  { j:true } - wait for journal commit if running with --journal\n"
+                 << "  { w:n } - await replication to n servers (including self) before returning\n"
+                 << "  { wtimeout:m} - timeout for w in m milliseconds";
         }
         bool run(const string& dbname, BSONObj& _cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             LastError *le = lastError.disableForCommand();
@@ -125,7 +126,17 @@ namespace mongo {
                 }
             }
 
-            if ( cmdObj["fsync"].trueValue() ) {
+            if ( cmdObj["j"].trueValue() ) { 
+                if( !getDur().awaitCommit() ) {
+                    // --journal is off
+                    result.append("jnote", "journaling not enabled on this server");
+                }
+                if( cmdObj["fsync"].trueValue() ) { 
+                    errmsg = "fsync and j options are not used together";
+                    return false;
+                }
+            }
+            else if ( cmdObj["fsync"].trueValue() ) {
                 Timer t;
                 if( !getDur().awaitCommit() ) {
                     // if get here, not running with --dur
@@ -1542,7 +1553,7 @@ namespace mongo {
             uassert( 13049, "godinsert must specify a collection", !coll.empty() );
             string ns = dbname + "." + coll;
             BSONObj obj = cmdObj[ "obj" ].embeddedObjectUserCheck();
-            DiskLoc loc = theDataFileMgr.insertWithObjMod( ns.c_str(), obj, true );
+            theDataFileMgr.insertWithObjMod( ns.c_str(), obj, true );
             return true;
         }
     } cmdGodInsert;
diff --git a/db/dur_commitjob.cpp b/db/dur_commitjob.cpp
index 2d57ffd..c67f37c 100644
--- a/db/dur_commitjob.cpp
+++ b/db/dur_commitjob.cpp
@@ -200,8 +200,17 @@ namespace mongo {
                                 log() << "debug nsincecommitifneeded:" << _nSinceCommitIfNeededCall << " bytes:" << _bytes << endl;
                         }
 #endif
-                        if ( _bytes >= UncommittedBytesLimit * 3 ) {
-                            wassert( ! "DR102 too much data written uncommitted" );
+                        if (_bytes > UncommittedBytesLimit * 3) {
+                            static time_t lastComplain;
+                            static unsigned nComplains;
+                            // throttle logging
+                            if( ++nComplains < 100 || time(0) - lastComplain >= 60 ) {
+                                lastComplain = time(0);
+                                log() << "replSet warning DR102 too much data written uncommitted " << _bytes/1000000.0 << "MB" << endl;
+                                if( nComplains < 10 || nComplains % 10 == 0 ) {
+                                    wassert(!"replSet warning DR102 too much data written uncommitted");
+                                }
+                            }
                         }
                     }
                 }
diff --git a/db/geo/2d.cpp b/db/geo/2d.cpp
index 934ee80..d6c97f6 100644
--- a/db/geo/2d.cpp
+++ b/db/geo/2d.cpp
@@ -1144,6 +1144,10 @@ namespace mongo {
 
         virtual long long nscanned() { return _nscanned; }
 
+        virtual CoveredIndexMatcher *matcher() const {
+        	return _s->_hopper->_matcher.get();
+        }
+
         shared_ptr<GeoSearch> _s;
         GeoHopper::Holder::iterator _cur;
         GeoHopper::Holder::iterator _end;
@@ -1212,6 +1216,9 @@ namespace mongo {
         virtual DiskLoc currLoc() { assert(ok()); return _cur._loc; }
         virtual BSONObj currKey() const { return _cur._key; }
 
+        virtual CoveredIndexMatcher *matcher() const {
+        	return _matcher.get();
+        }
 
         virtual bool moreToDo() = 0;
         virtual void fillStack() = 0;
diff --git a/db/index.cpp b/db/index.cpp
index c696e27..218ecb3 100644
--- a/db/index.cpp
+++ b/db/index.cpp
@@ -127,7 +127,6 @@ namespace mongo {
     void getIndexChanges(vector<IndexChanges>& v, NamespaceDetails& d, BSONObj newObj, BSONObj oldObj, bool &changedId) {
         int z = d.nIndexesBeingBuilt();
         v.resize(z);
-        NamespaceDetails::IndexIterator i = d.ii();
         for( int i = 0; i < z; i++ ) {
             IndexDetails& idx = d.idx(i);
             BSONObj idxKey = idx.info.obj().getObjectField("key"); // eg { ts : 1 }
diff --git a/db/instance.cpp b/db/instance.cpp
index 3b668ee..bb2d9a5 100644
--- a/db/instance.cpp
+++ b/db/instance.cpp
@@ -488,7 +488,7 @@ namespace mongo {
 
         writelock lk(ns);
         // if this ever moves to outside of lock, need to adjust check Client::Context::_finishInit
-        if ( ! broadcast & handlePossibleShardedMessage( m , 0 ) )
+        if ( ! broadcast && handlePossibleShardedMessage( m , 0 ) )
             return;
 
         Client::Context ctx(ns);
diff --git a/db/lasterror.cpp b/db/lasterror.cpp
index 4643aa9..240c84b 100644
--- a/db/lasterror.cpp
+++ b/db/lasterror.cpp
@@ -70,8 +70,10 @@ namespace mongo {
             b.appendBool( "updatedExisting", updatedExisting == True );
         if ( upsertedId.isSet() )
             b.append( "upserted" , upsertedId );
-        if ( writebackId.isSet() )
+        if ( writebackId.isSet() ) {
             b.append( "writeback" , writebackId );
+            b.append( "instanceIdent" , prettyHostName() ); // this can be any unique string
+        }
         b.appendNumber( "n", nObjects );
 
         return ! msg.empty();
diff --git a/db/query.cpp b/db/query.cpp
index 7f23ac8..671e714 100644
--- a/db/query.cpp
+++ b/db/query.cpp
@@ -1163,7 +1163,7 @@ namespace mongo {
                 cc = new ClientCursor(queryOptions, multi, ns, jsobj.getOwned());
             }
             else {
-                cursor->setMatcher( dqo.matcher() );
+                if( ! cursor->matcher() ) cursor->setMatcher( dqo.matcher() );
                 cc = new ClientCursor( queryOptions, cursor, ns, jsobj.getOwned() );
             }
             cursorid = cc->cursorid();
diff --git a/db/queryoptimizer.cpp b/db/queryoptimizer.cpp
index 0b9dce7..4eb2a99 100644
--- a/db/queryoptimizer.cpp
+++ b/db/queryoptimizer.cpp
@@ -914,7 +914,8 @@ doneCheckOrder:
         }
 
         if ( !id ) {
-            errmsg = (string)"no index found for specified keyPattern: " + keyPattern.toString();
+            errmsg = str::stream() << "no index found for specified keyPattern: " << keyPattern.toString() 
+                                   << " min: " << min << " max: " << max;
             return 0;
         }
 
diff --git a/db/queryoptimizer.h b/db/queryoptimizer.h
index cf3180a..ebd264e 100644
--- a/db/queryoptimizer.h
+++ b/db/queryoptimizer.h
@@ -449,7 +449,8 @@ namespace mongo {
             auto_ptr< FieldRangeSet > frs( new FieldRangeSet( ns, query ) );
             auto_ptr< FieldRangeSet > origFrs( new FieldRangeSet( *frs ) );
             shared_ptr< Cursor > ret = QueryPlanSet( ns, frs, origFrs, query, sort ).getBestGuess()->newCursor();
-            if ( !query.isEmpty() ) {
+            // If we don't already have a matcher, supply one.
+            if ( !query.isEmpty() && ! ret->matcher() ) {
                 shared_ptr< CoveredIndexMatcher > matcher( new CoveredIndexMatcher( query, ret->indexKeyPattern() ) );
                 ret->setMatcher( matcher );
             }
diff --git a/db/repl/consensus.cpp b/db/repl/consensus.cpp
index f764abe..dadb22e 100644
--- a/db/repl/consensus.cpp
+++ b/db/repl/consensus.cpp
@@ -154,6 +154,12 @@ namespace mongo {
             log() << "couldn't find member with id " << whoid << rsLog;
             vote = -10000;
         }
+        else if( primary && primary == rs._self && rs.lastOpTimeWritten >= hopeful->hbinfo().opTime ) {
+            // hbinfo is not updated, so we have to check the primary's last optime separately
+            log() << "I am already primary, " << hopeful->fullName()
+                  << " can try again once I've stepped down" << rsLog;
+            vote = -10000;
+        }
         else if( primary && primary->hbinfo().opTime >= hopeful->hbinfo().opTime ) {
             // other members might be aware of more up-to-date nodes
             log() << hopeful->fullName() << " is trying to elect itself but " <<
diff --git a/db/repl/manager.cpp b/db/repl/manager.cpp
index ed39c31..d2e0764 100644
--- a/db/repl/manager.cpp
+++ b/db/repl/manager.cpp
@@ -64,6 +64,9 @@ namespace mongo {
 
     void Manager::starting() {
         Client::initThread("rs Manager");
+        if (!noauth) {
+            cc().getAuthenticationInfo()->authorize("local");
+        }
     }
 
     void Manager::noteARemoteIsPrimary(const Member *m) {
diff --git a/db/repl/replset_commands.cpp b/db/repl/replset_commands.cpp
index dc8567a..1d110ac 100644
--- a/db/repl/replset_commands.cpp
+++ b/db/repl/replset_commands.cpp
@@ -274,7 +274,7 @@ namespace mongo {
                     s << p("Not using --replSet");
                 else  {
                     s << p("Still starting up, or else set is not yet " + a("http://www.mongodb.org/display/DOCS/Replica+Set+Configuration#InitialSetup", "", "initiated")
-                           + ".<br>" + ReplSet::startupStatusMsg);
+                           + ".<br>" + ReplSet::startupStatusMsg.get());
                 }
             }
             else {
@@ -305,7 +305,7 @@ namespace mongo {
                     s << p("Not using --replSet");
                 else  {
                     s << p("Still starting up, or else set is not yet " + a("http://www.mongodb.org/display/DOCS/Replica+Set+Configuration#InitialSetup", "", "initiated")
-                           + ".<br>" + ReplSet::startupStatusMsg);
+                           + ".<br>" + ReplSet::startupStatusMsg.get());
                 }
             }
             else {
diff --git a/db/repl/rs.cpp b/db/repl/rs.cpp
index 90ed9f4..bbfb057 100644
--- a/db/repl/rs.cpp
+++ b/db/repl/rs.cpp
@@ -321,7 +321,7 @@ namespace mongo {
     }
 
     ReplSetImpl::StartupStatus ReplSetImpl::startupStatus = PRESTART;
-    string ReplSetImpl::startupStatusMsg;
+    DiagStr ReplSetImpl::startupStatusMsg;
 
     extern BSONObj *getLastErrorDefault;
 
diff --git a/db/repl/rs.h b/db/repl/rs.h
index 1419ad6..ea9aef1 100644
--- a/db/repl/rs.h
+++ b/db/repl/rs.h
@@ -244,7 +244,7 @@ namespace mongo {
             EMPTYUNREACHABLE=4, STARTED=5, SOON=6
         };
         static StartupStatus startupStatus;
-        static string startupStatusMsg;
+        static DiagStr startupStatusMsg;
         static string stateAsHtml(MemberState state);
 
         /* todo thread */
@@ -420,7 +420,7 @@ namespace mongo {
             }
             if( theReplSet == 0 ) {
                 result.append("startupStatus", ReplSet::startupStatus);
-                errmsg = ReplSet::startupStatusMsg.empty() ? "replset unknown error 2" : ReplSet::startupStatusMsg;
+                errmsg = ReplSet::startupStatusMsg.empty() ? "replset unknown error 2" : ReplSet::startupStatusMsg.get();
                 if( ReplSet::startupStatus == 3 )
                     result.append("info", "run rs.initiate(...) if not yet done for the set");
                 return false;
diff --git a/db/repl/rs_member.h b/db/repl/rs_member.h
index 017b6ea..b685c04 100644
--- a/db/repl/rs_member.h
+++ b/db/repl/rs_member.h
@@ -19,6 +19,8 @@
 
 #pragma once
 
+#include "../../util/concurrency/value.h"
+
 namespace mongo {
 
 
@@ -74,7 +76,7 @@ namespace mongo {
         time_t upSince;
         long long downSince;
         time_t lastHeartbeat;
-        string lastHeartbeatMsg;
+        DiagStr lastHeartbeatMsg;
         OpTime opTime;
         int skew;
 
diff --git a/db/update.cpp b/db/update.cpp
index e79d3d5..e53f2af 100644
--- a/db/update.cpp
+++ b/db/update.cpp
@@ -987,7 +987,7 @@ namespace mongo {
                 BSONObj newObj = mss->createNewFromMods();
                 checkTooLarge(newObj);
                 assert(nsdt);
-                DiskLoc newLoc = theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , newObj.objdata(), newObj.objsize(), debug);
+                theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , newObj.objdata(), newObj.objsize(), debug);
             }
 
             if ( logop ) {