diff options
author | Antonin Kral <a.kral@bobek.cz> | 2011-04-07 07:35:46 +0200 |
---|---|---|
committer | Antonin Kral <a.kral@bobek.cz> | 2011-04-07 07:35:46 +0200 |
commit | 8d913e98e8058c6322975a858f884a769735bb43 (patch) | |
tree | 52da0129272024f7132594b8227d81ec72f83af1 | |
parent | 582fc32574a3b158c81e49cb00e6ae59205e66ba (diff) | |
download | mongodb-8d913e98e8058c6322975a858f884a769735bb43.tar.gz |
Imported Upstream version 1.8.1
40 files changed, 349 insertions, 296 deletions
diff --git a/client/dbclient.cpp b/client/dbclient.cpp index b4214ab..a68b1af 100644 --- a/client/dbclient.cpp +++ b/client/dbclient.cpp @@ -915,13 +915,14 @@ namespace mongo { void DBClientConnection::checkResponse( const char *data, int nReturned ) { /* check for errors. the only one we really care about at - this stage is "not master" */ + * this stage is "not master" + */ + if ( clientSet && nReturned ) { assert(data); BSONObj o(data); - BSONElement e = o.firstElement(); - if ( strcmp(e.fieldName(), "$err") == 0 && - e.type() == String && strncmp(e.valuestr(), "not master", 10) == 0 ) { + BSONElement e = o["$err"]; + if ( e.type() == String && str::contains( e.valuestr() , "not master" ) ) { clientSet->isntMaster(); } } diff --git a/client/dbclient_rs.cpp b/client/dbclient_rs.cpp index fd8ecec..b6ce776 100644 --- a/client/dbclient_rs.cpp +++ b/client/dbclient_rs.cpp @@ -52,13 +52,17 @@ namespace mongo { } protected: void run() { + log() << "starting" << endl; while ( ! inShutdown() ) { sleepsecs( 20 ); try { ReplicaSetMonitor::checkAll(); } catch ( std::exception& e ) { - error() << "ReplicaSetMonitorWatcher: check failed: " << e.what() << endl; + error() << "check failed: " << e.what() << endl; + } + catch ( ... ) { + error() << "unkown error" << endl; } } } @@ -121,6 +125,7 @@ namespace mongo { while ( true ) { ReplicaSetMonitorPtr m; { + scoped_lock lk( _setsLock ); for ( map<string,ReplicaSetMonitorPtr>::iterator i=_sets.begin(); i!=_sets.end(); ++i ) { string name = i->first; if ( seen.count( name ) ) @@ -175,8 +180,10 @@ namespace mongo { void ReplicaSetMonitor::notifyFailure( const HostAndPort& server ) { scoped_lock lk( _lock ); if ( _master >= 0 && _master < (int)_nodes.size() ) { - if ( server == _nodes[_master].addr ) + if ( server == _nodes[_master].addr ) { + _nodes[_master].ok = false; _master = -1; + } } } @@ -190,7 +197,7 @@ namespace mongo { } _check(); - + scoped_lock lk( _lock ); uassert( 10009 , str::stream() << "ReplicaSetMonitor no master found for set: " << _name , _master >= 0 ); return _nodes[_master].addr; @@ -427,7 +434,7 @@ namespace mongo { } _masterHost = _monitor->getMaster(); - _master.reset( new DBClientConnection( true ) ); + _master.reset( new DBClientConnection( true , this ) ); string errmsg; if ( ! _master->connect( _masterHost , errmsg ) ) { _monitor->notifyFailure( _masterHost ); @@ -447,7 +454,7 @@ namespace mongo { } _slaveHost = _monitor->getSlave(); - _slave.reset( new DBClientConnection( true ) ); + _slave.reset( new DBClientConnection( true , this ) ); _slave->connect( _slaveHost ); _auth( _slave.get() ); return _slave.get(); @@ -562,6 +569,11 @@ namespace mongo { assert(0); } + void DBClientReplicaSet::isntMaster() { + log() << "got not master for: " << _masterHost << endl; + _monitor->notifyFailure( _masterHost ); + _master.reset(); + } bool DBClientReplicaSet::call( Message &toSend, Message &response, bool assertOk , string * actualServer ) { if ( toSend.operation() == dbQuery ) { diff --git a/client/dbclient_rs.h b/client/dbclient_rs.h index 43bf561..fca6e6e 100644 --- a/client/dbclient_rs.h +++ b/client/dbclient_rs.h @@ -213,7 +213,7 @@ namespace mongo { /* this is the callback from our underlying connections to notify us that we got a "not master" error. */ - void isntMaster() { _master.reset(); } + void isntMaster(); // ----- status ------ @@ -319,6 +319,7 @@ namespace mongo { // 'end' has been found and removed, so break. break; } + getDur().commitIfNeeded(); // 'curr' will point to the newest document in the collection. DiskLoc curr = theCapExtent()->lastRecord; assert( !curr.isNull() ); @@ -808,7 +808,7 @@ int main(int argc, char* argv[]) { cmdLine.durOptions = params["durOptions"].as<int>(); } if (params.count("journalOptions")) { - cmdLine.durOptions = params["durOptions"].as<int>(); + cmdLine.durOptions = params["journalOptions"].as<int>(); } if (params.count("objcheck")) { objcheck = true; @@ -1116,6 +1116,19 @@ namespace mongo { dbexit( EXIT_ABRUPT ); } + void abruptQuitWithAddrSignal( int signal, siginfo_t *siginfo, void * ) { + ostringstream oss; + oss << "Invalid"; + if ( signal == SIGSEGV || signal == SIGBUS ) { + oss << " access"; + } else { + oss << " operation"; + } + oss << " at address: " << siginfo->si_addr << endl; + rawOut( oss.str() ); + abruptQuit( signal ); + } + sigset_t asyncSignals; // The above signals will be processed by this thread only, in order to // ensure the db and log mutexes aren't held. @@ -1138,10 +1151,18 @@ namespace mongo { void setupSignals_ignoreHelper( int signal ) {} void setupSignals( bool inFork ) { - assert( signal(SIGSEGV, abruptQuit) != SIG_ERR ); - assert( signal(SIGFPE, abruptQuit) != SIG_ERR ); + struct sigaction addrSignals; + memset( &addrSignals, 0, sizeof( struct sigaction ) ); + addrSignals.sa_sigaction = abruptQuitWithAddrSignal; + sigemptyset( &addrSignals.sa_mask ); + addrSignals.sa_flags = SA_SIGINFO; + + assert( sigaction(SIGSEGV, &addrSignals, 0) == 0 ); + assert( sigaction(SIGBUS, &addrSignals, 0) == 0 ); + assert( sigaction(SIGILL, &addrSignals, 0) == 0 ); + assert( sigaction(SIGFPE, &addrSignals, 0) == 0 ); + assert( signal(SIGABRT, abruptQuit) != SIG_ERR ); - assert( signal(SIGBUS, abruptQuit) != SIG_ERR ); assert( signal(SIGQUIT, abruptQuit) != SIG_ERR ); assert( signal(SIGPIPE, pipeSigHandler) != SIG_ERR ); diff --git a/db/dbhelpers.cpp b/db/dbhelpers.cpp index 75db430..3079aad 100644 --- a/db/dbhelpers.cpp +++ b/db/dbhelpers.cpp @@ -85,6 +85,7 @@ namespace mongo { } } virtual long long nscanned() { + // We don't support yielding, so will always have c_. assert( c_.get() ); return c_->nscanned(); } diff --git a/db/dur_commitjob.cpp b/db/dur_commitjob.cpp index aed38e8..2d57ffd 100644 --- a/db/dur_commitjob.cpp +++ b/db/dur_commitjob.cpp @@ -183,7 +183,7 @@ namespace mongo { // remember intent. we will journal it in a bit _wi.insertWriteIntent(p, len); wassert( _wi._writes.size() < 2000000 ); - assert( _wi._writes.size() < 20000000 ); + //assert( _wi._writes.size() < 20000000 ); { // a bit over conservative in counting pagebytes used @@ -200,7 +200,9 @@ namespace mongo { log() << "debug nsincecommitifneeded:" << _nSinceCommitIfNeededCall << " bytes:" << _bytes << endl; } #endif - uassert(13623, "DR102 too much data written uncommitted", _bytes < UncommittedBytesLimit * 3); + if ( _bytes >= UncommittedBytesLimit * 3 ) { + wassert( ! "DR102 too much data written uncommitted" ); + } } } } diff --git a/db/lasterror.cpp b/db/lasterror.cpp index ba52111..4643aa9 100644 --- a/db/lasterror.cpp +++ b/db/lasterror.cpp @@ -28,7 +28,6 @@ namespace mongo { LastError LastError::noError; LastErrorHolder lastError; - mongo::mutex LastErrorHolder::_idsmutex("LastErrorHolder"); bool isShell = false; void raiseError(int code , const char *msg) { @@ -79,22 +78,9 @@ namespace mongo { } LastErrorHolder::~LastErrorHolder() { - for ( IDMap::iterator i = _ids.begin(); i != _ids.end(); ++i ) { - delete i->second.lerr; - i->second.lerr = 0; - } - _ids.clear(); } - void LastErrorHolder::setID( int id ) { - _id.set( id ); - } - - int LastErrorHolder::getID() { - return _id.get(); - } - LastError * LastErrorHolder::disableForCommand() { LastError *le = _get(); assert( le ); @@ -111,77 +97,31 @@ namespace mongo { } LastError * LastErrorHolder::_get( bool create ) { - int id = _id.get(); - if ( id == 0 ) { - LastError * le = _tl.get(); - if ( ! le && create ) { - le = new LastError(); - _tl.reset( le ); - } - return le; - } - - scoped_lock lock(_idsmutex); - map<int,Status>::iterator i = _ids.find( id ); - if ( i == _ids.end() ) { - if ( ! create ) - return 0; - - LastError * le = new LastError(); - Status s; - s.time = time(0); - s.lerr = le; - _ids[id] = s; - return le; + LastError * le = _tl.get(); + if ( ! le && create ) { + le = new LastError(); + _tl.reset( le ); } - - Status &status = i->second; - status.time = time(0); - return status.lerr; - } - - void LastErrorHolder::remove( int id ) { - scoped_lock lock(_idsmutex); - map<int,Status>::iterator i = _ids.find( id ); - if ( i == _ids.end() ) - return; - - delete i->second.lerr; - _ids.erase( i ); + return le; } void LastErrorHolder::release() { - int id = _id.get(); - if ( id == 0 ) { - _tl.release(); - return; - } - - remove( id ); + _tl.release(); } /** ok to call more than once. */ void LastErrorHolder::initThread() { - if( _tl.get() ) return; - assert( _id.get() == 0 ); - _tl.reset( new LastError() ); + if( ! _tl.get() ) + _tl.reset( new LastError() ); } void LastErrorHolder::reset( LastError * le ) { - int id = _id.get(); - if ( id == 0 ) { - _tl.reset( le ); - return; - } - - scoped_lock lock(_idsmutex); - Status & status = _ids[id]; - status.time = time(0); - status.lerr = le; + _tl.reset( le ); } void prepareErrForNewRequest( Message &m, LastError * err ) { // a killCursors message shouldn't affect last error + assert( err ); if ( m.operation() == dbKillCursors ) { err->disabled = true; } @@ -191,60 +131,10 @@ namespace mongo { } } - LastError * LastErrorHolder::startRequest( Message& m , int clientId ) { - assert( clientId ); - setID( clientId ); - - LastError * le = _get( true ); + LastError * LastErrorHolder::startRequest( Message& m , LastError * le ) { + assert( le ); prepareErrForNewRequest( m, le ); return le; } - void LastErrorHolder::startRequest( Message& m , LastError * connectionOwned ) { - prepareErrForNewRequest( m, connectionOwned ); - } - - void LastErrorHolder::disconnect( int clientId ) { - if ( clientId ) - remove(clientId); - } - - struct LastErrorHolderTest : public UnitTest { - public: - - void test( int i ) { - _tl.set( i ); - assert( _tl.get() == i ); - } - - void tlmaptest() { - test( 1 ); - test( 12123123 ); - test( -123123 ); - test( numeric_limits<int>::min() ); - test( numeric_limits<int>::max() ); - } - - void run() { - tlmaptest(); - - LastError * a = new LastError(); - LastError * b = new LastError(); - - LastErrorHolder holder; - holder.reset( a ); - assert( a == holder.get() ); - holder.setID( 1 ); - assert( 0 == holder.get() ); - holder.reset( b ); - assert( b == holder.get() ); - holder.setID( 0 ); - assert( a == holder.get() ); - - holder.remove( 1 ); - } - - ThreadLocalValue<int> _tl; - } lastErrorHolderTest; - } // namespace mongo diff --git a/db/lasterror.h b/db/lasterror.h index c77ec74..86250e4 100644 --- a/db/lasterror.h +++ b/db/lasterror.h @@ -100,14 +100,14 @@ namespace mongo { extern class LastErrorHolder { public: - LastErrorHolder() : _id( 0 ) {} + LastErrorHolder(){} ~LastErrorHolder(); LastError * get( bool create = false ); LastError * getSafe() { LastError * le = get(false); if ( ! le ) { - log( LL_ERROR ) << " no LastError! id: " << getID() << endl; + error() << " no LastError!" << endl; assert( le ); } return le; @@ -120,18 +120,12 @@ namespace mongo { /** ok to call more than once. */ void initThread(); - /** - * id of 0 means should use thread local management - */ - void setID( int id ); int getID(); - - void remove( int id ); + void release(); /** when db receives a message/request, call this */ - void startRequest( Message& m , LastError * connectionOwned ); - LastError * startRequest( Message& m , int clientId ); + LastError * startRequest( Message& m , LastError * connectionOwned ); void disconnect( int clientId ); @@ -139,17 +133,12 @@ namespace mongo { // disable causes get() to return 0. LastError *disableForCommand(); // only call once per command invocation! private: - ThreadLocalValue<int> _id; boost::thread_specific_ptr<LastError> _tl; struct Status { time_t time; LastError *lerr; }; - typedef map<int,Status> IDMap; - - static mongo::mutex _idsmutex; - IDMap _ids; } lastError; void raiseError(int code , const char *msg); diff --git a/db/pdfile.cpp b/db/pdfile.cpp index 20a7423..663ae05 100644 --- a/db/pdfile.cpp +++ b/db/pdfile.cpp @@ -1198,8 +1198,10 @@ namespace mongo { log(1) << "\t fastBuildIndex dupsToDrop:" << dupsToDrop.size() << endl; - for( list<DiskLoc>::iterator i = dupsToDrop.begin(); i != dupsToDrop.end(); i++ ) + for( list<DiskLoc>::iterator i = dupsToDrop.begin(); i != dupsToDrop.end(); i++ ){ theDataFileMgr.deleteRecord( ns, i->rec(), *i, false, true ); + getDur().commitIfNeeded(); + } return n; } @@ -1254,6 +1256,8 @@ namespace mongo { n++; progress.hit(); + getDur().commitIfNeeded(); + if ( n % 128 == 0 && !cc->yield() ) { cc.release(); uasserted(12584, "cursor gone during bg index"); @@ -1287,7 +1291,7 @@ namespace mongo { prep(ns.c_str(), d); assert( idxNo == d->nIndexes ); try { - idx.head = BtreeBucket::addBucket(idx); + idx.head.writing() = BtreeBucket::addBucket(idx); n = addExistingToIndex(ns.c_str(), d, idx, idxNo); } catch(...) { diff --git a/db/query.cpp b/db/query.cpp index df09fce..7f23ac8 100644 --- a/db/query.cpp +++ b/db/query.cpp @@ -76,8 +76,7 @@ namespace mongo { } } virtual long long nscanned() { - assert( c_.get() ); - return c_->nscanned(); + return c_.get() ? c_->nscanned() : _nscanned; } virtual void next() { if ( !c_->ok() ) { @@ -416,6 +415,7 @@ namespace mongo { _ns(ns), _capped(false), _count(), _myCount(), _skip( spec["skip"].numberLong() ), _limit( spec["limit"].numberLong() ), + _nscanned(), _bc() { } @@ -430,19 +430,22 @@ namespace mongo { } virtual long long nscanned() { - assert( _c.get() ); - return _c->nscanned(); + return _c.get() ? _c->nscanned() : _nscanned; } virtual bool prepareToYield() { - if ( ! _cc ) { + if ( _c && !_cc ) { _cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , _c , _ns.c_str() ) ); } - return _cc->prepareToYield( _yieldData ); + if ( _cc ) { + return _cc->prepareToYield( _yieldData ); + } + // no active cursor - ok to yield + return true; } virtual void recoverFromYield() { - if ( !ClientCursor::recoverFromYield( _yieldData ) ) { + if ( _cc && !ClientCursor::recoverFromYield( _yieldData ) ) { _c.reset(); _cc.reset(); @@ -461,6 +464,7 @@ namespace mongo { return; } + _nscanned = _c->nscanned(); if ( _bc ) { if ( _firstMatch.isEmpty() ) { _firstMatch = _bc->currKeyNode().key.copy(); @@ -523,6 +527,7 @@ namespace mongo { long long _myCount; long long _skip; long long _limit; + long long _nscanned; shared_ptr<Cursor> _c; BSONObj _query; BtreeCursor * _bc; @@ -698,11 +703,15 @@ namespace mongo { return _findingStartCursor->prepareToYield(); } else { - if ( ! _cc ) { + if ( _c && !_cc ) { _cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , _c , _pq.ns() ) ); } - return _cc->prepareToYield( _yieldData ); + if ( _cc ) { + return _cc->prepareToYield( _yieldData ); + } } + // no active cursor - ok to yield + return true; } virtual void recoverFromYield() { @@ -711,7 +720,7 @@ namespace mongo { if ( _findingStartCursor.get() ) { _findingStartCursor->recoverFromYield(); } - else if ( ! ClientCursor::recoverFromYield( _yieldData ) ) { + else if ( _cc && !ClientCursor::recoverFromYield( _yieldData ) ) { _c.reset(); _cc.reset(); _so.reset(); @@ -732,8 +741,7 @@ namespace mongo { if ( _findingStartCursor.get() ) { return 0; // should only be one query plan, so value doesn't really matter. } - assert( _c.get() ); - return _c->nscanned(); + return _c.get() ? _c->nscanned() : _nscanned; } virtual void next() { @@ -850,6 +858,7 @@ namespace mongo { // this plan won, so set data for response broadly void finish( bool stop ) { + massert( 13638, "client cursor dropped during explain query yield", !_pq.isExplain() || _c.get() ); if ( _pq.isExplain() ) { _n = _inMemSort ? _so->size() : _n; @@ -871,7 +880,6 @@ namespace mongo { } if ( _pq.isExplain() ) { - massert( 13638, "client cursor dropped during explain query yield", _c.get() ); _eb.noteScan( _c.get(), _nscanned, _nscannedObjects, _n, scanAndOrderRequired(), _curop.elapsedMillis(), useHints && !_pq.getHint().eoo(), _nYields , _nChunkSkips, _keyFieldsOnly.get() > 0 ); diff --git a/db/repl/rs_config.cpp b/db/repl/rs_config.cpp index 5998f51..2341fe9 100644 --- a/db/repl/rs_config.cpp +++ b/db/repl/rs_config.cpp @@ -314,12 +314,12 @@ namespace mongo { } if( m.h.isLocalHost() ) localhosts++; - m.arbiterOnly = mobj.getBoolField("arbiterOnly"); + m.arbiterOnly = mobj["arbiterOnly"].trueValue(); m.slaveDelay = mobj["slaveDelay"].numberInt(); if( mobj.hasElement("hidden") ) - m.hidden = mobj.getBoolField("hidden"); + m.hidden = mobj["hidden"].trueValue(); if( mobj.hasElement("buildIndexes") ) - m.buildIndexes = mobj.getBoolField("buildIndexes"); + m.buildIndexes = mobj["buildIndexes"].trueValue(); if( mobj.hasElement("priority") ) m.priority = mobj["priority"].Number(); if( mobj.hasElement("votes") ) diff --git a/db/update.cpp b/db/update.cpp index 7de9bb1..e79d3d5 100644 --- a/db/update.cpp +++ b/db/update.cpp @@ -913,15 +913,14 @@ namespace mongo { } } virtual long long nscanned() { - assert( _c.get() ); - return _c->nscanned(); + return _c.get() ? _c->nscanned() : _nscanned; } virtual void next() { if ( ! _c->ok() ) { setComplete(); return; } - _nscanned++; + _nscanned = _c->nscanned(); if ( matcher()->matches(_c->currKey(), _c->currLoc(), &_details ) ) { setComplete(); return; diff --git a/doxygenConfig b/doxygenConfig index fcf10e7..90bf64e 100644 --- a/doxygenConfig +++ b/doxygenConfig @@ -3,7 +3,7 @@ #--------------------------------------------------------------------------- DOXYFILE_ENCODING = UTF-8 PROJECT_NAME = MongoDB -PROJECT_NUMBER = 1.8.0 +PROJECT_NUMBER = 1.8.1 OUTPUT_DIRECTORY = docs/doxygen CREATE_SUBDIRS = NO OUTPUT_LANGUAGE = English diff --git a/jstests/replsets/replset5.js b/jstests/replsets/replset5.js index 13ee5c9..6a7d8a5 100644 --- a/jstests/replsets/replset5.js +++ b/jstests/replsets/replset5.js @@ -49,6 +49,23 @@ doTest = function (signal) { var master1count = master.getDB(testDB).foo.count(); assert( master1count == docNum, "Master has " + master1count + " of " + docNum + " documents!"); + print("reconfigure with hidden=1"); + config = master.getDB("local").system.replset.findOne(); + config.version++; + config.members[2].priority = 0; + config.members[2].hidden = 1; + + try { + master.adminCommand({replSetReconfig : config}); + } + catch(e) { + print(e); + } + + config = master.getDB("local").system.replset.findOne(); + printjson(config); + assert.eq(config.members[2].hidden, true); + replTest.stopSet(signal);
} diff --git a/jstests/slowNightly/explain1.js b/jstests/slowNightly/explain1.js new file mode 100644 index 0000000..81baeb6 --- /dev/null +++ b/jstests/slowNightly/explain1.js @@ -0,0 +1,17 @@ +// SERVER-2662 - drop client cursor in a context where query will yield frequently + +t = db.jstests_slowNightly_explain1; +t.drop(); + +// Periodically drops the collection, invalidating client cursors for s2's operations. +s1 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 80; ++i ) { t.drop(); t.ensureIndex({x:1}); for( var j = 0; j < 1000; ++j ) { t.save( {x:j,y:1} ) }; sleep( 100 ); }" ); + +// Query repeatedly. +s2 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 500; ++i ) { try { z = t.find( {x:{$gt:0},y:1} ).explain(); t.count( {x:{$gt:0},y:1} ); } catch( e ) {} }" ); + +// Put pressure on s2 to yield more often. +s3 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 200; ++i ) { t.validate({scandata:true}); }" ); + +s1(); +s2(); +s3(); diff --git a/jstests/slowNightly/explain2.js b/jstests/slowNightly/explain2.js new file mode 100644 index 0000000..032f0fa --- /dev/null +++ b/jstests/slowNightly/explain2.js @@ -0,0 +1,18 @@ +// Test for race condition SERVER-2807. One cursor is dropped and another is not. + +collName = 'jstests_slowNightly_explain2'; + +t = db[ collName ]; +t.drop(); + +db.createCollection( collName, {capped:true,size:100000} ); +t = db[ collName ]; +t.ensureIndex( {x:1} ); + +a = startParallelShell( 'for( i = 0; i < 50000; ++i ) { db.' + collName + '.insert( {x:i,y:1} ); }' ); + +for( i = 0; i < 800; ++i ) { + t.find( {x:{$gt:-1},y:1} ).sort({x:-1}).explain(); +} + +a();
\ No newline at end of file diff --git a/jstests/slowNightly/explain3.js b/jstests/slowNightly/explain3.js new file mode 100644 index 0000000..af6fde7 --- /dev/null +++ b/jstests/slowNightly/explain3.js @@ -0,0 +1,17 @@ +// SERVER-2810 - similar to explain1 test, but with a scan and order find + +t = db.jstests_slowNightly_explain3; +t.drop(); + +// Periodically drops the collection, invalidating client cursors for s2's operations. +s1 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 80; ++i ) { t.drop(); t.ensureIndex({x:1}); for( var j = 0; j < 1000; ++j ) { t.save( {x:j,y:1} ) }; sleep( 100 ); }" ); + +// Query repeatedly. +s2 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 500; ++i ) { try { z = t.find( {x:{$gt:0},y:1} ).sort({x:1}).explain(); } catch( e ) {} }" ); + +// Put pressure on s2 to yield more often. +s3 = startParallelShell( "t = db.jstests_slowNightly_explain1; for( var i = 0; i < 200; ++i ) { t.validate({scandata:true}); }" ); + +s1(); +s2(); +s3(); diff --git a/jstests/slowWeekly/rollback4.js b/jstests/slowWeekly/rollback4.js new file mode 100644 index 0000000..968980d --- /dev/null +++ b/jstests/slowWeekly/rollback4.js @@ -0,0 +1,50 @@ +// Test a large rollback SERVER-2737 + +var replTest = new ReplSetTest({ name: 'unicomplex', nodes: 3, oplogSize: 2000 }); +var nodes = replTest.nodeList(); + +var conns = replTest.startSet(); +var r = replTest.initiate({ "_id": "unicomplex", + "members": [ + { "_id": 0, "host": nodes[0] }, + { "_id": 1, "host": nodes[1] }, + { "_id": 2, "host": nodes[2], arbiterOnly: true}] + }); + +// Make sure we have a master +var master = replTest.getMaster(); +b_conn = conns[1]; +b_conn.setSlaveOk(); +B = b_conn.getDB("admin"); + +// Make sure we have an arbiter +assert.soon(function () { + res = conns[2].getDB("admin").runCommand({ replSetGetStatus: 1 }); + return res.myState == 7; + }, "Arbiter failed to initialize."); + +// Wait for initial replication +replTest.awaitReplication(); + +// Insert into master +var big = { b:new Array( 1000 ).toString() }; +for( var i = 0; i < 1000000; ++i ) { + if ( i % 10000 == 0 ) { + print( i ); + } + master.getDB( 'db' ).c.insert( big ); +} + +// Stop master +replTest.stop( 0 ); + +// Wait for slave to take over +assert.soon(function () { return B.isMaster().ismaster; }); +master = replTest.getMaster(); + +// Save to new master, forcing rollback of old master +master.getDB( 'db' ).c.save( big ); + +// Restart old master +replTest.restart( 0 ); +replTest.awaitReplication(); @@ -72,6 +72,7 @@ #include <boost/any.hpp> #include "boost/thread/once.hpp" #include <boost/archive/iterators/transform_width.hpp> +#define BOOST_FILESYSTEM_VERSION 2 #include <boost/filesystem/convenience.hpp> #include <boost/filesystem/exception.hpp> #include <boost/filesystem/operations.hpp> diff --git a/rpm/mongo.spec b/rpm/mongo.spec index 5ef543b..1e0cd0d 100644 --- a/rpm/mongo.spec +++ b/rpm/mongo.spec @@ -1,5 +1,5 @@ Name: mongo -Version: 1.8.0 +Version: 1.8.1 Release: mongodb_1%{?dist} Summary: mongo client shell and tools License: AGPL 3.0 diff --git a/s/chunk.cpp b/s/chunk.cpp index 1c72535..b2ad03d 100644 --- a/s/chunk.cpp +++ b/s/chunk.cpp @@ -45,6 +45,8 @@ namespace mongo { string Chunk::chunkMetadataNS = "config.chunks"; int Chunk::MaxChunkSize = 1024 * 1024 * 64; + int Chunk::MaxObjectPerChunk = 250000; + Chunk::Chunk( ChunkManager * manager ) : _manager(manager), _lastmod(0) { _setDataWritten(); @@ -182,8 +184,7 @@ namespace mongo { if ( ! force ) { vector<BSONObj> candidates; const int maxPoints = 2; - const int maxObjs = 250000; - pickSplitVector( candidates , getManager()->getCurrentDesiredChunkSize() , maxPoints , maxObjs ); + pickSplitVector( candidates , getManager()->getCurrentDesiredChunkSize() , maxPoints , MaxObjectPerChunk ); if ( candidates.size() <= 1 ) { // no split points means there isn't enough data to split on // 1 split point means we have between half the chunk size to full chunk size @@ -174,7 +174,7 @@ namespace mongo { static string chunkMetadataNS; static int MaxChunkSize; - + static int MaxObjectPerChunk; // // accessors and helpers // @@ -189,10 +189,11 @@ namespace mongo { const char * getNS() { return "config.chunks"; } Shard getShard() const { return _shard; } const ChunkManager* getManager() const { return _manager; } + private: // main shard info - + ChunkManager * _manager; BSONObj _min; diff --git a/s/client.cpp b/s/client.cpp index b8559b6..95e3124 100644 --- a/s/client.cpp +++ b/s/client.cpp @@ -36,7 +36,7 @@ namespace mongo { - ClientInfo::ClientInfo( int clientId ) : _id( clientId ) { + ClientInfo::ClientInfo() { _cur = &_a; _prev = &_b; _autoSplitOk = true; @@ -44,13 +44,6 @@ namespace mongo { } ClientInfo::~ClientInfo() { - if ( _lastAccess ) { - scoped_lock lk( _clientsLock ); - Cache::iterator i = _clients.find( _id ); - if ( i != _clients.end() ) { - _clients.erase( i ); - } - } } void ClientInfo::addShard( const string& shard ) { @@ -79,49 +72,19 @@ namespace mongo { _cur->clear(); } - void ClientInfo::disconnect() { - _lastAccess = 0; - } - - ClientInfo * ClientInfo::get( int clientId , bool create ) { - - if ( ! clientId ) - clientId = getClientId(); - - if ( ! clientId ) { - ClientInfo * info = _tlInfo.get(); - if ( ! info ) { - info = new ClientInfo( 0 ); - _tlInfo.reset( info ); - } + ClientInfo * ClientInfo::get() { + ClientInfo * info = _tlInfo.get(); + if ( ! info ) { + info = new ClientInfo(); + _tlInfo.reset( info ); info->newRequest(); - return info; } - - scoped_lock lk( _clientsLock ); - Cache::iterator i = _clients.find( clientId ); - if ( i != _clients.end() ) - return i->second; - if ( ! create ) - return 0; - ClientInfo * info = new ClientInfo( clientId ); - _clients[clientId] = info; return info; } - void ClientInfo::disconnect( int clientId ) { - if ( ! clientId ) - return; - - scoped_lock lk( _clientsLock ); - Cache::iterator i = _clients.find( clientId ); - if ( i == _clients.end() ) - return; - - ClientInfo* ci = i->second; - ci->disconnect(); - delete ci; - _clients.erase( i ); + void ClientInfo::disconnect() { + // should be handled by TL cleanup + _lastAccess = 0; } void ClientInfo::_addWriteBack( vector<WBInfo>& all , const BSONObj& gle ) { @@ -142,14 +105,14 @@ namespace mongo { vector<BSONObj> ClientInfo::_handleWriteBacks( vector<WBInfo>& all , bool fromWriteBackListener ) { vector<BSONObj> res; + + if ( all.size() == 0 ) + return res; if ( fromWriteBackListener ) { LOG(1) << "not doing recusrive writeback" << endl; return res; } - - if ( all.size() == 0 ) - return res; for ( unsigned i=0; i<all.size(); i++ ) { res.push_back( WriteBackListener::waitFor( all[i].connectionId , all[i].id ) ); @@ -177,7 +140,21 @@ namespace mongo { ShardConnection conn( theShard , "" ); BSONObj res; - bool ok = conn->runCommand( "admin" , options , res ); + bool ok = false; + try{ + ok = conn->runCommand( "admin" , options , res ); + } + catch( std::exception &e ){ + + warning() << "Could not get last error." << e.what() << endl; + + // Catch everything that happens here, since we need to ensure we return our connection when we're + // finished. + conn.done(); + + return false; + } + res = res.getOwned(); conn.done(); @@ -205,6 +182,7 @@ namespace mongo { assert( v.size() == 1 ); result.appendElements( v[0] ); result.appendElementsUnique( res ); + result.append( "writebackGLE" , v[0] ); result.append( "initialGLEHost" , theShard ); } } @@ -217,8 +195,11 @@ namespace mongo { } BSONArrayBuilder bbb( result.subarrayStart( "shards" ) ); + BSONObjBuilder shardRawGLE; long long n = 0; + + int updatedExistingStat = 0; // 0 is none, -1 has but false, 1 has true // hit each shard vector<string> errors; @@ -228,7 +209,22 @@ namespace mongo { bbb.append( theShard ); ShardConnection conn( theShard , "" ); BSONObj res; - bool ok = conn->runCommand( "admin" , options , res ); + bool ok = false; + try { + ok = conn->runCommand( "admin" , options , res ); + shardRawGLE.append( theShard , res ); + } + catch( std::exception &e ){ + + // Safe to return here, since we haven't started any extra processing yet, just collecting + // responses. + + warning() << "Could not get last error." << e.what() << endl; + conn.done(); + + return false; + } + _addWriteBack( writebacks, res ); string temp = DBClientWithCommands::getLastErrorString( res ); @@ -236,13 +232,24 @@ namespace mongo { errors.push_back( temp ); errorObjects.push_back( res ); } + n += res["n"].numberLong(); + if ( res["updatedExisting"].type() ) { + if ( res["updatedExisting"].trueValue() ) + updatedExistingStat = 1; + else if ( updatedExistingStat == 0 ) + updatedExistingStat = -1; + } + conn.done(); } bbb.done(); + result.append( "shardRawGLE" , shardRawGLE.obj() ); result.appendNumber( "n" , n ); + if ( updatedExistingStat ) + result.appendBool( "updatedExisting" , updatedExistingStat > 0 ); // hit other machines just to block for ( set<string>::const_iterator i=sinceLastGetError().begin(); i!=sinceLastGetError().end(); ++i ) { @@ -285,8 +292,6 @@ namespace mongo { return true; } - ClientInfo::Cache& ClientInfo::_clients = *(new ClientInfo::Cache()); - mongo::mutex ClientInfo::_clientsLock("_clientsLock"); boost::thread_specific_ptr<ClientInfo> ClientInfo::_tlInfo; } // namespace mongo @@ -26,11 +26,8 @@ namespace mongo { * currently implemented with a thread local */ class ClientInfo { - - typedef map<int,ClientInfo*> Cache; - public: - ClientInfo( int clientId ); + ClientInfo(); ~ClientInfo(); /** new request from client, adjusts internal state */ @@ -54,7 +51,7 @@ namespace mongo { * gets shards used on the previous request */ set<string> * getPrev() const { return _prev; }; - + /** * gets all shards we've accessed since the last time we called clearSinceLastGetError */ @@ -65,6 +62,12 @@ namespace mongo { */ void clearSinceLastGetError() { _sinceLastGetError.clear(); } + + /** + * resets the list of shards using to process the current request + */ + void clearCurrentShards(){ _cur->clear(); } + /** * calls getLastError * resets shards since get last error @@ -77,8 +80,7 @@ namespace mongo { void noAutoSplit() { _autoSplitOk = false; } - static ClientInfo * get( int clientId = 0 , bool create = true ); - static void disconnect( int clientId ); + static ClientInfo * get(); private: @@ -111,8 +113,6 @@ namespace mongo { int _lastAccess; bool _autoSplitOk; - static mongo::mutex _clientsLock; - static Cache& _clients; static boost::thread_specific_ptr<ClientInfo> _tlInfo; }; diff --git a/s/commands_public.cpp b/s/commands_public.cpp index 02000a0..5b1ecaf 100644 --- a/s/commands_public.cpp +++ b/s/commands_public.cpp @@ -545,12 +545,8 @@ namespace mongo { bool ok = conn->runCommand( conf->getName() , cmdObj , res ); conn.done(); - if (ok || (strcmp(res["errmsg"].valuestrsafe(), "No matching object found") != 0)) { - result.appendElements(res); - return ok; - } - - return true; + result.appendElements(res); + return ok; } } findAndModifyCmd; diff --git a/s/d_split.cpp b/s/d_split.cpp index 0896803..66fe38e 100644 --- a/s/d_split.cpp +++ b/s/d_split.cpp @@ -269,7 +269,7 @@ namespace mongo { maxSplitPoints = maxSplitPointsElem.numberLong(); } - long long maxChunkObjects = 0; + long long maxChunkObjects = Chunk::MaxObjectPerChunk; BSONElement MaxChunkObjectsElem = jsobj[ "maxChunkObjects" ]; if ( MaxChunkObjectsElem.isNumber() ) { maxChunkObjects = MaxChunkObjectsElem.numberLong(); @@ -425,7 +425,8 @@ namespace mongo { currCount = 0; log() << "splitVector doing another cycle because of force, keyCount now: " << keyCount << endl; - c.reset( new BtreeCursor( d , d->idxNo(*idx) , *idx , min , max , false , 1 ) ); + bc = new BtreeCursor( d , d->idxNo(*idx) , *idx , min , max , false , 1 ); + c.reset( bc ); cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , c , ns ) ); } @@ -442,6 +443,7 @@ namespace mongo { // Remove the sentinel at the beginning before returning and add fieldnames. splitKeys.erase( splitKeys.begin() ); + assert( c.get() ); for ( vector<BSONObj>::iterator it = splitKeys.begin(); it != splitKeys.end() ; ++it ) { *it = bc->prettyKey( *it ); } diff --git a/s/request.cpp b/s/request.cpp index 52f2e54..32c17cc 100644 --- a/s/request.cpp +++ b/s/request.cpp @@ -41,8 +41,7 @@ namespace mongo { assert( _d.getns() ); _id = _m.header()->id; - _clientId = p ? p->getClientId() : 0; - _clientInfo = ClientInfo::get( _clientId ); + _clientInfo = ClientInfo::get(); _clientInfo->newRequest( p ); } @@ -74,7 +73,7 @@ namespace mongo { } _m.header()->id = _id; - + _clientInfo->clearCurrentShards(); } Shard Request::primaryShard() const { diff --git a/s/request.h b/s/request.h index 5b4c228..7c51e5c 100644 --- a/s/request.h +++ b/s/request.h @@ -66,9 +66,6 @@ namespace mongo { return _chunkManager; } - int getClientId() const { - return _clientId; - } ClientInfo * getClientInfo() const { return _clientInfo; } @@ -103,7 +100,6 @@ namespace mongo { DBConfigPtr _config; ChunkManagerPtr _chunkManager; - int _clientId; ClientInfo * _clientInfo; OpCounters* _counter; diff --git a/s/server.cpp b/s/server.cpp index 9bdeede..51f30f1 100644 --- a/s/server.cpp +++ b/s/server.cpp @@ -77,19 +77,19 @@ namespace mongo { public: virtual ~ShardedMessageHandler() {} - virtual void process( Message& m , AbstractMessagingPort* p ) { + virtual void connected( AbstractMessagingPort* p ) { + assert( ClientInfo::get() ); + } + + virtual void process( Message& m , AbstractMessagingPort* p , LastError * le) { assert( p ); Request r( m , p ); - LastError * le = lastError.startRequest( m , r.getClientId() ); - assert( le ); + assert( le ); + lastError.startRequest( m , le ); - if ( logLevel > 5 ) { - log(5) << "client id: " << hex << r.getClientId() << "\t" << r.getns() << "\t" << dec << r.op() << endl; - } try { r.init(); - setClientId( r.getClientId() ); r.process(); } catch ( AssertionException & e ) { @@ -119,8 +119,7 @@ namespace mongo { } virtual void disconnected( AbstractMessagingPort* p ) { - ClientInfo::disconnect( p->getClientId() ); - lastError.disconnect( p->getClientId() ); + // all things are thread local } }; diff --git a/s/shard_version.cpp b/s/shard_version.cpp index 0f3e80f..043b9bd 100644 --- a/s/shard_version.cpp +++ b/s/shard_version.cpp @@ -142,9 +142,10 @@ namespace mongo { checkShardVersion( conn , ns , 1 , tryNumber + 1 ); return true; } - - log() << " setShardVersion failed: " << result << endl; - massert( 10429 , (string)"setShardVersion failed! " + result.jsonString() , 0 ); + + string errmsg = str::stream() << "setShardVersion failed host[" << conn.getServerAddress() << "] " << result; + log() << " " << errmsg << endl; + massert( 10429 , errmsg , 0 ); return true; } @@ -105,7 +105,7 @@ namespace mongo { _combined = 0; break; default: - assert(0); + massert( 13657 , str::stream() << "unknown type for ShardChunkVersion: " << elem , 0 ); } return *this; } diff --git a/s/writeback_listener.cpp b/s/writeback_listener.cpp index 21d59d0..3051013 100644 --- a/s/writeback_listener.cpp +++ b/s/writeback_listener.cpp @@ -159,7 +159,7 @@ namespace mongo { DBConfigPtr db = grid.getDBConfig( ns ); ShardChunkVersion needVersion( data["version"] ); - log(1) << "connectionId: " << cid << " writebackId: " << wid << " needVersion : " << needVersion.toString() + LOG(1) << "connectionId: " << cid << " writebackId: " << wid << " needVersion : " << needVersion.toString() << " mine : " << db->getChunkManager( ns )->getVersion().toString() << endl;// TODO change to log(3) if ( logLevel ) log(1) << debugString( m ) << endl; diff --git a/shell/mongo_vstudio.cpp b/shell/mongo_vstudio.cpp index 08651d7..c3c3751 100644 --- a/shell/mongo_vstudio.cpp +++ b/shell/mongo_vstudio.cpp @@ -2253,8 +2253,8 @@ const StringData _jscode_raw_db = "}\n" "}\n" "\n" -"DB.prototype.printShardingStatus = function(){\n" -"printShardingStatus( this.getSiblingDB( \"config\" ) );\n" +"DB.prototype.printShardingStatus = function( verbose ){\n" +"printShardingStatus( this.getSiblingDB( \"config\" ) , verbose );\n" "}\n" "\n" "DB.autocomplete = function(obj){\n" diff --git a/util/message.cpp b/util/message.cpp index 653b562..916aa34 100644 --- a/util/message.cpp +++ b/util/message.cpp @@ -703,7 +703,6 @@ again: MSGID NextMsgId; - ThreadLocalValue<int> clientId; struct MsgStart { MsgStart() { @@ -721,14 +720,6 @@ again: return op == dbQuery || op == dbGetMore; } - void setClientId( int id ) { - clientId.set( id ); - } - - int getClientId() { - return clientId.get(); - } - const int DEFAULT_MAX_CONN = 20000; const int MAX_MAX_CONN = 20000; diff --git a/util/message.h b/util/message.h index 37e9433..f114445 100644 --- a/util/message.h +++ b/util/message.h @@ -85,12 +85,8 @@ namespace mongo { virtual HostAndPort remote() const = 0; virtual unsigned remotePort() const = 0; - virtual int getClientId() { - int x = remotePort(); - x = x << 16; - x |= ( ( 0xFF0 & (long long)this ) >> 8 ); // lowest byte in pointer often meaningless - return x; - } + private: + int _clientId; }; class MessagingPort : public AbstractMessagingPort { @@ -471,9 +467,6 @@ namespace mongo { MSGID nextMessageId(); - void setClientId( int id ); - int getClientId(); - extern TicketHolder connTicketHolder; class ElapsedTracker { diff --git a/util/message_server.h b/util/message_server.h index 39375c8..defae0b 100644 --- a/util/message_server.h +++ b/util/message_server.h @@ -29,7 +29,9 @@ namespace mongo { class MessageHandler { public: virtual ~MessageHandler() {} - virtual void process( Message& m , AbstractMessagingPort* p ) = 0; + + virtual void connected( AbstractMessagingPort* p ) = 0; + virtual void process( Message& m , AbstractMessagingPort* p , LastError * err ) = 0; virtual void disconnected( AbstractMessagingPort* p ) = 0; }; diff --git a/util/message_server_port.cpp b/util/message_server_port.cpp index 6d00628..76bd78d 100644 --- a/util/message_server_port.cpp +++ b/util/message_server_port.cpp @@ -23,6 +23,7 @@ #include "message_server.h" #include "../db/cmdline.h" +#include "../db/lasterror.h" #include "../db/stats/counters.h" namespace mongo { @@ -38,14 +39,19 @@ namespace mongo { setThreadName( "conn" ); - auto_ptr<MessagingPort> p( inPort ); + scoped_ptr<MessagingPort> p( inPort ); string otherSide; Message m; try { + LastError * le = new LastError(); + lastError.reset( le ); // lastError now has ownership + otherSide = p->farEnd.toString(); + handler->connected( p.get() ); + while ( 1 ) { m.reset(); p->clearCounters(); @@ -57,7 +63,7 @@ namespace mongo { break; } - handler->process( m , p.get() ); + handler->process( m , p.get() , le ); networkCounter.hit( p->getBytesIn() , p->getBytesOut() ); } } diff --git a/util/ntservice.cpp b/util/ntservice.cpp index ccf2981..4b21b8c 100644 --- a/util/ntservice.cpp +++ b/util/ntservice.cpp @@ -312,9 +312,21 @@ namespace mongo { SERVICE_STATUS ssStatus; + DWORD dwControlsAccepted; + switch ( reportState ) { + case SERVICE_START_PENDING: + case SERVICE_STOP_PENDING: + case SERVICE_STOPPED: + dwControlsAccepted = 0; + break; + default: + dwControlsAccepted = SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN; + break; + } + ssStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; ssStatus.dwServiceSpecificExitCode = 0; - ssStatus.dwControlsAccepted = reportState == SERVICE_START_PENDING ? 0 : SERVICE_ACCEPT_STOP; + ssStatus.dwControlsAccepted = dwControlsAccepted; ssStatus.dwCurrentState = reportState; ssStatus.dwWin32ExitCode = NO_ERROR; ssStatus.dwWaitHint = waitHint; @@ -340,6 +352,7 @@ namespace mongo { switch ( ctrlCode ) { case SERVICE_CONTROL_STOP: case SERVICE_CONTROL_SHUTDOWN: + reportStatus( SERVICE_STOP_PENDING ); shutdownServer(); reportStatus( SERVICE_STOPPED ); return; diff --git a/util/version.cpp b/util/version.cpp index 1755a92..f57e256 100644 --- a/util/version.cpp +++ b/util/version.cpp @@ -26,7 +26,7 @@ namespace mongo { - const char versionString[] = "1.8.0"; + const char versionString[] = "1.8.1"; string mongodVersion() { stringstream ss; |