diff options
Diffstat (limited to 'util')
102 files changed, 5532 insertions, 2950 deletions
diff --git a/util/admin_access.h b/util/admin_access.h new file mode 100644 index 0000000..bb882b2 --- /dev/null +++ b/util/admin_access.h @@ -0,0 +1,52 @@ +/** @file admin_access.h + */ + +/** +* Copyright (C) 2010 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#pragma once + +namespace mongo { + + /* + * An AdminAccess is an interface class used to determine if certain users have + * priviledges to a given resource. + * + */ + class AdminAccess { + public: + virtual ~AdminAccess() { } + + /** @return if there are any priviledge users. This should not + * block for long and throw if can't get a lock if needed. + */ + virtual bool haveAdminUsers() const = 0; + + /** @return priviledged user with this name. This should not block + * for long and throw if can't get a lock if needed + */ + virtual BSONObj getAdminUser( const string& username ) const = 0; + }; + + class NoAdminAccess : public AdminAccess { + public: + virtual ~NoAdminAccess() { } + + virtual bool haveAdminUsers() const { return false; } + virtual BSONObj getAdminUser( const string& username ) const { return BSONObj(); } + }; + +} // namespace mongo diff --git a/util/alignedbuilder.cpp b/util/alignedbuilder.cpp new file mode 100644 index 0000000..1734431 --- /dev/null +++ b/util/alignedbuilder.cpp @@ -0,0 +1,102 @@ +// @file alignedbuilder.cpp + +/** +* Copyright (C) 2009 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#include "pch.h" +#include "alignedbuilder.h" + +namespace mongo { + + AlignedBuilder::AlignedBuilder(unsigned initSize) { + _len = 0; + _malloc(initSize); + uassert(13584, "out of memory AlignedBuilder", _p._allocationAddress); + } + + BOOST_STATIC_ASSERT(sizeof(void*) == sizeof(size_t)); + + void AlignedBuilder::mallocSelfAligned(unsigned sz) { + assert( sz == _p._size ); + void *p = malloc(sz + Alignment - 1); + _p._allocationAddress = p; + size_t s = (size_t) p; + size_t sold = s; + s += Alignment - 1; + s = (s/Alignment)*Alignment; + assert( s >= sold ); // begining + assert( (s + sz) <= (sold + sz + Alignment - 1) ); //end + _p._data = (char *) s; + } + + /* "slow"/infrequent portion of 'grow()' */ + void NOINLINE_DECL AlignedBuilder::growReallocate(unsigned oldLen) { + unsigned a = _p._size; + assert( a ); + while( 1 ) { + a *= 2; + wassert( a <= 256*1024*1024 ); + assert( a <= 512*1024*1024 ); + if( _len < a ) + break; + } + _realloc(a, oldLen); + } + + void AlignedBuilder::_malloc(unsigned sz) { + _p._size = sz; +#if defined(_WIN32) + void *p = VirtualAlloc(0, sz, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + _p._allocationAddress = p; + _p._data = (char *) p; +#elif defined(__linux__) + // in theory #ifdef _POSIX_VERSION should work, but it doesn't on OS X 10.4, and needs to be testeed on solaris. + // so for now, linux only for this. + void *p = 0; + int res = posix_memalign(&p, Alignment, sz); + massert(13524, "out of memory AlignedBuilder", res == 0); + _p._allocationAddress = p; + _p._data = (char *) p; +#else + mallocSelfAligned(sz); + assert( ((size_t) _p._data) % Alignment == 0 ); +#endif + } + + void AlignedBuilder::_realloc(unsigned newSize, unsigned oldLen) { + // posix_memalign alignment is not maintained on reallocs, so we can't use realloc(). + AllocationInfo old = _p; + _malloc(newSize); + assert( oldLen <= _len ); + memcpy(_p._data, old._data, oldLen); + _free(old._allocationAddress); + } + + void AlignedBuilder::_free(void *p) { +#if defined(_WIN32) + VirtualFree(p, 0, MEM_RELEASE); +#else + free(p); +#endif + } + + void AlignedBuilder::kill() { + _free(_p._allocationAddress); + _p._allocationAddress = 0; + _p._data = 0; + } + +} diff --git a/util/alignedbuilder.h b/util/alignedbuilder.h new file mode 100644 index 0000000..452cec2 --- /dev/null +++ b/util/alignedbuilder.h @@ -0,0 +1,123 @@ +// @file alignedbuilder.h + +/** +* Copyright (C) 2009 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#pragma once + +#include "../bson/stringdata.h" + +namespace mongo { + + /** a page-aligned BufBuilder. */ + class AlignedBuilder { + public: + AlignedBuilder(unsigned init_size); + ~AlignedBuilder() { kill(); } + + /** reset for a re-use. shrinks if > 128MB */ + void reset() { + _len = 0; + const unsigned sizeCap = 128*1024*1024; + if (_p._size > sizeCap) + _realloc(sizeCap, _len); + } + + /** note this may be deallocated (realloced) if you keep writing or reset(). */ + const char* buf() const { return _p._data; } + + /** leave room for some stuff later + @return offset in the buffer that was our current position + */ + size_t skip(unsigned n) { + unsigned l = len(); + grow(n); + return l; + } + + char* atOfs(unsigned ofs) { return _p._data + ofs; } + + void appendChar(char j) { + *((char*)grow(sizeof(char))) = j; + } + void appendNum(char j) { + *((char*)grow(sizeof(char))) = j; + } + void appendNum(short j) { + *((short*)grow(sizeof(short))) = j; + } + void appendNum(int j) { + *((int*)grow(sizeof(int))) = j; + } + void appendNum(unsigned j) { + *((unsigned*)grow(sizeof(unsigned))) = j; + } + void appendNum(bool j) { + *((bool*)grow(sizeof(bool))) = j; + } + void appendNum(double j) { + *((double*)grow(sizeof(double))) = j; + } + void appendNum(long long j) { + *((long long*)grow(sizeof(long long))) = j; + } + void appendNum(unsigned long long j) { + *((unsigned long long*)grow(sizeof(unsigned long long))) = j; + } + + void appendBuf(const void *src, size_t len) { memcpy(grow((unsigned) len), src, len); } + + template<class T> + void appendStruct(const T& s) { appendBuf(&s, sizeof(T)); } + + void appendStr(const StringData &str , bool includeEOO = true ) { + const unsigned len = str.size() + ( includeEOO ? 1 : 0 ); + assert( len < (unsigned) BSONObjMaxUserSize ); + memcpy(grow(len), str.data(), len); + } + + /** @return the in-use length */ + unsigned len() const { return _len; } + + private: + static const unsigned Alignment = 8192; + + /** returns the pre-grow write position */ + inline char* grow(unsigned by) { + unsigned oldlen = _len; + _len += by; + if ( _len > _p._size ) { + growReallocate(oldlen); + } + return _p._data + oldlen; + } + + void growReallocate(unsigned oldLenInUse); + void kill(); + void mallocSelfAligned(unsigned sz); + void _malloc(unsigned sz); + void _realloc(unsigned newSize, unsigned oldLenInUse); + void _free(void*); + + struct AllocationInfo { + char *_data; + void *_allocationAddress; + unsigned _size; + } _p; + unsigned _len; // bytes in use + }; + +} diff --git a/util/allocator.h b/util/allocator.h index 2c07973..a642e7c 100644 --- a/util/allocator.h +++ b/util/allocator.h @@ -18,22 +18,22 @@ #pragma once namespace mongo { - + inline void * ourmalloc(size_t size) { void *x = malloc(size); if ( x == 0 ) dbexit( EXIT_OOM_MALLOC , "malloc fails"); return x; } - + inline void * ourrealloc(void *ptr, size_t size) { void *x = realloc(ptr, size); if ( x == 0 ) dbexit( EXIT_OOM_REALLOC , "realloc fails"); return x; } - + #define MONGO_malloc mongo::ourmalloc #define malloc MONGO_malloc #define MONGO_realloc mongo::ourrealloc #define realloc MONGO_realloc - + } // namespace mongo diff --git a/util/array.h b/util/array.h index 8da06fe..bf705a4 100644 --- a/util/array.h +++ b/util/array.h @@ -22,50 +22,50 @@ namespace mongo { class FastArray { public: FastArray( int capacity=10000 ) - : _capacity( capacity ) , _size(0) , _end(this,capacity){ + : _capacity( capacity ) , _size(0) , _end(this,capacity) { _data = new T[capacity]; } - ~FastArray(){ + ~FastArray() { delete[] _data; } - - void clear(){ + + void clear() { _size = 0; } - - T& operator[]( int x ){ + + T& operator[]( int x ) { assert( x >= 0 && x < _capacity ); return _data[x]; } - - T& getNext(){ + + T& getNext() { return _data[_size++]; } - - void push_back( const T& t ){ + + void push_back( const T& t ) { _data[_size++] = t; } - - void sort( int (*comp)(const void *, const void *) ){ + + void sort( int (*comp)(const void *, const void *) ) { qsort( _data , _size , sizeof(T) , comp ); } - - int size(){ + + int size() { return _size; } - - bool hasSpace(){ + + bool hasSpace() { return _size < _capacity; } class iterator { public: - iterator(){ + iterator() { _it = 0; _pos = 0; } - - iterator( FastArray * it , int pos=0 ){ + + iterator( FastArray * it , int pos=0 ) { _it = it; _pos = pos; } @@ -78,14 +78,14 @@ namespace mongo { return _pos != other._pos; } - void operator++(){ + void operator++() { _pos++; } - T& operator*(){ + T& operator*() { return _it->_data[_pos]; } - + string toString() const { stringstream ss; ss << _pos; @@ -97,13 +97,13 @@ namespace mongo { friend class FastArray; }; - - iterator begin(){ + + iterator begin() { return iterator(this); } - iterator end(){ + iterator end() { _end._pos = _size; return _end; } @@ -112,7 +112,7 @@ namespace mongo { private: int _capacity; int _size; - + iterator _end; T * _data; diff --git a/util/assert_util.cpp b/util/assert_util.cpp index faa18cb..47be5e9 100644 --- a/util/assert_util.cpp +++ b/util/assert_util.cpp @@ -18,7 +18,7 @@ #include "pch.h" #include "assert_util.h" #include "assert.h" -#include "file.h" +//#include "file.h" #include <cmath> using namespace std; @@ -33,12 +33,12 @@ using namespace std; namespace mongo { AssertionCount assertionCount; - + AssertionCount::AssertionCount() - : regular(0),warning(0),msg(0),user(0),rollovers(0){ + : regular(0),warning(0),msg(0),user(0),rollovers(0) { } - void AssertionCount::rollover(){ + void AssertionCount::rollover() { rollovers++; regular = 0; warning = 0; @@ -46,7 +46,7 @@ namespace mongo { user = 0; } - void AssertionCount::condrollover( int newvalue ){ + void AssertionCount::condrollover( int newvalue ) { static int max = (int)pow( 2.0 , 30 ); if ( newvalue >= max ) rollover(); @@ -57,22 +57,19 @@ namespace mongo { b.append( m , "unknown assertion" ); else b.append( m , msg ); - + if ( code ) b.append( c , code ); } - - string getDbContext(); - - Assertion lastAssert[4]; - - /* "warning" assert -- safe to continue, so we don't throw exception. */ + + string getDbContext(); + + /* "warning" assert -- safe to continue, so we don't throw exception. */ void wasserted(const char *msg, const char *file, unsigned line) { problem() << "Assertion failure " << msg << ' ' << file << ' ' << dec << line << endl; sayDbContext(); raiseError(0,msg && *msg ? msg : "wassertion failure"); - lastAssert[1].set(msg, getDbContext().c_str(), file, line); assertionCount.condrollover( ++assertionCount.warning ); } @@ -81,7 +78,6 @@ namespace mongo { problem() << "Assertion failure " << msg << ' ' << file << ' ' << dec << line << endl; sayDbContext(); raiseError(0,msg && *msg ? msg : "assertion failure"); - lastAssert[0].set(msg, getDbContext().c_str(), file, line); stringstream temp; temp << "assertion " << file << ":" << line; AssertionException e(temp.str(),0); @@ -90,13 +86,11 @@ namespace mongo { } void uassert_nothrow(const char *msg) { - lastAssert[3].set(msg, getDbContext().c_str(), "", 0); raiseError(0,msg); } void uasserted(int msgid, const char *msg) { assertionCount.condrollover( ++assertionCount.user ); - lastAssert[3].set(msg, getDbContext().c_str(), "", 0); raiseError(msgid,msg); throw UserException(msgid, msg); } @@ -104,7 +98,6 @@ namespace mongo { void msgasserted(int msgid, const char *msg) { assertionCount.condrollover( ++assertionCount.warning ); tlog() << "Assertion: " << msgid << ":" << msg << endl; - lastAssert[2].set(msg, getDbContext().c_str(), "", 0); raiseError(msgid,msg && *msg ? msg : "massert failure"); breakpoint(); printStackTrace(); @@ -114,40 +107,19 @@ namespace mongo { void msgassertedNoTrace(int msgid, const char *msg) { assertionCount.condrollover( ++assertionCount.warning ); log() << "Assertion: " << msgid << ":" << msg << endl; - lastAssert[2].set(msg, getDbContext().c_str(), "", 0); raiseError(msgid,msg && *msg ? msg : "massert failure"); throw MsgAssertionException(msgid, msg); } - void streamNotGood( int code , string msg , std::ios& myios ){ + void streamNotGood( int code , string msg , std::ios& myios ) { stringstream ss; // errno might not work on all systems for streams // if it doesn't for a system should deal with here ss << msg << " stream invalid: " << errnoWithDescription(); throw UserException( code , ss.str() ); } - - mongo::mutex *Assertion::_mutex = new mongo::mutex("Assertion"); - - string Assertion::toString() { - if( _mutex == 0 ) - return ""; - - scoped_lock lk(*_mutex); - - if ( !isSet() ) - return ""; - stringstream ss; - ss << msg << '\n'; - if ( *context ) - ss << context << '\n'; - if ( *file ) - ss << file << ' ' << line << '\n'; - return ss.str(); - } - - string errnoWithPrefix( const char * prefix ){ + string errnoWithPrefix( const char * prefix ) { stringstream ss; if ( prefix ) ss << prefix << ": "; @@ -155,23 +127,21 @@ namespace mongo { return ss.str(); } - - string demangleName( const type_info& typeinfo ){ + string demangleName( const type_info& typeinfo ) { #ifdef _WIN32 return typeinfo.name(); #else int status; - + char * niceName = abi::__cxa_demangle(typeinfo.name(), 0, 0, &status); if ( ! niceName ) return typeinfo.name(); - + string s = niceName; free(niceName); return s; #endif } - } diff --git a/util/assert_util.h b/util/assert_util.h index 018dc43..151e950 100644 --- a/util/assert_util.h +++ b/util/assert_util.h @@ -27,51 +27,6 @@ namespace mongo { StaleConfigInContextCode = 13388 }; - /* these are manipulated outside of mutexes, so be careful */ - struct Assertion { - Assertion() { - msg[0] = msg[127] = 0; - context[0] = context[127] = 0; - file = ""; - line = 0; - when = 0; - } - private: - static mongo::mutex *_mutex; - char msg[128]; - char context[128]; - const char *file; - unsigned line; - time_t when; - public: - void set(const char *m, const char *ctxt, const char *f, unsigned l) { - if( _mutex == 0 ) { - /* asserted during global variable initialization */ - return; - } - scoped_lock lk(*_mutex); - strncpy(msg, m, 127); - strncpy(context, ctxt, 127); - file = f; - line = l; - when = time(0); - } - std::string toString(); - bool isSet() { - return when != 0; - } - }; - - enum { - AssertRegular = 0, - AssertW = 1, - AssertMsg = 2, - AssertUser = 3 - }; - - /* last assert of diff types: regular, wassert, msgassert, uassert: */ - extern Assertion lastAssert[4]; - class AssertionCount { public: AssertionCount(); @@ -84,24 +39,20 @@ namespace mongo { int user; int rollovers; }; - + extern AssertionCount assertionCount; - + struct ExceptionInfo { - ExceptionInfo() : msg(""),code(-1){} + ExceptionInfo() : msg(""),code(-1) {} ExceptionInfo( const char * m , int c ) - : msg( m ) , code( c ){ + : msg( m ) , code( c ) { } ExceptionInfo( const string& m , int c ) - : msg( m ) , code( c ){ + : msg( m ) , code( c ) { } - void append( BSONObjBuilder& b , const char * m = "$err" , const char * c = "code" ) const ; - string toString() const { stringstream ss; ss << "exception: " << code << " " << msg; return ss.str(); } - bool empty() const { return msg.empty(); } - string msg; int code; @@ -109,69 +60,81 @@ namespace mongo { class DBException : public std::exception { public: - DBException( const ExceptionInfo& ei ) : _ei(ei){} - DBException( const char * msg , int code ) : _ei(msg,code){} - DBException( const string& msg , int code ) : _ei(msg,code){} + DBException( const ExceptionInfo& ei ) : _ei(ei) {} + DBException( const char * msg , int code ) : _ei(msg,code) {} + DBException( const string& msg , int code ) : _ei(msg,code) {} virtual ~DBException() throw() { } - - virtual const char* what() const throw(){ return _ei.msg.c_str(); } + + virtual const char* what() const throw() { return _ei.msg.c_str(); } virtual int getCode() const { return _ei.code; } - + virtual void appendPrefix( stringstream& ss ) const { } - + virtual string toString() const { stringstream ss; ss << getCode() << " " << what(); return ss.str(); return ss.str(); } - + const ExceptionInfo& getInfo() const { return _ei; } protected: ExceptionInfo _ei; }; - + class AssertionException : public DBException { public: - AssertionException( const ExceptionInfo& ei ) : DBException(ei){} - AssertionException( const char * msg , int code ) : DBException(msg,code){} - AssertionException( const string& msg , int code ) : DBException(msg,code){} + AssertionException( const ExceptionInfo& ei ) : DBException(ei) {} + AssertionException( const char * msg , int code ) : DBException(msg,code) {} + AssertionException( const string& msg , int code ) : DBException(msg,code) {} virtual ~AssertionException() throw() { } - + virtual bool severe() { return true; } virtual bool isUserAssertion() { return false; } /* true if an interrupted exception - see KillCurrentOp */ - bool interrupted() { + bool interrupted() { return _ei.code == 11600 || _ei.code == 11601; } }; - + /* UserExceptions are valid errors that a user can cause, like out of disk space or duplicate key */ class UserException : public AssertionException { public: - UserException(int c , const string& m) : AssertionException( m , c ){} + UserException(int c , const string& m) : AssertionException( m , c ) {} virtual bool severe() { return false; } virtual bool isUserAssertion() { return true; } virtual void appendPrefix( stringstream& ss ) const { ss << "userassert:"; } }; - + class MsgAssertionException : public AssertionException { public: - MsgAssertionException( const ExceptionInfo& ei ) : AssertionException( ei ){} - MsgAssertionException(int c, const string& m) : AssertionException( m , c ){} + MsgAssertionException( const ExceptionInfo& ei ) : AssertionException( ei ) {} + MsgAssertionException(int c, const string& m) : AssertionException( m , c ) {} virtual bool severe() { return false; } virtual void appendPrefix( stringstream& ss ) const { ss << "massert:"; } }; + void asserted(const char *msg, const char *file, unsigned line); void wasserted(const char *msg, const char *file, unsigned line); + + /** a "user assertion". throws UserAssertion. logs. typically used for errors that a user + could cause, such as dupliate key, disk full, etc. + */ void uasserted(int msgid, const char *msg); inline void uasserted(int msgid , string msg) { uasserted(msgid, msg.c_str()); } - void uassert_nothrow(const char *msg); // reported via lasterror, but don't throw exception + + /** reported via lasterror, but don't throw exception */ + void uassert_nothrow(const char *msg); + + /** msgassert and massert are for errors that are internal but have a well defined error text string. + a stack trace is logged. + */ void msgassertedNoTrace(int msgid, const char *msg); + inline void msgassertedNoTrace(int msgid, const string& msg) { msgassertedNoTrace( msgid , msg.c_str() ); } void msgasserted(int msgid, const char *msg); inline void msgasserted(int msgid, string msg) { msgasserted(msgid, msg.c_str()); } @@ -204,21 +167,21 @@ namespace mongo { #if defined(_DEBUG) # define MONGO_dassert assert #else -# define MONGO_dassert(x) +# define MONGO_dassert(x) #endif #define dassert MONGO_dassert // some special ids that we want to duplicate - + // > 10000 asserts // < 10000 UserException - + enum { ASSERT_ID_DUPKEY = 11000 }; /* throws a uassertion with an appropriate msg */ void streamNotGood( int code , string msg , std::ios& myios ); - inline void assertStreamGood(unsigned msgid, string msg, std::ios& myios) { + inline void assertStreamGood(unsigned msgid, string msg, std::ios& myios) { if( !myios.good() ) streamNotGood(msgid, msg, myios); } @@ -228,15 +191,15 @@ namespace mongo { #define BOOST_CHECK_EXCEPTION MONGO_BOOST_CHECK_EXCEPTION #define MONGO_BOOST_CHECK_EXCEPTION( expression ) \ - try { \ - expression; \ - } catch ( const std::exception &e ) { \ + try { \ + expression; \ + } catch ( const std::exception &e ) { \ stringstream ss; \ - ss << "caught boost exception: " << e.what(); \ + ss << "caught boost exception: " << e.what(); \ msgasserted( 13294 , ss.str() ); \ - } catch ( ... ) { \ - massert( 10437 , "unknown boost failed" , false ); \ - } + } catch ( ... ) { \ + massert( 10437 , "unknown boost failed" , false ); \ + } #define DESTRUCTOR_GUARD MONGO_DESTRUCTOR_GUARD #define MONGO_DESTRUCTOR_GUARD( expression ) \ diff --git a/util/background.cpp b/util/background.cpp index ec5483c..746d14c 100644 --- a/util/background.cpp +++ b/util/background.cpp @@ -1,4 +1,4 @@ -//background.cpp +// @file background.cpp /* Copyright 2009 10gen Inc. * @@ -16,103 +16,105 @@ */ #include "pch.h" -#include "goodies.h" + +#include "concurrency/mutex.h" + #include "background.h" -#include <list> + +#include "mongoutils/str.h" namespace mongo { - BackgroundJob *BackgroundJob::grab = 0; - mongo::mutex BackgroundJob::mutex("BackgroundJob"); + // both the BackgroundJob and the internal thread point to JobStatus + struct BackgroundJob::JobStatus { + JobStatus( bool delFlag ) + : deleteSelf(delFlag), m("backgroundJob"), state(NotStarted) { } - /* static */ - void BackgroundJob::thr() { - assert( grab ); - BackgroundJob *us = grab; - assert( us->state == NotStarted ); - us->state = Running; - grab = 0; + const bool deleteSelf; + + mongo::mutex m; // protects state below + boost::condition finished; // means _state == Done + State state; + }; + + BackgroundJob::BackgroundJob( bool selfDelete ) { + _status.reset( new JobStatus( selfDelete ) ); + } + // Background object can be only be destroyed after jobBody() ran + void BackgroundJob::jobBody( boost::shared_ptr<JobStatus> status ) { + LOG(1) << "BackgroundJob starting: " << name() << endl; { - string nm = us->name(); - setThreadName(nm.c_str()); + scoped_lock l( status->m ); + massert( 13643 , mongoutils::str::stream() << "backgroundjob already started: " << name() , status->state == NotStarted ); + status->state = Running; } + const string threadName = name(); + if( ! threadName.empty() ) + setThreadName( threadName.c_str() ); + try { - us->run(); + run(); } - catch ( std::exception& e ){ - log( LL_ERROR ) << "backgroundjob error: " << e.what() << endl; + catch ( std::exception& e ) { + log( LL_ERROR ) << "backgroundjob " << name() << "error: " << e.what() << endl; } catch(...) { - log( LL_ERROR ) << "uncaught exception in BackgroundJob" << endl; + log( LL_ERROR ) << "uncaught exception in BackgroundJob " << name() << endl; + } + + { + scoped_lock l( status->m ); + status->state = Done; + status->finished.notify_all(); } - us->state = Done; - bool delSelf = us->deleteSelf; - us->ending(); - if( delSelf ) - delete us; + + if( status->deleteSelf ) + delete this; } BackgroundJob& BackgroundJob::go() { - scoped_lock bl(mutex); - assert( grab == 0 ); - grab = this; - boost::thread t(thr); - while ( grab ) - sleepmillis(2); + boost::thread t( boost::bind( &BackgroundJob::jobBody , this, _status ) ); return *this; } - bool BackgroundJob::wait(int msMax, unsigned maxsleep) { - unsigned ms = 1; - Date_t start = jsTime(); - while ( state != Done ) { - sleepmillis(ms); - if( ms*2<maxsleep ) ms*=2; - if ( msMax && ( int( jsTime() - start ) > msMax) ) - return false; + bool BackgroundJob::wait( unsigned msTimeOut ) { + scoped_lock l( _status->m ); + while ( _status->state != Done ) { + if ( msTimeOut ) { + // add msTimeOut millisecond to current time + boost::xtime xt; + boost::xtime_get( &xt, boost::TIME_UTC ); + + unsigned long long ns = msTimeOut * 1000000ULL; // milli to nano + if ( xt.nsec + ns < 1000000000 ) { + xt.nsec = (xtime::xtime_nsec_t) (xt.nsec + ns); + } + else { + xt.sec += 1 + ns / 1000000000; + xt.nsec = ( ns + xt.nsec ) % 1000000000; + } + + if ( ! _status->finished.timed_wait( l.boost() , xt ) ) + return false; + + } + else { + _status->finished.wait( l.boost() ); + } } return true; } - void BackgroundJob::go(list<BackgroundJob*>& L) { - for( list<BackgroundJob*>::iterator i = L.begin(); i != L.end(); i++ ) - (*i)->go(); + BackgroundJob::State BackgroundJob::getState() const { + scoped_lock l( _status->m); + return _status->state; } - /* wait for several jobs to finish. */ - void BackgroundJob::wait(list<BackgroundJob*>& L, unsigned maxsleep) { - unsigned ms = 1; - { - x: - sleepmillis(ms); - if( ms*2<maxsleep ) ms*=2; - for( list<BackgroundJob*>::iterator i = L.begin(); i != L.end(); i++ ) { - assert( (*i)->state != NotStarted ); - if( (*i)->state != Done ) - goto x; - } - } - } - - void PeriodicBackgroundJob::run(){ - // want to handle first one differently so inShutdown is obeyed nicely - sleepmillis( _millis ); - - while ( ! inShutdown() ){ - try { - runLoop(); - } - catch ( std::exception& e ){ - log( LL_ERROR ) << "PeriodicBackgroundJob [" << name() << "] error: " << e.what() << endl; - } - catch ( ... ){ - log( LL_ERROR ) << "PeriodicBackgroundJob [" << name() << "] unknown error" << endl; - } - - sleepmillis( _millis ); - } + bool BackgroundJob::running() const { + scoped_lock l( _status->m); + return _status->state == Running; } } // namespace mongo diff --git a/util/background.h b/util/background.h index ee59455..861df9b 100644 --- a/util/background.h +++ b/util/background.h @@ -1,4 +1,4 @@ -// background.h +// @file background.h /* Copyright 2009 10gen Inc. * @@ -19,95 +19,88 @@ namespace mongo { - /** object-orienty background thread dispatching. + /** + * Background thread dispatching. + * subclass and define run() + * + * It is ok to call go(), that is, run the job, more than once -- if the + * previous invocation has finished. Thus one pattern of use is to embed + * a backgroundjob in your object and reuse it (or same thing with + * inheritance). Each go() call spawns a new thread. + * + * Thread safety: + * note when job destructs, the thread is not terminated if still running. + * generally if the thread could still be running, allocate the job dynamically + * and set deleteSelf to true. + * + * go() and wait() are not thread safe + * run() will be executed on the background thread + * BackgroundJob object must exist for as long the background thread is running + */ - subclass and define run() - - It is ok to call go(), that is, run the job, more than once -- if the - previous invocation has finished. Thus one pattern of use is to embed - a backgroundjob in your object and reuse it (or same thing with - inheritance). Each go() call spawns a new thread. - - note when job destructs, the thread is not terminated if still running. - generally if the thread could still be running, allocate the job dynamically - and set deleteSelf to true. - */ - /* example - class ConnectBG : public BackgroundJob { - public: - int sock; - int res; - SockAddr farEnd; - void run() { - res = ::connect(sock, farEnd.raw(), farEnd.addressSize); - } - }; - */ class BackgroundJob : boost::noncopyable { protected: - /** define this to do your work. - after this returns, state is set to done. - after this returns, deleted if deleteSelf true. - */ + /** + * sub-class must intantiate the BackgrounJob + * + * @param selfDelete if set to true, object will destruct itself after the run() finished + * @note selfDelete instantes cannot be wait()-ed upon + */ + explicit BackgroundJob(bool selfDelete = false); + + virtual string name() const = 0; + + /** + * define this to do your work. + * after this returns, state is set to done. + * after this returns, deleted if deleteSelf true. + * + * NOTE: + * if run() throws, the exception will be caught within 'this' object and will ultimately lead to the + * BackgroundJob's thread being finished, as if run() returned. + * + */ virtual void run() = 0; - virtual string name() = 0; - virtual void ending() { } // hook for post processing if desired after everything else done. not called when deleteSelf=true + public: enum State { NotStarted, Running, Done }; - State getState() const { return state; } - bool running() const { return state == Running; } - - bool deleteSelf; // delete self when Done? - BackgroundJob() { - deleteSelf = false; - state = NotStarted; - } virtual ~BackgroundJob() { } - // starts job. returns once it is "dispatched" + /** + * starts job. + * returns immediatelly after dispatching. + * + * @note the BackgroundJob object must live for as long the thread is still running, ie + * until getState() returns Done. + */ BackgroundJob& go(); - // wait for completion. this spins with sleep() so not terribly efficient. - // returns true if did not time out. - // - // note you can call wait() more than once if the first call times out. - bool wait(int msMax = 0, unsigned maxSleepInterval=1000); - - /* start several */ - static void go(list<BackgroundJob*>&); - - /* wait for several jobs to finish. */ - static void wait(list<BackgroundJob*>&, unsigned maxSleepInterval=1000); + /** + * wait for completion. + * + * @param msTimeOut maximum amount of time to wait in millisecons + * @return true if did not time out. false otherwise. + * + * @note you can call wait() more than once if the first call times out. + * but you cannot call wait on a self-deleting job. + */ + bool wait( unsigned msTimeOut = 0 ); + + // accessors + State getState() const; + bool running() const; private: - static BackgroundJob *grab; - static mongo::mutex mutex; - static void thr(); - volatile State state; - }; - - class PeriodicBackgroundJob : public BackgroundJob { - public: - PeriodicBackgroundJob( int millisToSleep ) - : _millis( millisToSleep ){ - } - - virtual ~PeriodicBackgroundJob(){} - - /** this gets called every millisToSleep ms */ - virtual void runLoop() = 0; - - virtual void run(); + struct JobStatus; + boost::shared_ptr<JobStatus> _status; // shared between 'this' and body() thread + void jobBody( boost::shared_ptr<JobStatus> status ); - private: - int _millis; - }; } // namespace mongo diff --git a/util/base64.cpp b/util/base64.cpp index 35a3aba..aff06e2 100644 --- a/util/base64.cpp +++ b/util/base64.cpp @@ -21,20 +21,20 @@ namespace mongo { namespace base64 { - + Alphabet alphabet; - void encode( stringstream& ss , const char * data , int size ){ - for ( int i=0; i<size; i+=3 ){ + void encode( stringstream& ss , const char * data , int size ) { + for ( int i=0; i<size; i+=3 ) { int left = size - i; const unsigned char * start = (const unsigned char*)data + i; - + // byte 0 ss << alphabet.e(start[0]>>2); - + // byte 1 unsigned char temp = ( start[0] << 4 ); - if ( left == 1 ){ + if ( left == 1 ) { ss << alphabet.e(temp); break; } @@ -43,7 +43,7 @@ namespace mongo { // byte 2 temp = ( start[1] & 0xF ) << 2; - if ( left == 2 ){ + if ( left == 2 ) { ss << alphabet.e(temp); break; } @@ -55,50 +55,50 @@ namespace mongo { } int mod = size % 3; - if ( mod == 1 ){ + if ( mod == 1 ) { ss << "=="; } - else if ( mod == 2 ){ + else if ( mod == 2 ) { ss << "="; } } - string encode( const char * data , int size ){ + string encode( const char * data , int size ) { stringstream ss; encode( ss , data ,size ); return ss.str(); } - - string encode( const string& s ){ + + string encode( const string& s ) { return encode( s.c_str() , s.size() ); } - void decode( stringstream& ss , const string& s ){ + void decode( stringstream& ss , const string& s ) { uassert( 10270 , "invalid base64" , s.size() % 4 == 0 ); const unsigned char * data = (const unsigned char*)s.c_str(); int size = s.size(); - + unsigned char buf[3]; - for ( int i=0; i<size; i+=4){ + for ( int i=0; i<size; i+=4) { const unsigned char * start = data + i; buf[0] = ( ( alphabet.decode[start[0]] << 2 ) & 0xFC ) | ( ( alphabet.decode[start[1]] >> 4 ) & 0x3 ); buf[1] = ( ( alphabet.decode[start[1]] << 4 ) & 0xF0 ) | ( ( alphabet.decode[start[2]] >> 2 ) & 0xF ); buf[2] = ( ( alphabet.decode[start[2]] << 6 ) & 0xC0 ) | ( ( alphabet.decode[start[3]] & 0x3F ) ); - + int len = 3; - if ( start[3] == '=' ){ + if ( start[3] == '=' ) { len = 2; - if ( start[2] == '=' ){ + if ( start[2] == '=' ) { len = 1; } } ss.write( (const char*)buf , len ); } } - - string decode( const string& s ){ + + string decode( const string& s ) { stringstream ss; decode( ss , s ); return ss.str(); diff --git a/util/base64.h b/util/base64.h index c113eed..505b5d7 100644 --- a/util/base64.h +++ b/util/base64.h @@ -24,45 +24,44 @@ namespace mongo { public: Alphabet() : encode((unsigned char*) - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789" - "+/") - , decode(new unsigned char[257]) - { + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789" + "+/") + , decode(new unsigned char[257]) { memset( decode.get() , 0 , 256 ); - for ( int i=0; i<64; i++ ){ + for ( int i=0; i<64; i++ ) { decode[ encode[i] ] = i; } test(); } - void test(){ + void test() { assert( strlen( (char*)encode ) == 64 ); for ( int i=0; i<26; i++ ) assert( encode[i] == toupper( encode[i+26] ) ); } - char e( int x ){ + char e( int x ) { return encode[x&0x3f]; } - + private: const unsigned char * encode; public: boost::scoped_array<unsigned char> decode; }; - + extern Alphabet alphabet; void encode( stringstream& ss , const char * data , int size ); string encode( const char * data , int size ); string encode( const string& s ); - + void decode( stringstream& ss , const string& s ); string decode( const string& s ); - + void testAlphabet(); } diff --git a/util/bufreader.h b/util/bufreader.h new file mode 100644 index 0000000..a0dcefa --- /dev/null +++ b/util/bufreader.h @@ -0,0 +1,98 @@ +// @file bufreader.h parse a memory region into usable pieces + +/** +* Copyright (C) 2009 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#pragma once + +namespace mongo { + + /** helper to read and parse a block of memory + methods throw the eof exception if the operation would pass the end of the + buffer with which we are working. + */ + class BufReader : boost::noncopyable { + public: + class eof : public std::exception { + public: + virtual const char * what() { return "BufReader eof"; } + }; + + BufReader(const void *p, unsigned len) : _start(p), _pos(p), _end(((char *)_pos)+len) { } + + bool atEof() const { return _pos == _end; } + + /** read in the object specified, and advance buffer pointer */ + template <typename T> + void read(T &t) { + T* cur = (T*) _pos; + T *next = cur + 1; + if( _end < next ) throw eof(); + t = *cur; + _pos = next; + } + + /** verify we can look at t, but do not advance */ + template <typename T> + void peek(T &t) { + T* cur = (T*) _pos; + T *next = cur + 1; + if( _end < next ) throw eof(); + t = *cur; + } + + /** return current offset into buffer */ + unsigned offset() const { return (char*)_pos - (char*)_start; } + + /** return remaining bytes */ + unsigned remaining() const { return (char*)_end -(char*)_pos; } + + /** back up by nbytes */ + void rewind(unsigned nbytes) { + _pos = ((char *) _pos) - nbytes; + assert( _pos >= _start ); + } + + /** return current position pointer, and advance by len */ + const void* skip(unsigned len) { + const char *nxt = ((char *) _pos) + len; + if( _end < nxt ) throw eof(); + const void *p = _pos; + _pos = nxt; + return p; + } + + void readStr(string& s) { + StringBuilder b; + while( 1 ) { + char ch; + read(ch); + if( ch == 0 ) + break; + b << ch; + } + s = b.str(); + } + + const void* pos() { return _pos; } + + private: + const void *_start; + const void *_pos; + const void *_end; + }; + +} diff --git a/util/concurrency/readme.txt b/util/concurrency/README index 6f308f5..1d72a1c 100644 --- a/util/concurrency/readme.txt +++ b/util/concurrency/README @@ -1,11 +1,10 @@ -util/concurrency/ files
-
-list.h - a list class that is lock-free for reads
-rwlock.h - read/write locks (RWLock)
-msg.h - message passing between threads
-task.h - an abstraction around threads
-mutex.h - small enhancements that wrap boost::mutex
-thread_pool.h
+util/concurrency/ files + +list.h - a list class that is lock-free for reads +rwlock.h - read/write locks (RWLock) +msg.h - message passing between threads +task.h - an abstraction around threads +mutex.h - small enhancements that wrap boost::mutex mvar.h This is based on haskell's MVar synchronization primitive: http://www.haskell.org/ghc/docs/latest/html/libraries/base-4.2.0.0/Control-Concurrent-MVar.html @@ -13,3 +12,8 @@ mvar.h You can also think of it as a box that can be either full or empty. value.h Atomic wrapper for values/objects that are copy constructable / assignable +thread_pool.h +spinlock.h +synchronization.h + A class to establish a sinchronization point between two threads. One thread is the waiter and one + is the notifier. After the notification event, both proceed normally. diff --git a/util/concurrency/list.h b/util/concurrency/list.h index 968ff4d..e5eaec6 100644 --- a/util/concurrency/list.h +++ b/util/concurrency/list.h @@ -18,64 +18,64 @@ #pragma once -namespace mongo { +namespace mongo { -/* this class uses a mutex for writes, but not for reads. - we can get fancier later... + /* this class uses a mutex for writes, but not for reads. + we can get fancier later... - struct Member : public List1<Member>::Base { - const char *host; - int port; - }; - List1<Member> _members; - _members.head()->next(); + struct Member : public List1<Member>::Base { + const char *host; + int port; + }; + List1<Member> _members; + _members.head()->next(); -*/ -template<typename T> -class List1 : boost::noncopyable { -public: - /* next() and head() return 0 at end of list */ + */ + template<typename T> + class List1 : boost::noncopyable { + public: + /* next() and head() return 0 at end of list */ - List1() : _head(0), _m("List1"), _orphans(0) { } + List1() : _head(0), _m("List1"), _orphans(0) { } - class Base { - friend class List1; - T *_next; - public: - T* next() const { return _next; } - }; + class Base { + friend class List1; + T *_next; + public: + T* next() const { return _next; } + }; - T* head() const { return _head; } + T* head() const { return _head; } - void push(T* t) { - scoped_lock lk(_m); - t->_next = _head; - _head = t; - } + void push(T* t) { + scoped_lock lk(_m); + t->_next = _head; + _head = t; + } - // intentionally leak. - void orphanAll() { - _head = 0; - } + // intentionally leak. + void orphanAll() { + _head = 0; + } - /* t is not deleted, but is removed from the list. (orphaned) */ - void orphan(T* t) { - scoped_lock lk(_m); - T *&prev = _head; - T *n = prev; - while( n != t ) { - prev = n->_next; - n = prev; + /* t is not deleted, but is removed from the list. (orphaned) */ + void orphan(T* t) { + scoped_lock lk(_m); + T *&prev = _head; + T *n = prev; + while( n != t ) { + prev = n->_next; + n = prev; + } + prev = t->_next; + if( ++_orphans > 500 ) + log() << "warning orphans=" << _orphans << '\n'; } - prev = t->_next; - if( ++_orphans > 500 ) - log() << "warning orphans=" << _orphans << '\n'; - } -private: - T *_head; - mutex _m; - int _orphans; -}; + private: + T *_head; + mongo::mutex _m; + int _orphans; + }; }; diff --git a/util/concurrency/msg.h b/util/concurrency/msg.h index a5b07d3..f7c6788 100644 --- a/util/concurrency/msg.h +++ b/util/concurrency/msg.h @@ -21,14 +21,14 @@ #include <deque> #include "task.h" -namespace mongo { +namespace mongo { - namespace task { + namespace task { typedef boost::function<void()> lam; /** typical usage is: task::fork( new Server("threadname") ); */ - class Server : public Task { + class Server : public Task { public: /** send a message to the port */ void send(lam); @@ -47,7 +47,7 @@ namespace mongo { private: virtual bool initClient() { return true; } - virtual string name() { return _name; } + virtual string name() const { return _name; } void doWork(); deque<lam> d; boost::mutex m; diff --git a/util/concurrency/mutex.h b/util/concurrency/mutex.h index 797ab77..c463498 100644 --- a/util/concurrency/mutex.h +++ b/util/concurrency/mutex.h @@ -20,13 +20,29 @@ #include <map> #include <set> -namespace mongo { +#include "../heapcheck.h" + +namespace mongo { - extern bool __destroyingStatics; class mutex; - // only used on _DEBUG builds: - class MutexDebugger { + inline boost::xtime incxtimemillis( long long s ) { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + xt.sec += (int)( s / 1000 ); + xt.nsec += (int)(( s % 1000 ) * 1000000); + if ( xt.nsec >= 1000000000 ) { + xt.nsec -= 1000000000; + xt.sec++; + } + return xt; + } + + /** only used on _DEBUG builds. + MutexDebugger checks that we always acquire locks for multiple mutexes in a consistant (acyclic) order. + If we were inconsistent we could deadlock. + */ + class MutexDebugger { typedef const char * mid; // mid = mutex ID typedef map<mid,int> Preceeding; map< mid, int > maxNest; @@ -34,36 +50,41 @@ namespace mongo { map< mid, set<mid> > followers; boost::mutex &x; unsigned magic; + + void aBreakPoint() { } // for debugging public: // set these to create an assert that // b must never be locked before a - // so + // so // a.lock(); b.lock(); is fine // b.lock(); alone is fine too // only checked on _DEBUG builds. string a,b; - - void aBreakPoint(){} + + /** outputs some diagnostic info on mutexes (on _DEBUG builds) */ void programEnding(); + MutexDebugger(); + void entering(mid m) { - if( magic != 0x12345678 ) return; + if( this == 0 ) return; + assert( magic == 0x12345678 ); Preceeding *_preceeding = us.get(); if( _preceeding == 0 ) us.reset( _preceeding = new Preceeding() ); Preceeding &preceeding = *_preceeding; - if( a == m ) { + if( a == m ) { aBreakPoint(); if( preceeding[b.c_str()] ) { - cout << "mutex problem " << b << " was locked before " << a << endl; + cout << "****** MutexDebugger error! warning " << b << " was locked before " << a << endl; assert(false); } } preceeding[m]++; - if( preceeding[m] > 1 ) { + if( preceeding[m] > 1 ) { // recursive re-locking. if( preceeding[m] > maxNest[m] ) maxNest[m] = preceeding[m]; @@ -75,19 +96,19 @@ namespace mongo { { boost::mutex::scoped_lock lk(x); followers[m]; - for( Preceeding::iterator i = preceeding.begin(); i != preceeding.end(); i++ ) { + for( Preceeding::iterator i = preceeding.begin(); i != preceeding.end(); i++ ) { if( m != i->first && i->second > 0 ) { followers[i->first].insert(m); - if( followers[m].count(i->first) != 0 ){ + if( followers[m].count(i->first) != 0 ) { failed = true; stringstream ss; mid bad = i->first; ss << "mutex problem" << - "\n when locking " << m << - "\n " << bad << " was already locked and should not be." - "\n set a and b above to debug.\n"; + "\n when locking " << m << + "\n " << bad << " was already locked and should not be." + "\n set a and b above to debug.\n"; stringstream q; - for( Preceeding::iterator i = preceeding.begin(); i != preceeding.end(); i++ ) { + for( Preceeding::iterator i = preceeding.begin(); i != preceeding.end(); i++ ) { if( i->first != m && i->first != bad && i->second > 0 ) q << " " << i->first << '\n'; } @@ -105,8 +126,8 @@ namespace mongo { assert( 0 ); } } - void leaving(mid m) { - if( magic != 0x12345678 ) return; + void leaving(mid m) { + if( this == 0 ) return; // still in startup pre-main() Preceeding& preceeding = *us.get(); preceeding[m]--; if( preceeding[m] < 0 ) { @@ -116,38 +137,67 @@ namespace mongo { } }; extern MutexDebugger &mutexDebugger; - + // If you create a local static instance of this class, that instance will be destroyed - // before all global static objects are destroyed, so __destroyingStatics will be set + // before all global static objects are destroyed, so _destroyingStatics will be set // to true before the global static variables are destroyed. class StaticObserver : boost::noncopyable { public: - ~StaticObserver() { __destroyingStatics = true; } + static bool _destroyingStatics; + ~StaticObserver() { _destroyingStatics = true; } }; - // On pthread systems, it is an error to destroy a mutex while held. Static global - // mutexes may be held upon shutdown in our implementation, and this way we avoid - // destroying them. + /** On pthread systems, it is an error to destroy a mutex while held. Static global + * mutexes may be held upon shutdown in our implementation, and this way we avoid + * destroying them. + * NOT recursive. + */ class mutex : boost::noncopyable { public: #if defined(_DEBUG) - const char *_name; + const char * const _name; #endif #if defined(_DEBUG) - mutex(const char *name) - : _name(name) + mutex(const char *name) + : _name(name) #else - mutex(const char *) + mutex(const char *) #endif - { - _m = new boost::mutex(); + { + _m = new boost::timed_mutex(); + IGNORE_OBJECT( _m ); // Turn-off heap checking on _m } ~mutex() { - if( !__destroyingStatics ) { + if( !StaticObserver::_destroyingStatics ) { + UNIGNORE_OBJECT( _m ); delete _m; } } + + class try_lock : boost::noncopyable { + public: + try_lock( mongo::mutex &m , int millis = 0 ) + : _l( m.boost() , incxtimemillis( millis ) ) , +#if BOOST_VERSION >= 103500 + ok( _l.owns_lock() ) +#else + ok( _l.locked() ) +#endif + { + } + + ~try_lock() { + } + + private: + boost::timed_mutex::scoped_timed_lock _l; + + public: + const bool ok; + }; + + class scoped_lock : boost::noncopyable { #if defined(_DEBUG) mongo::mutex *mut; @@ -159,20 +209,23 @@ namespace mongo { mutexDebugger.entering(mut->_name); #endif } - ~scoped_lock() { + ~scoped_lock() { #if defined(_DEBUG) mutexDebugger.leaving(mut->_name); #endif } - boost::mutex::scoped_lock &boost() { return _l; } + boost::timed_mutex::scoped_lock &boost() { return _l; } private: - boost::mutex::scoped_lock _l; + boost::timed_mutex::scoped_lock _l; }; + + private: - boost::mutex &boost() { return *_m; } - boost::mutex *_m; + + boost::timed_mutex &boost() { return *_m; } + boost::timed_mutex *_m; }; - + typedef mutex::scoped_lock scoped_lock; typedef boost::recursive_mutex::scoped_lock recursive_scoped_lock; diff --git a/util/concurrency/mvar.h b/util/concurrency/mvar.h index 7d17051..9c7a505 100644 --- a/util/concurrency/mvar.h +++ b/util/concurrency/mvar.h @@ -31,18 +31,18 @@ namespace mongo { // create an empty MVar MVar() - : _state(EMPTY) + : _state(EMPTY) {} // creates a full MVar MVar(const T& val) - : _state(FULL) - , _value(val) + : _state(FULL) + , _value(val) {} // puts val into the MVar and returns true or returns false if full // never blocks - bool tryPut(const T& val){ + bool tryPut(const T& val) { // intentionally repeat test before and after lock if (_state == FULL) return false; Mutex::scoped_lock lock(_mutex); @@ -59,17 +59,17 @@ namespace mongo { // puts val into the MVar // will block if the MVar is already full - void put(const T& val){ + void put(const T& val) { Mutex::scoped_lock lock(_mutex); - while (!tryPut(val)){ - // unlocks lock while waiting and relocks before returning + while (!tryPut(val)) { + // unlocks lock while waiting and relocks before returning _condition.wait(lock); - } + } } // takes val out of the MVar and returns true or returns false if empty // never blocks - bool tryTake(T& out){ + bool tryTake(T& out) { // intentionally repeat test before and after lock if (_state == EMPTY) return false; Mutex::scoped_lock lock(_mutex); @@ -86,14 +86,14 @@ namespace mongo { // takes val out of the MVar // will block if the MVar is empty - T take(){ + T take() { T ret = T(); Mutex::scoped_lock lock(_mutex); - while (!tryTake(ret)){ - // unlocks lock while waiting and relocks before returning + while (!tryTake(ret)) { + // unlocks lock while waiting and relocks before returning _condition.wait(lock); - } + } return ret; } @@ -102,7 +102,7 @@ namespace mongo { // Note: this is fast because there is no locking, but state could // change before you get a chance to act on it. // Mainly useful for sanity checks / asserts. - State getState(){ return _state; } + State getState() { return _state; } private: diff --git a/util/concurrency/race.h b/util/concurrency/race.h new file mode 100644 index 0000000..0b8338c --- /dev/null +++ b/util/concurrency/race.h @@ -0,0 +1,72 @@ +#pragma once + +#include "../goodies.h" // printStackTrace + +namespace mongo { + + /** some self-testing of synchronization and attempts to catch race conditions. + + use something like: + + CodeBlock myBlock; + + void foo() { + CodeBlock::Within w(myBlock); + ... + } + + In _DEBUG builds, will (sometimes/maybe) fail if two threads are in the same code block at + the same time. Also detects and disallows recursion. + */ + +#if defined(_DEBUG) + + class CodeBlock { + volatile int n; + unsigned tid; + void fail() { + log() << "synchronization (race condition) failure" << endl; + printStackTrace(); + abort(); + } + void enter() { + if( ++n != 1 ) fail(); +#if defined(_WIN32) + tid = GetCurrentThreadId(); +#endif + } + void leave() { + if( --n != 0 ) fail(); + } + public: + CodeBlock() : n(0) { } + + class Within { + CodeBlock& _s; + public: + Within(CodeBlock& s) : _s(s) { _s.enter(); } + ~Within() { _s.leave(); } + }; + + void assertWithin() { + assert( n == 1 ); +#if defined(_WIN32) + assert( GetCurrentThreadId() == tid ); +#endif + } + }; + +#else + + class CodeBlock{ + public: + class Within { + public: + Within(CodeBlock&) { } + }; + void assertWithin() { } + }; + +#endif + +} diff --git a/util/concurrency/rwlock.h b/util/concurrency/rwlock.h index 75169b2..ca81a9f 100644 --- a/util/concurrency/rwlock.h +++ b/util/concurrency/rwlock.h @@ -1,4 +1,4 @@ -// rwlock.h +// @file rwlock.h generic reader-writer lock (cross platform support) /* * Copyright (C) 2010 10gen Inc. @@ -19,30 +19,79 @@ #pragma once #include "mutex.h" +#include "../time_support.h" + +// this requires Vista+ to work +// it works better than sharable_mutex under high contention +//#define MONGO_USE_SRW_ON_WINDOWS 1 + +#if !defined(MONGO_USE_SRW_ON_WINDOWS) #if BOOST_VERSION >= 103500 - #define BOOST_RWLOCK +# define BOOST_RWLOCK #else +# if defined(_WIN32) +# error need boost >= 1.35 for windows +# endif +# include <pthread.h> +#endif - #if defined(_WIN32) - #error need boost >= 1.35 for windows - #endif - - #include <pthread.h> - +#if defined(_WIN32) +# include "shared_mutex_win.hpp" +namespace mongo { + typedef boost::modified_shared_mutex shared_mutex; +} +# undef assert +# define assert MONGO_assert +#elif defined(BOOST_RWLOCK) +# include <boost/thread/shared_mutex.hpp> +# undef assert +# define assert MONGO_assert #endif -#ifdef BOOST_RWLOCK -#include <boost/thread/shared_mutex.hpp> -#undef assert -#define assert MONGO_assert #endif namespace mongo { -#ifdef BOOST_RWLOCK +#if defined(MONGO_USE_SRW_ON_WINDOWS) && defined(_WIN32) + class RWLock { - boost::shared_mutex _m; + public: + RWLock(const char *) { InitializeSRWLock(&_lock); } + ~RWLock() { } + void lock() { AcquireSRWLockExclusive(&_lock); } + void unlock() { ReleaseSRWLockExclusive(&_lock); } + void lock_shared() { AcquireSRWLockShared(&_lock); } + void unlock_shared() { ReleaseSRWLockShared(&_lock); } + bool lock_shared_try( int millis ) { + unsigned long long end = curTimeMicros64() + millis*1000; + while( 1 ) { + if( TryAcquireSRWLockShared(&_lock) ) + return true; + if( curTimeMicros64() >= end ) + break; + Sleep(1); + } + return false; + } + bool lock_try( int millis = 0 ) { + unsigned long long end = curTimeMicros64() + millis*1000; + while( 1 ) { + if( TryAcquireSRWLockExclusive(&_lock) ) + return true; + if( curTimeMicros64() >= end ) + break; + Sleep(1); + } + return false; + } + private: + SRWLOCK _lock; + }; + +#elif defined(BOOST_RWLOCK) + class RWLock { + shared_mutex _m; public: #if defined(_DEBUG) const char *_name; @@ -50,40 +99,40 @@ namespace mongo { #else RWLock(const char *) { } #endif - void lock(){ + void lock() { _m.lock(); #if defined(_DEBUG) mutexDebugger.entering(_name); #endif } - void unlock(){ + void unlock() { #if defined(_DEBUG) mutexDebugger.leaving(_name); #endif _m.unlock(); } - - void lock_shared(){ + + void lock_shared() { _m.lock_shared(); } - - void unlock_shared(){ + + void unlock_shared() { _m.unlock_shared(); } - bool lock_shared_try( int millis ){ + bool lock_shared_try( int millis ) { boost::system_time until = get_system_time(); until += boost::posix_time::milliseconds(millis); - if( _m.timed_lock_shared( until ) ) { + if( _m.timed_lock_shared( until ) ) { return true; } return false; } - bool lock_try( int millis = 0 ){ + bool lock_try( int millis = 0 ) { boost::system_time until = get_system_time(); until += boost::posix_time::milliseconds(millis); - if( _m.timed_lock( until ) ) { + if( _m.timed_lock( until ) ) { #if defined(_DEBUG) mutexDebugger.entering(_name); #endif @@ -98,7 +147,7 @@ namespace mongo { class RWLock { pthread_rwlock_t _lock; - inline void check( int x ){ + inline void check( int x ) { if( x == 0 ) return; log() << "pthread rwlock failed: " << x << endl; @@ -114,40 +163,40 @@ namespace mongo { #endif check( pthread_rwlock_init( &_lock , 0 ) ); } - - ~RWLock(){ - if ( ! __destroyingStatics ){ + + ~RWLock() { + if ( ! StaticObserver::_destroyingStatics ) { check( pthread_rwlock_destroy( &_lock ) ); } } - void lock(){ + void lock() { check( pthread_rwlock_wrlock( &_lock ) ); #if defined(_DEBUG) mutexDebugger.entering(_name); #endif } - void unlock(){ + void unlock() { #if defined(_DEBUG) mutexDebugger.leaving(_name); #endif check( pthread_rwlock_unlock( &_lock ) ); } - - void lock_shared(){ + + void lock_shared() { check( pthread_rwlock_rdlock( &_lock ) ); } - - void unlock_shared(){ + + void unlock_shared() { check( pthread_rwlock_unlock( &_lock ) ); } - - bool lock_shared_try( int millis ){ + + bool lock_shared_try( int millis ) { return _try( millis , false ); } - bool lock_try( int millis = 0 ){ - if( _try( millis , true ) ) { + bool lock_try( int millis = 0 ) { + if( _try( millis , true ) ) { #if defined(_DEBUG) mutexDebugger.entering(_name); #endif @@ -156,65 +205,66 @@ namespace mongo { return false; } - bool _try( int millis , bool write ){ + bool _try( int millis , bool write ) { while ( true ) { - int x = write ? - pthread_rwlock_trywrlock( &_lock ) : - pthread_rwlock_tryrdlock( &_lock ); - + int x = write ? + pthread_rwlock_trywrlock( &_lock ) : + pthread_rwlock_tryrdlock( &_lock ); + if ( x <= 0 ) { return true; } - + if ( millis-- <= 0 ) return false; - - if ( x == EBUSY ){ + + if ( x == EBUSY ) { sleepmillis(1); continue; } check(x); - } - + } + return false; } }; - #endif + /** throws on failure to acquire in the specified time period. */ class rwlock_try_write { - RWLock& _l; public: struct exception { }; rwlock_try_write(RWLock& l, int millis = 0) : _l(l) { - if( !l.lock_try(millis) ) throw exception(); + if( !l.lock_try(millis) ) + throw exception(); } ~rwlock_try_write() { _l.unlock(); } + private: + RWLock& _l; }; - /* scoped lock */ - struct rwlock { + /* scoped lock for RWLock */ + class rwlock { + public: rwlock( const RWLock& lock , bool write , bool alreadyHaveLock = false ) - : _lock( (RWLock&)lock ) , _write( write ){ - - if ( ! alreadyHaveLock ){ + : _lock( (RWLock&)lock ) , _write( write ) { + if ( ! alreadyHaveLock ) { if ( _write ) _lock.lock(); else _lock.lock_shared(); } } - - ~rwlock(){ + ~rwlock() { if ( _write ) _lock.unlock(); else _lock.unlock_shared(); } - + private: RWLock& _lock; - bool _write; + const bool _write; }; } diff --git a/util/concurrency/shared_mutex_win.hpp b/util/concurrency/shared_mutex_win.hpp new file mode 100755 index 0000000..5356cf2 --- /dev/null +++ b/util/concurrency/shared_mutex_win.hpp @@ -0,0 +1,573 @@ +#ifndef BOOST_THREAD_WIN32_SHARED_MUTEX_HPP_MODIFIED
+#define BOOST_THREAD_WIN32_SHARED_MUTEX_HPP_MODIFIED
+
+// (C) Copyright 2006-8 Anthony Williams
+//
+// Distributed under the Boost Software License, Version 1.0. (See
+// accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// MongoDB :
+//
+// Slightly modified boost file to not die above 127 pending writes
+//
+
+#include <boost/assert.hpp>
+#include <boost/detail/interlocked.hpp>
+#include <boost/thread/win32/thread_primitives.hpp>
+#include <boost/static_assert.hpp>
+#include <limits.h>
+#include <boost/utility.hpp>
+#include <boost/thread/thread_time.hpp>
+
+#include <boost/config/abi_prefix.hpp>
+
+namespace boost
+{
+ class modified_shared_mutex:
+ private boost::noncopyable
+ {
+ private:
+ struct state_data
+ {
+ unsigned shared_count:11,
+ shared_waiting:11,
+ exclusive:1,
+ upgrade:1,
+ exclusive_waiting:7,
+ exclusive_waiting_blocked:1;
+
+ friend bool operator==(state_data const& lhs,state_data const& rhs)
+ {
+ return *reinterpret_cast<unsigned const*>(&lhs)==*reinterpret_cast<unsigned const*>(&rhs);
+ }
+ };
+
+
+ template<typename T>
+ T interlocked_compare_exchange(T* target,T new_value,T comparand)
+ {
+ BOOST_STATIC_ASSERT(sizeof(T)==sizeof(long));
+ long const res=BOOST_INTERLOCKED_COMPARE_EXCHANGE(reinterpret_cast<long*>(target),
+ *reinterpret_cast<long*>(&new_value),
+ *reinterpret_cast<long*>(&comparand));
+ return *reinterpret_cast<T const*>(&res);
+ }
+
+ state_data state;
+ detail::win32::handle semaphores[2];
+ detail::win32::handle &unlock_sem;
+ detail::win32::handle &exclusive_sem;
+ detail::win32::handle upgrade_sem;
+
+ void release_waiters(state_data old_state)
+ {
+ if(old_state.exclusive_waiting)
+ {
+ BOOST_VERIFY(detail::win32::ReleaseSemaphore(exclusive_sem,1,0)!=0);
+ }
+
+ if(old_state.shared_waiting || old_state.exclusive_waiting)
+ {
+ BOOST_VERIFY(detail::win32::ReleaseSemaphore(unlock_sem,old_state.shared_waiting + (old_state.exclusive_waiting?1:0),0)!=0);
+ }
+ }
+
+
+ public:
+ modified_shared_mutex():
+ unlock_sem(semaphores[0]),
+ exclusive_sem(semaphores[1])
+ {
+ unlock_sem=detail::win32::create_anonymous_semaphore(0,LONG_MAX);
+ exclusive_sem=detail::win32::create_anonymous_semaphore(0,LONG_MAX);
+ upgrade_sem=detail::win32::create_anonymous_semaphore(0,LONG_MAX);
+ state_data state_={0};
+ state=state_;
+ }
+
+ ~modified_shared_mutex()
+ {
+ detail::win32::CloseHandle(upgrade_sem);
+ detail::win32::CloseHandle(unlock_sem);
+ detail::win32::CloseHandle(exclusive_sem);
+ }
+
+ bool try_lock_shared()
+ {
+ state_data old_state=state;
+ for(;;)
+ {
+ state_data new_state=old_state;
+ if(!new_state.exclusive && !new_state.exclusive_waiting_blocked)
+ {
+ ++new_state.shared_count;
+ }
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ break;
+ }
+ old_state=current_state;
+ }
+ return !(old_state.exclusive| old_state.exclusive_waiting_blocked);
+ }
+
+ void lock_shared()
+ {
+ BOOST_VERIFY(timed_lock_shared(::boost::detail::get_system_time_sentinel()));
+ }
+
+ template<typename TimeDuration>
+ bool timed_lock_shared(TimeDuration const & relative_time)
+ {
+ return timed_lock_shared(get_system_time()+relative_time);
+ }
+
+ bool timed_lock_shared(boost::system_time const& wait_until)
+ {
+ for(;;)
+ {
+ state_data old_state=state;
+ for(;;)
+ {
+ state_data new_state=old_state;
+ if(new_state.exclusive || new_state.exclusive_waiting_blocked)
+ {
+ ++new_state.shared_waiting;
+ }
+ else
+ {
+ ++new_state.shared_count;
+ }
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ break;
+ }
+ old_state=current_state;
+ }
+
+ if(!(old_state.exclusive| old_state.exclusive_waiting_blocked))
+ {
+ return true;
+ }
+
+ unsigned long const res=detail::win32::WaitForSingleObject(unlock_sem,::boost::detail::get_milliseconds_until(wait_until));
+ if(res==detail::win32::timeout)
+ {
+ for(;;)
+ {
+ state_data new_state=old_state;
+ if(new_state.exclusive || new_state.exclusive_waiting_blocked)
+ {
+ if(new_state.shared_waiting)
+ {
+ --new_state.shared_waiting;
+ }
+ }
+ else
+ {
+ ++new_state.shared_count;
+ }
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ break;
+ }
+ old_state=current_state;
+ }
+
+ if(!(old_state.exclusive| old_state.exclusive_waiting_blocked))
+ {
+ return true;
+ }
+ return false;
+ }
+
+ BOOST_ASSERT(res==0);
+ }
+ }
+
+ void unlock_shared()
+ {
+ state_data old_state=state;
+ for(;;)
+ {
+ state_data new_state=old_state;
+ bool const last_reader=!--new_state.shared_count;
+
+ if(last_reader)
+ {
+ if(new_state.upgrade)
+ {
+ new_state.upgrade=false;
+ new_state.exclusive=true;
+ }
+ else
+ {
+ if(new_state.exclusive_waiting)
+ {
+ --new_state.exclusive_waiting;
+ new_state.exclusive_waiting_blocked=false;
+ }
+ new_state.shared_waiting=0;
+ }
+ }
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ if(last_reader)
+ {
+ if(old_state.upgrade)
+ {
+ BOOST_VERIFY(detail::win32::ReleaseSemaphore(upgrade_sem,1,0)!=0);
+ }
+ else
+ {
+ release_waiters(old_state);
+ }
+ }
+ break;
+ }
+ old_state=current_state;
+ }
+ }
+
+ void lock()
+ {
+ BOOST_VERIFY(timed_lock(::boost::detail::get_system_time_sentinel()));
+ }
+
+ template<typename TimeDuration>
+ bool timed_lock(TimeDuration const & relative_time)
+ {
+ return timed_lock(get_system_time()+relative_time);
+ }
+
+ bool try_lock()
+ {
+ state_data old_state=state;
+ for(;;)
+ {
+ state_data new_state=old_state;
+ if(new_state.shared_count || new_state.exclusive)
+ {
+ return false;
+ }
+ else
+ {
+ new_state.exclusive=true;
+ }
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ break;
+ }
+ old_state=current_state;
+ }
+ return true;
+ }
+
+
+ bool timed_lock(boost::system_time const& wait_until)
+ {
+ for(;;)
+ {
+ state_data old_state=state;
+
+ for(;;)
+ {
+ state_data new_state=old_state;
+ if(new_state.shared_count || new_state.exclusive)
+ {
+ if( new_state.exclusive_waiting == 127 ) // the maximum already!
+ break;
+ ++new_state.exclusive_waiting;
+ new_state.exclusive_waiting_blocked=true;
+ }
+ else
+ {
+ new_state.exclusive=true;
+ }
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ break;
+ }
+ old_state=current_state;
+ }
+
+ if(!old_state.shared_count && !old_state.exclusive)
+ {
+ return true;
+ }
+ unsigned long const wait_res=detail::win32::WaitForMultipleObjects(2,semaphores,true,::boost::detail::get_milliseconds_until(wait_until));
+ if(wait_res==detail::win32::timeout)
+ {
+ for(;;)
+ {
+ state_data new_state=old_state;
+ if(new_state.shared_count || new_state.exclusive)
+ {
+ if(new_state.exclusive_waiting)
+ {
+ if(!--new_state.exclusive_waiting)
+ {
+ new_state.exclusive_waiting_blocked=false;
+ }
+ }
+ }
+ else
+ {
+ new_state.exclusive=true;
+ }
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ break;
+ }
+ old_state=current_state;
+ }
+ if(!old_state.shared_count && !old_state.exclusive)
+ {
+ return true;
+ }
+ return false;
+ }
+ BOOST_ASSERT(wait_res<2);
+ }
+ }
+
+ void unlock()
+ {
+ state_data old_state=state;
+ for(;;)
+ {
+ state_data new_state=old_state;
+ new_state.exclusive=false;
+ if(new_state.exclusive_waiting)
+ {
+ --new_state.exclusive_waiting;
+ new_state.exclusive_waiting_blocked=false;
+ }
+ new_state.shared_waiting=0;
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ break;
+ }
+ old_state=current_state;
+ }
+ release_waiters(old_state);
+ }
+
+ void lock_upgrade()
+ {
+ for(;;)
+ {
+ state_data old_state=state;
+ for(;;)
+ {
+ state_data new_state=old_state;
+ if(new_state.exclusive || new_state.exclusive_waiting_blocked || new_state.upgrade)
+ {
+ ++new_state.shared_waiting;
+ }
+ else
+ {
+ ++new_state.shared_count;
+ new_state.upgrade=true;
+ }
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ break;
+ }
+ old_state=current_state;
+ }
+
+ if(!(old_state.exclusive|| old_state.exclusive_waiting_blocked|| old_state.upgrade))
+ {
+ return;
+ }
+
+ BOOST_VERIFY(!detail::win32::WaitForSingleObject(unlock_sem,detail::win32::infinite));
+ }
+ }
+
+ bool try_lock_upgrade()
+ {
+ state_data old_state=state;
+ for(;;)
+ {
+ state_data new_state=old_state;
+ if(new_state.exclusive || new_state.exclusive_waiting_blocked || new_state.upgrade)
+ {
+ return false;
+ }
+ else
+ {
+ ++new_state.shared_count;
+ new_state.upgrade=true;
+ }
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ break;
+ }
+ old_state=current_state;
+ }
+ return true;
+ }
+
+ void unlock_upgrade()
+ {
+ state_data old_state=state;
+ for(;;)
+ {
+ state_data new_state=old_state;
+ new_state.upgrade=false;
+ bool const last_reader=!--new_state.shared_count;
+
+ if(last_reader)
+ {
+ if(new_state.exclusive_waiting)
+ {
+ --new_state.exclusive_waiting;
+ new_state.exclusive_waiting_blocked=false;
+ }
+ new_state.shared_waiting=0;
+ }
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ if(last_reader)
+ {
+ release_waiters(old_state);
+ }
+ break;
+ }
+ old_state=current_state;
+ }
+ }
+
+ void unlock_upgrade_and_lock()
+ {
+ state_data old_state=state;
+ for(;;)
+ {
+ state_data new_state=old_state;
+ bool const last_reader=!--new_state.shared_count;
+
+ if(last_reader)
+ {
+ new_state.upgrade=false;
+ new_state.exclusive=true;
+ }
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ if(!last_reader)
+ {
+ BOOST_VERIFY(!detail::win32::WaitForSingleObject(upgrade_sem,detail::win32::infinite));
+ }
+ break;
+ }
+ old_state=current_state;
+ }
+ }
+
+ void unlock_and_lock_upgrade()
+ {
+ state_data old_state=state;
+ for(;;)
+ {
+ state_data new_state=old_state;
+ new_state.exclusive=false;
+ new_state.upgrade=true;
+ ++new_state.shared_count;
+ if(new_state.exclusive_waiting)
+ {
+ --new_state.exclusive_waiting;
+ new_state.exclusive_waiting_blocked=false;
+ }
+ new_state.shared_waiting=0;
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ break;
+ }
+ old_state=current_state;
+ }
+ release_waiters(old_state);
+ }
+
+ void unlock_and_lock_shared()
+ {
+ state_data old_state=state;
+ for(;;)
+ {
+ state_data new_state=old_state;
+ new_state.exclusive=false;
+ ++new_state.shared_count;
+ if(new_state.exclusive_waiting)
+ {
+ --new_state.exclusive_waiting;
+ new_state.exclusive_waiting_blocked=false;
+ }
+ new_state.shared_waiting=0;
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ break;
+ }
+ old_state=current_state;
+ }
+ release_waiters(old_state);
+ }
+
+ void unlock_upgrade_and_lock_shared()
+ {
+ state_data old_state=state;
+ for(;;)
+ {
+ state_data new_state=old_state;
+ new_state.upgrade=false;
+ if(new_state.exclusive_waiting)
+ {
+ --new_state.exclusive_waiting;
+ new_state.exclusive_waiting_blocked=false;
+ }
+ new_state.shared_waiting=0;
+
+ state_data const current_state=interlocked_compare_exchange(&state,new_state,old_state);
+ if(current_state==old_state)
+ {
+ break;
+ }
+ old_state=current_state;
+ }
+ release_waiters(old_state);
+ }
+
+ };
+}
+
+#include <boost/config/abi_suffix.hpp>
+
+#endif
diff --git a/util/concurrency/spin_lock.cpp b/util/concurrency/spin_lock.cpp index b3e689a..0f33609 100644 --- a/util/concurrency/spin_lock.cpp +++ b/util/concurrency/spin_lock.cpp @@ -16,18 +16,29 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include "pch.h" #include <time.h> #include "spin_lock.h" namespace mongo { - SpinLock::SpinLock() : _locked( false ){} - - SpinLock::~SpinLock(){} + SpinLock::~SpinLock() { +#if defined(_WIN32) + DeleteCriticalSection(&_cs); +#endif + } - void SpinLock::lock(){ + SpinLock::SpinLock() #if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) + : _locked( false ) { } +#elif defined(_WIN32) + { InitializeCriticalSectionAndSpinCount(&_cs, 4000); } +#else + : _mutex( "SpinLock" ) { } +#endif + void SpinLock::lock() { +#if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) // fast path if (!_locked && !__sync_lock_test_and_set(&_locked, true)) { return; @@ -44,21 +55,28 @@ namespace mongo { while (__sync_lock_test_and_set(&_locked, true)) { nanosleep(&t, NULL); } +#elif defined(_WIN32) + EnterCriticalSection(&_cs); #else - - // WARNING "TODO Missing spin lock in this platform." + // WARNING Missing spin lock in this platform. This can potentially + // be slow. + _mutex.lock(); #endif } - void SpinLock::unlock(){ + void SpinLock::unlock() { #if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) __sync_lock_release(&_locked); +#elif defined(WIN32) + + LeaveCriticalSection(&_cs); + #else - // WARNING "TODO Missing spin lock in this platform." + _mutex.unlock(); #endif } diff --git a/util/concurrency/spin_lock.h b/util/concurrency/spin_lock.h index 110290d..d5360f7 100644 --- a/util/concurrency/spin_lock.h +++ b/util/concurrency/spin_lock.h @@ -16,18 +16,18 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#ifndef CONCURRENCY_SPINLOCK_HEADER -#define CONCURRENCY_SPINLOCK_HEADER +#pragma once + +#include "pch.h" +#include "rwlock.h" namespace mongo { /** - * BIG WARNING - COMPILES, BUT NOT READY FOR USE - BIG WARNING - * - * The spinlock currently requires late GCC support - * routines. Support for other platforms will be added soon. + * The spinlock currently requires late GCC support routines to be efficient. + * Other platforms default to a mutex implemenation. */ - class SpinLock{ + class SpinLock { public: SpinLock(); ~SpinLock(); @@ -36,13 +36,19 @@ namespace mongo { void unlock(); private: - bool _locked; +#if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) + volatile bool _locked; +#elif defined(_WIN32) + CRITICAL_SECTION _cs; +#else + // default to a scoped mutex if not implemented + RWLock _mutex; +#endif // Non-copyable, non-assignable SpinLock(SpinLock&); SpinLock& operator=(SpinLock&); - }; + }; } // namespace mongo -#endif // CONCURRENCY_SPINLOCK_HEADER diff --git a/util/concurrency/synchronization.cpp b/util/concurrency/synchronization.cpp new file mode 100644 index 0000000..12e2894 --- /dev/null +++ b/util/concurrency/synchronization.cpp @@ -0,0 +1,56 @@ +// synchronization.cpp + +/* Copyright 2010 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pch.h" +#include "synchronization.h" + +namespace mongo { + + Notification::Notification() : _mutex ( "Notification" ) , _notified( false ) { } + + Notification::~Notification() { } + + void Notification::waitToBeNotified() { + scoped_lock lock( _mutex ); + while ( ! _notified ) + _condition.wait( lock.boost() ); + } + + void Notification::notifyOne() { + scoped_lock lock( _mutex ); + assert( !_notified ); + _notified = true; + _condition.notify_one(); + } + + NotifyAll::NotifyAll() : _mutex("NotifyAll"), _counter(0) { } + + void NotifyAll::wait() { + scoped_lock lock( _mutex ); + unsigned long long old = _counter; + while( old == _counter ) { + _condition.wait( lock.boost() ); + } + } + + void NotifyAll::notifyAll() { + scoped_lock lock( _mutex ); + ++_counter; + _condition.notify_all(); + } + +} // namespace mongo diff --git a/util/concurrency/synchronization.h b/util/concurrency/synchronization.h new file mode 100644 index 0000000..ac2fcab --- /dev/null +++ b/util/concurrency/synchronization.h @@ -0,0 +1,73 @@ +// synchronization.h + +/* Copyright 2010 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <boost/thread/condition.hpp> +#include "mutex.h" + +namespace mongo { + + /* + * A class to establish a synchronization point between two threads. One thread is the waiter and one is + * the notifier. After the notification event, both proceed normally. + * + * This class is thread-safe. + */ + class Notification { + public: + Notification(); + ~Notification(); + + /* + * Blocks until the method 'notifyOne()' is called. + */ + void waitToBeNotified(); + + /* + * Notifies the waiter of '*this' that it can proceed. Can only be called once. + */ + void notifyOne(); + + private: + mongo::mutex _mutex; // protects state below + bool _notified; // was notifyOne() issued? + boost::condition _condition; // cond over _notified being true + }; + + /** establishes a synchronization point between threads. N threads are waits and one is notifier. + threadsafe. + */ + class NotifyAll : boost::noncopyable { + public: + NotifyAll(); + + /** awaits the next notifyAll() call by another thread. notifications that precede this + call are ignored -- we are looking for a fresh event. + */ + void wait(); + + /** may be called multiple times. notifies all waiters */ + void notifyAll(); + + private: + mongo::mutex _mutex; + unsigned long long _counter; + boost::condition _condition; + }; + +} // namespace mongo diff --git a/util/concurrency/task.cpp b/util/concurrency/task.cpp index 99288bb..d84cd71 100644 --- a/util/concurrency/task.cpp +++ b/util/concurrency/task.cpp @@ -17,16 +17,19 @@ */ #include "pch.h" + +#include <boost/thread/condition.hpp> + #include "task.h" #include "../goodies.h" #include "../unittest.h" -#include "boost/thread/condition.hpp" +#include "../time_support.h" -namespace mongo { +namespace mongo { - namespace task { + namespace task { - /*void foo() { + /*void foo() { boost::mutex m; boost::mutex::scoped_lock lk(m); boost::condition cond; @@ -34,21 +37,21 @@ namespace mongo { cond.notify_one(); }*/ - Task::Task() { + Task::Task() + : BackgroundJob( true /* deleteSelf */ ) { n = 0; repeat = 0; - deleteSelf = true; } void Task::halt() { repeat = 0; } - void Task::run() { + void Task::run() { assert( n == 0 ); while( 1 ) { n++; - try { + try { doWork(); - } + } catch(...) { } if( repeat == 0 ) break; @@ -62,11 +65,11 @@ namespace mongo { go(); } - void fork(Task *t) { + void fork(Task *t) { t->begin(); } - void repeat(Task *t, unsigned millis) { + void repeat(Task *t, unsigned millis) { t->repeat = millis; t->begin(); } @@ -107,7 +110,7 @@ namespace mongo { } } - void Server::send( lam msg ) { + void Server::send( lam msg ) { { boost::mutex::scoped_lock lk(m); d.push_back(msg); @@ -115,9 +118,9 @@ namespace mongo { c.notify_one(); } - void Server::doWork() { + void Server::doWork() { starting(); - while( 1 ) { + while( 1 ) { lam f; try { boost::mutex::scoped_lock lk(m); @@ -126,7 +129,7 @@ namespace mongo { f = d.front(); d.pop_front(); } - catch(...) { + catch(...) { log() << "ERROR exception in Server:doWork?" << endl; } try { @@ -138,27 +141,28 @@ namespace mongo { d.push_back(f); } } - } catch(std::exception& e) { - log() << "Server::doWork task:" << name() << " exception:" << e.what() << endl; - } - catch(const char *p) { - log() << "Server::doWork task:" << name() << " unknown c exception:" << - ((p&&strlen(p)<800)?p:"?") << endl; - } - catch(...) { - log() << "Server::doWork unknown exception task:" << name() << endl; + } + catch(std::exception& e) { + log() << "Server::doWork task:" << name() << " exception:" << e.what() << endl; + } + catch(const char *p) { + log() << "Server::doWork task:" << name() << " unknown c exception:" << + ((p&&strlen(p)<800)?p:"?") << endl; + } + catch(...) { + log() << "Server::doWork unknown exception task:" << name() << endl; } } } static Server *s; - static void abc(int i) { + static void abc(int i) { cout << "Hello " << i << endl; s->requeue(); } class TaskUnitTest : public mongo::UnitTest { public: - virtual void run() { + virtual void run() { lam f = boost::bind(abc, 3); //f(); diff --git a/util/concurrency/task.h b/util/concurrency/task.h index b3a2ece..d7b45ee 100644 --- a/util/concurrency/task.h +++ b/util/concurrency/task.h @@ -20,9 +20,9 @@ #include "../background.h" -namespace mongo { +namespace mongo { - namespace task { + namespace task { /** abstraction around threads. simpler than BackgroundJob which is used behind the scenes. allocate the Task dynamically. when the thread terminates, the Task object will delete itself. @@ -30,11 +30,11 @@ namespace mongo { class Task : private BackgroundJob { protected: virtual void doWork() = 0; // implement the task here. - virtual string name() = 0; // name the threada + virtual string name() const = 0; // name the threada public: Task(); - /** for a repeating task, stop after current invocation ends. can be called by other threads + /** for a repeating task, stop after current invocation ends. can be called by other threads as long as the Task is still in scope. */ void halt(); @@ -43,7 +43,7 @@ namespace mongo { friend void fork(Task* t); friend void repeat(Task* t, unsigned millis); virtual void run(); - virtual void ending() { } + //virtual void ending() { } void begin(); }; @@ -54,8 +54,8 @@ namespace mongo { void repeat(Task *t, unsigned millis); /*** Example *** - inline void sample() { - class Sample : public Task { + inline void sample() { + class Sample : public Task { public: int result; virtual void doWork() { result = 1234; } diff --git a/util/concurrency/thread_pool.cpp b/util/concurrency/thread_pool.cpp index 2caac1f..1c25884 100644 --- a/util/concurrency/thread_pool.cpp +++ b/util/concurrency/thread_pool.cpp @@ -20,8 +20,8 @@ #include "thread_pool.h" #include "mvar.h" -namespace mongo{ - namespace threadpool{ +namespace mongo { + namespace threadpool { // Worker thread class Worker : boost::noncopyable { @@ -34,12 +34,12 @@ namespace mongo{ // destructor will block until current operation is completed // Acts as a "join" on this thread - ~Worker(){ + ~Worker() { _task.put(Task()); _thread.join(); } - void set_task(Task& func){ + void set_task(Task& func) { assert(!func.empty()); assert(_is_done); _is_done = false; @@ -47,13 +47,13 @@ namespace mongo{ _task.put(func); } - private: + private: ThreadPool& _owner; MVar<Task> _task; bool _is_done; // only used for error detection boost::thread _thread; - void loop(){ + void loop() { while (true) { Task task = _task.take(); if (task.empty()) @@ -61,9 +61,11 @@ namespace mongo{ try { task(); - } catch (std::exception e){ + } + catch (std::exception e) { log() << "Unhandled exception in worker thread: " << e.what() << endl;; - } catch (...){ + } + catch (...) { log() << "Unhandled non-exception in worker thread" << endl; } _is_done = true; @@ -74,16 +76,15 @@ namespace mongo{ ThreadPool::ThreadPool(int nThreads) : _mutex("ThreadPool"), _tasksRemaining(0) - , _nThreads(nThreads) - { + , _nThreads(nThreads) { scoped_lock lock(_mutex); - while (nThreads-- > 0){ + while (nThreads-- > 0) { Worker* worker = new Worker(*this); _freeWorkers.push_front(worker); } } - ThreadPool::~ThreadPool(){ + ThreadPool::~ThreadPool() { join(); assert(_tasks.empty()); @@ -91,40 +92,42 @@ namespace mongo{ // O(n) but n should be small assert(_freeWorkers.size() == (unsigned)_nThreads); - while(!_freeWorkers.empty()){ + while(!_freeWorkers.empty()) { delete _freeWorkers.front(); _freeWorkers.pop_front(); } } - void ThreadPool::join(){ + void ThreadPool::join() { scoped_lock lock(_mutex); - while(_tasksRemaining){ + while(_tasksRemaining) { _condition.wait(lock.boost()); } } - void ThreadPool::schedule(Task task){ + void ThreadPool::schedule(Task task) { scoped_lock lock(_mutex); _tasksRemaining++; - if (!_freeWorkers.empty()){ + if (!_freeWorkers.empty()) { _freeWorkers.front()->set_task(task); _freeWorkers.pop_front(); - }else{ + } + else { _tasks.push_back(task); } } // should only be called by a worker from the worker thread - void ThreadPool::task_done(Worker* worker){ + void ThreadPool::task_done(Worker* worker) { scoped_lock lock(_mutex); - if (!_tasks.empty()){ + if (!_tasks.empty()) { worker->set_task(_tasks.front()); _tasks.pop_front(); - }else{ + } + else { _freeWorkers.push_front(worker); } diff --git a/util/concurrency/thread_pool.h b/util/concurrency/thread_pool.h index f0fe8f1..b348ed1 100644 --- a/util/concurrency/thread_pool.h +++ b/util/concurrency/thread_pool.h @@ -15,6 +15,8 @@ * limitations under the License. */ +#pragma once + #include <boost/function.hpp> #include <boost/bind.hpp> #undef assert @@ -22,59 +24,59 @@ namespace mongo { -namespace threadpool { - class Worker; - - typedef boost::function<void(void)> Task; //nullary function or functor - - // exported to the mongo namespace - class ThreadPool : boost::noncopyable{ - public: - explicit ThreadPool(int nThreads=8); - - // blocks until all tasks are complete (tasks_remaining() == 0) - // You should not call schedule while in the destructor - ~ThreadPool(); - - // blocks until all tasks are complete (tasks_remaining() == 0) - // does not prevent new tasks from being scheduled so could wait forever. - // Also, new tasks could be scheduled after this returns. - void join(); - - // task will be copied a few times so make sure it's relatively cheap - void schedule(Task task); - - // Helpers that wrap schedule and boost::bind. - // Functor and args will be copied a few times so make sure it's relatively cheap - template<typename F, typename A> - void schedule(F f, A a){ schedule(boost::bind(f,a)); } - template<typename F, typename A, typename B> - void schedule(F f, A a, B b){ schedule(boost::bind(f,a,b)); } - template<typename F, typename A, typename B, typename C> - void schedule(F f, A a, B b, C c){ schedule(boost::bind(f,a,b,c)); } - template<typename F, typename A, typename B, typename C, typename D> - void schedule(F f, A a, B b, C c, D d){ schedule(boost::bind(f,a,b,c,d)); } - template<typename F, typename A, typename B, typename C, typename D, typename E> - void schedule(F f, A a, B b, C c, D d, E e){ schedule(boost::bind(f,a,b,c,d,e)); } - - int tasks_remaining() { return _tasksRemaining; } - - private: - mongo::mutex _mutex; - boost::condition _condition; - - list<Worker*> _freeWorkers; //used as LIFO stack (always front) - list<Task> _tasks; //used as FIFO queue (push_back, pop_front) - int _tasksRemaining; // in queue + currently processing - int _nThreads; // only used for sanity checking. could be removed in the future. - - // should only be called by a worker from the worker's thread - void task_done(Worker* worker); - friend class Worker; - }; - -} //namespace threadpool - -using threadpool::ThreadPool; + namespace threadpool { + class Worker; + + typedef boost::function<void(void)> Task; //nullary function or functor + + // exported to the mongo namespace + class ThreadPool : boost::noncopyable { + public: + explicit ThreadPool(int nThreads=8); + + // blocks until all tasks are complete (tasks_remaining() == 0) + // You should not call schedule while in the destructor + ~ThreadPool(); + + // blocks until all tasks are complete (tasks_remaining() == 0) + // does not prevent new tasks from being scheduled so could wait forever. + // Also, new tasks could be scheduled after this returns. + void join(); + + // task will be copied a few times so make sure it's relatively cheap + void schedule(Task task); + + // Helpers that wrap schedule and boost::bind. + // Functor and args will be copied a few times so make sure it's relatively cheap + template<typename F, typename A> + void schedule(F f, A a) { schedule(boost::bind(f,a)); } + template<typename F, typename A, typename B> + void schedule(F f, A a, B b) { schedule(boost::bind(f,a,b)); } + template<typename F, typename A, typename B, typename C> + void schedule(F f, A a, B b, C c) { schedule(boost::bind(f,a,b,c)); } + template<typename F, typename A, typename B, typename C, typename D> + void schedule(F f, A a, B b, C c, D d) { schedule(boost::bind(f,a,b,c,d)); } + template<typename F, typename A, typename B, typename C, typename D, typename E> + void schedule(F f, A a, B b, C c, D d, E e) { schedule(boost::bind(f,a,b,c,d,e)); } + + int tasks_remaining() { return _tasksRemaining; } + + private: + mongo::mutex _mutex; + boost::condition _condition; + + list<Worker*> _freeWorkers; //used as LIFO stack (always front) + list<Task> _tasks; //used as FIFO queue (push_back, pop_front) + int _tasksRemaining; // in queue + currently processing + int _nThreads; // only used for sanity checking. could be removed in the future. + + // should only be called by a worker from the worker's thread + void task_done(Worker* worker); + friend class Worker; + }; + + } //namespace threadpool + + using threadpool::ThreadPool; } //namespace mongo diff --git a/util/concurrency/value.h b/util/concurrency/value.h index dabeb95..08d5306 100644 --- a/util/concurrency/value.h +++ b/util/concurrency/value.h @@ -20,11 +20,11 @@ #pragma once -namespace mongo { +namespace mongo { extern mutex _atomicMutex; - /** atomic wrapper for a value. enters a mutex on each access. must + /** atomic wrapper for a value. enters a mutex on each access. must be copyable. */ template<typename T> @@ -33,20 +33,22 @@ namespace mongo { public: Atomic<T>() { } - void operator=(const T& a) { + void operator=(const T& a) { scoped_lock lk(_atomicMutex); - val = a; } + val = a; + } - operator T() const { + operator T() const { scoped_lock lk(_atomicMutex); - return val; } - + return val; + } + /** example: Atomic<int> q; ... { Atomic<int>::tran t(q); - if( q.ref() > 0 ) + if( q.ref() > 0 ) q.ref()--; } */ @@ -58,11 +60,11 @@ namespace mongo { }; }; - /** this string COULD be mangled but with the double buffering, assuming writes - are infrequent, it's unlikely. thus, this is reasonable for lockless setting of + /** this string COULD be mangled but with the double buffering, assuming writes + are infrequent, it's unlikely. thus, this is reasonable for lockless setting of diagnostic strings, where their content isn't critical. */ - class DiagStr { + class DiagStr { char buf1[256]; char buf2[256]; char *p; diff --git a/util/concurrency/vars.cpp b/util/concurrency/vars.cpp index 8863a27..3d057a4 100644 --- a/util/concurrency/vars.cpp +++ b/util/concurrency/vars.cpp @@ -20,28 +20,28 @@ #include "value.h" #include "mutex.h" -namespace mongo { +namespace mongo { - mutex _atomicMutex("_atomicMutex"); + mongo::mutex _atomicMutex("_atomicMutex"); // intentional leak. otherwise destructor orders can be problematic at termination. MutexDebugger &mutexDebugger = *(new MutexDebugger()); - MutexDebugger::MutexDebugger() : - x( *(new boost::mutex()) ), magic(0x12345678) { - // optional way to debug lock order - /* - a = "a_lock"; - b = "b_lock"; - */ + MutexDebugger::MutexDebugger() : + x( *(new boost::mutex()) ), magic(0x12345678) { + // optional way to debug lock order + /* + a = "a_lock"; + b = "b_lock"; + */ } - void MutexDebugger::programEnding() { + void MutexDebugger::programEnding() { if( logLevel>=1 && followers.size() ) { std::cout << followers.size() << " mutexes in program" << endl; - for( map< mid, set<mid> >::iterator i = followers.begin(); i != followers.end(); i++ ) { + for( map< mid, set<mid> >::iterator i = followers.begin(); i != followers.end(); i++ ) { cout << i->first; - if( maxNest[i->first] > 1 ) + if( maxNest[i->first] > 1 ) cout << " maxNest:" << maxNest[i->first]; cout << '\n'; for( set<mid>::iterator j = i->second.begin(); j != i->second.end(); j++ ) diff --git a/util/debug_util.cpp b/util/debug_util.cpp index f0a916d..8ba6534 100644 --- a/util/debug_util.cpp +++ b/util/debug_util.cpp @@ -29,7 +29,7 @@ namespace mongo { * 2) You have run "handle SIGSTOP noprint" in gdb * 3) cmdLine.port + 2000 is free */ - void launchGDB(int){ + void launchGDB(int) { // Don't come back here signal(SIGTRAP, SIG_IGN); @@ -38,18 +38,19 @@ namespace mongo { string pidToDebug = BSONObjBuilder::numStr(getpid()); cout << "\n\n\t**** Launching gdbserver on " << newPortStr << " ****" << endl << endl; - if (fork() == 0){ + if (fork() == 0) { //child execlp("gdbserver", "gdbserver", "--attach", newPortStr.c_str(), pidToDebug.c_str(), NULL); perror(NULL); - }else{ + } + else { //parent raise(SIGSTOP); // pause all threads until gdb connects and continues raise(SIGTRAP); // break inside gdbserver } } - void setupSIGTRAPforGDB(){ + void setupSIGTRAPforGDB() { assert( signal(SIGTRAP , launchGDB ) != SIG_ERR ); } #else diff --git a/util/debug_util.h b/util/debug_util.h index 7686ecc..abed8d9 100644 --- a/util/debug_util.h +++ b/util/debug_util.h @@ -62,7 +62,7 @@ namespace mongo { #define MONGO_RARELY SOMETIMES( rarely, 128 ) #define RARELY MONGO_RARELY -#define MONGO_ONCE for( static bool undone = true; undone; undone = false ) +#define MONGO_ONCE for( static bool undone = true; undone; undone = false ) #define ONCE MONGO_ONCE #if defined(_WIN32) @@ -74,30 +74,33 @@ namespace mongo { void setupSIGTRAPforGDB(); extern int tlogLevel; - - inline void breakpoint(){ + + inline void breakpoint() { if ( tlogLevel < 0 ) return; +#ifdef _WIN32 + //DEV DebugBreak(); +#endif #ifndef _WIN32 // code to raise a breakpoint in GDB ONCE { //prevent SIGTRAP from crashing the program if default action is specified and we are not in gdb struct sigaction current; sigaction(SIGTRAP, NULL, ¤t); - if (current.sa_handler == SIG_DFL){ + if (current.sa_handler == SIG_DFL) { signal(SIGTRAP, SIG_IGN); } } - + raise(SIGTRAP); #endif } - + // conditional breakpoint - inline void breakif(bool test){ + inline void breakif(bool test) { if (test) breakpoint(); } - + } // namespace mongo diff --git a/util/embedded_builder.h b/util/embedded_builder.h index 8ca47e5..abf518e 100644 --- a/util/embedded_builder.h +++ b/util/embedded_builder.h @@ -29,10 +29,10 @@ namespace mongo { // parameter in lex ascending order. void prepareContext( string &name ) { int i = 1, n = _builders.size(); - while( i < n && - name.substr( 0, _builders[ i ].first.length() ) == _builders[ i ].first && - ( name[ _builders[i].first.length() ] == '.' || name[ _builders[i].first.length() ] == 0 ) - ){ + while( i < n && + name.substr( 0, _builders[ i ].first.length() ) == _builders[ i ].first && + ( name[ _builders[i].first.length() ] == '.' || name[ _builders[i].first.length() ] == 0 ) + ) { name = name.substr( _builders[ i ].first.length() + 1 ); ++i; } @@ -54,7 +54,7 @@ namespace mongo { } BufBuilder &subarrayStartAs( string name ) { prepareContext( name ); - return back()->subarrayStart( name.c_str() ); + return back()->subarrayStart( name ); } void done() { while( ! _builderStorage.empty() ) @@ -72,7 +72,7 @@ namespace mongo { private: void addBuilder( const string &name ) { - shared_ptr< BSONObjBuilder > newBuilder( new BSONObjBuilder( back()->subobjStart( name.c_str() ) ) ); + shared_ptr< BSONObjBuilder > newBuilder( new BSONObjBuilder( back()->subobjStart( name ) ) ); _builders.push_back( make_pair( name, newBuilder.get() ) ); _builderStorage.push_back( newBuilder ); } @@ -83,10 +83,10 @@ namespace mongo { } BSONObjBuilder *back() { return _builders.back().second; } - + vector< pair< string, BSONObjBuilder * > > _builders; vector< shared_ptr< BSONObjBuilder > > _builderStorage; }; - + } //namespace mongo diff --git a/util/file.h b/util/file.h index 0302290..0a973e3 100644 --- a/util/file.h +++ b/util/file.h @@ -1,4 +1,4 @@ -// file.h +// file.h cross platform basic file class. supports 64 bit offsets and such. /* Copyright 2009 10gen Inc. * @@ -29,139 +29,140 @@ #include "text.h" -namespace mongo { +namespace mongo { #ifndef __sunos__ -typedef uint64_t fileofs; + typedef uint64_t fileofs; #else -typedef boost::uint64_t fileofs; + typedef boost::uint64_t fileofs; #endif -class FileInterface { -public: - void open(const char *fn) {} - void write(fileofs o, const char *data, unsigned len) {} - void read(fileofs o, char *data, unsigned len) {} - bool bad() {return false;} - bool is_open() {return false;} - fileofs len() { return 0; } -}; - -#if defined(_WIN32) + class FileInterface { + public: + void open(const char *fn) {} + void write(fileofs o, const char *data, unsigned len) {} + void read(fileofs o, char *data, unsigned len) {} + bool bad() {return false;} + bool is_open() {return false;} + fileofs len() { return 0; } + void fsync() { assert(false); } + }; + +#if defined(_WIN32) #include <io.h> -class File : public FileInterface { - HANDLE fd; - bool _bad; - void err(BOOL b=false) { /* false = error happened */ - if( !b && !_bad ) { + class File : public FileInterface { + HANDLE fd; + bool _bad; + void err(BOOL b=false) { /* false = error happened */ + if( !b && !_bad ) { + _bad = true; + log() << "File I/O error " << GetLastError() << '\n'; + } + } + public: + File() { + fd = INVALID_HANDLE_VALUE; _bad = true; - log() << "File I/O error " << GetLastError() << '\n'; } - } -public: - File() { - fd = INVALID_HANDLE_VALUE; - _bad = true; - } - ~File() { - if( is_open() ) CloseHandle(fd); - fd = INVALID_HANDLE_VALUE; - } - void open(const char *filename, bool readOnly=false ) { - fd = CreateFile( - toNativeString(filename).c_str(), - ( readOnly ? 0 : GENERIC_WRITE ) | GENERIC_READ, FILE_SHARE_READ, - NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); - if( !is_open() ) { - out() << "CreateFile failed " << filename << endl; + ~File() { + if( is_open() ) CloseHandle(fd); + fd = INVALID_HANDLE_VALUE; } - else - _bad = false; - } - void write(fileofs o, const char *data, unsigned len) { - LARGE_INTEGER li; - li.QuadPart = o; - SetFilePointerEx(fd, li, NULL, FILE_BEGIN); - DWORD written; - err( WriteFile(fd, data, len, &written, NULL) ); - } - void read(fileofs o, char *data, unsigned len) { - DWORD read; - LARGE_INTEGER li; - li.QuadPart = o; - SetFilePointerEx(fd, li, NULL, FILE_BEGIN); - int ok = ReadFile(fd, data, len, &read, 0); - if( !ok ) - err(ok); - else - massert( 10438 , "ReadFile error - truncated file?", read == len); - } - bool bad() { return _bad; } - bool is_open() { return fd != INVALID_HANDLE_VALUE; } - fileofs len() { - LARGE_INTEGER li; - li.LowPart = GetFileSize(fd, (DWORD *) &li.HighPart); - if( li.HighPart == 0 && li.LowPart == INVALID_FILE_SIZE ) { - err( false ); - return 0; + void open(const char *filename, bool readOnly=false ) { + fd = CreateFile( + toNativeString(filename).c_str(), + ( readOnly ? 0 : GENERIC_WRITE ) | GENERIC_READ, FILE_SHARE_WRITE|FILE_SHARE_READ, + NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if( !is_open() ) { + DWORD e = GetLastError(); + log() << "Create/Open File failed " << filename << ' ' << errnoWithDescription(e) << endl; + } + else + _bad = false; + } + void write(fileofs o, const char *data, unsigned len) { + LARGE_INTEGER li; + li.QuadPart = o; + SetFilePointerEx(fd, li, NULL, FILE_BEGIN); + DWORD written; + err( WriteFile(fd, data, len, &written, NULL) ); } - return li.QuadPart; - } - void fsync() { FlushFileBuffers(fd); } -}; + void read(fileofs o, char *data, unsigned len) { + DWORD read; + LARGE_INTEGER li; + li.QuadPart = o; + SetFilePointerEx(fd, li, NULL, FILE_BEGIN); + int ok = ReadFile(fd, data, len, &read, 0); + if( !ok ) + err(ok); + else + massert( 10438 , "ReadFile error - truncated file?", read == len); + } + bool bad() { return _bad; } + bool is_open() { return fd != INVALID_HANDLE_VALUE; } + fileofs len() { + LARGE_INTEGER li; + li.LowPart = GetFileSize(fd, (DWORD *) &li.HighPart); + if( li.HighPart == 0 && li.LowPart == INVALID_FILE_SIZE ) { + err( false ); + return 0; + } + return li.QuadPart; + } + void fsync() { FlushFileBuffers(fd); } + }; #else -class File : public FileInterface { - int fd; - bool _bad; - void err(bool ok) { - if( !ok && !_bad ) { + class File : public FileInterface { + public: + int fd; + private: + bool _bad; + void err(bool ok) { + if( !ok && !_bad ) { + _bad = true; + log() << "File I/O " << errnoWithDescription() << '\n'; + } + } + public: + File() { + fd = -1; _bad = true; - log() << "File I/O " << errnoWithDescription() << '\n'; } - } -public: - File() { - fd = -1; - _bad = true; - } - ~File() { - if( is_open() ) ::close(fd); - fd = -1; - } + ~File() { + if( is_open() ) ::close(fd); + fd = -1; + } #ifndef O_NOATIME #define O_NOATIME 0 -#define lseek64 lseek #endif - void open(const char *filename, bool readOnly=false ) { - fd = ::open(filename, - O_CREAT | ( readOnly ? 0 : ( O_RDWR | O_NOATIME ) ) , - S_IRUSR | S_IWUSR); - if ( fd <= 0 ) { - out() << "couldn't open " << filename << ' ' << errnoWithDescription() << endl; - return; + void open(const char *filename, bool readOnly=false ) { + fd = ::open(filename, + O_CREAT | ( readOnly ? 0 : ( O_RDWR | O_NOATIME ) ) , + S_IRUSR | S_IWUSR); + if ( fd <= 0 ) { + out() << "couldn't open " << filename << ' ' << errnoWithDescription() << endl; + return; + } + _bad = false; + } + void write(fileofs o, const char *data, unsigned len) { + err( ::pwrite(fd, data, len, o) == (int) len ); + } + void read(fileofs o, char *data, unsigned len) { + err( ::pread(fd, data, len, o) == (int) len ); + } + bool bad() { return _bad; } + bool is_open() { return fd > 0; } + fileofs len() { + return lseek(fd, 0, SEEK_END); } - _bad = false; - } - void write(fileofs o, const char *data, unsigned len) { - lseek64(fd, o, SEEK_SET); - err( ::write(fd, data, len) == (int) len ); - } - void read(fileofs o, char *data, unsigned len) { - lseek(fd, o, SEEK_SET); - err( ::read(fd, data, len) == (int) len ); - } - bool bad() { return _bad; } - bool is_open() { return fd > 0; } - fileofs len() { - return lseek(fd, 0, SEEK_END); - } - void fsync() { ::fsync(fd); } -}; + void fsync() { ::fsync(fd); } + }; #endif diff --git a/util/file_allocator.cpp b/util/file_allocator.cpp new file mode 100644 index 0000000..54590ed --- /dev/null +++ b/util/file_allocator.cpp @@ -0,0 +1,282 @@ +// @file file_allocator.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pch.h" +#include <fcntl.h> +#include <errno.h> + +#if defined(__freebsd__) || defined(__openbsd__) +#include <sys/stat.h> +#endif + +#include "timer.h" +#include "mongoutils/str.h" +using namespace mongoutils; + +#ifndef O_NOATIME +#define O_NOATIME (0) +#endif + +#include "file_allocator.h" + +namespace mongo { + + void ensureParentDirCreated(const boost::filesystem::path& p){ + const boost::filesystem::path parent = p.branch_path(); + + if (! boost::filesystem::exists(parent)){ + ensureParentDirCreated(parent); + log() << "creating directory " << parent.string() << endl; + boost::filesystem::create_directory(parent); + } + + assert(boost::filesystem::is_directory(parent)); + } + +#if defined(_WIN32) + FileAllocator::FileAllocator() { + } + + void FileAllocator::start() { + } + + void FileAllocator::requestAllocation( const string &name, long &size ) { + /* Some of the system calls in the file allocator don't work in win, + so no win support - 32 or 64 bit. Plus we don't seem to need preallocation + on windows anyway as we don't have to pre-zero the file there. + */ + } + + void FileAllocator::allocateAsap( const string &name, unsigned long long &size ) { + // no-op + } + + void FileAllocator::waitUntilFinished() const { + // no-op + } + + void FileAllocator::ensureLength(int fd , long size) { + // we don't zero on windows + // TODO : we should to avoid fragmentation + } + +#else + + FileAllocator::FileAllocator() + : _pendingMutex("FileAllocator"), _failed() { + } + + + void FileAllocator::start() { + boost::thread t( boost::bind( &FileAllocator::run , this ) ); + } + + void FileAllocator::requestAllocation( const string &name, long &size ) { + scoped_lock lk( _pendingMutex ); + if ( _failed ) + return; + long oldSize = prevSize( name ); + if ( oldSize != -1 ) { + size = oldSize; + return; + } + _pending.push_back( name ); + _pendingSize[ name ] = size; + _pendingUpdated.notify_all(); + } + + void FileAllocator::allocateAsap( const string &name, unsigned long long &size ) { + scoped_lock lk( _pendingMutex ); + long oldSize = prevSize( name ); + if ( oldSize != -1 ) { + size = oldSize; + if ( !inProgress( name ) ) + return; + } + checkFailure(); + _pendingSize[ name ] = size; + if ( _pending.size() == 0 ) + _pending.push_back( name ); + else if ( _pending.front() != name ) { + _pending.remove( name ); + list< string >::iterator i = _pending.begin(); + ++i; + _pending.insert( i, name ); + } + _pendingUpdated.notify_all(); + while( inProgress( name ) ) { + checkFailure(); + _pendingUpdated.wait( lk.boost() ); + } + + } + + void FileAllocator::waitUntilFinished() const { + if ( _failed ) + return; + scoped_lock lk( _pendingMutex ); + while( _pending.size() != 0 ) + _pendingUpdated.wait( lk.boost() ); + } + + void FileAllocator::ensureLength(int fd , long size) { +#if defined(__linux__) + int ret = posix_fallocate(fd,0,size); + if ( ret == 0 ) + return; + + log() << "FileAllocator: posix_fallocate failed: " << errnoWithDescription( ret ) << " falling back" << endl; +#endif + + off_t filelen = lseek(fd, 0, SEEK_END); + if ( filelen < size ) { + if (filelen != 0) { + stringstream ss; + ss << "failure creating new datafile; lseek failed for fd " << fd << " with errno: " << errnoWithDescription(); + uassert( 10440 , ss.str(), filelen == 0 ); + } + // Check for end of disk. + + uassert( 10441 , str::stream() << "Unable to allocate new file of size " << size << ' ' << errnoWithDescription(), + size - 1 == lseek(fd, size - 1, SEEK_SET) ); + uassert( 10442 , str::stream() << "Unable to allocate new file of size " << size << ' ' << errnoWithDescription(), + 1 == write(fd, "", 1) ); + lseek(fd, 0, SEEK_SET); + + const long z = 256 * 1024; + const boost::scoped_array<char> buf_holder (new char[z]); + char* buf = buf_holder.get(); + memset(buf, 0, z); + long left = size; + while ( left > 0 ) { + long towrite = left; + if ( towrite > z ) + towrite = z; + + int written = write( fd , buf , towrite ); + uassert( 10443 , errnoWithPrefix("FileAllocator: file write failed" ), written > 0 ); + left -= written; + } + } + } + + void FileAllocator::checkFailure() { + if (_failed) { + // we want to log the problem (diskfull.js expects it) but we do not want to dump a stack tracke + msgassertedNoTrace( 12520, "new file allocation failure" ); + } + } + + long FileAllocator::prevSize( const string &name ) const { + if ( _pendingSize.count( name ) > 0 ) + return _pendingSize[ name ]; + if ( boost::filesystem::exists( name ) ) + return boost::filesystem::file_size( name ); + return -1; + } + + // caller must hold _pendingMutex lock. + bool FileAllocator::inProgress( const string &name ) const { + for( list< string >::const_iterator i = _pending.begin(); i != _pending.end(); ++i ) + if ( *i == name ) + return true; + return false; + } + + void FileAllocator::run( FileAllocator * fa ) { + setThreadName( "FileAllocator" ); + while( 1 ) { + { + scoped_lock lk( fa->_pendingMutex ); + if ( fa->_pending.size() == 0 ) + fa->_pendingUpdated.wait( lk.boost() ); + } + while( 1 ) { + string name; + long size; + { + scoped_lock lk( fa->_pendingMutex ); + if ( fa->_pending.size() == 0 ) + break; + name = fa->_pending.front(); + size = fa->_pendingSize[ name ]; + } + try { + log() << "allocating new datafile " << name << ", filling with zeroes..." << endl; + ensureParentDirCreated(name); + long fd = open(name.c_str(), O_CREAT | O_RDWR | O_NOATIME, S_IRUSR | S_IWUSR); + if ( fd <= 0 ) { + stringstream ss; + ss << "FileAllocator: couldn't open " << name << ' ' << errnoWithDescription(); + uassert( 10439 , ss.str(), fd <= 0 ); + } + +#if defined(POSIX_FADV_DONTNEED) + if( posix_fadvise(fd, 0, size, POSIX_FADV_DONTNEED) ) { + log() << "warning: posix_fadvise fails " << name << ' ' << errnoWithDescription() << endl; + } +#endif + + Timer t; + + /* make sure the file is the full desired length */ + ensureLength( fd , size ); + + log() << "done allocating datafile " << name << ", " + << "size: " << size/1024/1024 << "MB, " + << " took " << ((double)t.millis())/1000.0 << " secs" + << endl; + + close( fd ); + + } + catch ( ... ) { + log() << "error failed to allocate new file: " << name + << " size: " << size << ' ' << errnoWithDescription() << endl; + try { + BOOST_CHECK_EXCEPTION( boost::filesystem::remove( name ) ); + } + catch ( ... ) { + } + scoped_lock lk( fa->_pendingMutex ); + fa->_failed = true; + // not erasing from pending + fa->_pendingUpdated.notify_all(); + return; // no more allocation + } + + { + scoped_lock lk( fa->_pendingMutex ); + fa->_pendingSize.erase( name ); + fa->_pending.pop_front(); + fa->_pendingUpdated.notify_all(); + } + } + } + } + +#endif + + FileAllocator* FileAllocator::_instance = 0; + + FileAllocator* FileAllocator::get(){ + if ( ! _instance ) + _instance = new FileAllocator(); + return _instance; + } + +} // namespace mongo diff --git a/util/file_allocator.h b/util/file_allocator.h index b0267d9..6cc7b2d 100644 --- a/util/file_allocator.h +++ b/util/file_allocator.h @@ -16,246 +16,74 @@ */ #include "../pch.h" -#include <fcntl.h> -#include <errno.h> -#if defined(__freebsd__) || defined(__openbsd__) -#include <sys/stat.h> -#endif - -#ifndef O_NOATIME -#define O_NOATIME 0 -#endif namespace mongo { - /* Handles allocation of contiguous files on disk. Allocation may be - requested asynchronously or synchronously. - */ - class FileAllocator { - /* The public functions may not be called concurrently. The allocation - functions may be called multiple times per file, but only the first - size specified per file will be used. + /* + * Handles allocation of contiguous files on disk. Allocation may be + * requested asynchronously or synchronously. + * singleton + */ + class FileAllocator : boost::noncopyable { + /* + * The public functions may not be called concurrently. The allocation + * functions may be called multiple times per file, but only the first + * size specified per file will be used. */ public: -#if !defined(_WIN32) - FileAllocator() : pendingMutex_("FileAllocator"), failed_() {} -#endif - void start() { -#if !defined(_WIN32) - Runner r( *this ); - boost::thread t( r ); -#endif - } - // May be called if file exists. If file exists, or its allocation has - // been requested, size is updated to match existing file size. - void requestAllocation( const string &name, long &size ) { - /* Some of the system calls in the file allocator don't work in win, - so no win support - 32 or 64 bit. Plus we don't seem to need preallocation - on windows anyway as we don't have to pre-zero the file there. - */ -#if !defined(_WIN32) - scoped_lock lk( pendingMutex_ ); - if ( failed_ ) - return; - long oldSize = prevSize( name ); - if ( oldSize != -1 ) { - size = oldSize; - return; - } - pending_.push_back( name ); - pendingSize_[ name ] = size; - pendingUpdated_.notify_all(); -#endif - } - // Returns when file has been allocated. If file exists, size is - // updated to match existing file size. - void allocateAsap( const string &name, long &size ) { -#if !defined(_WIN32) - scoped_lock lk( pendingMutex_ ); - long oldSize = prevSize( name ); - if ( oldSize != -1 ) { - size = oldSize; - if ( !inProgress( name ) ) - return; - } - checkFailure(); - pendingSize_[ name ] = size; - if ( pending_.size() == 0 ) - pending_.push_back( name ); - else if ( pending_.front() != name ) { - pending_.remove( name ); - list< string >::iterator i = pending_.begin(); - ++i; - pending_.insert( i, name ); - } - pendingUpdated_.notify_all(); - while( inProgress( name ) ) { - checkFailure(); - pendingUpdated_.wait( lk.boost() ); - } -#endif - } + void start(); - void waitUntilFinished() const { -#if !defined(_WIN32) - if ( failed_ ) - return; - scoped_lock lk( pendingMutex_ ); - while( pending_.size() != 0 ) - pendingUpdated_.wait( lk.boost() ); -#endif - } - - static void ensureLength( int fd , long size ){ + /** + * May be called if file exists. If file exists, or its allocation has + * been requested, size is updated to match existing file size. + */ + void requestAllocation( const string &name, long &size ); -#if defined(_WIN32) - // we don't zero on windows - // TODO : we should to avoid fragmentation -#else -#if defined(__linux__) - int ret = posix_fallocate(fd,0,size); - if ( ret == 0 ) - return; - - log() << "posix_fallocate failed: " << errnoWithDescription( ret ) << " falling back" << endl; -#endif - - off_t filelen = lseek(fd, 0, SEEK_END); - if ( filelen < size ) { - if (filelen != 0) { - stringstream ss; - ss << "failure creating new datafile; lseek failed for fd " << fd << " with errno: " << errnoWithDescription(); - massert( 10440 , ss.str(), filelen == 0 ); - } - // Check for end of disk. - massert( 10441 , "Unable to allocate file of desired size", - size - 1 == lseek(fd, size - 1, SEEK_SET) ); - massert( 10442 , "Unable to allocate file of desired size", - 1 == write(fd, "", 1) ); - lseek(fd, 0, SEEK_SET); - - const long z = 256 * 1024; - const boost::scoped_array<char> buf_holder (new char[z]); - char* buf = buf_holder.get(); - memset(buf, 0, z); - long left = size; - while ( left > 0 ) { - long towrite = left; - if ( towrite > z ) - towrite = z; - - int written = write( fd , buf , towrite ); - massert( 10443 , errnoWithPrefix("write failed" ), written > 0 ); - left -= written; - } - } -#endif - } - + /** + * Returns when file has been allocated. If file exists, size is + * updated to match existing file size. + */ + void allocateAsap( const string &name, unsigned long long &size ); + + void waitUntilFinished() const; + + static void ensureLength(int fd , long size); + + /** @return the singletone */ + static FileAllocator * get(); + private: + + FileAllocator(); + #if !defined(_WIN32) - void checkFailure() { - massert( 12520, "file allocation failure", !failed_ ); - } - - // caller must hold pendingMutex_ lock. Returns size if allocated or + void checkFailure(); + + // caller must hold pendingMutex_ lock. Returns size if allocated or // allocation requested, -1 otherwise. - long prevSize( const string &name ) const { - if ( pendingSize_.count( name ) > 0 ) - return pendingSize_[ name ]; - if ( boost::filesystem::exists( name ) ) - return boost::filesystem::file_size( name ); - return -1; - } - + long prevSize( const string &name ) const; + // caller must hold pendingMutex_ lock. - bool inProgress( const string &name ) const { - for( list< string >::const_iterator i = pending_.begin(); i != pending_.end(); ++i ) - if ( *i == name ) - return true; - return false; - } + bool inProgress( const string &name ) const; - mutable mongo::mutex pendingMutex_; - mutable boost::condition pendingUpdated_; - list< string > pending_; - mutable map< string, long > pendingSize_; - bool failed_; - - struct Runner { - Runner( FileAllocator &allocator ) : a_( allocator ) {} - FileAllocator &a_; - void operator()() { - while( 1 ) { - { - scoped_lock lk( a_.pendingMutex_ ); - if ( a_.pending_.size() == 0 ) - a_.pendingUpdated_.wait( lk.boost() ); - } - while( 1 ) { - string name; - long size; - { - scoped_lock lk( a_.pendingMutex_ ); - if ( a_.pending_.size() == 0 ) - break; - name = a_.pending_.front(); - size = a_.pendingSize_[ name ]; - } - try { - log() << "allocating new datafile " << name << ", filling with zeroes..." << endl; - long fd = open(name.c_str(), O_CREAT | O_RDWR | O_NOATIME, S_IRUSR | S_IWUSR); - if ( fd <= 0 ) { - stringstream ss; - ss << "couldn't open " << name << ' ' << errnoWithDescription(); - massert( 10439 , ss.str(), fd <= 0 ); - } + /** called from the worked thread */ + static void run( FileAllocator * fa ); -#if defined(POSIX_FADV_DONTNEED) - if( posix_fadvise(fd, 0, size, POSIX_FADV_DONTNEED) ) { - log() << "warning: posix_fadvise fails " << name << ' ' << errnoWithDescription() << endl; - } -#endif - - Timer t; - - /* make sure the file is the full desired length */ - ensureLength( fd , size ); + mutable mongo::mutex _pendingMutex; + mutable boost::condition _pendingUpdated; + + list< string > _pending; + mutable map< string, long > _pendingSize; - log() << "done allocating datafile " << name << ", " - << "size: " << size/1024/1024 << "MB, " - << " took " << ((double)t.millis())/1000.0 << " secs" - << endl; + bool _failed; +#endif + + static FileAllocator* _instance; - close( fd ); - - } catch ( ... ) { - problem() << "Failed to allocate new file: " << name - << ", size: " << size << ", aborting." << endl; - try { - BOOST_CHECK_EXCEPTION( boost::filesystem::remove( name ) ); - } catch ( ... ) { - } - scoped_lock lk( a_.pendingMutex_ ); - a_.failed_ = true; - // not erasing from pending - a_.pendingUpdated_.notify_all(); - return; // no more allocation - } - - { - scoped_lock lk( a_.pendingMutex_ ); - a_.pendingSize_.erase( name ); - a_.pending_.pop_front(); - a_.pendingUpdated_.notify_all(); - } - } - } - } - }; -#endif }; - - FileAllocator &theFileAllocator(); + + /** like "mkdir -p" but on parent dir of p rather than p itself */ + void ensureParentDirCreated(const boost::filesystem::path& p); + } // namespace mongo diff --git a/util/goodies.h b/util/goodies.h index 7b73996..53a74c2 100644 --- a/util/goodies.h +++ b/util/goodies.h @@ -1,5 +1,5 @@ // @file goodies.h -// miscellaneous junk +// miscellaneous /* Copyright 2009 10gen Inc. * @@ -23,11 +23,18 @@ namespace mongo { - void setThreadName(const char * name); + /* @return a dump of the buffer as hex byte ascii output */ + string hexdump(const char *data, unsigned len); + + /** + * @return if this name has an increasing counter associated, return the value + * otherwise 0 + */ + unsigned setThreadName(const char * name); string getThreadName(); - + template<class T> - inline string ToString(const T& t) { + inline string ToString(const T& t) { stringstream s; s << t; return s.str(); @@ -49,17 +56,16 @@ namespace mongo { /* use "addr2line -CFe <exe>" to parse. */ inline void printStackTrace( ostream &o = cout ) { void *b[20]; - size_t size; + + int size = backtrace(b, 20); + for (int i = 0; i < size; i++) + o << hex << b[i] << dec << ' '; + o << endl; + char **strings; - size_t i; - size = backtrace(b, 20); strings = backtrace_symbols(b, size); - - for (i = 0; i < size; i++) - o << hex << b[i] << dec << ' '; - o << '\n'; - for (i = 0; i < size; i++) + for (int i = 0; i < size; i++) o << ' ' << strings[i] << '\n'; o.flush(); free (strings); @@ -68,23 +74,6 @@ namespace mongo { inline void printStackTrace( ostream &o = cout ) { } #endif - /* set to TRUE if we are exiting */ - extern bool goingAway; - - /* find the multimap member which matches a particular key and value. - - note this can be slow if there are a lot with the same key. - */ - template<class C,class K,class V> inline typename C::iterator kv_find(C& c, const K& k,const V& v) { - pair<typename C::iterator,typename C::iterator> p = c.equal_range(k); - - for ( typename C::iterator it=p.first; it!=p.second; ++it) - if ( it->second == v ) - return it; - - return c.end(); - } - bool isPrime(int n); int nextPrime(int n); @@ -109,7 +98,8 @@ namespace mongo { cout << endl; len -= 16; } - } catch (...) { + } + catch (...) { } } @@ -147,173 +137,15 @@ namespace mongo { } }; - inline void time_t_to_Struct(time_t t, struct tm * buf , bool local = false ) { -#if defined(_WIN32) - if ( local ) - localtime_s( buf , &t ); - else - gmtime_s(buf, &t); -#else - if ( local ) - localtime_r(&t, buf); - else - gmtime_r(&t, buf); -#endif - } - - // uses ISO 8601 dates without trailing Z - // colonsOk should be false when creating filenames - inline string terseCurrentTime(bool colonsOk=true){ - struct tm t; - time_t_to_Struct( time(0) , &t ); - - const char* fmt = (colonsOk ? "%Y-%m-%dT%H:%M:%S" : "%Y-%m-%dT%H-%M-%S"); - char buf[32]; - assert(strftime(buf, sizeof(buf), fmt, &t) == 19); - return buf; - } - -#define MONGO_asctime _asctime_not_threadsafe_ -#define asctime MONGO_asctime -#define MONGO_gmtime _gmtime_not_threadsafe_ -#define gmtime MONGO_gmtime -#define MONGO_localtime _localtime_not_threadsafe_ -#define localtime MONGO_localtime -#define MONGO_ctime _ctime_is_not_threadsafe_ -#define ctime MONGO_ctime - -#if defined(_WIN32) || defined(__sunos__) - inline void sleepsecs(int s) { - boost::xtime xt; - boost::xtime_get(&xt, boost::TIME_UTC); - xt.sec += s; - boost::thread::sleep(xt); - } - inline void sleepmillis(long long s) { - boost::xtime xt; - boost::xtime_get(&xt, boost::TIME_UTC); - xt.sec += (int)( s / 1000 ); - xt.nsec += (int)(( s % 1000 ) * 1000000); - if ( xt.nsec >= 1000000000 ) { - xt.nsec -= 1000000000; - xt.sec++; - } - boost::thread::sleep(xt); - } - inline void sleepmicros(long long s) { - if ( s <= 0 ) - return; - boost::xtime xt; - boost::xtime_get(&xt, boost::TIME_UTC); - xt.sec += (int)( s / 1000000 ); - xt.nsec += (int)(( s % 1000000 ) * 1000); - if ( xt.nsec >= 1000000000 ) { - xt.nsec -= 1000000000; - xt.sec++; - } - boost::thread::sleep(xt); - } -#else - inline void sleepsecs(int s) { - struct timespec t; - t.tv_sec = s; - t.tv_nsec = 0; - if ( nanosleep( &t , 0 ) ){ - cout << "nanosleep failed" << endl; - } - } - inline void sleepmicros(long long s) { - if ( s <= 0 ) - return; - struct timespec t; - t.tv_sec = (int)(s / 1000000); - t.tv_nsec = 1000 * ( s % 1000000 ); - struct timespec out; - if ( nanosleep( &t , &out ) ){ - cout << "nanosleep failed" << endl; - } - } - inline void sleepmillis(long long s) { - sleepmicros( s * 1000 ); - } -#endif - - // note this wraps - inline int tdiff(unsigned told, unsigned tnew) { - return WrappingInt::diff(tnew, told); - } - inline unsigned curTimeMillis() { - boost::xtime xt; - boost::xtime_get(&xt, boost::TIME_UTC); - unsigned t = xt.nsec / 1000000; - return (xt.sec & 0xfffff) * 1000 + t; - } - - inline Date_t jsTime() { - boost::xtime xt; - boost::xtime_get(&xt, boost::TIME_UTC); - unsigned long long t = xt.nsec / 1000000; - return ((unsigned long long) xt.sec * 1000) + t; - } - - inline unsigned long long curTimeMicros64() { - boost::xtime xt; - boost::xtime_get(&xt, boost::TIME_UTC); - unsigned long long t = xt.nsec / 1000; - return (((unsigned long long) xt.sec) * 1000000) + t; - } - -// measures up to 1024 seconds. or, 512 seconds with tdiff that is... - inline unsigned curTimeMicros() { - boost::xtime xt; - boost::xtime_get(&xt, boost::TIME_UTC); - unsigned t = xt.nsec / 1000; - unsigned secs = xt.sec % 1024; - return secs*1000000 + t; - } - -// simple scoped timer - class Timer { - public: - Timer() { - reset(); - } - Timer( unsigned long long start ) { - old = start; - } - int seconds() const { - return (int)(micros() / 1000000); - } - int millis() const { - return (long)(micros() / 1000); - } - unsigned long long micros() const { - unsigned long long n = curTimeMicros64(); - return n - old; - } - unsigned long long micros(unsigned long long & n) const { // returns cur time in addition to timer result - n = curTimeMicros64(); - return n - old; - } - unsigned long long startTime(){ - return old; - } - void reset() { - old = curTimeMicros64(); - } - private: - unsigned long long old; - }; - /* class DebugMutex : boost::noncopyable { - friend class lock; - mongo::mutex m; - int locked; + friend class lock; + mongo::mutex m; + int locked; public: - DebugMutex() : locked(0); { } - bool isLocked() { return locked; } + DebugMutex() : locked(0); { } + bool isLocked() { return locked; } }; */ @@ -351,19 +183,20 @@ namespace mongo { return swapEndian(x); } #endif - + #if !defined(_WIN32) typedef int HANDLE; inline void strcpy_s(char *dst, unsigned len, const char *src) { + assert( strlen(src) < len ); strcpy(dst, src); } #else typedef void *HANDLE; #endif - + /* thread local "value" rather than a pointer good for things which have copy constructors (and the copy constructor is fast enough) - e.g. + e.g. ThreadLocalValue<int> myint; */ template<class T> @@ -371,7 +204,7 @@ namespace mongo { public: ThreadLocalValue( T def = 0 ) : _default( def ) { } - T get() { + T get() const { T * val = _val.get(); if ( val ) return *val; @@ -380,7 +213,7 @@ namespace mongo { void set( const T& i ) { T *v = _val.get(); - if( v ) { + if( v ) { *v = i; return; } @@ -389,21 +222,21 @@ namespace mongo { } private: - T _default; boost::thread_specific_ptr<T> _val; + const T _default; }; class ProgressMeter : boost::noncopyable { public: - ProgressMeter( long long total , int secondsBetween = 3 , int checkInterval = 100 ){ + ProgressMeter( unsigned long long total , int secondsBetween = 3 , int checkInterval = 100 ) { reset( total , secondsBetween , checkInterval ); } - ProgressMeter(){ + ProgressMeter() { _active = 0; } - - void reset( long long total , int secondsBetween = 3 , int checkInterval = 100 ){ + + void reset( unsigned long long total , int secondsBetween = 3 , int checkInterval = 100 ) { _total = total; _secondsBetween = secondsBetween; _checkInterval = checkInterval; @@ -415,29 +248,33 @@ namespace mongo { _active = 1; } - void finished(){ + void finished() { _active = 0; } - bool isActive(){ + bool isActive() { return _active; } - - bool hit( int n = 1 ){ - if ( ! _active ){ + + /** + * @return if row was printed + */ + bool hit( int n = 1 ) { + if ( ! _active ) { cout << "warning: hit on in-active ProgressMeter" << endl; + return false; } _done += n; _hits++; if ( _hits % _checkInterval ) return false; - + int t = (int) time(0); if ( t - _lastTime < _secondsBetween ) return false; - - if ( _total > 0 ){ + + if ( _total > 0 ) { int per = (int)( ( (double)_done * 100.0 ) / (double)_total ); cout << "\t\t" << _done << "/" << _total << "\t" << per << "%" << endl; } @@ -445,11 +282,11 @@ namespace mongo { return true; } - long long done(){ + unsigned long long done() { return _done; } - - long long hits(){ + + unsigned long long hits() { return _hits; } @@ -467,42 +304,42 @@ namespace mongo { private: bool _active; - - long long _total; + + unsigned long long _total; int _secondsBetween; int _checkInterval; - long long _done; - long long _hits; + unsigned long long _done; + unsigned long long _hits; int _lastTime; }; class ProgressMeterHolder : boost::noncopyable { public: ProgressMeterHolder( ProgressMeter& pm ) - : _pm( pm ){ + : _pm( pm ) { } - - ~ProgressMeterHolder(){ + + ~ProgressMeterHolder() { _pm.finished(); } - ProgressMeter* operator->(){ + ProgressMeter* operator->() { return &_pm; } - bool hit( int n = 1 ){ + bool hit( int n = 1 ) { return _pm.hit( n ); } - void finished(){ + void finished() { _pm.finished(); } - - bool operator==( const ProgressMeter& other ){ + + bool operator==( const ProgressMeter& other ) { return _pm == other; } - + private: ProgressMeter& _pm; }; @@ -513,11 +350,11 @@ namespace mongo { _outof = num; _num = num; } - - bool tryAcquire(){ + + bool tryAcquire() { scoped_lock lk( _mutex ); - if ( _num <= 0 ){ - if ( _num < 0 ){ + if ( _num <= 0 ) { + if ( _num < 0 ) { cerr << "DISASTER! in TicketHolder" << endl; } return false; @@ -525,20 +362,20 @@ namespace mongo { _num--; return true; } - - void release(){ + + void release() { scoped_lock lk( _mutex ); _num++; } - void resize( int newSize ){ - scoped_lock lk( _mutex ); + void resize( int newSize ) { + scoped_lock lk( _mutex ); int used = _outof - _num; - if ( used > newSize ){ + if ( used > newSize ) { cout << "ERROR: can't resize since we're using (" << used << ") more than newSize(" << newSize << ")" << endl; return; } - + _outof = newSize; _num = _outof - used; } @@ -561,11 +398,11 @@ namespace mongo { class TicketHolderReleaser { public: - TicketHolderReleaser( TicketHolder * holder ){ + TicketHolderReleaser( TicketHolder * holder ) { _holder = holder; } - - ~TicketHolderReleaser(){ + + ~TicketHolderReleaser() { _holder->release(); } private: @@ -580,26 +417,26 @@ namespace mongo { class ThreadSafeString { public: ThreadSafeString( size_t size=256 ) - : _size( 256 ) , _buf( new char[256] ){ + : _size( 256 ) , _buf( new char[256] ) { memset( _buf , 0 , _size ); } ThreadSafeString( const ThreadSafeString& other ) - : _size( other._size ) , _buf( new char[_size] ){ + : _size( other._size ) , _buf( new char[_size] ) { strncpy( _buf , other._buf , _size ); } - ~ThreadSafeString(){ + ~ThreadSafeString() { delete[] _buf; _buf = 0; } - + string toString() const { string s = _buf; return s; } - ThreadSafeString& operator=( const char * str ){ + ThreadSafeString& operator=( const char * str ) { size_t s = strlen(str); if ( s >= _size - 2 ) s = _size - 2; @@ -607,7 +444,7 @@ namespace mongo { _buf[s] = 0; return *this; } - + bool operator==( const ThreadSafeString& other ) const { return strcmp( _buf , other._buf ) == 0; } @@ -626,7 +463,7 @@ namespace mongo { private: size_t _size; - char * _buf; + char * _buf; }; ostream& operator<<( ostream &s, const ThreadSafeString &o ); @@ -648,7 +485,7 @@ namespace mongo { } return x; } - + // for convenience, '{' is greater than anything and stops number parsing inline int lexNumCmp( const char *s1, const char *s2 ) { //cout << "START : " << s1 << "\t" << s2 << endl; @@ -661,10 +498,10 @@ namespace mongo { return 1; if ( p2 && !p1 ) return -1; - + bool n1 = isNumber( *s1 ); bool n2 = isNumber( *s2 ); - + if ( n1 && n2 ) { // get rid of leading 0s while ( *s1 == '0' ) s1++; @@ -678,8 +515,8 @@ namespace mongo { while ( isNumber (*e1) ) e1++; while ( isNumber (*e2) ) e2++; - int len1 = e1-s1; - int len2 = e2-s2; + int len1 = (int)(e1-s1); + int len2 = (int)(e2-s2); int result; // if one is longer than the other, return @@ -698,24 +535,24 @@ namespace mongo { s1 = e1; s2 = e2; continue; - } - - if ( n1 ) + } + + if ( n1 ) return 1; - - if ( n2 ) + + if ( n2 ) return -1; - + if ( *s1 > *s2 ) return 1; - + if ( *s2 > *s1 ) return -1; - + s1++; s2++; } - - if ( *s1 ) + + if ( *s1 ) return 1; if ( *s2 ) return -1; @@ -729,8 +566,8 @@ namespace mongo { * ptr<const T> => T const * or const T* */ template <typename T> - struct ptr{ - + struct ptr { + ptr() : _p(NULL) {} // convert to ptr<T> @@ -740,7 +577,7 @@ namespace mongo { template<typename U> ptr(const boost::shared_ptr<U>& p) : _p(p.get()) {} template<typename U> ptr(const boost::scoped_ptr<U>& p) : _p(p.get()) {} //template<typename U> ptr(const auto_ptr<U>& p) : _p(p.get()) {} - + // assign to ptr<T> ptr& operator= (T* p) { _p = p; return *this; } // needed for NULL template<typename U> ptr& operator= (U* p) { _p = p; return *this; } diff --git a/util/hashtab.h b/util/hashtab.h index 16c5483..6818bef 100644 --- a/util/hashtab.h +++ b/util/hashtab.h @@ -24,6 +24,7 @@ #include "../pch.h" #include <map> +#include "../db/dur.h" namespace mongo { @@ -36,9 +37,8 @@ namespace mongo { template < class Key, - class Type, - class PTR - > + class Type + > class HashTable : boost::noncopyable { public: const char *name; @@ -53,12 +53,13 @@ namespace mongo { hash = 0; } }; - PTR _buf; + void* _buf; int n; int maxChain; Node& nodes(int i) { - return *((Node*) _buf.at(i * sizeof(Node), sizeof(Node))); + Node *nodes = (Node *) _buf; + return nodes[i]; } int _find(const Key& k, bool& found) { @@ -87,10 +88,10 @@ namespace mongo { out() << "error: hashtable " << name << " is full n:" << n << endl; return -1; } - if( chain >= maxChain ) { + if( chain >= maxChain ) { if ( firstNonUsed >= 0 ) return firstNonUsed; - out() << "error: hashtable " << name << " max chain n:" << n << endl; + out() << "error: hashtable " << name << " max chain reached:" << maxChain << endl; return -1; } } @@ -98,7 +99,7 @@ namespace mongo { public: /* buf must be all zeroes on initialization. */ - HashTable(PTR buf, int buflen, const char *_name) : name(_name) { + HashTable(void* buf, int buflen, const char *_name) : name(_name) { int m = sizeof(Node); // out() << "hashtab init, buflen:" << buflen << " m:" << m << endl; n = buflen / m; @@ -108,7 +109,7 @@ namespace mongo { _buf = buf; //nodes = (Node *) buf; - if ( sizeof(Node) != 628 ){ + if ( sizeof(Node) != 628 ) { out() << "HashTable() " << _name << " sizeof(node):" << sizeof(Node) << " n:" << n << " sizeof(Key): " << sizeof(Key) << " sizeof(Type):" << sizeof(Type) << endl; assert( sizeof(Node) == 628 ); } @@ -127,41 +128,34 @@ namespace mongo { bool found; int i = _find(k, found); if ( i >= 0 && found ) { - Node& n = nodes(i); - n.k.kill(); - n.setUnused(); + Node* n = &nodes(i); + n = getDur().writing(n); + n->k.kill(); + n->setUnused(); } } -/* - void drop(const Key& k) { - bool found; - int i = _find(k, found); - if ( i >= 0 && found ) { - nodes[i].setUnused(); - } - } -*/ + /** returns false if too full */ bool put(const Key& k, const Type& value) { bool found; int i = _find(k, found); if ( i < 0 ) return false; - Node& n = nodes(i); + Node* n = getDur().writing( &nodes(i) ); if ( !found ) { - n.k = k; - n.hash = k.hash(); + n->k = k; + n->hash = k.hash(); } else { - assert( n.hash == k.hash() ); + assert( n->hash == k.hash() ); } - n.value = value; + n->value = value; return true; } - + typedef void (*IteratorCallback)( const Key& k , Type& v ); - void iterAll( IteratorCallback callback ){ - for ( int i=0; i<n; i++ ){ + void iterAll( IteratorCallback callback ) { + for ( int i=0; i<n; i++ ) { if ( ! nodes(i).inUse() ) continue; callback( nodes(i).k , nodes(i).value ); @@ -170,14 +164,14 @@ namespace mongo { // TODO: should probably use boost::bind for this, but didn't want to look at it typedef void (*IteratorCallback2)( const Key& k , Type& v , void * extra ); - void iterAll( IteratorCallback2 callback , void * extra ){ - for ( int i=0; i<n; i++ ){ + void iterAll( IteratorCallback2 callback , void * extra ) { + for ( int i=0; i<n; i++ ) { if ( ! nodes(i).inUse() ) continue; callback( nodes(i).k , nodes(i).value , extra ); } } - + }; #pragma pack() diff --git a/util/heapcheck.h b/util/heapcheck.h new file mode 100644 index 0000000..95da953 --- /dev/null +++ b/util/heapcheck.h @@ -0,0 +1,33 @@ +// @file heapcheck.h + +/** +* Copyright (C) 2010 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#pragma once + +#if defined(HEAP_CHECKING) + +#include <google/heap-checker.h> + +#define IGNORE_OBJECT( a ) HeapLeakChecker::IgnoreObject( a ) +#define UNIGNORE_OBJECT( a ) HeapLeakChecker::UnIgnoreObject( a ) + +#else + +#define IGNORE_OBJECT( a ) +#define UNIGNORE_OBJECT( a ) + +#endif @@ -30,15 +30,15 @@ namespace mongo { return 0xff; } inline char fromHex( const char *c ) { - return ( fromHex( c[ 0 ] ) << 4 ) | fromHex( c[ 1 ] ); + return (char)(( fromHex( c[ 0 ] ) << 4 ) | fromHex( c[ 1 ] )); } - inline string toHex(const void* inRaw, int len){ + inline string toHex(const void* inRaw, int len) { static const char hexchars[] = "0123456789ABCDEF"; StringBuilder out; const char* in = reinterpret_cast<const char*>(inRaw); - for (int i=0; i<len; ++i){ + for (int i=0; i<len; ++i) { char c = in[i]; char hi = hexchars[(c & 0xF0) >> 4]; char lo = hexchars[(c & 0x0F)]; @@ -48,13 +48,13 @@ namespace mongo { return out.str(); } - - inline string toHexLower(const void* inRaw, int len){ + + inline string toHexLower(const void* inRaw, int len) { static const char hexchars[] = "0123456789abcdef"; StringBuilder out; const char* in = reinterpret_cast<const char*>(inRaw); - for (int i=0; i<len; ++i){ + for (int i=0; i<len; ++i) { char c = in[i]; char hi = hexchars[(c & 0xF0) >> 4]; char lo = hexchars[(c & 0x0F)]; diff --git a/util/histogram.cpp b/util/histogram.cpp index 4541dfd..17a8505 100644 --- a/util/histogram.cpp +++ b/util/histogram.cpp @@ -28,30 +28,31 @@ namespace mongo { using std::setfill; using std::setw; - Histogram::Histogram( const Options& opts ) + Histogram::Histogram( const Options& opts ) : _initialValue( opts.initialValue ) , _numBuckets( opts.numBuckets ) , _boundaries( new uint32_t[_numBuckets] ) - , _buckets( new uint64_t[_numBuckets] ){ + , _buckets( new uint64_t[_numBuckets] ) { // TODO more sanity checks // + not too few buckets // + initialBucket and bucketSize fit within 32 bit ints // _boundaries store the maximum value falling in that bucket. - if ( opts.exponential ){ + if ( opts.exponential ) { uint32_t twoPow = 1; // 2^0 - for ( uint32_t i = 0; i < _numBuckets - 1; i++){ + for ( uint32_t i = 0; i < _numBuckets - 1; i++) { _boundaries[i] = _initialValue + opts.bucketSize * twoPow; twoPow *= 2; // 2^i+1 } - } else { + } + else { _boundaries[0] = _initialValue + opts.bucketSize; - for ( uint32_t i = 1; i < _numBuckets - 1; i++ ){ + for ( uint32_t i = 1; i < _numBuckets - 1; i++ ) { _boundaries[i] = _boundaries[ i-1 ] + opts.bucketSize; } } - _boundaries[ _numBuckets-1 ] = std::numeric_limits<uint32_t>::max(); + _boundaries[ _numBuckets-1 ] = std::numeric_limits<uint32_t>::max(); for ( uint32_t i = 0; i < _numBuckets; i++ ) { _buckets[i] = 0; @@ -63,16 +64,16 @@ namespace mongo { delete [] _buckets; } - void Histogram::insert( uint32_t element ){ + void Histogram::insert( uint32_t element ) { if ( element < _initialValue) return; _buckets[ _findBucket(element) ] += 1; } - string Histogram::toHTML() const{ + string Histogram::toHTML() const { uint64_t max = 0; - for ( uint32_t i = 0; i < _numBuckets; i++ ){ - if ( _buckets[i] > max ){ + for ( uint32_t i = 0; i < _numBuckets; i++ ) { + if ( _buckets[i] > max ) { max = _buckets[i]; } } @@ -83,10 +84,10 @@ namespace mongo { // normalize buckets to max const int maxBar = 20; ostringstream ss; - for ( uint32_t i = 0; i < _numBuckets; i++ ){ + for ( uint32_t i = 0; i < _numBuckets; i++ ) { int barSize = _buckets[i] * maxBar / max; - ss << string( barSize,'*' ) - << setfill(' ') << setw( maxBar-barSize + 12 ) + ss << string( barSize,'*' ) + << setfill(' ') << setw( maxBar-barSize + 12 ) << _boundaries[i] << '\n'; } @@ -109,21 +110,22 @@ namespace mongo { return _numBuckets; } - uint32_t Histogram::_findBucket( uint32_t element ) const{ + uint32_t Histogram::_findBucket( uint32_t element ) const { // TODO assert not too small a value? uint32_t low = 0; uint32_t high = _numBuckets - 1; - while ( low < high ){ + while ( low < high ) { // low + ( (high - low) / 2 ); uint32_t mid = ( low + high ) >> 1; - if ( element > _boundaries[ mid ] ){ + if ( element > _boundaries[ mid ] ) { low = mid + 1; - } else { + } + else { high = mid; } } return low; - } + } } // namespace mongo diff --git a/util/histogram.h b/util/histogram.h index d4a6fa7..40ec562 100644 --- a/util/histogram.h +++ b/util/histogram.h @@ -65,12 +65,12 @@ namespace mongo { // use exponential buckets? bool exponential; - - Options() + + Options() : numBuckets(0) , bucketSize(0) , initialValue(0) - , exponential(false){} + , exponential(false) {} }; explicit Histogram( const Options& opts ); ~Histogram(); @@ -103,7 +103,7 @@ namespace mongo { * Return the number of buckets in this histogram. */ boost::uint32_t getBucketsNum() const; - + private: /** * Returns the bucket where 'element' should fall diff --git a/util/hostandport.h b/util/hostandport.h index 6124570..fd27296 100644 --- a/util/hostandport.h +++ b/util/hostandport.h @@ -20,17 +20,17 @@ #include "sock.h" #include "../db/cmdline.h" #include "mongoutils/str.h" - -namespace mongo { + +namespace mongo { using namespace mongoutils; - /** helper for manipulating host:port connection endpoints. + /** helper for manipulating host:port connection endpoints. */ - struct HostAndPort { + struct HostAndPort { HostAndPort() : _port(-1) { } - /** From a string hostname[:portnumber] + /** From a string hostname[:portnumber] Throws user assertion if bad config string or bad port #. */ HostAndPort(string s); @@ -38,11 +38,11 @@ namespace mongo { /** @param p port number. -1 is ok to use default. */ HostAndPort(string h, int p /*= -1*/) : _host(h), _port(p) { } - HostAndPort(const SockAddr& sock ) - : _host( sock.getAddr() ) , _port( sock.getPort() ){ + HostAndPort(const SockAddr& sock ) + : _host( sock.getAddr() ) , _port( sock.getPort() ) { } - static HostAndPort me() { + static HostAndPort me() { return HostAndPort("localhost", cmdLine.port); } @@ -50,7 +50,7 @@ namespace mongo { static HostAndPort Me(); bool operator<(const HostAndPort& r) const { - if( _host < r._host ) + if( _host < r._host ) return true; if( _host == r._host ) return port() < r.port(); @@ -61,13 +61,17 @@ namespace mongo { return _host == r._host && port() == r.port(); } + bool operator!=(const HostAndPort& r) const { + return _host != r._host || port() != r.port(); + } + /* returns true if the host/port combo identifies this process instance. */ bool isSelf() const; // defined in message.cpp bool isLocalHost() const; // @returns host:port - string toString() const; + string toString() const; operator string() const { return toString(); } @@ -84,15 +88,42 @@ namespace mongo { }; /** returns true if strings seem to be the same hostname. - "nyc1" and "nyc1.acme.com" are treated as the same. - in fact "nyc1.foo.com" and "nyc1.acme.com" are treated the same - - we oly look up to the first period. + "nyc1", "nyc1.acme", and "nyc1.acme.com" are treated as the same. */ inline bool sameHostname(const string& a, const string& b) { - return str::before(a, '.') == str::before(b, '.'); + size_t prefixLen = str::shareCommonPrefix(a.c_str(), b.c_str()); + + if (prefixLen == a.size()) { // (a == b) or (a isPrefixOf b) + if ( b[prefixLen] == '.' || b[prefixLen] == '\0') + return true; + } + else if(prefixLen == b.size()) { // (b isPrefixOf a) + if ( a[prefixLen] == '.') // can't be '\0' + return true; + } + + return false; } - inline HostAndPort HostAndPort::Me() { + inline HostAndPort HostAndPort::Me() { + const char* ips = cmdLine.bind_ip.c_str(); + while(*ips) { + string ip; + const char * comma = strchr(ips, ','); + if (comma) { + ip = string(ips, comma - ips); + ips = comma + 1; + } + else { + ip = string(ips); + ips = ""; + } + HostAndPort h = HostAndPort(ip, cmdLine.port); + if (!h.isLocalHost()) { + return h; + } + } + string h = getHostName(); assert( !h.empty() ); assert( h != "localhost" ); @@ -102,10 +133,10 @@ namespace mongo { inline string HostAndPort::toString() const { stringstream ss; ss << _host; - if ( _port != -1 ){ + if ( _port != -1 ) { ss << ':'; #if defined(_DEBUG) - if( _port >= 44000 && _port < 44100 ) { + if( _port >= 44000 && _port < 44100 ) { log() << "warning: special debug port 44xxx used" << endl; ss << _port+1; } @@ -118,7 +149,7 @@ namespace mongo { return ss.str(); } - inline bool HostAndPort::isLocalHost() const { + inline bool HostAndPort::isLocalHost() const { return _host == "localhost" || startsWith(_host.c_str(), "127.") || _host == "::1"; } diff --git a/util/httpclient.cpp b/util/httpclient.cpp index 4f78029..61d5671 100644 --- a/util/httpclient.cpp +++ b/util/httpclient.cpp @@ -27,20 +27,20 @@ namespace mongo { #define HD(x) - int HttpClient::get( string url , Result * result ){ + int HttpClient::get( string url , Result * result ) { return _go( "GET" , url , 0 , result ); } - int HttpClient::post( string url , string data , Result * result ){ + int HttpClient::post( string url , string data , Result * result ) { return _go( "POST" , url , data.c_str() , result ); - } + } - int HttpClient::_go( const char * command , string url , const char * body , Result * result ){ + int HttpClient::_go( const char * command , string url , const char * body , Result * result ) { uassert( 10271 , "invalid url" , url.find( "http://" ) == 0 ); url = url.substr( 7 ); - + string host , path; - if ( url.find( "/" ) == string::npos ){ + if ( url.find( "/" ) == string::npos ) { host = url; path = "/"; } @@ -49,15 +49,15 @@ namespace mongo { path = url.substr( url.find( "/" ) ); } - + HD( "host [" << host << "]" ); HD( "path [" << path << "]" ); string server = host; int port = 80; - + string::size_type idx = host.find( ":" ); - if ( idx != string::npos ){ + if ( idx != string::npos ) { server = host.substr( 0 , idx ); string t = host.substr( idx + 1 ); port = atoi( t.c_str() ); @@ -65,7 +65,7 @@ namespace mongo { HD( "server [" << server << "]" ); HD( "port [" << port << "]" ); - + string req; { stringstream ss; @@ -83,20 +83,20 @@ namespace mongo { req = ss.str(); } - + SockAddr addr( server.c_str() , port ); HD( "addr: " << addr.toString() ); - + MessagingPort p; if ( ! p.connect( addr ) ) return -1; - - { + + { const char * out = req.c_str(); int toSend = req.size(); p.send( out , toSend, "_go" ); } - + char buf[4096]; int got = p.unsafe_recv( buf , 4096 ); buf[got] = 0; @@ -105,46 +105,46 @@ namespace mongo { char version[32]; assert( sscanf( buf , "%s %d" , version , &rc ) == 2 ); HD( "rc: " << rc ); - + StringBuilder sb; if ( result ) sb << buf; - - while ( ( got = p.unsafe_recv( buf , 4096 ) ) > 0){ + + while ( ( got = p.unsafe_recv( buf , 4096 ) ) > 0) { if ( result ) sb << buf; } - if ( result ){ + if ( result ) { result->_init( rc , sb.str() ); } return rc; } - void HttpClient::Result::_init( int code , string entire ){ + void HttpClient::Result::_init( int code , string entire ) { _code = code; _entireResponse = entire; - while ( true ){ + while ( true ) { size_t i = entire.find( '\n' ); - if ( i == string::npos ){ + if ( i == string::npos ) { // invalid break; } - + string h = entire.substr( 0 , i ); entire = entire.substr( i + 1 ); - + if ( h.size() && h[h.size()-1] == '\r' ) h = h.substr( 0 , h.size() - 1 ); if ( h.size() == 0 ) break; } - + _body = entire; } - + } diff --git a/util/httpclient.h b/util/httpclient.h index 8b9da97..d66544e 100644 --- a/util/httpclient.h +++ b/util/httpclient.h @@ -20,28 +20,28 @@ #include "../pch.h" namespace mongo { - + class HttpClient { public: - + class Result { public: - Result(){} - + Result() {} + const string& getEntireResponse() const { return _entireResponse; } - + const map<string,string> getHeaders() const { return _headers; } - + const string& getBody() const { return _body; } - + private: - + void _init( int code , string entire ); int _code; @@ -49,10 +49,10 @@ namespace mongo { map<string,string> _headers; string _body; - + friend class HttpClient; }; - + /** * @return response code */ @@ -65,7 +65,7 @@ namespace mongo { private: int _go( const char * command , string url , const char * body , Result * result ); - + }; } diff --git a/util/log.cpp b/util/log.cpp index 334c66b..eb1cbae 100644 --- a/util/log.cpp +++ b/util/log.cpp @@ -19,7 +19,7 @@ #include "pch.h" #include "assert_util.h" #include "assert.h" -#include "file.h" +//#include "file.h" #include <cmath> using namespace std; @@ -42,49 +42,54 @@ namespace mongo { LoggingManager() : _enabled(0) , _file(0) { } - - void start( const string& lp , bool append ){ + + void start( const string& lp , bool append ) { uassert( 10268 , "LoggingManager already started" , ! _enabled ); _append = append; // test path FILE * test = fopen( lp.c_str() , _append ? "a" : "w" ); - if ( ! test ){ - cout << "can't open [" << lp << "] for log file: " << errnoWithDescription() << endl; + if ( ! test ) { + if (boost::filesystem::is_directory(lp)) { + cout << "logpath [" << lp << "] should be a file name not a directory" << endl; + } + else { + cout << "can't open [" << lp << "] for log file: " << errnoWithDescription() << endl; + } dbexit( EXIT_BADOPTIONS ); assert( 0 ); } fclose( test ); - + _path = lp; _enabled = 1; rotate(); } - - void rotate(){ - if ( ! _enabled ){ + + void rotate() { + if ( ! _enabled ) { cout << "LoggingManager not enabled" << endl; return; } - if ( _file ){ + if ( _file ) { #ifdef _WIN32 cout << "log rotation doesn't work on windows" << endl; return; #else struct tm t; localtime_r( &_opened , &t ); - + stringstream ss; ss << _path << "." << terseCurrentTime(false); string s = ss.str(); rename( _path.c_str() , s.c_str() ); #endif } - - + + FILE* tmp = freopen(_path.c_str(), (_append ? "a" : "w"), stdout); - if (!tmp){ + if (!tmp) { cerr << "can't open: " << _path.c_str() << " for log file" << endl; dbexit( EXIT_BADOPTIONS ); assert(0); @@ -95,24 +100,24 @@ namespace mongo { _file = tmp; _opened = time(0); } - + private: - + bool _enabled; string _path; bool _append; - + FILE * _file; time_t _opened; - + } loggingManager; - void initLogging( const string& lp , bool append ){ + void initLogging( const string& lp , bool append ) { cout << "all output going to: " << lp << endl; loggingManager.start( lp , append ); } - void rotateLogs( int signal ){ + void rotateLogs( int signal ) { loggingManager.rotate(); } @@ -28,24 +28,24 @@ namespace mongo { enum LogLevel { LL_DEBUG , LL_INFO , LL_NOTICE , LL_WARNING , LL_ERROR , LL_SEVERE }; - - inline const char * logLevelToString( LogLevel l ){ - switch ( l ){ + + inline const char * logLevelToString( LogLevel l ) { + switch ( l ) { case LL_DEBUG: - case LL_INFO: + case LL_INFO: case LL_NOTICE: return ""; - case LL_WARNING: - return "warning" ; - case LL_ERROR: + case LL_WARNING: + return "warning" ; + case LL_ERROR: return "ERROR"; - case LL_SEVERE: + case LL_SEVERE: return "SEVERE"; default: return "UNKNOWN"; } } - + class LazyString { public: virtual ~LazyString() {} @@ -62,15 +62,15 @@ namespace mongo { const T& t_; }; - class Tee { + class Tee { public: - virtual ~Tee(){} + virtual ~Tee() {} virtual void write(LogLevel level , const string& str) = 0; }; class Nullstream { public: - virtual Nullstream& operator<< (Tee* tee) { + virtual Nullstream& operator<< (Tee* tee) { return *this; } virtual ~Nullstream() {} @@ -80,6 +80,9 @@ namespace mongo { virtual Nullstream& operator<<(const string& ) { return *this; } + virtual Nullstream& operator<<(const StringData& ) { + return *this; + } virtual Nullstream& operator<<(char *) { return *this; } @@ -125,53 +128,72 @@ namespace mongo { template< class T > Nullstream& operator<<(T *t) { return operator<<( static_cast<void*>( t ) ); - } + } template< class T > Nullstream& operator<<(const T *t) { return operator<<( static_cast<const void*>( t ) ); - } + } template< class T > - Nullstream& operator<<(const shared_ptr<T> p ){ + Nullstream& operator<<(const shared_ptr<T> p ) { + T * t = p.get(); + if ( ! t ) + *this << "null"; + else + *this << *t; return *this; } template< class T > Nullstream& operator<<(const T &t) { return operator<<( static_cast<const LazyString&>( LazyStringImpl< T >( t ) ) ); } + virtual Nullstream& operator<< (ostream& ( *endl )(ostream&)) { return *this; } virtual Nullstream& operator<< (ios_base& (*hex)(ios_base&)) { return *this; } + virtual void flush(Tee *t = 0) {} }; extern Nullstream nullstream; - + class Logstream : public Nullstream { static mongo::mutex mutex; static int doneSetup; stringstream ss; + int indent; LogLevel logLevel; static FILE* logfile; static boost::scoped_ptr<ostream> stream; static vector<Tee*> * globalTees; public: - inline static void logLockless( const StringData& s ); - - static void setLogFile(FILE* f){ + + static void setLogFile(FILE* f) { scoped_lock lk(mutex); logfile = f; } - static int magicNumber(){ + static int magicNumber() { return 1717; } + static int getLogDesc() { + int fd = -1; + if (logfile != NULL) +#if defined(_WIN32) + // the ISO C++ conformant name is _fileno + fd = _fileno( logfile ); +#else + fd = fileno( logfile ); +#endif + return fd; + } + inline void flush(Tee *t = 0); - - inline Nullstream& setLogLevel(LogLevel l){ + + inline Nullstream& setLogLevel(LogLevel l) { logLevel = l; return *this; } @@ -179,6 +201,7 @@ namespace mongo { /** note these are virtual */ Logstream& operator<<(const char *x) { ss << x; return *this; } Logstream& operator<<(const string& x) { ss << x; return *this; } + Logstream& operator<<(const StringData& x) { ss << x.data(); return *this; } Logstream& operator<<(char *x) { ss << x; return *this; } Logstream& operator<<(char x) { ss << x; return *this; } Logstream& operator<<(int x) { ss << x; return *this; } @@ -197,7 +220,7 @@ namespace mongo { ss << x.val(); return *this; } - Nullstream& operator<< (Tee* tee) { + Nullstream& operator<< (Tee* tee) { ss << '\n'; flush(tee); return *this; @@ -212,32 +235,27 @@ namespace mongo { return *this; } - template< class T > - Nullstream& operator<<(const shared_ptr<T> p ){ - T * t = p.get(); - if ( ! t ) - *this << "null"; - else - *this << *t; - return *this; - } - Logstream& prolog() { return *this; } - - void addGlobalTee( Tee * t ){ + + void addGlobalTee( Tee * t ) { if ( ! globalTees ) globalTees = new vector<Tee*>(); globalTees->push_back( t ); } + + void indentInc(){ indent++; } + void indentDec(){ indent--; } + int getIndent() const { return indent; } private: static thread_specific_ptr<Logstream> tsp; - Logstream(){ + Logstream() { + indent = 0; _init(); } - void _init(){ + void _init() { ss.str(""); logLevel = LL_INFO; } @@ -258,16 +276,16 @@ namespace mongo { return nullstream; return Logstream::get(); } - - /* flush the log stream if the log level is + + /* flush the log stream if the log level is at the specified level or higher. */ - inline void logflush(int level = 0) { + inline void logflush(int level = 0) { if( level > logLevel ) Logstream::get().flush(0); } /* without prolog */ - inline Nullstream& _log( int level = 0 ){ + inline Nullstream& _log( int level = 0 ) { if ( level > logLevel ) return nullstream; return Logstream::get(); @@ -287,6 +305,9 @@ namespace mongo { return Logstream::get().prolog(); } +#define MONGO_LOG(level) if ( logLevel >= (level) ) log( level ) +#define LOG MONGO_LOG + inline Nullstream& log( LogLevel l ) { return Logstream::get().prolog().setLogLevel( l ); } @@ -295,7 +316,7 @@ namespace mongo { inline Nullstream& log() { return Logstream::get().prolog(); } - + inline Nullstream& error() { return log( LL_ERROR ); } @@ -317,7 +338,7 @@ namespace mongo { /** log to a file rather than stdout - defined in assert_util.cpp + defined in assert_util.cpp */ void initLogging( const string& logpath , bool append ); void rotateLogs( int signal = 0 ); @@ -333,7 +354,7 @@ namespace mongo { FormatMessage( FORMAT_MESSAGE_FROM_SYSTEM |FORMAT_MESSAGE_ALLOCATE_BUFFER - |FORMAT_MESSAGE_IGNORE_INSERTS, + |FORMAT_MESSAGE_IGNORE_INSERTS, NULL, x, 0, (LPTSTR) &errorText, // output @@ -342,7 +363,7 @@ namespace mongo { if( errorText ) { string x = toUtf8String(errorText); for( string::iterator i = x.begin(); i != x.end(); i++ ) { - if( *i == '\n' || *i == '\r' ) + if( *i == '\n' || *i == '\r' ) break; s << *i; } @@ -351,11 +372,11 @@ namespace mongo { else s << strerror(x); /* - DWORD n = FormatMessage( - FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | + DWORD n = FormatMessage( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, x, + NULL, x, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR) &lpMsgBuf, 0, NULL); */ @@ -365,22 +386,28 @@ namespace mongo { return s.str(); } - /** output the error # and error message with prefix. + /** output the error # and error message with prefix. handy for use as parm in uassert/massert. */ string errnoWithPrefix( const char * prefix ); - void Logstream::logLockless( const StringData& s ){ - if ( doneSetup == 1717 ){ - if(fwrite(s.data(), s.size(), 1, logfile)){ + void Logstream::logLockless( const StringData& s ) { + + if ( s.size() == 0 ) + return; + + if ( doneSetup == 1717 ) { + if (fwrite(s.data(), s.size(), 1, logfile)) { fflush(logfile); - }else{ + } + else { int x = errno; - cout << "Failed to write to logfile: " << errnoWithDescription(x) << ": " << out << endl; + cout << "Failed to write to logfile: " << errnoWithDescription(x) << endl; } } else { - cout << s.data() << endl; + cout << s.data(); + cout.flush(); } } @@ -391,23 +418,28 @@ namespace mongo { string threadName = getThreadName(); const char * type = logLevelToString(logLevel); - int spaceNeeded = msg.size() + 64 + threadName.size(); + int spaceNeeded = (int)(msg.size() + 64 + threadName.size()); int bufSize = 128; while ( bufSize < spaceNeeded ) bufSize += 128; BufBuilder b(bufSize); time_t_to_String( time(0) , b.grow(20) ); - if (!threadName.empty()){ + if (!threadName.empty()) { b.appendChar( '[' ); b.appendStr( threadName , false ); b.appendChar( ']' ); b.appendChar( ' ' ); } - if ( type[0] ){ + + for ( int i=0; i<indent; i++ ) + b.appendChar( '\t' ); + + if ( type[0] ) { b.appendStr( type , false ); b.appendStr( ": " , false ); } + b.appendStr( msg ); string out( b.buf() , b.len() - 1); @@ -415,7 +447,7 @@ namespace mongo { scoped_lock lk(mutex); if( t ) t->write(logLevel,out); - if ( globalTees ){ + if ( globalTees ) { for ( unsigned i=0; i<globalTees->size(); i++ ) (*globalTees)[i]->write(logLevel,out); } @@ -423,9 +455,10 @@ namespace mongo { #ifndef _WIN32 //syslog( LOG_INFO , "%s" , cc ); #endif - if(fwrite(out.data(), out.size(), 1, logfile)){ + if(fwrite(out.data(), out.size(), 1, logfile)) { fflush(logfile); - }else{ + } + else { int x = errno; cout << "Failed to write to logfile: " << errnoWithDescription(x) << ": " << out << endl; } @@ -433,4 +466,13 @@ namespace mongo { _init(); } + struct LogIndentLevel { + LogIndentLevel(){ + Logstream::get().indentInc(); + } + ~LogIndentLevel(){ + Logstream::get().indentDec(); + } + }; + } // namespace mongo diff --git a/util/logfile.cpp b/util/logfile.cpp new file mode 100644 index 0000000..0386a59 --- /dev/null +++ b/util/logfile.cpp @@ -0,0 +1,157 @@ +// @file logfile.cpp simple file log writing / journaling + +/** +* Copyright (C) 2008 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#include "pch.h" +#include "logfile.h" +#include "text.h" +#include "mongoutils/str.h" +#include "unittest.h" + +using namespace mongoutils; + +namespace mongo { + struct LogfileTest : public UnitTest { + LogfileTest() { } + void run() { + if( 0 && debug ) { + try { + LogFile f("logfile_test"); + void *p = malloc(16384); + char *buf = (char*) p; + buf += 4095; + buf = (char*) (((size_t)buf)&(~0xfff)); + memset(buf, 'z', 8192); + buf[8190] = '\n'; + buf[8191] = 'B'; + buf[0] = 'A'; + f.synchronousAppend(buf, 8192); + f.synchronousAppend(buf, 8192); + free(p); + } + catch(DBException& e ) { + log() << "logfile.cpp test failed : " << e.what() << endl; + throw; + } + } + } + } __test; +} + +#if defined(_WIN32) + +namespace mongo { + + LogFile::LogFile(string name) : _name(name) { + _fd = CreateFile( + toNativeString(name.c_str()).c_str(), + GENERIC_WRITE, + FILE_SHARE_READ, + NULL, + OPEN_ALWAYS, + FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH, + NULL); + if( _fd == INVALID_HANDLE_VALUE ) { + DWORD e = GetLastError(); + uasserted(13518, str::stream() << "couldn't open file " << name << " for writing " << errnoWithDescription(e)); + } + SetFilePointer(_fd, 0, 0, FILE_BEGIN); + } + + LogFile::~LogFile() { + if( _fd != INVALID_HANDLE_VALUE ) + CloseHandle(_fd); + } + + void LogFile::synchronousAppend(const void *buf, size_t len) { + assert(_fd); + DWORD written; + if( !WriteFile(_fd, buf, len, &written, NULL) ) { + DWORD e = GetLastError(); + if( e == 87 ) + massert(13519, "error appending to file - misaligned direct write?", false); + else + uasserted(13517, str::stream() << "error appending to file " << errnoWithDescription(e)); + } + else { + dassert( written == len ); + } + } + +} + +#else + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +namespace mongo { + + LogFile::LogFile(string name) : _name(name) { + _fd = open(name.c_str(), + O_CREAT + | O_WRONLY +#if defined(O_DIRECT) + | O_DIRECT +#endif +#if defined(O_NOATIME) + | O_NOATIME +#endif + , + S_IRUSR | S_IWUSR); + if( _fd < 0 ) { + uasserted(13516, str::stream() << "couldn't open file " << name << " for writing " << errnoWithDescription()); + } + + } + + LogFile::~LogFile() { + if( _fd >= 0 ) + close(_fd); + _fd = -1; + } + + void LogFile::synchronousAppend(const void *b, size_t len) { + const char *buf = (char *) b; + assert(_fd); + assert(((size_t)buf)%4096==0); // aligned + if( len % 4096 != 0 ) { + log() << len << ' ' << len % 4096 << endl; + assert(false); + } + ssize_t written = write(_fd, buf, len); + if( written != (ssize_t) len ) { + log() << "write fails written:" << written << " len:" << len << " buf:" << buf << ' ' << errnoWithDescription() << endl; + uasserted(13515, str::stream() << "error appending to file " << _fd << ' ' << errnoWithDescription()); + } + + if( +#if defined(__linux__) + fdatasync(_fd) < 0 +#else + fsync(_fd) +#endif + ) { + uasserted(13514, str::stream() << "error appending to file on fsync " << ' ' << errnoWithDescription()); + } + + } + +} + +#endif diff --git a/util/logfile.h b/util/logfile.h new file mode 100644 index 0000000..9085161 --- /dev/null +++ b/util/logfile.h @@ -0,0 +1,50 @@ +// @file logfile.h simple file log writing / journaling + +/** +* Copyright (C) 2010 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#pragma once + +namespace mongo { + + class LogFile { + public: + /** create the file and open. must not already exist. + throws UserAssertion on i/o error + */ + LogFile(string name); + + /** closes */ + ~LogFile(); + + /** append to file. does not return until sync'd. uses direct i/o when possible. + throws UserAssertion on an i/o error + note direct i/o may have alignment requirements + */ + void synchronousAppend(const void *buf, size_t len); + + const string _name; + + private: +#if defined(_WIN32) + typedef HANDLE fd_type; +#else + typedef int fd_type; +#endif + fd_type _fd; + }; + +} diff --git a/util/lruishmap.h b/util/lruishmap.h index fe8b1dc..ba91bf6 100644 --- a/util/lruishmap.h +++ b/util/lruishmap.h @@ -23,8 +23,8 @@ namespace mongo { /* Your K object must define: - int hash() - must always return > 0. - operator== + int hash() - must always return > 0. + operator== */ template <class K, class V, int MaxChain> @@ -27,7 +27,7 @@ This code implements the MD5 Algorithm defined in RFC 1321, whose text is available at - http://www.ietf.org/rfc/rfc1321.txt + http://www.ietf.org/rfc/rfc1321.txt The code is derived from the text of the RFC, including the test suite (section A.5) but excluding the rest of Appendix A. It does not include any code or documentation that is identified in the RFC as being @@ -38,12 +38,12 @@ that follows (in reverse chronological order): 2002-04-13 lpd Removed support for non-ANSI compilers; removed - references to Ghostscript; clarified derivation from RFC 1321; - now handles byte order either statically or dynamically. + references to Ghostscript; clarified derivation from RFC 1321; + now handles byte order either statically or dynamically. 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5); - added conditionalization for C++ compilation from Martin - Purschke <purschke@bnl.gov>. + added conditionalization for C++ compilation from Martin + Purschke <purschke@bnl.gov>. 1999-05-03 lpd Original version. */ @@ -65,9 +65,9 @@ typedef unsigned int md5_word_t; /* 32-bit word */ /* Define the state of the MD5 Algorithm. */ typedef struct md5_state_s { - md5_word_t count[2]; /* message length in bits, lsw first */ - md5_word_t abcd[4]; /* digest buffer */ - md5_byte_t buf[64]; /* accumulate block */ + md5_word_t count[2]; /* message length in bits, lsw first */ + md5_word_t abcd[4]; /* digest buffer */ + md5_byte_t buf[64]; /* accumulate block */ } md5_state_t; #ifdef __cplusplus diff --git a/util/md5.hpp b/util/md5.hpp index d955910..dc06171 100644 --- a/util/md5.hpp +++ b/util/md5.hpp @@ -44,10 +44,15 @@ namespace mongo { return ss.str(); } - inline std::string md5simpledigest( string s ){ + inline std::string md5simpledigest( const void* buf, int nbytes){ md5digest d; - md5( s.c_str() , d ); + md5( buf, nbytes , d ); return digestToString( d ); } + inline std::string md5simpledigest( string s ){ + return md5simpledigest(s.data(), s.size()); + } + + } // namespace mongo diff --git a/util/md5main.cpp b/util/md5main.cpp index 9c56f91..9995fee 100644 --- a/util/md5main.cpp +++ b/util/md5main.cpp @@ -27,7 +27,7 @@ This code implements the MD5 Algorithm defined in RFC 1321, whose text is available at - http://www.ietf.org/rfc/rfc1321.txt + http://www.ietf.org/rfc/rfc1321.txt The code is derived from the text of the RFC, including the test suite (section A.5) but excluding the rest of Appendix A. It does not include any code or documentation that is identified in the RFC as being @@ -49,7 +49,7 @@ /* * This file builds an executable that performs various functions related * to the MD5 library. Typical compilation: - * gcc -o md5main -lm md5main.c md5.c + * gcc -o md5main -lm md5main.c md5.c */ static const char *const usage = "\ Usage:\n\ @@ -63,62 +63,61 @@ static const char *const version = "2002-04-13"; /* Run the self-test. */ /*static*/ int //do_test(void) -do_md5_test(void) -{ +do_md5_test(void) { static const char *const test[7*2] = { - "", "d41d8cd98f00b204e9800998ecf8427e", - "a", "0cc175b9c0f1b6a831c399e269772661", - "abc", "900150983cd24fb0d6963f7d28e17f72", - "message digest", "f96b697d7cb7938d525a2f31aaf161d0", - "abcdefghijklmnopqrstuvwxyz", "c3fcd3d76192e4007dfb496cca67e13b", - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", - "d174ab98d277d9f5a5611c2c9f419d9f", - "12345678901234567890123456789012345678901234567890123456789012345678901234567890", "57edf4a22be3c955ac49da2e2107b67a" + "", "d41d8cd98f00b204e9800998ecf8427e", + "a", "0cc175b9c0f1b6a831c399e269772661", + "abc", "900150983cd24fb0d6963f7d28e17f72", + "message digest", "f96b697d7cb7938d525a2f31aaf161d0", + "abcdefghijklmnopqrstuvwxyz", "c3fcd3d76192e4007dfb496cca67e13b", + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + "d174ab98d277d9f5a5611c2c9f419d9f", + "12345678901234567890123456789012345678901234567890123456789012345678901234567890", "57edf4a22be3c955ac49da2e2107b67a" }; int i; int status = 0; for (i = 0; i < 7*2; i += 2) { - md5_state_t state; - md5_byte_t digest[16]; - char hex_output[16*2 + 1]; - int di; - - md5_init(&state); - md5_append(&state, (const md5_byte_t *)test[i], strlen(test[i])); - md5_finish(&state, digest); - for (di = 0; di < 16; ++di) - sprintf(hex_output + di * 2, "%02x", digest[di]); - if (strcmp(hex_output, test[i + 1])) { - printf("MD5 (\"%s\") = ", test[i]); - puts(hex_output); - printf("**** ERROR, should be: %s\n", test[i + 1]); - status = 1; - } + md5_state_t state; + md5_byte_t digest[16]; + char hex_output[16*2 + 1]; + int di; + + md5_init(&state); + md5_append(&state, (const md5_byte_t *)test[i], strlen(test[i])); + md5_finish(&state, digest); + for (di = 0; di < 16; ++di) + sprintf(hex_output + di * 2, "%02x", digest[di]); + if (strcmp(hex_output, test[i + 1])) { + printf("MD5 (\"%s\") = ", test[i]); + puts(hex_output); + printf("**** ERROR, should be: %s\n", test[i + 1]); + status = 1; + } } // if (status == 0) -/*modified commented out: puts("md5 self-test completed successfully."); */ + /*modified commented out: puts("md5 self-test completed successfully."); */ return status; } /* Print the T values. */ static int -do_t_values(void) -{ +do_t_values(void) { int i; for (i = 1; i <= 64; ++i) { - unsigned long v = (unsigned long)(4294967296.0 * fabs(sin((double)i))); - - /* - * The following nonsense is only to avoid compiler warnings about - * "integer constant is unsigned in ANSI C, signed with -traditional". - */ - if (v >> 31) { - printf("#define T%d /* 0x%08lx */ (T_MASK ^ 0x%08lx)\n", i, - v, (unsigned long)(unsigned int)(~v)); - } else { - printf("#define T%d 0x%08lx\n", i, v); - } + unsigned long v = (unsigned long)(4294967296.0 * fabs(sin((double)i))); + + /* + * The following nonsense is only to avoid compiler warnings about + * "integer constant is unsigned in ANSI C, signed with -traditional". + */ + if (v >> 31) { + printf("#define T%d /* 0x%08lx */ (T_MASK ^ 0x%08lx)\n", i, + v, (unsigned long)(unsigned int)(~v)); + } + else { + printf("#define T%d 0x%08lx\n", i, v); + } } return 0; } @@ -126,17 +125,16 @@ do_t_values(void) /* modified from original code changed function name main->md5main */ /* Main program */ int -md5main(int argc, char *argv[]) -{ +md5main(int argc, char *argv[]) { if (argc == 2) { - if (!strcmp(argv[1], "--test")) - return do_md5_test(); - if (!strcmp(argv[1], "--t-values")) - return do_t_values(); - if (!strcmp(argv[1], "--version")) { - puts(version); - return 0; - } + if (!strcmp(argv[1], "--test")) + return do_md5_test(); + if (!strcmp(argv[1], "--t-values")) + return do_t_values(); + if (!strcmp(argv[1], "--version")) { + puts(version); + return 0; + } } puts(usage); return 0; diff --git a/util/message.cpp b/util/message.cpp index cd19bd5..653b562 100644 --- a/util/message.cpp +++ b/util/message.cpp @@ -27,9 +27,14 @@ #include <errno.h> #include "../db/cmdline.h" #include "../client/dbclient.h" +#include "../util/time_support.h" #ifndef _WIN32 -#include <sys/resource.h> +# ifndef __sunos__ +# include <ifaddrs.h> +# endif +# include <sys/resource.h> +# include <sys/stat.h> #else // errno doesn't work for winsock. @@ -45,7 +50,7 @@ namespace mongo { bool objcheck = false; void checkTicketNumbers(); - + // if you want trace output: #define mmm(x) @@ -59,9 +64,23 @@ namespace mongo { const Listener* Listener::_timeTracker; - vector<SockAddr> ipToAddrs(const char* ips, int port){ + string SocketException::toString() const { + stringstream ss; + ss << _ei.code << " socket exception [" << _type << "] "; + + if ( _server.size() ) + ss << "server [" << _server << "] "; + + if ( _extra.size() ) + ss << _extra; + + return ss.str(); + } + + + vector<SockAddr> ipToAddrs(const char* ips, int port) { vector<SockAddr> out; - if (*ips == '\0'){ + if (*ips == '\0') { out.push_back(SockAddr("0.0.0.0", port)); // IPv4 all if (IPv6Enabled()) @@ -73,13 +92,14 @@ namespace mongo { return out; } - while(*ips){ + while(*ips) { string ip; const char * comma = strchr(ips, ','); - if (comma){ + if (comma) { ip = string(ips, comma - ips); ips = comma + 1; - }else{ + } + else { ip = string(ips); ips = ""; } @@ -104,7 +124,7 @@ namespace mongo { vector<int> socks; SOCKET maxfd = 0; // needed for select() - for (vector<SockAddr>::iterator it=mine.begin(), end=mine.end(); it != end; ++it){ + for (vector<SockAddr>::iterator it=mine.begin(), end=mine.end(); it != end; ++it) { SockAddr& me = *it; SOCKET sock = ::socket(me.getType(), SOCK_STREAM, 0); @@ -112,17 +132,18 @@ namespace mongo { log() << "ERROR: listen(): invalid socket? " << errnoWithDescription() << endl; } - if (me.getType() == AF_UNIX){ + if (me.getType() == AF_UNIX) { #if !defined(_WIN32) - if (unlink(me.getAddr().c_str()) == -1){ + if (unlink(me.getAddr().c_str()) == -1) { int x = errno; - if (x != ENOENT){ + if (x != ENOENT) { log() << "couldn't unlink socket file " << me << errnoWithDescription(x) << " skipping" << endl; continue; } } #endif - } else if (me.getType() == AF_INET6) { + } + else if (me.getType() == AF_INET6) { // IPv6 can also accept IPv4 connections as mapped addresses (::ffff:127.0.0.1) // That causes a conflict if we don't do set it to IPV6_ONLY const int one = 1; @@ -130,7 +151,7 @@ namespace mongo { } prebindOptions( sock ); - + if ( ::bind(sock, me.raw(), me.addressSize) != 0 ) { int x = errno; log() << "listen(): bind() failed " << errnoWithDescription(x) << " for socket: " << me.toString() << endl; @@ -140,6 +161,16 @@ namespace mongo { return; } +#if !defined(_WIN32) + if (me.getType() == AF_UNIX) { + if (chmod(me.getAddr().c_str(), 0777) == -1) { + log() << "couldn't chmod socket file " << me << errnoWithDescription() << endl; + } + + ListeningSockets::get()->addPath( me.getAddr() ); + } +#endif + if ( ::listen(sock, 128) != 0 ) { log() << "listen(): listen() failed " << errnoWithDescription() << endl; closesocket(sock); @@ -159,15 +190,15 @@ namespace mongo { fd_set fds[1]; FD_ZERO(fds); - for (vector<int>::iterator it=socks.begin(), end=socks.end(); it != end; ++it){ + for (vector<int>::iterator it=socks.begin(), end=socks.end(); it != end; ++it) { FD_SET(*it, fds); } maxSelectTime.tv_sec = 0; maxSelectTime.tv_usec = 10000; const int ret = select(maxfd+1, fds, NULL, NULL, &maxSelectTime); - - if (ret == 0){ + + if (ret == 0) { #if defined(__linux__) _elapsedTime += ( 10000 - maxSelectTime.tv_usec ) / 1000; #else @@ -176,11 +207,11 @@ namespace mongo { continue; } _elapsedTime += ret; // assume 1ms to grab connection. very rough - - if (ret < 0){ + + if (ret < 0) { int x = errno; #ifdef EINTR - if ( x == EINTR ){ + if ( x == EINTR ) { log() << "select() signal caught, continuing" << endl; continue; } @@ -190,7 +221,7 @@ namespace mongo { return; } - for (vector<int>::iterator it=socks.begin(), end=socks.end(); it != end; ++it){ + for (vector<int>::iterator it=socks.begin(), end=socks.end(); it != end; ++it) { if (! (FD_ISSET(*it, fds))) continue; @@ -201,24 +232,24 @@ namespace mongo { if ( x == ECONNABORTED || x == EBADF ) { log() << "Listener on port " << _port << " aborted" << endl; return; - } + } if ( x == 0 && inShutdown() ) { return; // socket closed } if( !inShutdown() ) log() << "Listener: accept() returns " << s << " " << errnoWithDescription(x) << endl; continue; - } + } if (from.getType() != AF_UNIX) disableNagle(s); - if ( _logConnect && ! cmdLine.quiet ) + if ( _logConnect && ! cmdLine.quiet ) log() << "connection accepted from " << from.toString() << " #" << ++connNumber << endl; accepted(s, from); } } } - void Listener::accepted(int sock, const SockAddr& from){ + void Listener::accepted(int sock, const SockAddr& from) { accepted( new MessagingPort(sock, from) ); } @@ -265,7 +296,7 @@ namespace mongo { char * _cur; }; - class Ports { + class Ports { set<MessagingPort*> ports; mongo::mutex m; public: @@ -278,11 +309,11 @@ namespace mongo { (*i)->shutdown(); } } - void insert(MessagingPort* p) { + void insert(MessagingPort* p) { scoped_lock bl(m); ports.insert(p); } - void erase(MessagingPort* p) { + void erase(MessagingPort* p) { scoped_lock bl(m); ports.erase(p); } @@ -296,12 +327,12 @@ namespace mongo { ports.closeAll(mask); } - MessagingPort::MessagingPort(int _sock, const SockAddr& _far) : sock(_sock), piggyBackData(0), farEnd(_far), _timeout(), tag(0) { + MessagingPort::MessagingPort(int _sock, const SockAddr& _far) : sock(_sock), piggyBackData(0), _bytesIn(0), _bytesOut(0), farEnd(_far), _timeout(), tag(0) { _logLevel = 0; ports.insert(this); } - MessagingPort::MessagingPort( double timeout, int ll ) : tag(0) { + MessagingPort::MessagingPort( double timeout, int ll ) : _bytesIn(0), _bytesOut(0), tag(0) { _logLevel = ll; ports.insert(this); sock = -1; @@ -325,17 +356,19 @@ namespace mongo { class ConnectBG : public BackgroundJob { public: - int sock; - int res; - SockAddr farEnd; - void run() { - res = ::connect(sock, farEnd.raw(), farEnd.addressSize); - } - string name() { return "ConnectBG"; } + ConnectBG(int sock, SockAddr farEnd) : _sock(sock), _farEnd(farEnd) { } + + void run() { _res = ::connect(_sock, _farEnd.raw(), _farEnd.addressSize); } + string name() const { return ""; /* too short lived to need to name */ } + int inError() const { return _res; } + + private: + int _sock; + int _res; + SockAddr _farEnd; }; - bool MessagingPort::connect(SockAddr& _far) - { + bool MessagingPort::connect(SockAddr& _far) { farEnd = _far; sock = socket(farEnd.getType(), SOCK_STREAM, 0); @@ -347,14 +380,11 @@ namespace mongo { if ( _timeout > 0 ) { setSockTimeouts( sock, _timeout ); } - - ConnectBG bg; - bg.sock = sock; - bg.farEnd = farEnd; - bg.go(); + ConnectBG bg(sock, farEnd); + bg.go(); if ( bg.wait(5000) ) { - if ( bg.res ) { + if ( bg.inError() ) { closesocket(sock); sock = -1; return false; @@ -377,28 +407,37 @@ namespace mongo { setsockopt(sock, SOL_SOCKET, SO_NOSIGPIPE, &one, sizeof(int)); #endif + /* + // SO_LINGER is bad + #ifdef SO_LINGER + struct linger ling; + ling.l_onoff = 1; + ling.l_linger = 0; + setsockopt(sock, SOL_SOCKET, SO_LINGER, (char *) &ling, sizeof(ling)); + #endif + */ return true; } bool MessagingPort::recv(Message& m) { try { - again: +again: mmm( log() << "* recv() sock:" << this->sock << endl; ) int len = -1; - + char *lenbuf = (char *) &len; int lft = 4; recv( lenbuf, lft ); - - if ( len < 16 || len > 16000000 ) { // messages must be large enough for headers + + if ( len < 16 || len > 48000000 ) { // messages must be large enough for headers if ( len == -1 ) { - // Endian check from the database, after connecting, to see what mode server is running in. + // Endian check from the client, after connecting, to see what mode server is running in. unsigned foo = 0x10203040; send( (char *) &foo, 4, "endian" ); goto again; } - - if ( len == 542393671 ){ + + if ( len == 542393671 ) { // an http GET log(_logLevel) << "looks like you're trying to access db over http on native driver port. please add 1000 for webserver" << endl; string msg = "You are trying to access MongoDB on the native driver port. For http diagnostic access, add 1000 to the port number\n"; @@ -408,36 +447,39 @@ namespace mongo { send( s.c_str(), s.size(), "http" ); return false; } - log(_logLevel) << "bad recv() len: " << len << '\n'; + log(0) << "recv(): message len " << len << " is too large" << len << endl; return false; } - + int z = (len+1023)&0xfffffc00; assert(z>=len); MsgData *md = (MsgData *) malloc(z); assert(md); md->len = len; - + char *p = (char *) &md->id; int left = len -4; try { recv( p, left ); - } catch (...) { + } + catch (...) { free(md); throw; } - + + _bytesIn += len; m.setData(md, true); return true; - - } catch ( const SocketException & e ) { - log(_logLevel + (e.shouldPrint() ? 0 : 1) ) << "SocketException: " << e << endl; + + } + catch ( const SocketException & e ) { + log(_logLevel + (e.shouldPrint() ? 0 : 1) ) << "SocketException: remote: " << remote() << " error: " << e << endl; m.reset(); return false; } } - + void MessagingPort::reply(Message& received, Message& response) { say(/*received.from, */response, received.header()->id); } @@ -448,8 +490,11 @@ namespace mongo { bool MessagingPort::call(Message& toSend, Message& response) { mmm( log() << "*call()" << endl; ) - MSGID old = toSend.header()->id; - say(/*to,*/ toSend); + say(toSend); + return recv( toSend , response ); + } + + bool MessagingPort::recv( const Message& toSend , Message& response ) { while ( 1 ) { bool ok = recv(response); if ( !ok ) @@ -457,13 +502,13 @@ namespace mongo { //log() << "got response: " << response.data->responseTo << endl; if ( response.header()->responseTo == toSend.header()->id ) break; - log() << "********************" << endl; - log() << "ERROR: MessagingPort::call() wrong id got:" << hex << (unsigned)response.header()->responseTo << " expect:" << (unsigned)toSend.header()->id << endl; - log() << " toSend op: " << toSend.operation() << " old id:" << (unsigned)old << endl; - log() << " response msgid:" << (unsigned)response.header()->id << endl; - log() << " response len: " << (unsigned)response.header()->len << endl; - log() << " response op: " << response.operation() << endl; - log() << " farEnd: " << farEnd << endl; + error() << "MessagingPort::call() wrong id got:" << hex << (unsigned)response.header()->responseTo << " expect:" << (unsigned)toSend.header()->id << '\n' + << dec + << " toSend op: " << (unsigned)toSend.operation() << '\n' + << " response msgid:" << (unsigned)response.header()->id << '\n' + << " response len: " << (unsigned)response.header()->len << '\n' + << " response op: " << response.operation() << '\n' + << " farEnd: " << farEnd << endl; assert(false); response.reset(); } @@ -493,30 +538,37 @@ namespace mongo { toSend.send( *this, "say" ); } - // sends all data or throws an exception + // sends all data or throws an exception void MessagingPort::send( const char * data , int len, const char *context ) { + _bytesOut += len; while( len > 0 ) { int ret = ::send( sock , data , len , portSendFlags ); if ( ret == -1 ) { if ( errno != EAGAIN || _timeout == 0 ) { + SocketException::Type t = SocketException::SEND_ERROR; +#if defined(_WINDOWS) + if( e == WSAETIMEDOUT ) t = SocketException::SEND_TIMEOUT; +#endif log(_logLevel) << "MessagingPort " << context << " send() " << errnoWithDescription() << ' ' << farEnd.toString() << endl; - throw SocketException( SocketException::SEND_ERROR ); - } else { + throw SocketException( t ); + } + else { if ( !serverAlive( farEnd.toString() ) ) { log(_logLevel) << "MessagingPort " << context << " send() remote dead " << farEnd.toString() << endl; - throw SocketException( SocketException::SEND_ERROR ); + throw SocketException( SocketException::SEND_ERROR ); } } - } else { + } + else { assert( ret <= len ); len -= ret; data += ret; } - } + } } - + // sends all data or throws an exception - void MessagingPort::send( const vector< pair< char *, int > > &data, const char *context ){ + void MessagingPort::send( const vector< pair< char *, int > > &data, const char *context ) { #if defined(_WIN32) // TODO use scatter/gather api for( vector< pair< char *, int > >::const_iterator i = data.begin(); i != data.end(); ++i ) { @@ -538,27 +590,30 @@ namespace mongo { memset( &meta, 0, sizeof( meta ) ); meta.msg_iov = &d[ 0 ]; meta.msg_iovlen = d.size(); - + while( meta.msg_iovlen > 0 ) { int ret = ::sendmsg( sock , &meta , portSendFlags ); if ( ret == -1 ) { if ( errno != EAGAIN || _timeout == 0 ) { log(_logLevel) << "MessagingPort " << context << " send() " << errnoWithDescription() << ' ' << farEnd.toString() << endl; - throw SocketException( SocketException::SEND_ERROR ); - } else { + throw SocketException( SocketException::SEND_ERROR ); + } + else { if ( !serverAlive( farEnd.toString() ) ) { log(_logLevel) << "MessagingPort " << context << " send() remote dead " << farEnd.toString() << endl; - throw SocketException( SocketException::SEND_ERROR ); + throw SocketException( SocketException::SEND_ERROR ); } } - } else { + } + else { struct iovec *& i = meta.msg_iov; while( ret > 0 ) { if ( i->iov_len > unsigned( ret ) ) { i->iov_len -= ret; i->iov_base = (char*)(i->iov_base) + ret; ret = 0; - } else { + } + else { ret -= i->iov_len; ++i; --(meta.msg_iovlen); @@ -569,25 +624,42 @@ namespace mongo { #endif } - void MessagingPort::recv( char * buf , int len ){ + void MessagingPort::recv( char * buf , int len ) { + unsigned retries = 0; while( len > 0 ) { int ret = ::recv( sock , buf , len , portRecvFlags ); if ( ret == 0 ) { log(3) << "MessagingPort recv() conn closed? " << farEnd.toString() << endl; throw SocketException( SocketException::CLOSED ); } - if ( ret == -1 ) { + if ( ret < 0 ) { int e = errno; - if ( e != EAGAIN || _timeout == 0 ) { +#if defined(EINTR) && !defined(_WIN32) + if( e == EINTR ) { + if( ++retries == 1 ) { + log() << "EINTR retry" << endl; + continue; + } + } +#endif + if ( e != EAGAIN || _timeout == 0 ) { + SocketException::Type t = SocketException::RECV_ERROR; +#if defined(_WINDOWS) + if( e == WSAETIMEDOUT ) t = SocketException::RECV_TIMEOUT; +#else + /* todo: what is the error code on an SO_RCVTIMEO on linux? EGAIN? EWOULDBLOCK? */ +#endif log(_logLevel) << "MessagingPort recv() " << errnoWithDescription(e) << " " << farEnd.toString() <<endl; - throw SocketException( SocketException::RECV_ERROR ); - } else { + throw SocketException(t); + } + else { if ( !serverAlive( farEnd.toString() ) ) { log(_logLevel) << "MessagingPort recv() remote dead " << farEnd.toString() << endl; - throw SocketException( SocketException::RECV_ERROR ); + throw SocketException( SocketException::RECV_ERROR ); } } - } else { + } + else { if ( len <= 4 && ret != len ) log(_logLevel) << "MessagingPort recv() got " << ret << " bytes wanted len=" << len << endl; assert( ret <= len ); @@ -598,9 +670,9 @@ namespace mongo { } int MessagingPort::unsafe_recv( char *buf, int max ) { - return ::recv( sock , buf , max , portRecvFlags ); + return ::recv( sock , buf , max , portRecvFlags ); } - + void MessagingPort::piggyBack( Message& toSend , int responseTo ) { if ( toSend.header()->len > 1300 ) { @@ -624,7 +696,9 @@ namespace mongo { } HostAndPort MessagingPort::remote() const { - return farEnd; + if ( _farEndParsed.port() == -1 ) + _farEndParsed = HostAndPort( farEnd ); + return _farEndParsed; } @@ -637,80 +711,66 @@ namespace mongo { assert(MsgDataHeaderSize == 16); } } msgstart; - - MSGID nextMessageId(){ + + MSGID nextMessageId() { MSGID msgid = NextMsgId++; return msgid; } - bool doesOpGetAResponse( int op ){ + bool doesOpGetAResponse( int op ) { return op == dbQuery || op == dbGetMore; } - - void setClientId( int id ){ + + void setClientId( int id ) { clientId.set( id ); } - - int getClientId(){ + + int getClientId() { return clientId.get(); } - - int getMaxConnections(){ + + const int DEFAULT_MAX_CONN = 20000; + const int MAX_MAX_CONN = 20000; + + int getMaxConnections() { #ifdef _WIN32 - return 20000; + return DEFAULT_MAX_CONN; #else struct rlimit limit; assert( getrlimit(RLIMIT_NOFILE,&limit) == 0 ); int max = (int)(limit.rlim_cur * .8); - log(1) << "fd limit" - << " hard:" << limit.rlim_max - << " soft:" << limit.rlim_cur + log(1) << "fd limit" + << " hard:" << limit.rlim_max + << " soft:" << limit.rlim_cur << " max conn: " << max << endl; - - if ( max > 20000 ) - max = 20000; + + if ( max > MAX_MAX_CONN ) + max = MAX_MAX_CONN; return max; #endif } - void checkTicketNumbers(){ - connTicketHolder.resize( getMaxConnections() ); - } - - TicketHolder connTicketHolder(20000); - - namespace { - map<string, bool> isSelfCache; // host, isSelf - } - - bool HostAndPort::isSelf() const { - int p = _port == -1 ? CmdLine::DefaultDBPort : _port; - - if( p != cmdLine.port ){ - return false; - } else if (sameHostname(getHostName(), _host) || isLocalHost()) { - return true; - } else { - map<string, bool>::const_iterator it = isSelfCache.find(_host); - if (it != isSelfCache.end()){ - return it->second; + void checkTicketNumbers() { + int want = getMaxConnections(); + int current = connTicketHolder.outof(); + if ( current != DEFAULT_MAX_CONN ) { + if ( current < want ) { + // they want fewer than they can handle + // which is fine + log(1) << " only allowing " << current << " connections" << endl; + return; + } + if ( current > want ) { + log() << " --maxConns too high, can only handle " << want << endl; } - - SockAddr addr (_host.c_str(), 0); // port 0 is dynamically assigned - SOCKET sock = ::socket(addr.getType(), SOCK_STREAM, 0); - assert(sock != INVALID_SOCKET); - - bool ret = (::bind(sock, addr.raw(), addr.addressSize) == 0); - isSelfCache[_host] = ret; - - closesocket(sock); - - return ret; } + connTicketHolder.resize( want ); } + TicketHolder connTicketHolder(DEFAULT_MAX_CONN); + } // namespace mongo diff --git a/util/message.h b/util/message.h index 9651141..37e9433 100644 --- a/util/message.h +++ b/util/message.h @@ -32,7 +32,7 @@ namespace mongo { class Listener : boost::noncopyable { public: - Listener(const string &ip, int p, bool logConnect=true ) : _port(p), _ip(ip), _logConnect(logConnect), _elapsedTime(0){ } + Listener(const string &ip, int p, bool logConnect=true ) : _port(p), _ip(ip), _logConnect(logConnect), _elapsedTime(0) { } virtual ~Listener() { if ( _timeTracker == this ) _timeTracker = 0; @@ -41,28 +41,30 @@ namespace mongo { /* spawn a thread, etc., then return */ virtual void accepted(int sock, const SockAddr& from); - virtual void accepted(MessagingPort *mp){ + virtual void accepted(MessagingPort *mp) { assert(!"You must overwrite one of the accepted methods"); } const int _port; - + /** * @return a rough estimate of elepased time since the server started */ long long getMyElapsedTimeMillis() const { return _elapsedTime; } - void setAsTimeTracker(){ + void setAsTimeTracker() { _timeTracker = this; } - static const Listener* getTimeTracker(){ + static const Listener* getTimeTracker() { return _timeTracker; } - - static long long getElapsedTimeMillis() { + + static long long getElapsedTimeMillis() { if ( _timeTracker ) return _timeTracker->getMyElapsedTimeMillis(); + + // should this assert or throw? seems like callers may not expect to get zero back, certainly not forever. return 0; } @@ -79,11 +81,11 @@ namespace mongo { virtual ~AbstractMessagingPort() { } virtual void reply(Message& received, Message& response, MSGID responseTo) = 0; // like the reply below, but doesn't rely on received.data still being available virtual void reply(Message& received, Message& response) = 0; - + virtual HostAndPort remote() const = 0; virtual unsigned remotePort() const = 0; - virtual int getClientId(){ + virtual int getClientId() { int x = remotePort(); x = x << 16; x |= ( ( 0xFF0 & (long long)this ) >> 8 ); // lowest byte in pointer often meaningless @@ -98,12 +100,12 @@ namespace mongo { // in some cases the timeout will actually be 2x this value - eg we do a partial send, // then the timeout fires, then we try to send again, then the timeout fires again with // no data sent, then we detect that the other side is down - MessagingPort(double timeout = 0, int logLevel = 0 ); + MessagingPort(double so_timeout = 0, int logLevel = 0 ); virtual ~MessagingPort(); void shutdown(); - + bool connect(SockAddr& farEnd); /* it's assumed if you reuse a message object, that it doesn't cross MessagingPort's. @@ -113,8 +115,20 @@ namespace mongo { void reply(Message& received, Message& response, MSGID responseTo); void reply(Message& received, Message& response); bool call(Message& toSend, Message& response); + void say(Message& toSend, int responseTo = -1); + /** + * this is used for doing 'async' queries + * instead of doing call( to , from ) + * you would do + * say( to ) + * recv( from ) + * Note: if you fail to call recv and someone else uses this port, + * horrible things will happend + */ + bool recv( const Message& sent , Message& response ); + void piggyBack( Message& toSend , int responseTo = -1 ); virtual unsigned remotePort() const; @@ -126,11 +140,23 @@ namespace mongo { // recv len or throw SocketException void recv( char * data , int len ); - + int unsafe_recv( char *buf, int max ); + + void clearCounters() { _bytesIn = 0; _bytesOut = 0; } + long long getBytesIn() const { return _bytesIn; } + long long getBytesOut() const { return _bytesOut; } private: int sock; PiggyBackData * piggyBackData; + + long long _bytesIn; + long long _bytesOut; + + // this is the parsed version of farEnd + // mutable because its initialized only on call to remote() + mutable HostAndPort _farEndParsed; + public: SockAddr farEnd; double _timeout; @@ -158,8 +184,8 @@ namespace mongo { bool doesOpGetAResponse( int op ); - inline const char * opToString( int op ){ - switch ( op ){ + inline const char * opToString( int op ) { + switch ( op ) { case 0: return "none"; case opReply: return "reply"; case dbMsg: return "msg"; @@ -169,54 +195,54 @@ namespace mongo { case dbGetMore: return "getmore"; case dbDelete: return "remove"; case dbKillCursors: return "killcursors"; - default: + default: PRINT(op); - assert(0); + assert(0); return ""; } } - - inline bool opIsWrite( int op ){ - switch ( op ){ - - case 0: - case opReply: - case dbMsg: - case dbQuery: - case dbGetMore: - case dbKillCursors: + + inline bool opIsWrite( int op ) { + switch ( op ) { + + case 0: + case opReply: + case dbMsg: + case dbQuery: + case dbGetMore: + case dbKillCursors: return false; - - case dbUpdate: - case dbInsert: - case dbDelete: + + case dbUpdate: + case dbInsert: + case dbDelete: return false; - default: + default: PRINT(op); - assert(0); + assert(0); return ""; } - + } #pragma pack(1) -/* see http://www.mongodb.org/display/DOCS/Mongo+Wire+Protocol -*/ -struct MSGHEADER { - int messageLength; // total message size, including this - int requestID; // identifier for this message - int responseTo; // requestID from the original request - // (used in reponses from db) - int opCode; -}; -struct OP_GETMORE : public MSGHEADER { - MSGHEADER header; // standard message header - int ZERO_or_flags; // 0 - reserved for future use - //cstring fullCollectionName; // "dbname.collectionname" - //int32 numberToReturn; // number of documents to return - //int64 cursorID; // cursorID from the OP_REPLY -}; + /* see http://www.mongodb.org/display/DOCS/Mongo+Wire+Protocol + */ + struct MSGHEADER { + int messageLength; // total message size, including this + int requestID; // identifier for this message + int responseTo; // requestID from the original request + // (used in reponses from db) + int opCode; + }; + struct OP_GETMORE : public MSGHEADER { + MSGHEADER header; // standard message header + int ZERO_or_flags; // 0 - reserved for future use + //cstring fullCollectionName; // "dbname.collectionname" + //int32 numberToReturn; // number of documents to return + //int64 cursorID; // cursorID from the OP_REPLY + }; #pragma pack() #pragma pack(1) @@ -225,11 +251,15 @@ struct OP_GETMORE : public MSGHEADER { int len; /* len of the msg, including this field */ MSGID id; /* request/reply id's match... */ MSGID responseTo; /* id of the message we are responding to */ - int _operation; + short _operation; + char _flags; + char _version; int operation() const { return _operation; } void setOperation(int o) { + _flags = 0; + _version = 0; _operation = o; } char _data[4]; @@ -237,16 +267,16 @@ struct OP_GETMORE : public MSGHEADER { int& dataAsInt() { return *((int *) _data); } - - bool valid(){ - if ( len <= 0 || len > ( 1024 * 1024 * 10 ) ) + + bool valid() { + if ( len <= 0 || len > ( 4 * BSONObjMaxInternalSize ) ) return false; - if ( _operation < 0 || _operation > 100000 ) + if ( _operation < 0 || _operation > 30000 ) return false; return true; } - long long getCursor(){ + long long getCursor() { assert( responseTo > 0 ); assert( _operation == opReply ); long long * l = (long long *)(_data + 4); @@ -269,13 +299,13 @@ struct OP_GETMORE : public MSGHEADER { _buf( 0 ), _data( 0 ), _freeIt( false ) { _setData( reinterpret_cast< MsgData* >( data ), freeIt ); }; - Message(Message& r) : _buf( 0 ), _data( 0 ), _freeIt( false ) { + Message(Message& r) : _buf( 0 ), _data( 0 ), _freeIt( false ) { *this = r; } ~Message() { reset(); } - + SockAddr _from; MsgData *header() const { @@ -283,7 +313,7 @@ struct OP_GETMORE : public MSGHEADER { return _buf ? _buf : reinterpret_cast< MsgData* > ( _data[ 0 ].first ); } int operation() const { return header()->operation(); } - + MsgData *singleData() const { massert( 13273, "single data buffer expected", _buf ); return header(); @@ -291,25 +321,28 @@ struct OP_GETMORE : public MSGHEADER { bool empty() const { return !_buf && _data.empty(); } - int size() const{ + int size() const { int res = 0; - if ( _buf ){ + if ( _buf ) { res = _buf->len; - } else { - for (MsgVec::const_iterator it = _data.begin(); it != _data.end(); ++it){ + } + else { + for (MsgVec::const_iterator it = _data.begin(); it != _data.end(); ++it) { res += it->second; } } return res; } - + + int dataSize() const { return size() - sizeof(MSGHEADER); } + // concat multiple buffers - noop if <2 buffers already, otherwise can be expensive copy // can get rid of this if we make response handling smarter void concat() { if ( _buf || empty() ) { return; } - + assert( _freeIt ); int totalSize = 0; for( vector< pair< char *, int > >::const_iterator i = _data.begin(); i != _data.end(); ++i ) { @@ -324,7 +357,7 @@ struct OP_GETMORE : public MSGHEADER { reset(); _setData( (MsgData*)buf, true ); } - + // vector swap() so this is fast Message& operator=(Message& r) { assert( empty() ); @@ -373,7 +406,7 @@ struct OP_GETMORE : public MSGHEADER { _data.push_back( make_pair( d, size ) ); header()->len += size; } - + // use to set first buffer if empty void setData(MsgData *d, bool freeIt) { assert( empty() ); @@ -402,7 +435,8 @@ struct OP_GETMORE : public MSGHEADER { } if ( _buf != 0 ) { p.send( (char*)_buf, _buf->len, context ); - } else { + } + else { p.send( _data, context ); } } @@ -422,13 +456,17 @@ struct OP_GETMORE : public MSGHEADER { class SocketException : public DBException { public: - enum Type { CLOSED , RECV_ERROR , SEND_ERROR } type; - SocketException( Type t ) : DBException( "socket exception" , 9001 ) , type(t){} - - bool shouldPrint() const { - return type != CLOSED; - } + const enum Type { CLOSED , RECV_ERROR , SEND_ERROR, RECV_TIMEOUT, SEND_TIMEOUT, FAILED_STATE, CONNECT_ERROR } _type; + SocketException( Type t , string server="" , int code = 9001 , string extra="" ) : DBException( "socket exception" , code ) , _type(t) , _server(server), _extra(extra){ } + virtual ~SocketException() throw() {} + + bool shouldPrint() const { return _type != CLOSED; } + virtual string toString() const; + + private: + string _server; + string _extra; }; MSGID nextMessageId(); @@ -441,7 +479,7 @@ struct OP_GETMORE : public MSGHEADER { class ElapsedTracker { public: ElapsedTracker( int hitsBetweenMarks , int msBetweenMarks ) - : _h( hitsBetweenMarks ) , _ms( msBetweenMarks ) , _pings(0){ + : _h( hitsBetweenMarks ) , _ms( msBetweenMarks ) , _pings(0) { _last = Listener::getElapsedTimeMillis(); } @@ -449,18 +487,18 @@ struct OP_GETMORE : public MSGHEADER { * call this for every iteration * returns true if one of the triggers has gone off */ - bool ping(){ - if ( ( ++_pings % _h ) == 0 ){ + bool ping() { + if ( ( ++_pings % _h ) == 0 ) { _last = Listener::getElapsedTimeMillis(); return true; } - + long long now = Listener::getElapsedTimeMillis(); - if ( now - _last > _ms ){ + if ( now - _last > _ms ) { _last = now; return true; } - + return false; } @@ -471,7 +509,7 @@ struct OP_GETMORE : public MSGHEADER { unsigned long long _pings; long long _last; - + }; - + } // namespace mongo diff --git a/util/message_server.h b/util/message_server.h index 9d6a8f2..39375c8 100644 --- a/util/message_server.h +++ b/util/message_server.h @@ -25,10 +25,10 @@ #include "../pch.h" namespace mongo { - + class MessageHandler { public: - virtual ~MessageHandler(){} + virtual ~MessageHandler() {} virtual void process( Message& m , AbstractMessagingPort* p ) = 0; virtual void disconnected( AbstractMessagingPort* p ) = 0; }; @@ -39,14 +39,14 @@ namespace mongo { int port; // port to bind to string ipList; // addresses to bind to - Options() : port(0), ipList(""){} + Options() : port(0), ipList("") {} }; - virtual ~MessageServer(){} + virtual ~MessageServer() {} virtual void run() = 0; virtual void setAsTimeTracker() = 0; }; - // TODO use a factory here to decide between port and asio variations + // TODO use a factory here to decide between port and asio variations MessageServer * createServer( const MessageServer::Options& opts , MessageHandler * handler ); } diff --git a/util/message_server_asio.cpp b/util/message_server_asio.cpp index 0c9479c..0c6a7d9 100644 --- a/util/message_server_asio.cpp +++ b/util/message_server_asio.cpp @@ -37,29 +37,29 @@ namespace mongo { class MessageServerSession; namespace { - class StickyThread{ + class StickyThread { public: StickyThread() : _thread(boost::ref(*this)) {} - ~StickyThread(){ + ~StickyThread() { _mss.put(boost::shared_ptr<MessageServerSession>()); _thread.join(); } - void ready(boost::shared_ptr<MessageServerSession> mss){ + void ready(boost::shared_ptr<MessageServerSession> mss) { _mss.put(mss); } - void operator() (){ + void operator() () { boost::shared_ptr<MessageServerSession> mss; - while((mss = _mss.take())){ // intentionally not using == + while((mss = _mss.take())) { // intentionally not using == task(mss.get()); mss.reset(); } } - + private: boost::thread _thread; inline void task(MessageServerSession* mss); // must be defined after MessageServerSession @@ -79,34 +79,34 @@ namespace mongo { , _portCache(0) { } - ~MessageServerSession(){ + ~MessageServerSession() { cout << "disconnect from: " << _socket.remote_endpoint() << endl; } - tcp::socket& socket(){ + tcp::socket& socket() { return _socket; } - void start(){ + void start() { cout << "MessageServerSession start from:" << _socket.remote_endpoint() << endl; _startHeaderRead(); } - - void handleReadHeader( const boost::system::error_code& error ){ + + void handleReadHeader( const boost::system::error_code& error ) { if ( _inHeader.len == 0 ) return; - if ( ! _inHeader.valid() ){ + if ( ! _inHeader.valid() ) { cout << " got invalid header from: " << _socket.remote_endpoint() << " closing connected" << endl; return; } - + char * raw = (char*)malloc( _inHeader.len ); - + MsgData * data = (MsgData*)raw; memcpy( data , &_inHeader , sizeof( _inHeader ) ); assert( data->len == _inHeader.len ); - + uassert( 10273 , "_cur not empty! pipelining requests not supported" , ! _cur.data ); _cur.setData( data , true ); @@ -114,11 +114,11 @@ namespace mongo { buffer( raw + sizeof( _inHeader ) , _inHeader.len - sizeof( _inHeader ) ) , boost::bind( &MessageServerSession::handleReadBody , shared_from_this() , boost::asio::placeholders::error ) ); } - - void handleReadBody( const boost::system::error_code& error ){ - if (!_myThread){ + + void handleReadBody( const boost::system::error_code& error ) { + if (!_myThread) { mongo::mutex::scoped_lock(tp_mutex); - if (!thread_pool.empty()){ + if (!thread_pool.empty()) { _myThread = thread_pool.back(); thread_pool.pop_back(); } @@ -132,20 +132,21 @@ namespace mongo { _myThread->ready(shared_from_this()); } - void process(){ + void process() { _handler->process( _cur , this ); - if (_reply.data){ + if (_reply.data) { async_write( _socket , buffer( (char*)_reply.data , _reply.data->len ) , boost::bind( &MessageServerSession::handleWriteDone , shared_from_this() , boost::asio::placeholders::error ) ); - } else { + } + else { _cur.reset(); _startHeaderRead(); } } - - void handleWriteDone( const boost::system::error_code& error ){ + + void handleWriteDone( const boost::system::error_code& error ) { { // return thread to pool after we have sent data to the client mongo::mutex::scoped_lock(tp_mutex); @@ -157,12 +158,12 @@ namespace mongo { _reply.reset(); _startHeaderRead(); } - - virtual void reply( Message& received, Message& response ){ + + virtual void reply( Message& received, Message& response ) { reply( received , response , received.data->id ); } - - virtual void reply( Message& query , Message& toSend, MSGID responseTo ){ + + virtual void reply( Message& query , Message& toSend, MSGID responseTo ) { _reply = toSend; _reply.data->id = nextMessageId(); @@ -170,22 +171,22 @@ namespace mongo { uassert( 10274 , "pipelining requests doesn't work yet" , query.data->id == _cur.data->id ); } - - virtual unsigned remotePort(){ + + virtual unsigned remotePort() { if (!_portCache) _portCache = _socket.remote_endpoint().port(); //this is expensive return _portCache; } - - private: - - void _startHeaderRead(){ + + private: + + void _startHeaderRead() { _inHeader.len = 0; - async_read( _socket , + async_read( _socket , buffer( &_inHeader , sizeof( _inHeader ) ) , boost::bind( &MessageServerSession::handleReadHeader , shared_from_this() , boost::asio::placeholders::error ) ); } - + MessageHandler * _handler; tcp::socket _socket; MsgData _inHeader; @@ -197,10 +198,10 @@ namespace mongo { boost::shared_ptr<StickyThread> _myThread; }; - void StickyThread::task(MessageServerSession* mss){ + void StickyThread::task(MessageServerSession* mss) { mss->process(); } - + class AsyncMessageServer : public MessageServer { public: @@ -209,39 +210,38 @@ namespace mongo { : _port( opts.port ) , _handler(handler) , _endpoint( tcp::v4() , opts.port ) - , _acceptor( _ioservice , _endpoint ) - { + , _acceptor( _ioservice , _endpoint ) { _accept(); } - virtual ~AsyncMessageServer(){ - + virtual ~AsyncMessageServer() { + } - void run(){ + void run() { cout << "AsyncMessageServer starting to listen on: " << _port << endl; boost::thread other(boost::bind(&io_service::run, &_ioservice)); _ioservice.run(); cout << "AsyncMessageServer done listening on: " << _port << endl; } - - void handleAccept( shared_ptr<MessageServerSession> session , - const boost::system::error_code& error ){ - if ( error ){ + + void handleAccept( shared_ptr<MessageServerSession> session , + const boost::system::error_code& error ) { + if ( error ) { cout << "handleAccept error!" << endl; return; } session->start(); _accept(); } - - void _accept( ){ + + void _accept( ) { shared_ptr<MessageServerSession> session( new MessageServerSession( _handler , _ioservice ) ); _acceptor.async_accept( session->socket() , - boost::bind( &AsyncMessageServer::handleAccept, - this, - session, - boost::asio::placeholders::error ) - ); + boost::bind( &AsyncMessageServer::handleAccept, + this, + session, + boost::asio::placeholders::error ) + ); } private: @@ -252,9 +252,9 @@ namespace mongo { tcp::acceptor _acceptor; }; - MessageServer * createServer( const MessageServer::Options& opts , MessageHandler * handler ){ + MessageServer * createServer( const MessageServer::Options& opts , MessageHandler * handler ) { return new AsyncMessageServer( opts , handler ); - } + } } diff --git a/util/message_server_port.cpp b/util/message_server_port.cpp index 9649e45..6d00628 100644 --- a/util/message_server_port.cpp +++ b/util/message_server_port.cpp @@ -23,29 +23,32 @@ #include "message_server.h" #include "../db/cmdline.h" +#include "../db/stats/counters.h" namespace mongo { namespace pms { MessageHandler * handler; - - void threadRun( MessagingPort * inPort){ - assert( inPort ); + + void threadRun( MessagingPort * inPort) { + TicketHolderReleaser connTicketReleaser( &connTicketHolder ); + assert( inPort ); + setThreadName( "conn" ); - TicketHolderReleaser connTicketReleaser( &connTicketHolder ); auto_ptr<MessagingPort> p( inPort ); - + string otherSide; - + Message m; try { otherSide = p->farEnd.toString(); - while ( 1 ){ + while ( 1 ) { m.reset(); + p->clearCounters(); if ( ! p->recv(m) ) { if( !cmdLine.quiet ) @@ -53,20 +56,21 @@ namespace mongo { p->shutdown(); break; } - + handler->process( m , p.get() ); + networkCounter.hit( p->getBytesIn() , p->getBytesOut() ); } } - catch ( const SocketException& ){ + catch ( const SocketException& ) { log() << "unclean socket shutdown from: " << otherSide << endl; } - catch ( const std::exception& e ){ + catch ( const std::exception& e ) { problem() << "uncaught exception (" << e.what() << ")(" << demangleName( typeid(e) ) <<") in PortMessageServer::threadRun, closing connection" << endl; } - catch ( ... ){ + catch ( ... ) { problem() << "uncaught exception in PortMessageServer::threadRun, closing connection" << endl; - } - + } + handler->disconnected( p.get() ); } @@ -74,16 +78,16 @@ namespace mongo { class PortMessageServer : public MessageServer , public Listener { public: - PortMessageServer( const MessageServer::Options& opts, MessageHandler * handler ) : - Listener( opts.ipList, opts.port ){ - + PortMessageServer( const MessageServer::Options& opts, MessageHandler * handler ) : + Listener( opts.ipList, opts.port ) { + uassert( 10275 , "multiple PortMessageServer not supported" , ! pms::handler ); pms::handler = handler; } - + virtual void accepted(MessagingPort * p) { - - if ( ! connTicketHolder.tryAcquire() ){ + + if ( ! connTicketHolder.tryAcquire() ) { log() << "connection refused because too many open connections: " << connTicketHolder.used() << endl; // TODO: would be nice if we notified them... @@ -97,7 +101,8 @@ namespace mongo { try { boost::thread thr( boost::bind( &pms::threadRun , p ) ); } - catch ( boost::thread_resource_error& ){ + catch ( boost::thread_resource_error& ) { + connTicketHolder.release(); log() << "can't create new thread, closing connection" << endl; p->shutdown(); @@ -106,21 +111,21 @@ namespace mongo { sleepmillis(2); } } - - virtual void setAsTimeTracker(){ + + virtual void setAsTimeTracker() { Listener::setAsTimeTracker(); } - void run(){ + void run() { initAndListen(); } }; - MessageServer * createServer( const MessageServer::Options& opts , MessageHandler * handler ){ + MessageServer * createServer( const MessageServer::Options& opts , MessageHandler * handler ) { return new PortMessageServer( opts , handler ); - } + } } diff --git a/util/miniwebserver.cpp b/util/miniwebserver.cpp index 0193c5d..e700112 100644 --- a/util/miniwebserver.cpp +++ b/util/miniwebserver.cpp @@ -55,10 +55,10 @@ namespace mongo { void MiniWebServer::parseParams( BSONObj & params , string query ) { if ( query.size() == 0 ) return; - + BSONObjBuilder b; while ( query.size() ) { - + string::size_type amp = query.find( "&" ); string cur; @@ -77,7 +77,7 @@ namespace mongo { b.append( urlDecode(cur.substr(0,eq)) , urlDecode(cur.substr(eq+1) ) ); } - + params = b.obj(); } @@ -132,16 +132,16 @@ namespace mongo { string responseMsg; int responseCode = 599; vector<string> headers; - + try { doRequest(buf, parseURL( buf ), responseMsg, responseCode, headers, from); } - catch ( std::exception& e ){ + catch ( std::exception& e ) { responseCode = 500; responseMsg = "error loading page: "; responseMsg += e.what(); } - catch ( ... ){ + catch ( ... ) { responseCode = 500; responseMsg = "unknown error loading page"; } @@ -168,32 +168,34 @@ namespace mongo { ::send(s, response.c_str(), response.size(), 0); closesocket(s); } - - string MiniWebServer::getHeader( const char * req , string wanted ){ + + string MiniWebServer::getHeader( const char * req , string wanted ) { const char * headers = strchr( req , '\n' ); if ( ! headers ) return ""; pcrecpp::StringPiece input( headers + 1 ); - + string name; string val; pcrecpp::RE re("([\\w\\-]+): (.*?)\r?\n"); - while ( re.Consume( &input, &name, &val) ){ + while ( re.Consume( &input, &name, &val) ) { if ( name == wanted ) return val; } return ""; } - string MiniWebServer::urlDecode(const char* s){ + string MiniWebServer::urlDecode(const char* s) { stringstream out; - while(*s){ - if (*s == '+'){ + while(*s) { + if (*s == '+') { out << ' '; - }else if (*s == '%'){ + } + else if (*s == '%') { out << fromHex(s+1); s+=2; - }else{ + } + else { out << *s; } s++; diff --git a/util/miniwebserver.h b/util/miniwebserver.h index bbd1ba2..b385afc 100644 --- a/util/miniwebserver.h +++ b/util/miniwebserver.h @@ -41,7 +41,7 @@ namespace mongo { // --- static helpers ---- static void parseParams( BSONObj & params , string query ); - + static string parseURL( const char * buf ); static string parseMethod( const char * headers ); static string getHeader( const char * headers , string name ); diff --git a/util/mmap.cpp b/util/mmap.cpp index b9c1994..18edc34 100644 --- a/util/mmap.cpp +++ b/util/mmap.cpp @@ -19,48 +19,59 @@ #include "mmap.h" #include "processinfo.h" #include "concurrency/rwlock.h" +#include "../db/namespace.h" namespace mongo { - /*static*/ void MemoryMappedFile::updateLength( const char *filename, long &length ) { + set<MongoFile*> MongoFile::mmfiles; + map<string,MongoFile*> MongoFile::pathToFile; + + /* Create. Must not exist. + @param zero fill file with zeros when true + */ + void* MemoryMappedFile::create(string filename, unsigned long long len, bool zero) { + uassert( 13468, string("can't create file already exists ") + filename, !exists(filename) ); + void *p = map(filename.c_str(), len); + if( p && zero ) { + size_t sz = (size_t) len; + assert( len == sz ); + memset(p, 0, sz); + } + return p; + } + + /*static*/ void MemoryMappedFile::updateLength( const char *filename, unsigned long long &length ) { if ( !boost::filesystem::exists( filename ) ) return; // make sure we map full length if preexisting file. boost::uintmax_t l = boost::filesystem::file_size( filename ); - assert( l <= 0x7fffffff ); - length = (long) l; + length = l; } void* MemoryMappedFile::map(const char *filename) { - boost::uintmax_t l = boost::filesystem::file_size( filename ); - assert( l <= 0x7fffffff ); - long i = (long)l; - return map( filename , i ); + unsigned long long l = boost::filesystem::file_size( filename ); + return map( filename , l ); } - - void printMemInfo( const char * where ){ - cout << "mem info: "; - if ( where ) - cout << where << " "; - ProcessInfo pi; - if ( ! pi.supported() ){ - cout << " not supported" << endl; - return; - } - - cout << "vsize: " << pi.getVirtualMemorySize() << " resident: " << pi.getResidentSize() << " mapped: " << ( MemoryMappedFile::totalMappedLength() / ( 1024 * 1024 ) ) << endl; + void* MemoryMappedFile::mapWithOptions(const char *filename, int options) { + unsigned long long l = boost::filesystem::file_size( filename ); + return map( filename , l, options ); } /* --- MongoFile ------------------------------------------------- - this is the administrative stuff + this is the administrative stuff */ - static set<MongoFile*> mmfiles; - static RWLock mmmutex("rw:mmmutex"); + RWLock MongoFile::mmmutex("rw:mmmutex"); + /* subclass must call in destructor (or at close). + removes this from pathToFile and other maps + safe to call more than once, albeit might be wasted work + ideal to call close to the close, if the close is well before object destruction + */ void MongoFile::destroyed() { rwlock lk( mmmutex , true ); mmfiles.erase(this); + pathToFile.erase( filename() ); } /*static*/ @@ -75,17 +86,17 @@ namespace mongo { rwlock lk( mmmutex , true ); ProgressMeter pm( mmfiles.size() , 2 , 1 ); - for ( set<MongoFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ){ + for ( set<MongoFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ) { (*i)->close(); pm.hit(); } - message << " closeAllFiles() finished" << endl; + message << "closeAllFiles() finished"; --closingAllFiles; } - /*static*/ long long MongoFile::totalMappedLength(){ + /*static*/ long long MongoFile::totalMappedLength() { unsigned long long total = 0; - + rwlock lk( mmmutex , false ); for ( set<MongoFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ) total += (*i)->length(); @@ -93,28 +104,41 @@ namespace mongo { return total; } - /*static*/ int MongoFile::flushAll( bool sync ){ - if ( ! sync ){ + void nullFunc() { } + + // callback notifications + void (*MongoFile::notifyPreFlush)() = nullFunc; + void (*MongoFile::notifyPostFlush)() = nullFunc; + + /*static*/ int MongoFile::flushAll( bool sync ) { + notifyPreFlush(); + int x = _flushAll(sync); + notifyPostFlush(); + return x; + } + + /*static*/ int MongoFile::_flushAll( bool sync ) { + if ( ! sync ) { int num = 0; rwlock lk( mmmutex , false ); - for ( set<MongoFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ){ + for ( set<MongoFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ) { num++; MongoFile * mmf = *i; if ( ! mmf ) continue; - + mmf->flush( sync ); } return num; } - + // want to do it sync set<MongoFile*> seen; - while ( true ){ + while ( true ) { auto_ptr<Flushable> f; { rwlock lk( mmmutex , false ); - for ( set<MongoFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ){ + for ( set<MongoFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ) { MongoFile * mmf = *i; if ( ! mmf ) continue; @@ -127,34 +151,41 @@ namespace mongo { } if ( ! f.get() ) break; - + f->flush(); } return seen.size(); } - void MongoFile::created(){ + void MongoFile::created() { rwlock lk( mmmutex , true ); mmfiles.insert(this); } -#ifdef _DEBUG + void MongoFile::setFilename(string fn) { + rwlock( mmmutex, true ); + assert( _filename.empty() ); + _filename = fn; + MongoFile *&ptf = pathToFile[fn]; + massert(13617, "MongoFile : multiple opens of same filename", ptf == 0); + ptf = this; + } - void MongoFile::lockAll() { +#if defined(_DEBUG) + void MongoFile::markAllWritable() { rwlock lk( mmmutex , false ); - for ( set<MongoFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ){ + for ( set<MongoFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ) { MongoFile * mmf = *i; if (mmf) mmf->_lock(); } } - void MongoFile::unlockAll() { + void MongoFile::unmarkAllWritable() { rwlock lk( mmmutex , false ); - for ( set<MongoFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ){ + for ( set<MongoFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ) { MongoFile * mmf = *i; if (mmf) mmf->_unlock(); } } #endif - } // namespace mongo diff --git a/util/mmap.h b/util/mmap.h index c954ef7..2ef4176 100644 --- a/util/mmap.h +++ b/util/mmap.h @@ -16,20 +16,63 @@ */ #pragma once +#include <boost/thread/xtime.hpp> +#include "concurrency/rwlock.h" namespace mongo { - + /* the administrative-ish stuff here */ - class MongoFile : boost::noncopyable { - + class MongoFile : boost::noncopyable { public: /** Flushable has to fail nicely if the underlying object gets killed */ class Flushable { public: - virtual ~Flushable(){} + virtual ~Flushable() {} virtual void flush() = 0; }; - + + virtual ~MongoFile() {} + + enum Options { + SEQUENTIAL = 1, // hint - e.g. FILE_FLAG_SEQUENTIAL_SCAN on windows + READONLY = 2 // not contractually guaranteed, but if specified the impl has option to fault writes + }; + + /** @param fun is called for each MongoFile. + calledl from within a mutex that MongoFile uses. so be careful not to deadlock. + */ + template < class F > + static void forEach( F fun ); + + /** note: you need to be in mmmutex when using this. forEach (above) handles that for you automatically. */ + static set<MongoFile*>& getAllFiles() { return mmfiles; } + + // callbacks if you need them + static void (*notifyPreFlush)(); + static void (*notifyPostFlush)(); + + static int flushAll( bool sync ); // returns n flushed + static long long totalMappedLength(); + static void closeAllFiles( stringstream &message ); + +#if defined(_DEBUG) + static void markAllWritable(); + static void unmarkAllWritable(); +#else + static void markAllWritable() { } + static void unmarkAllWritable() { } +#endif + + static bool exists(boost::filesystem::path p) { return boost::filesystem::exists(p); } + + virtual bool isMongoMMF() { return false; } + + string filename() const { return _filename; } + void setFilename(string fn); + + private: + string _filename; + static int _flushAll( bool sync ); // returns n flushed protected: virtual void close() = 0; virtual void flush(bool sync) = 0; @@ -38,164 +81,178 @@ namespace mongo { * Flushable has to fail nicely if the underlying object gets killed */ virtual Flushable * prepareFlush() = 0; - + void created(); /* subclass must call after create */ - void destroyed(); /* subclass must call in destructor */ + + /* subclass must call in destructor (or at close). + removes this from pathToFile and other maps + safe to call more than once, albeit might be wasted work + ideal to call close to the close, if the close is well before object destruction + */ + void destroyed(); + + virtual unsigned long long length() const = 0; // only supporting on posix mmap virtual void _lock() {} virtual void _unlock() {} + static set<MongoFile*> mmfiles; public: - virtual ~MongoFile() {} - virtual long length() = 0; - - enum Options { - SEQUENTIAL = 1 // hint - e.g. FILE_FLAG_SEQUENTIAL_SCAN on windows - }; - - static int flushAll( bool sync ); // returns n flushed - static long long totalMappedLength(); - static void closeAllFiles( stringstream &message ); + static map<string,MongoFile*> pathToFile; + static RWLock mmmutex; + }; - // Locking allows writes. Reads are always allowed - static void lockAll(); - static void unlockAll(); + /** look up a MMF by filename. scoped mutex locking convention. + example: + MMFFinderByName finder; + MongoMMF *a = finder.find("file_name_a"); + MongoMMF *b = finder.find("file_name_b"); + */ + class MongoFileFinder : boost::noncopyable { + public: + MongoFileFinder() : _lk(MongoFile::mmmutex,false) { } - /* can be "overriden" if necessary */ - static bool exists(boost::filesystem::path p) { - return boost::filesystem::exists(p); + /** @return The MongoFile object associated with the specified file name. If no file is open + with the specified name, returns null. + */ + MongoFile* findByPath(string path) { + map<string,MongoFile*>::iterator i = MongoFile::pathToFile.find(path); + return i == MongoFile::pathToFile.end() ? NULL : i->second; } - }; - -#ifndef _DEBUG - // no-ops in production - inline void MongoFile::lockAll() {} - inline void MongoFile::unlockAll() {} -#endif + private: + rwlock _lk; + }; struct MongoFileAllowWrites { - MongoFileAllowWrites(){ - MongoFile::lockAll(); + MongoFileAllowWrites() { + MongoFile::markAllWritable(); } - ~MongoFileAllowWrites(){ - MongoFile::unlockAll(); + ~MongoFileAllowWrites() { + MongoFile::unmarkAllWritable(); } }; - /** template for what a new storage engine's class definition must implement - PRELIMINARY - subject to change. - */ - class StorageContainerTemplate : public MongoFile { - protected: - virtual void close(); - virtual void flush(bool sync); - public: - virtual long length(); - - /** pointer to a range of space in this storage unit */ - class Pointer { - public: - /** retried address of buffer at offset 'offset' withing the storage unit. returned range is a contiguous - buffer reflecting what is in storage. caller will not read or write past 'len'. - - note calls may be received that are at different points in a range and different lengths. however - for now assume that on writes, if a call is made, previously returned addresses are no longer valid. i.e. - p = at(10000, 500); - q = at(10000, 600); - after the second call it is ok if p is invalid. - */ - void* at(int offset, int len); - - /** indicate that we wrote to the range (from a previous at() call) and that it needs - flushing to disk. - */ - void written(int offset, int len); - - bool isNull() const; - }; - - /** commit written() calls from above. */ - void commit(); - - Pointer open(const char *filename); - Pointer open(const char *_filename, long &length, int options=0); - }; - class MemoryMappedFile : public MongoFile { public: - class Pointer { - char *_base; - public: - Pointer() : _base(0) { } - Pointer(void *p) : _base((char*) p) { } - void* at(int offset, int maxLen) { return _base + offset; } - void grow(int offset, int len) { /* no action required with mem mapped file */ } - bool isNull() const { return _base == 0; } - }; - MemoryMappedFile(); - ~MemoryMappedFile() { - destroyed(); + + virtual ~MemoryMappedFile() { + destroyed(); // cleans up from the master list of mmaps close(); } - void close(); - - // Throws exception if file doesn't exist. (dm may2010: not sure if this is always true?) - void* map( const char *filename ); - /*To replace map(): - - Pointer open( const char *filename ) { - void *p = map(filename); - uassert(13077, "couldn't open/map file", p); - return Pointer(p); - }*/ + virtual void close(); + + // Throws exception if file doesn't exist. (dm may2010: not sure if this is always true?) + void* map(const char *filename); + void* mapWithOptions(const char *filename, int options); /* Creates with length if DNE, otherwise uses existing file length, passed length. + @param options MongoFile::Options bits */ - void* map(const char *filename, long &length, int options = 0 ); + void* map(const char *filename, unsigned long long &length, int options = 0 ); + + /* Create. Must not exist. + @param zero fill file with zeros when true + */ + void* create(string filename, unsigned long long len, bool zero); void flush(bool sync); virtual Flushable * prepareFlush(); - /*void* viewOfs() { - return view; - }*/ + long shortLength() const { return (long) len; } + unsigned long long length() const { return len; } - long length() { - return len; - } + /** create a new view with the specified properties. + automatically cleaned up upon close/destruction of the MemoryMappedFile object. + */ + void* createReadOnlyMap(); + void* createPrivateMap(); - string filename() const { return _filename; } + /** make the private map range writable (necessary for our windows implementation) */ + static void makeWritable(void *, unsigned len) +#if defined(_WIN32) + ; +#else + { } +#endif private: - static void updateLength( const char *filename, long &length ); - + static void updateLength( const char *filename, unsigned long long &length ); + HANDLE fd; HANDLE maphandle; - void *view; - long len; - string _filename; + vector<void *> views; + unsigned long long len; + +#ifdef _WIN32 + boost::shared_ptr<mutex> _flushMutex; + void clearWritableBits(void *privateView); + public: + static const unsigned ChunkSize = 64 * 1024 * 1024; + static const unsigned NChunks = 1024 * 1024; +#else + void clearWritableBits(void *privateView) { } +#endif protected: // only posix mmap implementations will support this virtual void _lock(); virtual void _unlock(); + /** close the current private view and open a new replacement */ + void* remapPrivateView(void *oldPrivateAddr); }; - void printMemInfo( const char * where ); + typedef MemoryMappedFile MMF; -#include "ramstore.h" + /** p is called from within a mutex that MongoFile uses. so be careful not to deadlock. */ + template < class F > + inline void MongoFile::forEach( F p ) { + rwlock lk( mmmutex , false ); + for ( set<MongoFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ) + p(*i); + } + +#if defined(_WIN32) + class ourbitset { + volatile unsigned bits[MemoryMappedFile::NChunks]; // volatile as we are doing double check locking + public: + ourbitset() { + memset((void*) bits, 0, sizeof(bits)); + } + bool get(unsigned i) const { + unsigned x = i / 32; + assert( x < MemoryMappedFile::NChunks ); + return bits[x] & (1 << (i%32)); + } + void set(unsigned i) { + unsigned x = i / 32; + assert( x < MemoryMappedFile::NChunks ); + bits[x] |= (1 << (i%32)); + } + void clear(unsigned i) { + unsigned x = i / 32; + assert( x < MemoryMappedFile::NChunks ); + bits[x] &= ~(1 << (i%32)); + } + }; + extern ourbitset writable; + void makeChunkWritable(size_t chunkno);
+ inline void MemoryMappedFile::makeWritable(void *_p, unsigned len) {
+ size_t p = (size_t) _p;
+ unsigned a = p/ChunkSize;
+ unsigned b = (p+len)/ChunkSize;
+ for( unsigned i = a; i <= b; i++ ) {
+ if( !writable.get(i) ) {
+ makeChunkWritable(i);
+ }
+ }
+ } -//#define _RAMSTORE -#if defined(_RAMSTORE) - typedef RamStoreFile MMF; -#else - typedef MemoryMappedFile MMF; #endif } // namespace mongo diff --git a/util/mmap_mm.cpp b/util/mmap_mm.cpp index 3cbb0d2..ec2400e 100644 --- a/util/mmap_mm.cpp +++ b/util/mmap_mm.cpp @@ -44,9 +44,9 @@ namespace mongo { void MemoryMappedFile::flush(bool sync) { } - + void MemoryMappedFile::_lock() {} void MemoryMappedFile::_unlock() {} -} +} diff --git a/util/mmap_posix.cpp b/util/mmap_posix.cpp index af1592c..f47a06f 100644 --- a/util/mmap_posix.cpp +++ b/util/mmap_posix.cpp @@ -26,20 +26,23 @@ #include <sys/stat.h> #include <fcntl.h> +#include "mongoutils/str.h" +using namespace mongoutils; + namespace mongo { MemoryMappedFile::MemoryMappedFile() { fd = 0; maphandle = 0; - view = 0; len = 0; created(); } void MemoryMappedFile::close() { - if ( view ) - munmap(view, len); - view = 0; + for( vector<void*>::iterator i = views.begin(); i != views.end(); i++ ) { + munmap(*i,len); + } + views.clear(); if ( fd ) ::close(fd); @@ -47,76 +50,129 @@ namespace mongo { } #ifndef O_NOATIME -#define O_NOATIME 0 +#define O_NOATIME (0) +#endif + +#ifndef MAP_NORESERVE +#define MAP_NORESERVE (0) #endif - void* MemoryMappedFile::map(const char *filename, long &length, int options) { + void* MemoryMappedFile::map(const char *filename, unsigned long long &length, int options) { // length may be updated by callee. - _filename = filename; - theFileAllocator().allocateAsap( filename, length ); + setFilename(filename); + FileAllocator::get()->allocateAsap( filename, length ); len = length; - massert( 10446 , (string)"mmap() can't map area of size 0 [" + filename + "]" , length > 0 ); + massert( 10446 , str::stream() << "mmap: can't map area of size 0 file: " << filename, length > 0 ); - fd = open(filename, O_RDWR | O_NOATIME); if ( fd <= 0 ) { - out() << "couldn't open " << filename << ' ' << errnoWithDescription() << endl; + log() << "couldn't open " << filename << ' ' << errnoWithDescription() << endl; return 0; } - off_t filelen = lseek(fd, 0, SEEK_END); - if ( filelen != length ){ - cout << "wanted length: " << length << " filelen: " << filelen << endl; - cout << sizeof(size_t) << endl; - massert( 10447 , "file size allocation failed", filelen == length ); - } + unsigned long long filelen = lseek(fd, 0, SEEK_END); + uassert(10447, str::stream() << "map file alloc failed, wanted: " << length << " filelen: " << filelen << ' ' << sizeof(size_t), filelen == length ); lseek( fd, 0, SEEK_SET ); - - view = mmap(NULL, length, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); + + void * view = mmap(NULL, length, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); if ( view == MAP_FAILED ) { - out() << " mmap() failed for " << filename << " len:" << length << " " << errnoWithDescription() << endl; - if ( errno == ENOMEM ){ - out() << " mmap failed with out of memory, if you're using 32-bits, then you probably need to upgrade to 64" << endl; + error() << " mmap() failed for " << filename << " len:" << length << " " << errnoWithDescription() << endl; + if ( errno == ENOMEM ) { + if( sizeof(void*) == 4 ) + error() << "mmap failed with out of memory. You are using a 32-bit build and probably need to upgrade to 64" << endl; + else + error() << "mmap failed with out of memory. (64 bit build)" << endl; } return 0; } + #if defined(__sunos__) #warning madvise not supported on solaris yet #else - if ( options & SEQUENTIAL ){ - if ( madvise( view , length , MADV_SEQUENTIAL ) ){ - out() << " madvise failed for " << filename << " " << errnoWithDescription() << endl; + if ( options & SEQUENTIAL ) { + if ( madvise( view , length , MADV_SEQUENTIAL ) ) { + warning() << "map: madvise failed for " << filename << ' ' << errnoWithDescription() << endl; } } #endif - DEV if (! dbMutex.info().isLocked()){ + views.push_back( view ); + + DEV if (! dbMutex.info().isLocked()) { _unlock(); } return view; } - + + void* MemoryMappedFile::createReadOnlyMap() { + void * x = mmap( /*start*/0 , len , PROT_READ , MAP_SHARED , fd , 0 ); + if( x == MAP_FAILED ) { + if ( errno == ENOMEM ) { + if( sizeof(void*) == 4 ) + error() << "mmap ro failed with out of memory. You are using a 32-bit build and probably need to upgrade to 64" << endl; + else + error() << "mmap ro failed with out of memory. (64 bit build)" << endl; + } + return 0; + } + return x; + } + + void* MemoryMappedFile::createPrivateMap() { + void * x = mmap( /*start*/0 , len , PROT_READ|PROT_WRITE , MAP_PRIVATE|MAP_NORESERVE , fd , 0 ); + if( x == MAP_FAILED ) { + if ( errno == ENOMEM ) { + if( sizeof(void*) == 4 ) { + error() << "mmap private failed with out of memory. You are using a 32-bit build and probably need to upgrade to 64" << endl; + } + else { + error() << "mmap private failed with out of memory. (64 bit build)" << endl; + } + } + else { + error() << "mmap private failed " << errnoWithDescription() << endl; + } + return 0; + } + + views.push_back(x); + return x; + } + + void* MemoryMappedFile::remapPrivateView(void *oldPrivateAddr) { + // don't unmap, just mmap over the old region + void * x = mmap( oldPrivateAddr, len , PROT_READ|PROT_WRITE , MAP_PRIVATE|MAP_NORESERVE|MAP_FIXED , fd , 0 ); + if( x == MAP_FAILED ) { + int err = errno; + error() << "13601 Couldn't remap private view: " << errnoWithDescription(err) << endl; + log() << "aborting" << endl; + abort(); + } + assert( x == oldPrivateAddr ); + return x; + } + void MemoryMappedFile::flush(bool sync) { - if ( view == 0 || fd == 0 ) + if ( views.empty() || fd == 0 ) return; - if ( msync(view, len, sync ? MS_SYNC : MS_ASYNC) ) + if ( msync(views[0], len, sync ? MS_SYNC : MS_ASYNC) ) problem() << "msync " << errnoWithDescription() << endl; } - + class PosixFlushable : public MemoryMappedFile::Flushable { public: PosixFlushable( void * view , HANDLE fd , long len ) - : _view( view ) , _fd( fd ) , _len(len){ + : _view( view ) , _fd( fd ) , _len(len) { } - void flush(){ + void flush() { if ( _view && _fd ) if ( msync(_view, _len, MS_SYNC ) ) problem() << "msync " << errnoWithDescription() << endl; - + } void * _view; @@ -124,16 +180,18 @@ namespace mongo { long _len; }; - MemoryMappedFile::Flushable * MemoryMappedFile::prepareFlush(){ - return new PosixFlushable( view , fd , len ); + MemoryMappedFile::Flushable * MemoryMappedFile::prepareFlush() { + return new PosixFlushable( views.empty() ? 0 : views[0] , fd , len ); } void MemoryMappedFile::_lock() { - if (view) assert(mprotect(view, len, PROT_READ | PROT_WRITE) == 0); + if (! views.empty() && isMongoMMF() ) + assert(mprotect(views[0], len, PROT_READ | PROT_WRITE) == 0); } void MemoryMappedFile::_unlock() { - if (view) assert(mprotect(view, len, PROT_READ) == 0); + if (! views.empty() && isMongoMMF() ) + assert(mprotect(views[0], len, PROT_READ) == 0); } } // namespace mongo diff --git a/util/mmap_win.cpp b/util/mmap_win.cpp index 97e1589..0b0b834 100644 --- a/util/mmap_win.cpp +++ b/util/mmap_win.cpp @@ -19,21 +19,36 @@ #include "mmap.h" #include "text.h" #include <windows.h> +#include "../db/mongommf.h" +#include "../db/concurrency.h" namespace mongo { - MemoryMappedFile::MemoryMappedFile() { + mutex mapViewMutex("mapView"); + ourbitset writable; + + /** notification on unmapping so we can clear writable bits */ + void MemoryMappedFile::clearWritableBits(void *p) { + for( unsigned i = ((size_t)p)/ChunkSize; i <= (((size_t)p)+len)/ChunkSize; i++ ) { + writable.clear(i); + assert( !writable.get(i) ); + } + } + + MemoryMappedFile::MemoryMappedFile() + : _flushMutex(new mutex("flushMutex")) { fd = 0; maphandle = 0; - view = 0; len = 0; created(); } void MemoryMappedFile::close() { - if ( view ) - UnmapViewOfFile(view); - view = 0; + for( vector<void*>::iterator i = views.begin(); i != views.end(); i++ ) { + clearWritableBits(*i); + UnmapViewOfFile(*i); + } + views.clear(); if ( maphandle ) CloseHandle(maphandle); maphandle = 0; @@ -41,22 +56,37 @@ namespace mongo { CloseHandle(fd); fd = 0; } - - unsigned mapped = 0; - void* MemoryMappedFile::map(const char *filenameIn, long &length, int options) { - _filename = filenameIn; + unsigned long long mapped = 0; + + void* MemoryMappedFile::createReadOnlyMap() { + assert( maphandle ); + scoped_lock lk(mapViewMutex); + void *p = MapViewOfFile(maphandle, FILE_MAP_READ, /*f ofs hi*/0, /*f ofs lo*/ 0, /*dwNumberOfBytesToMap 0 means to eof*/0); + if ( p == 0 ) { + DWORD e = GetLastError(); + log() << "FILE_MAP_READ MapViewOfFile failed " << filename() << " " << errnoWithDescription(e) << endl; + } + else { + views.push_back(p); + } + return p; + } + + void* MemoryMappedFile::map(const char *filenameIn, unsigned long long &length, int options) { + assert( fd == 0 && len == 0 ); // can't open more than once + setFilename(filenameIn); /* big hack here: Babble uses db names with colons. doesn't seem to work on windows. temporary perhaps. */ char filename[256]; strncpy(filename, filenameIn, 255); filename[255] = 0; - { + { size_t len = strlen( filename ); - for ( size_t i=len-1; i>=0; i-- ){ + for ( size_t i=len-1; i>=0; i-- ) { if ( filename[i] == '/' || - filename[i] == '\\' ) + filename[i] == '\\' ) break; - + if ( filename[i] == ':' ) filename[i] = '_'; } @@ -64,78 +94,104 @@ namespace mongo { updateLength( filename, length ); - DWORD createOptions = FILE_ATTRIBUTE_NORMAL; - if ( options & SEQUENTIAL ) - createOptions |= FILE_FLAG_SEQUENTIAL_SCAN; - - fd = CreateFile( - toNativeString(filename).c_str(), - GENERIC_WRITE | GENERIC_READ, FILE_SHARE_READ, - NULL, OPEN_ALWAYS, createOptions , NULL); - if ( fd == INVALID_HANDLE_VALUE ) { - out() << "Create/OpenFile failed " << filename << ' ' << GetLastError() << endl; - return 0; + { + DWORD createOptions = FILE_ATTRIBUTE_NORMAL; + if ( options & SEQUENTIAL ) + createOptions |= FILE_FLAG_SEQUENTIAL_SCAN; + DWORD rw = GENERIC_READ | GENERIC_WRITE; + fd = CreateFile( + toNativeString(filename).c_str(), + rw, // desired access + FILE_SHARE_WRITE | FILE_SHARE_READ, // share mode + NULL, // security + OPEN_ALWAYS, // create disposition + createOptions , // flags + NULL); // hTempl + if ( fd == INVALID_HANDLE_VALUE ) { + DWORD e = GetLastError(); + log() << "Create/OpenFile failed " << filename << " errno:" << e << endl; + return 0; + } } mapped += length; - maphandle = CreateFileMapping(fd, NULL, PAGE_READWRITE, 0, length, NULL); - if ( maphandle == NULL ) { - out() << "CreateFileMapping failed " << filename << ' ' << GetLastError() << endl; - return 0; + { + DWORD flProtect = PAGE_READWRITE; //(options & READONLY)?PAGE_READONLY:PAGE_READWRITE; + maphandle = CreateFileMapping(fd, NULL, flProtect, + length >> 32 /*maxsizehigh*/, + (unsigned) length /*maxsizelow*/, + NULL/*lpName*/); + if ( maphandle == NULL ) { + DWORD e = GetLastError(); // log() call was killing lasterror before we get to that point in the stream + log() << "CreateFileMapping failed " << filename << ' ' << errnoWithDescription(e) << endl; + close(); + return 0; + } } - view = MapViewOfFile(maphandle, FILE_MAP_ALL_ACCESS, 0, 0, 0); + void *view = 0; + { + scoped_lock lk(mapViewMutex); + DWORD access = (options&READONLY)? FILE_MAP_READ : FILE_MAP_ALL_ACCESS; + view = MapViewOfFile(maphandle, access, /*f ofs hi*/0, /*f ofs lo*/ 0, /*dwNumberOfBytesToMap 0 means to eof*/0); + } if ( view == 0 ) { - out() << "MapViewOfFile failed " << filename << " " << errnoWithDescription() << " "; - out() << GetLastError(); - out() << endl; + DWORD e = GetLastError(); + log() << "MapViewOfFile failed " << filename << " " << errnoWithDescription(e) << endl; + close(); + } + else { + views.push_back(view); } len = length; + return view; } class WindowsFlushable : public MemoryMappedFile::Flushable { public: - WindowsFlushable( void * view , HANDLE fd , string filename ) - : _view(view) , _fd(fd) , _filename(filename){ - - } - - void flush(){ - if (!_view || !_fd) + WindowsFlushable( void * view , HANDLE fd , string filename , boost::shared_ptr<mutex> flushMutex ) + : _view(view) , _fd(fd) , _filename(filename) , _flushMutex(flushMutex) + {} + + void flush() { + if (!_view || !_fd) return; + scoped_lock lk(*_flushMutex); + bool success = FlushViewOfFile(_view, 0); // 0 means whole mapping - if (!success){ + if (!success) { int err = GetLastError(); out() << "FlushViewOfFile failed " << err << " file: " << _filename << endl; } - + success = FlushFileBuffers(_fd); - if (!success){ + if (!success) { int err = GetLastError(); out() << "FlushFileBuffers failed " << err << " file: " << _filename << endl; } } - + void * _view; HANDLE _fd; string _filename; - + boost::shared_ptr<mutex> _flushMutex; }; - + void MemoryMappedFile::flush(bool sync) { uassert(13056, "Async flushing not supported on windows", sync); - - WindowsFlushable f( view , fd , _filename ); - f.flush(); + if( !views.empty() ) { + WindowsFlushable f( views[0] , fd , filename() , _flushMutex); + f.flush(); + } } - MemoryMappedFile::Flushable * MemoryMappedFile::prepareFlush(){ - return new WindowsFlushable( view , fd , _filename ); + MemoryMappedFile::Flushable * MemoryMappedFile::prepareFlush() { + return new WindowsFlushable( views.empty() ? 0 : views[0] , fd , filename() , _flushMutex ); } void MemoryMappedFile::_lock() {} void MemoryMappedFile::_unlock() {} -} +} diff --git a/util/mongoutils/README b/util/mongoutils/README index ab614b6..fd2a589 100755 --- a/util/mongoutils/README +++ b/util/mongoutils/README @@ -3,5 +3,11 @@ (1) code is not database specific, rather, true utilities (2) are cross platform (3) may require boost headers, but not libs - (4) are clean and easy to use in any c++ project without pulling in lots of other stuff + (4) are clean and easy to use in any c++ project without pulling in lots of other stuff. + specifically, does not use anything in the mongo namespace! (5) apache license + (6) must be documented! if you aren't going to bother (but don't do that), stick it in util. + (7) ideally header only (in the spirit of #3) + + So basically, easy to use, general purpose stuff, with no arduous dependencies to drop into + any new project. diff --git a/util/mongoutils/checksum.h b/util/mongoutils/checksum.h index 6beb7f4..ea3d051 100644 --- a/util/mongoutils/checksum.h +++ b/util/mongoutils/checksum.h @@ -1,4 +1,4 @@ -/** @checksum.h */ +/** @file checksum.h */ /* Copyright 2009 10gen Inc. * @@ -22,7 +22,7 @@ namespace mongoutils { /** * this is a silly temporary implementation */ - inline int checksum( const char* x , int size ){ + inline int checksum( const char* x , int size ) { int ck = 0; for ( int i=0; i<size; i++ ) ck += ( (int)x[i] * ( i + 1 ) ); diff --git a/util/mongoutils/hash.h b/util/mongoutils/hash.h new file mode 100644 index 0000000..49f30b3 --- /dev/null +++ b/util/mongoutils/hash.h @@ -0,0 +1,41 @@ +/** @file hash.h */ + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace mongoutils { + + /** @return hash of a pointer to an unsigned. so you get a 32 bit hash out, regardless of whether + pointers are 32 or 64 bit on the particular platform. + + is there a faster way to impl this that hashes just as well? + */ + inline unsigned hashPointer(void *v) { + unsigned x = 0; + unsigned char *p = (unsigned char *) &v; + for( unsigned i = 0; i < sizeof(void*); i++ ) { + x = x * 131 + p[i]; + } + return x; + } + + inline unsigned hash(unsigned u) { + unsigned char *p = (unsigned char *) &u; + return (((((p[3] * 131) + p[2]) * 131) + p[1]) * 131) + p[0]; + } + +} diff --git a/util/mongoutils/html.h b/util/mongoutils/html.h index e8502ec..f79e6ca 100644 --- a/util/mongoutils/html.h +++ b/util/mongoutils/html.h @@ -2,7 +2,7 @@ #pragma once -/* Things in the mongoutils namespace +/* Things in the mongoutils namespace (1) are not database specific, rather, true utilities (2) are cross platform (3) may require boost headers, but not libs @@ -37,41 +37,41 @@ namespace mongoutils { inline string _tr() { return "</tr>\n"; } inline string tr() { return "<tr>"; } - inline string tr(string a, string b) { + inline string tr(string a, string b) { stringstream ss; ss << "<tr><td>" << a << "</td><td>" << b << "</td></tr>\n"; return ss.str(); } template <class T> - inline string td(T x) { + inline string td(T x) { stringstream ss; ss << "<td>" << x << "</td>"; return ss.str(); } - inline string td(string x) { + inline string td(string x) { return "<td>" + x + "</td>"; } - inline string th(string x) { + inline string th(string x) { return "<th>" + x + "</th>"; } - inline void tablecell( stringstream& ss , bool b ){ + inline void tablecell( stringstream& ss , bool b ) { ss << "<td>" << (b ? "<b>X</b>" : "") << "</td>"; } - template< typename T> - inline void tablecell( stringstream& ss , const T& t ){ + template< typename T> + inline void tablecell( stringstream& ss , const T& t ) { ss << "<td>" << t << "</td>"; } - - inline string table(const char *headers[] = 0, bool border = true) { + + inline string table(const char *headers[] = 0, bool border = true) { stringstream ss; - ss << "\n<table " - << (border?"border=1 ":"") - << "cellpadding=2 cellspacing=0>\n"; - if( headers ) { + ss << "\n<table " + << (border?"border=1 ":"") + << "cellpadding=2 cellspacing=0>\n"; + if( headers ) { ss << "<tr>"; - while( *headers ) { + while( *headers ) { ss << "<th>" << *headers << "</th>"; headers++; } @@ -80,18 +80,18 @@ namespace mongoutils { return ss.str(); } - inline string start(string title) { + inline string start(string title) { stringstream ss; ss << "<html><head>\n<title>"; ss << title; ss << "</title>\n"; ss << "<style type=\"text/css\" media=\"screen\">" - "body { font-family: helvetica, arial, san-serif }\n" - "table { border-collapse:collapse; border-color:#999; margin-top:.5em }\n" - "th { background-color:#bbb; color:#000 }\n" - "td,th { padding:.25em }\n" - "</style>\n"; + "body { font-family: helvetica, arial, san-serif }\n" + "table { border-collapse:collapse; border-color:#999; margin-top:.5em }\n" + "th { background-color:#bbb; color:#000 }\n" + "td,th { padding:.25em }\n" + "</style>\n"; ss << "</head>\n<body>\n"; return ss.str(); @@ -141,7 +141,7 @@ namespace mongoutils { } /* does NOT escape the strings. */ - inline string a(string href, string title="", string contentHtml = "") { + inline string a(string href, string title="", string contentHtml = "") { stringstream ss; ss << "<a"; if( !href.empty() ) ss << " href=\"" << href << '"'; diff --git a/util/mongoutils/mongoutils.vcxproj b/util/mongoutils/mongoutils.vcxproj index f8919cd..f6ec093 100755 --- a/util/mongoutils/mongoutils.vcxproj +++ b/util/mongoutils/mongoutils.vcxproj @@ -42,6 +42,7 @@ <ClCompile>
<WarningLevel>Level3</WarningLevel>
<Optimization>Disabled</Optimization>
+ <AdditionalIncludeDirectories>c:\boost;\boost</AdditionalIncludeDirectories>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -53,6 +54,7 @@ <Optimization>MaxSpeed</Optimization>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
+ <AdditionalIncludeDirectories>c:\boost;\boost</AdditionalIncludeDirectories>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
diff --git a/util/mongoutils/str.h b/util/mongoutils/str.h index 2028264..ea8f938 100644 --- a/util/mongoutils/str.h +++ b/util/mongoutils/str.h @@ -17,36 +17,37 @@ #pragma once -/* Things in the mongoutils namespace +/* Things in the mongoutils namespace (1) are not database specific, rather, true utilities (2) are cross platform (3) may require boost headers, but not libs (4) are clean and easy to use in any c++ project without pulling in lots of other stuff - Note: within this module, we use int for all offsets -- there are no unsigned offsets - and no size_t's. If you need 3 gigabyte long strings, don't use this module. + Note: within this module, we use int for all offsets -- there are no unsigned offsets + and no size_t's. If you need 3 gigabyte long strings, don't use this module. */ #include <string> #include <sstream> +#include "../../bson/util/builder.h" namespace mongoutils { namespace str { - using namespace std; + typedef std::string string; - /** the idea here is to make one liners easy. e.g.: + /** the idea here is to make one liners easy. e.g.: return str::stream() << 1 << ' ' << 2; since the following doesn't work: - + (stringstream() << 1).str(); */ class stream { public: - stringstream ss; + mongo::StringBuilder ss; template<class T> stream& operator<<(const T& v) { @@ -60,7 +61,7 @@ namespace mongoutils { inline bool startsWith(const char *str, const char *prefix) { const char *s = str; const char *p = prefix; - while( *p ) { + while( *p ) { if( *p != *s ) return false; p++; s++; } @@ -68,37 +69,56 @@ namespace mongoutils { } inline bool startsWith(string s, string p) { return startsWith(s.c_str(), p.c_str()); } - inline bool endsWith(string s, string p) { + inline bool endsWith(string s, string p) { int l = p.size(); int x = s.size(); if( x < l ) return false; return strncmp(s.c_str()+x-l, p.c_str(), l) == 0; } + inline bool equals( const char * a , const char * b ) { return strcmp( a , b ) == 0; } + /** find char x, and return rest of string thereafter, or "" if not found */ inline const char * after(const char *s, char x) { const char *p = strchr(s, x); - return (p != 0) ? p+1 : ""; } + return (p != 0) ? p+1 : ""; + } inline string after(const string& s, char x) { const char *p = strchr(s.c_str(), x); - return (p != 0) ? string(p+1) : ""; } + return (p != 0) ? string(p+1) : ""; + } + /** find string x, and return rest of string thereafter, or "" if not found */ inline const char * after(const char *s, const char *x) { const char *p = strstr(s, x); - return (p != 0) ? p+strlen(x) : ""; } + return (p != 0) ? p+strlen(x) : ""; + } inline string after(string s, string x) { const char *p = strstr(s.c_str(), x.c_str()); - return (p != 0) ? string(p+x.size()) : ""; } + return (p != 0) ? string(p+x.size()) : ""; + } - inline bool contains(string s, string x) { - return strstr(s.c_str(), x.c_str()) != 0; } + /** @return true if s contains x */ + inline bool contains(string s, string x) { + return strstr(s.c_str(), x.c_str()) != 0; + } + inline bool contains(string s, char x) { + return strchr(s.c_str(), x) != 0; + } /** @return everything befor the character x, else entire string */ inline string before(const string& s, char x) { const char *p = strchr(s.c_str(), x); - return (p != 0) ? s.substr(0, p-s.c_str()) : s; } + return (p != 0) ? s.substr(0, p-s.c_str()) : s; + } - /** check if if strings share a common starting prefix + /** @return everything befor the string x, else entire string */ + inline string before(const string& s, const string& x) { + const char *p = strstr(s.c_str(), x.c_str()); + return (p != 0) ? s.substr(0, p-s.c_str()) : s; + } + + /** check if if strings share a common starting prefix @return offset of divergence (or length if equal). 0=nothing in common. */ inline int shareCommonPrefix(const char *p, const char *q) { int ofs = 0; @@ -109,10 +129,80 @@ namespace mongoutils { break; p++; q++; ofs++; } - return ofs; } + return ofs; + } inline int shareCommonPrefix(const string &a, const string &b) { return shareCommonPrefix(a.c_str(), b.c_str()); } + /** string to unsigned. zero if not a number. can end with non-num chars */ + inline unsigned toUnsigned(const string& a) { + unsigned x = 0; + const char *p = a.c_str(); + while( 1 ) { + if( !isdigit(*p) ) + break; + x = x * 10 + (*p - '0'); + p++; + } + return x; + } + + /** split a string on a specific char. We don't split N times, just once + on the first occurrence. If char not present entire string is in L + and R is empty. + @return true if char found + */ + inline bool splitOn(const string &s, char c, string& L, string& R) { + const char *start = s.c_str(); + const char *p = strchr(start, c); + if( p == 0 ) { + L = s; R.clear(); + return false; + } + L = string(start, p-start); + R = string(p+1); + return true; + } + /** split scanning reverse direction. Splits ONCE ONLY. */ + inline bool rSplitOn(const string &s, char c, string& L, string& R) { + const char *start = s.c_str(); + const char *p = strrchr(start, c); + if( p == 0 ) { + L = s; R.clear(); + return false; + } + L = string(start, p-start); + R = string(p+1); + return true; + } + + /** @return number of occurrences of c in s */ + inline unsigned count( const string& s , char c ) { + unsigned n=0; + for ( unsigned i=0; i<s.size(); i++ ) + if ( s[i] == c ) + n++; + return n; + } + + /** trim leading spaces. spaces only, not tabs etc. */ + inline string ltrim(const string& s) { + const char *p = s.c_str(); + while( *p == ' ' ) p++; + return p; + } + + /** remove trailing chars in place */ + inline void stripTrailing(string& s, const char *chars) { + string::iterator i = s.end(); + while( s.begin() != i ) { + i--; + if( contains(chars, *i) ) { + s.erase(i); + } + } + } + } } diff --git a/util/mongoutils/test.cpp b/util/mongoutils/test.cpp index 0420624..d8ee461 100755..100644 --- a/util/mongoutils/test.cpp +++ b/util/mongoutils/test.cpp @@ -1,34 +1,45 @@ -/* @file test.cpp
- utils/mongoutils/test.cpp
- unit tests for mongoutils
-*/
-
-/*
- * Copyright 2010 10gen Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "str.h"
-#include "html.h"
-#include <assert.h>
-
-using namespace std;
-using namespace mongoutils;
-
-int main() {
- string x = str::after("abcde", 'c');
- assert( x == "de" );
- assert( str::after("abcde", 'x') == "" );
- return 0;
-}
+/* @file test.cpp + utils/mongoutils/test.cpp + unit tests for mongoutils +*/ + +/* + * Copyright 2010 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "str.h" +#include "html.h" +#include <assert.h> + +using namespace std; +using namespace mongoutils; + +int main() { + { + string s = "abcde"; + str::stripTrailing(s, "ef"); + assert( s == "abcd" ); + str::stripTrailing(s, "abcd"); + assert( s.empty() ); + s = "abcddd"; + str::stripTrailing(s, "d"); + assert( s == "abc" ); + } + + string x = str::after("abcde", 'c'); + assert( x == "de" ); + assert( str::after("abcde", 'x') == "" ); + return 0; +} diff --git a/util/moveablebuffer.h b/util/moveablebuffer.h new file mode 100644 index 0000000..e01f2d8 --- /dev/null +++ b/util/moveablebuffer.h @@ -0,0 +1,51 @@ +/* moveablebuffer.h +*/ + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace mongo { + + /** this is a sort of smart pointer class where we can move where something is and all the pointers will adjust. + not threadsafe. + */ + struct MoveableBuffer { + MoveableBuffer(); + MoveableBuffer(void *); + MoveableBuffer& operator=(const MoveableBuffer&); + ~MoveableBuffer(); + + void *p; + }; + + /* implementation (inlines) below */ + + // this is a temp stub implementation...not really done yet - just having everything compile & such for checkpointing into git + + inline MoveableBuffer::MoveableBuffer() : p(0) { } + + inline MoveableBuffer::MoveableBuffer(void *_p) : p(_p) { } + + inline MoveableBuffer& MoveableBuffer::operator=(const MoveableBuffer& r) { + p = r.p; + return *this; + } + + inline MoveableBuffer::~MoveableBuffer() { + } + +} diff --git a/util/ntservice.cpp b/util/ntservice.cpp index 22f83a5..ccf2981 100644 --- a/util/ntservice.cpp +++ b/util/ntservice.cpp @@ -25,15 +25,107 @@ namespace mongo { - void shutdown(); + void shutdownServer(); - SERVICE_STATUS_HANDLE ServiceController::_statusHandle = NULL; - std::wstring ServiceController::_serviceName; - ServiceCallback ServiceController::_serviceCallback = NULL; + SERVICE_STATUS_HANDLE ServiceController::_statusHandle = NULL; + std::wstring ServiceController::_serviceName; + ServiceCallback ServiceController::_serviceCallback = NULL; - ServiceController::ServiceController() { + ServiceController::ServiceController() {} + + bool initService(); + + // returns true if the service is started. + bool serviceParamsCheck( program_options::variables_map& params, const std::string dbpath, int argc, char* argv[] ) { + bool installService = false; + bool removeService = false; + bool reinstallService = false; + bool startService = false; + + std::wstring windowsServiceName = L"MongoDB"; + std::wstring windowsServiceDisplayName = L"Mongo DB"; + std::wstring windowsServiceDescription = L"Mongo DB Server"; + std::wstring windowsServiceUser = L""; + std::wstring windowsServicePassword = L""; + + if (params.count("install")) { + if ( ! params.count( "logpath" ) ) { + cerr << "--install has to be used with --logpath" << endl; + ::exit(-1); + } + installService = true; + } + if (params.count("reinstall")) { + if ( ! params.count( "logpath" ) ) { + cerr << "--reinstall has to be used with --logpath" << endl; + ::exit(-1); + } + reinstallService = true; + } + if (params.count("remove")) { + removeService = true; + } + if (params.count("service")) { + startService = true; + } + + if (params.count("serviceName")) { + string x = params["serviceName"].as<string>(); + windowsServiceName = wstring(x.size(),L' '); + for ( size_t i=0; i<x.size(); i++) { + windowsServiceName[i] = x[i]; + } + } + if (params.count("serviceDisplayName")) { + string x = params["serviceDisplayName"].as<string>(); + windowsServiceDisplayName = wstring(x.size(),L' '); + for ( size_t i=0; i<x.size(); i++) { + windowsServiceDisplayName[i] = x[i]; + } + } + if (params.count("serviceDescription")) { + string x = params["serviceDescription"].as<string>(); + windowsServiceDescription = wstring(x.size(),L' '); + for ( size_t i=0; i<x.size(); i++) { + windowsServiceDescription[i] = x[i]; + } + } + if (params.count("serviceUser")) { + string x = params["serviceUser"].as<string>(); + windowsServiceUser = wstring(x.size(),L' '); + for ( size_t i=0; i<x.size(); i++) { + windowsServiceUser[i] = x[i]; + } + } + if (params.count("servicePassword")) { + string x = params["servicePassword"].as<string>(); + windowsServicePassword = wstring(x.size(),L' '); + for ( size_t i=0; i<x.size(); i++) { + windowsServicePassword[i] = x[i]; + } + } + + if ( reinstallService ) { + ServiceController::removeService( windowsServiceName ); + } + if ( installService || reinstallService ) { + if ( !ServiceController::installService( windowsServiceName , windowsServiceDisplayName, windowsServiceDescription, windowsServiceUser, windowsServicePassword, dbpath, argc, argv ) ) + dbexit( EXIT_NTSERVICE_ERROR ); + dbexit( EXIT_CLEAN ); + } + else if ( removeService ) { + if ( !ServiceController::removeService( windowsServiceName ) ) + dbexit( EXIT_NTSERVICE_ERROR ); + dbexit( EXIT_CLEAN ); + } + else if ( startService ) { + if ( !ServiceController::startService( windowsServiceName , mongo::initService ) ) + dbexit( EXIT_NTSERVICE_ERROR ); + return true; + } + return false; } - + bool ServiceController::installService( const std::wstring& serviceName, const std::wstring& displayName, const std::wstring& serviceDesc, const std::wstring& serviceUser, const std::wstring& servicePassword, const std::string dbpath, int argc, char* argv[] ) { assert(argc >= 1); @@ -41,26 +133,30 @@ namespace mongo { if ( strchr(argv[0], ':') ) { // a crude test for fully qualified path commandLine << '"' << argv[0] << "\" "; - } else { + } + else { char buffer[256]; assert( _getcwd(buffer, 256) ); commandLine << '"' << buffer << '\\' << argv[0] << "\" "; } - + for ( int i = 1; i < argc; i++ ) { std::string arg( argv[ i ] ); // replace install command to indicate process is being started as a service if ( arg == "--install" || arg == "--reinstall" ) { arg = "--service"; - } else if ( arg == "--dbpath" && i + 1 < argc ) { + } + else if ( arg == "--dbpath" && i + 1 < argc ) { commandLine << arg << " \"" << dbpath << "\" "; i++; continue; - } else if ( arg == "--logpath" && i + 1 < argc ) { + } + else if ( arg == "--logpath" && i + 1 < argc ) { commandLine << arg << " \"" << argv[i+1] << "\" "; i++; continue; - } else if ( arg.length() > 9 && arg.substr(0, 9) == "--service" ) { + } + else if ( arg.length() > 9 && arg.substr(0, 9) == "--service" ) { // Strip off --service(Name|User|Password) arguments i++; continue; @@ -75,25 +171,25 @@ namespace mongo { return false; } - // Make sure servise doesn't already exist. - // TODO: Check to see if service is in "Deleting" status, suggest the user close down Services MMC snap-ins. - SC_HANDLE schService = ::OpenService( schSCManager, serviceName.c_str(), SERVICE_ALL_ACCESS ); - if ( schService != NULL ) { - cerr << "There is already a service named " << toUtf8String(serviceName) << ". Aborting" << endl; - ::CloseServiceHandle( schService ); - ::CloseServiceHandle( schSCManager ); - return false; - } - std::basic_ostringstream< TCHAR > commandLineWide; - commandLineWide << commandLine.str().c_str(); - - cerr << "Creating service " << toUtf8String(serviceName) << "." << endl; - - // create new service - schService = ::CreateService( schSCManager, serviceName.c_str(), displayName.c_str(), - SERVICE_ALL_ACCESS, SERVICE_WIN32_OWN_PROCESS, - SERVICE_AUTO_START, SERVICE_ERROR_NORMAL, - commandLineWide.str().c_str(), NULL, NULL, L"\0\0", NULL, NULL ); + // Make sure servise doesn't already exist. + // TODO: Check to see if service is in "Deleting" status, suggest the user close down Services MMC snap-ins. + SC_HANDLE schService = ::OpenService( schSCManager, serviceName.c_str(), SERVICE_ALL_ACCESS ); + if ( schService != NULL ) { + cerr << "There is already a service named " << toUtf8String(serviceName) << ". Aborting" << endl; + ::CloseServiceHandle( schService ); + ::CloseServiceHandle( schSCManager ); + return false; + } + std::basic_ostringstream< TCHAR > commandLineWide; + commandLineWide << commandLine.str().c_str(); + + cerr << "Creating service " << toUtf8String(serviceName) << "." << endl; + + // create new service + schService = ::CreateService( schSCManager, serviceName.c_str(), displayName.c_str(), + SERVICE_ALL_ACCESS, SERVICE_WIN32_OWN_PROCESS, + SERVICE_AUTO_START, SERVICE_ERROR_NORMAL, + commandLineWide.str().c_str(), NULL, NULL, L"\0\0", NULL, NULL ); if ( schService == NULL ) { DWORD err = ::GetLastError(); cerr << "Error creating service: " << GetWinErrMsg(err) << endl; @@ -101,56 +197,56 @@ namespace mongo { return false; } - cerr << "Service creation successful." << endl; - cerr << "Service can be started from the command line via 'net start \"" << toUtf8String(serviceName) << "\"'." << endl; - - bool serviceInstalled; - - // TODO: If neccessary grant user "Login as a Service" permission. - if ( !serviceUser.empty() ) { - std::wstring actualServiceUser; - if ( serviceUser.find(L"\\") == string::npos ) { - actualServiceUser = L".\\" + serviceUser; - } - else { - actualServiceUser = serviceUser; - } - - cerr << "Setting service login credentials. User: " << toUtf8String(actualServiceUser) << endl; - serviceInstalled = ::ChangeServiceConfig( schService, SERVICE_NO_CHANGE, SERVICE_NO_CHANGE, SERVICE_NO_CHANGE, NULL, NULL, NULL, NULL, actualServiceUser.c_str(), servicePassword.c_str(), NULL ); - if ( !serviceInstalled ) { - cerr << "Setting service login failed. Service has 'LocalService' permissions." << endl; - } - } - - // set the service description - SERVICE_DESCRIPTION serviceDescription; - serviceDescription.lpDescription = (LPTSTR)serviceDesc.c_str(); - serviceInstalled = ::ChangeServiceConfig2( schService, SERVICE_CONFIG_DESCRIPTION, &serviceDescription ); - - - if ( serviceInstalled ) { - SC_ACTION aActions[ 3 ] = { { SC_ACTION_RESTART, 0 }, { SC_ACTION_RESTART, 0 }, { SC_ACTION_RESTART, 0 } }; - - SERVICE_FAILURE_ACTIONS serviceFailure; - ZeroMemory( &serviceFailure, sizeof( SERVICE_FAILURE_ACTIONS ) ); - serviceFailure.cActions = 3; - serviceFailure.lpsaActions = aActions; - - // set service recovery options - serviceInstalled = ::ChangeServiceConfig2( schService, SERVICE_CONFIG_FAILURE_ACTIONS, &serviceFailure ); - - } - else { - cerr << "Could not set service description. Check the event log for more details." << endl; - } - - ::CloseServiceHandle( schService ); - ::CloseServiceHandle( schSCManager ); - - return serviceInstalled; + cerr << "Service creation successful." << endl; + cerr << "Service can be started from the command line via 'net start \"" << toUtf8String(serviceName) << "\"'." << endl; + + bool serviceInstalled; + + // TODO: If neccessary grant user "Login as a Service" permission. + if ( !serviceUser.empty() ) { + std::wstring actualServiceUser; + if ( serviceUser.find(L"\\") == string::npos ) { + actualServiceUser = L".\\" + serviceUser; + } + else { + actualServiceUser = serviceUser; + } + + cerr << "Setting service login credentials. User: " << toUtf8String(actualServiceUser) << endl; + serviceInstalled = ::ChangeServiceConfig( schService, SERVICE_NO_CHANGE, SERVICE_NO_CHANGE, SERVICE_NO_CHANGE, NULL, NULL, NULL, NULL, actualServiceUser.c_str(), servicePassword.c_str(), NULL ); + if ( !serviceInstalled ) { + cerr << "Setting service login failed. Service has 'LocalService' permissions." << endl; + } + } + + // set the service description + SERVICE_DESCRIPTION serviceDescription; + serviceDescription.lpDescription = (LPTSTR)serviceDesc.c_str(); + serviceInstalled = ::ChangeServiceConfig2( schService, SERVICE_CONFIG_DESCRIPTION, &serviceDescription ); + + + if ( serviceInstalled ) { + SC_ACTION aActions[ 3 ] = { { SC_ACTION_RESTART, 0 }, { SC_ACTION_RESTART, 0 }, { SC_ACTION_RESTART, 0 } }; + + SERVICE_FAILURE_ACTIONS serviceFailure; + ZeroMemory( &serviceFailure, sizeof( SERVICE_FAILURE_ACTIONS ) ); + serviceFailure.cActions = 3; + serviceFailure.lpsaActions = aActions; + + // set service recovery options + serviceInstalled = ::ChangeServiceConfig2( schService, SERVICE_CONFIG_FAILURE_ACTIONS, &serviceFailure ); + + } + else { + cerr << "Could not set service description. Check the event log for more details." << endl; + } + + ::CloseServiceHandle( schService ); + ::CloseServiceHandle( schSCManager ); + + return serviceInstalled; } - + bool ServiceController::removeService( const std::wstring& serviceName ) { SC_HANDLE schSCManager = ::OpenSCManager( NULL, NULL, SC_MANAGER_ALL_ACCESS ); if ( schSCManager == NULL ) { @@ -159,96 +255,96 @@ namespace mongo { return false; } - SC_HANDLE schService = ::OpenService( schSCManager, serviceName.c_str(), SERVICE_ALL_ACCESS ); - if ( schService == NULL ) { - cerr << "Could not find a service named " << toUtf8String(serviceName) << " to uninstall." << endl; - ::CloseServiceHandle( schSCManager ); - return false; - } - - SERVICE_STATUS serviceStatus; - - // stop service if its running - if ( ::ControlService( schService, SERVICE_CONTROL_STOP, &serviceStatus ) ) { - cerr << "Service " << toUtf8String(serviceName) << " is currently running. Stopping service." << endl; - while ( ::QueryServiceStatus( schService, &serviceStatus ) ) { - if ( serviceStatus.dwCurrentState == SERVICE_STOP_PENDING ) - { - Sleep( 1000 ); - } - else { break; } - } - cerr << "Service stopped." << endl; - } - - cerr << "Deleting service " << toUtf8String(serviceName) << "." << endl; - bool serviceRemoved = ::DeleteService( schService ); - - ::CloseServiceHandle( schService ); - ::CloseServiceHandle( schSCManager ); - - if (serviceRemoved) { - cerr << "Service deleted successfully." << endl; - } - else { - cerr << "Failed to delete service." << endl; - } - - return serviceRemoved; + SC_HANDLE schService = ::OpenService( schSCManager, serviceName.c_str(), SERVICE_ALL_ACCESS ); + if ( schService == NULL ) { + cerr << "Could not find a service named " << toUtf8String(serviceName) << " to uninstall." << endl; + ::CloseServiceHandle( schSCManager ); + return false; + } + + SERVICE_STATUS serviceStatus; + + // stop service if its running + if ( ::ControlService( schService, SERVICE_CONTROL_STOP, &serviceStatus ) ) { + cerr << "Service " << toUtf8String(serviceName) << " is currently running. Stopping service." << endl; + while ( ::QueryServiceStatus( schService, &serviceStatus ) ) { + if ( serviceStatus.dwCurrentState == SERVICE_STOP_PENDING ) { + Sleep( 1000 ); + } + else { break; } + } + cerr << "Service stopped." << endl; + } + + cerr << "Deleting service " << toUtf8String(serviceName) << "." << endl; + bool serviceRemoved = ::DeleteService( schService ); + + ::CloseServiceHandle( schService ); + ::CloseServiceHandle( schSCManager ); + + if (serviceRemoved) { + cerr << "Service deleted successfully." << endl; + } + else { + cerr << "Failed to delete service." << endl; + } + + return serviceRemoved; } - + bool ServiceController::startService( const std::wstring& serviceName, ServiceCallback startService ) { _serviceName = serviceName; - _serviceCallback = startService; - + _serviceCallback = startService; + SERVICE_TABLE_ENTRY dispTable[] = { - { (LPTSTR)serviceName.c_str(), (LPSERVICE_MAIN_FUNCTION)ServiceController::initService }, - { NULL, NULL } - }; + { (LPTSTR)serviceName.c_str(), (LPSERVICE_MAIN_FUNCTION)ServiceController::initService }, + { NULL, NULL } + }; - return StartServiceCtrlDispatcher( dispTable ); + return StartServiceCtrlDispatcher( dispTable ); } - + bool ServiceController::reportStatus( DWORD reportState, DWORD waitHint ) { - if ( _statusHandle == NULL ) - return false; - - static DWORD checkPoint = 1; - - SERVICE_STATUS ssStatus; - - ssStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; - ssStatus.dwServiceSpecificExitCode = 0; - ssStatus.dwControlsAccepted = reportState == SERVICE_START_PENDING ? 0 : SERVICE_ACCEPT_STOP; - ssStatus.dwCurrentState = reportState; - ssStatus.dwWin32ExitCode = NO_ERROR; - ssStatus.dwWaitHint = waitHint; - ssStatus.dwCheckPoint = ( reportState == SERVICE_RUNNING || reportState == SERVICE_STOPPED ) ? 0 : checkPoint++; - - return SetServiceStatus( _statusHandle, &ssStatus ); - } - + if ( _statusHandle == NULL ) + return false; + + static DWORD checkPoint = 1; + + SERVICE_STATUS ssStatus; + + ssStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; + ssStatus.dwServiceSpecificExitCode = 0; + ssStatus.dwControlsAccepted = reportState == SERVICE_START_PENDING ? 0 : SERVICE_ACCEPT_STOP; + ssStatus.dwCurrentState = reportState; + ssStatus.dwWin32ExitCode = NO_ERROR; + ssStatus.dwWaitHint = waitHint; + ssStatus.dwCheckPoint = ( reportState == SERVICE_RUNNING || reportState == SERVICE_STOPPED ) ? 0 : checkPoint++; + + return SetServiceStatus( _statusHandle, &ssStatus ); + } + void WINAPI ServiceController::initService( DWORD argc, LPTSTR *argv ) { - _statusHandle = RegisterServiceCtrlHandler( _serviceName.c_str(), serviceCtrl ); - if ( !_statusHandle ) - return; - - reportStatus( SERVICE_START_PENDING, 1000 ); - - _serviceCallback(); - - reportStatus( SERVICE_STOPPED ); - } - - void WINAPI ServiceController::serviceCtrl( DWORD ctrlCode ) { - switch ( ctrlCode ) { - case SERVICE_CONTROL_STOP: - case SERVICE_CONTROL_SHUTDOWN: - shutdown(); - reportStatus( SERVICE_STOPPED ); - return; - } - } + _statusHandle = RegisterServiceCtrlHandler( _serviceName.c_str(), serviceCtrl ); + if ( !_statusHandle ) + return; + + reportStatus( SERVICE_START_PENDING, 1000 ); + + _serviceCallback(); + dbexit( EXIT_CLEAN ); + + reportStatus( SERVICE_STOPPED ); + } + + void WINAPI ServiceController::serviceCtrl( DWORD ctrlCode ) { + switch ( ctrlCode ) { + case SERVICE_CONTROL_STOP: + case SERVICE_CONTROL_SHUTDOWN: + shutdownServer(); + reportStatus( SERVICE_STOPPED ); + return; + } + } } // namespace mongo diff --git a/util/ntservice.h b/util/ntservice.h index 271e7d7..4958d03 100644 --- a/util/ntservice.h +++ b/util/ntservice.h @@ -22,25 +22,26 @@ namespace mongo { - typedef bool ( *ServiceCallback )( void ); + typedef bool ( *ServiceCallback )( void ); + bool serviceParamsCheck( program_options::variables_map& params, const std::string dbpath, int argc, char* argv[] ); class ServiceController { public: ServiceController(); virtual ~ServiceController() {} - + static bool installService( const std::wstring& serviceName, const std::wstring& displayName, const std::wstring& serviceDesc, const std::wstring& serviceUser, const std::wstring& servicePassword, const std::string dbpath, int argc, char* argv[] ); static bool removeService( const std::wstring& serviceName ); static bool startService( const std::wstring& serviceName, ServiceCallback startService ); static bool reportStatus( DWORD reportState, DWORD waitHint = 0 ); - + static void WINAPI initService( DWORD argc, LPTSTR *argv ); - static void WINAPI serviceCtrl( DWORD ctrlCode ); - + static void WINAPI serviceCtrl( DWORD ctrlCode ); + protected: - static std::wstring _serviceName; - static SERVICE_STATUS_HANDLE _statusHandle; - static ServiceCallback _serviceCallback; + static std::wstring _serviceName; + static SERVICE_STATUS_HANDLE _statusHandle; + static ServiceCallback _serviceCallback; }; } // namespace mongo diff --git a/util/optime.h b/util/optime.h index 9214479..7e6be4d 100644 --- a/util/optime.h +++ b/util/optime.h @@ -21,9 +21,9 @@ namespace mongo { void exitCleanly( ExitCode code ); - + struct ClockSkewException : public DBException { - ClockSkewException() : DBException( "clock skew exception" , 20001 ){} + ClockSkewException() : DBException( "clock skew exception" , 20001 ) {} }; /* replsets use RSOpTime. @@ -56,14 +56,17 @@ namespace mongo { secs = a; i = b; } + OpTime( const OpTime& other ) { + secs = other.secs; + i = other.i; + } OpTime() { secs = 0; i = 0; } static OpTime now() { unsigned t = (unsigned) time(0); -// DEV assertInWriteLock(); - if ( t < last.secs ){ + if ( t < last.secs ) { bool toLog = false; ONCE toLog = true; RARELY toLog = true; @@ -82,13 +85,13 @@ namespace mongo { return last; } last = OpTime(t, 1); - return last; + return last; } - + /* We store OpTime's in the database as BSON Date datatype -- we needed some sort of 64 bit "container" for these values. While these are not really "Dates", that seems a better choice for now than say, Number, which is floating point. Note the BinData type - is perhaps the cleanest choice, lacking a true unsigned64 datatype, but BinData has 5 + is perhaps the cleanest choice, lacking a true unsigned64 datatype, but BinData has 5 bytes of overhead. */ unsigned long long asDate() const { @@ -97,9 +100,9 @@ namespace mongo { long long asLL() const { return reinterpret_cast<const long long*>(&i)[0]; } - + bool isNull() const { return secs == 0; } - + string toStringLong() const { char buf[64]; time_t_to_String(secs, buf); @@ -108,13 +111,13 @@ namespace mongo { ss << hex << secs << ':' << i; return ss.str(); } - + string toStringPretty() const { stringstream ss; ss << time_t_to_String_short(secs) << ':' << hex << i; return ss.str(); } - + string toString() const { stringstream ss; ss << hex << secs << ':' << i; @@ -132,10 +135,10 @@ namespace mongo { return secs < r.secs; return i < r.i; } - bool operator<=(const OpTime& r) const { + bool operator<=(const OpTime& r) const { return *this < r || *this == r; } - bool operator>(const OpTime& r) const { + bool operator>(const OpTime& r) const { return !(*this <= r); } bool operator>=(const OpTime& r) const { @@ -143,5 +146,5 @@ namespace mongo { } }; #pragma pack() - + } // namespace mongo diff --git a/util/password.h b/util/password.h index 18294b2..519f712 100644 --- a/util/password.h +++ b/util/password.h @@ -39,7 +39,7 @@ namespace mongo { return false; } - void xparse( boost::any& value_store, + void xparse( boost::any& value_store, const std::vector<std::string>& new_tokens ) const { if ( !value_store.empty() ) #if BOOST_VERSION >= 104200 @@ -49,7 +49,7 @@ namespace mongo { #endif else if ( !new_tokens.empty() ) boost::program_options::typed_value<std::string>::xparse - (value_store, new_tokens); + (value_store, new_tokens); else value_store = std::string(); } diff --git a/util/paths.h b/util/paths.h new file mode 100644 index 0000000..ce0a378 --- /dev/null +++ b/util/paths.h @@ -0,0 +1,79 @@ +// @file paths.h +// file paths and directory handling + +/* Copyright 2010 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "mongoutils/str.h" + +using namespace mongoutils; + +namespace mongo { + + extern string dbpath; + + /** this is very much like a boost::path. however, we define a new type to get some type + checking. if you want to say 'my param MUST be a relative path", use this. + */ + struct RelativePath { + string _p; + + bool empty() const { return _p.empty(); } + + static RelativePath fromRelativePath(string f) { + RelativePath rp; + rp._p = f; + return rp; + } + + /** from a full path */ + static RelativePath fromFullPath(path f) { + path dbp(dbpath); // normalizes / and backslash + string fullpath = f.string(); + string relative = str::after(fullpath, dbp.string()); + if( relative.empty() ) { + log() << "warning file is not under db path? " << fullpath << ' ' << dbp.string() << endl; + RelativePath rp; + rp._p = fullpath; + return rp; + } + /*uassert(13600, + str::stream() << "file path is not under the db path? " << fullpath << ' ' << dbpath, + relative != fullpath);*/ + if( str::startsWith(relative, "/") || str::startsWith(relative, "\\") ) { + relative.erase(0, 1); + } + RelativePath rp; + rp._p = relative; + return rp; + } + + string toString() const { return _p; } + + bool operator!=(const RelativePath& r) const { return _p != r._p; } + bool operator==(const RelativePath& r) const { return _p == r._p; } + bool operator<(const RelativePath& r) const { return _p < r._p; } + + string asFullPath() const { + path x(dbpath); + x /= _p; + return x.string(); + } + + }; + +} diff --git a/util/processinfo.cpp b/util/processinfo.cpp index 3257b5e..2d5b6e3 100644 --- a/util/processinfo.cpp +++ b/util/processinfo.cpp @@ -17,31 +17,46 @@ #include "pch.h" #include "processinfo.h" +#include "mmap.h" #include <iostream> using namespace std; namespace mongo { - + class PidFileWiper { public: - ~PidFileWiper(){ + ~PidFileWiper() { ofstream out( path.c_str() , ios_base::out ); - out.close(); + out.close(); } - - void write( const string& p ){ + + void write( const string& p ) { path = p; ofstream out( path.c_str() , ios_base::out ); out << getpid() << endl; out.close(); } - + string path; } pidFileWiper; - - void writePidFile( const string& path ){ + + void writePidFile( const string& path ) { pidFileWiper.write( path ); - } + } + + void printMemInfo( const char * where ) { + cout << "mem info: "; + if ( where ) + cout << where << " "; + ProcessInfo pi; + if ( ! pi.supported() ) { + cout << " not supported" << endl; + return; + } + + cout << "vsize: " << pi.getVirtualMemorySize() << " resident: " << pi.getResidentSize() << " mapped: " << ( MemoryMappedFile::totalMappedLength() / ( 1024 * 1024 ) ) << endl; + } + } diff --git a/util/processinfo.h b/util/processinfo.h index 8e20beb..b10e6fe 100644 --- a/util/processinfo.h +++ b/util/processinfo.h @@ -30,12 +30,12 @@ int getpid(); namespace mongo { class BSONObjBuilder; - + class ProcessInfo { public: ProcessInfo( pid_t pid = getpid() ); ~ProcessInfo(); - + /** * @return mbytes */ @@ -50,7 +50,7 @@ namespace mongo { * Append platform-specific data to obj */ void getExtraInfo(BSONObjBuilder& info); - + bool supported(); bool blockCheckSupported(); @@ -59,7 +59,9 @@ namespace mongo { private: pid_t _pid; }; - + void writePidFile( const std::string& path ); - + + void printMemInfo( const char * where ); + } diff --git a/util/processinfo_darwin.cpp b/util/processinfo_darwin.cpp index cb54bed..c1190ae 100644 --- a/util/processinfo_darwin.cpp +++ b/util/processinfo_darwin.cpp @@ -36,58 +36,58 @@ using namespace std; namespace mongo { - - ProcessInfo::ProcessInfo( pid_t pid ) : _pid( pid ){ + + ProcessInfo::ProcessInfo( pid_t pid ) : _pid( pid ) { } - ProcessInfo::~ProcessInfo(){ + ProcessInfo::~ProcessInfo() { } - bool ProcessInfo::supported(){ + bool ProcessInfo::supported() { return true; } - - int ProcessInfo::getVirtualMemorySize(){ + + int ProcessInfo::getVirtualMemorySize() { task_t result; - + mach_port_t task; - - if ( ( result = task_for_pid( mach_task_self() , _pid , &task) ) != KERN_SUCCESS ){ + + if ( ( result = task_for_pid( mach_task_self() , _pid , &task) ) != KERN_SUCCESS ) { cout << "error getting task\n"; return 0; } - + #if !defined(__LP64__) task_basic_info_32 ti; #else task_basic_info_64 ti; #endif mach_msg_type_number_t count = TASK_BASIC_INFO_COUNT; - if ( ( result = task_info( task , TASK_BASIC_INFO , (task_info_t)&ti, &count ) ) != KERN_SUCCESS ){ + if ( ( result = task_info( task , TASK_BASIC_INFO , (task_info_t)&ti, &count ) ) != KERN_SUCCESS ) { cout << "error getting task_info: " << result << endl; return 0; } return (int)((double)ti.virtual_size / (1024.0 * 1024 ) ); } - - int ProcessInfo::getResidentSize(){ + + int ProcessInfo::getResidentSize() { task_t result; - + mach_port_t task; - - if ( ( result = task_for_pid( mach_task_self() , _pid , &task) ) != KERN_SUCCESS ){ + + if ( ( result = task_for_pid( mach_task_self() , _pid , &task) ) != KERN_SUCCESS ) { cout << "error getting task\n"; return 0; } - - + + #if !defined(__LP64__) task_basic_info_32 ti; #else task_basic_info_64 ti; #endif mach_msg_type_number_t count = TASK_BASIC_INFO_COUNT; - if ( ( result = task_info( task , TASK_BASIC_INFO , (task_info_t)&ti, &count ) ) != KERN_SUCCESS ){ + if ( ( result = task_info( task , TASK_BASIC_INFO , (task_info_t)&ti, &count ) ) != KERN_SUCCESS ) { cout << "error getting task_info: " << result << endl; return 0; } @@ -96,18 +96,18 @@ namespace mongo { void ProcessInfo::getExtraInfo(BSONObjBuilder& info) {} - bool ProcessInfo::blockCheckSupported(){ + bool ProcessInfo::blockCheckSupported() { return true; } - - bool ProcessInfo::blockInMemory( char * start ){ + + bool ProcessInfo::blockInMemory( char * start ) { static long pageSize = 0; - if ( pageSize == 0 ){ + if ( pageSize == 0 ) { pageSize = sysconf( _SC_PAGESIZE ); } start = start - ( (unsigned long long)start % pageSize ); char x = 0; - if ( mincore( start , 128 , &x ) ){ + if ( mincore( start , 128 , &x ) ) { log() << "mincore failed: " << errnoWithDescription() << endl; return 1; } diff --git a/util/processinfo_linux2.cpp b/util/processinfo_linux2.cpp index 02a7ad0..e82e2d1 100644 --- a/util/processinfo_linux2.cpp +++ b/util/processinfo_linux2.cpp @@ -30,188 +30,189 @@ using namespace std; #define KLF "l" namespace mongo { - + class LinuxProc { public: - LinuxProc( pid_t pid = getpid() ){ + LinuxProc( pid_t pid = getpid() ) { char name[128]; sprintf( name , "/proc/%d/stat" , pid ); - + FILE * f = fopen( name , "r"); - if ( ! f ){ + if ( ! f ) { stringstream ss; ss << "couldn't open [" << name << "] " << errnoWithDescription(); string s = ss.str(); - msgassertedNoTrace( 13276 , s.c_str() ); + // help the assert# control uasserted( 13538 , s.c_str() ); + msgassertedNoTrace( 13538 , s.c_str() ); } int found = fscanf(f, - "%d %s %c " - "%d %d %d %d %d " - "%lu %lu %lu %lu %lu " - "%lu %lu %ld %ld " /* utime stime cutime cstime */ - "%ld %ld " - "%ld " - "%ld " - "%lu " /* start_time */ - "%lu " - "%ld " // rss - "%lu %"KLF"u %"KLF"u %"KLF"u %"KLF"u %"KLF"u " - /* - "%*s %*s %*s %*s " - "%"KLF"u %*lu %*lu " - "%d %d " - "%lu %lu" - */ - - , - - &_pid, - _comm, - &_state, - &_ppid, &_pgrp, &_session, &_tty, &_tpgid, - &_flags, &_min_flt, &_cmin_flt, &_maj_flt, &_cmaj_flt, - &_utime, &_stime, &_cutime, &_cstime, - &_priority, &_nice, - &_alarm, - &_nlwp, - &_start_time, - &_vsize, - &_rss, - &_rss_rlim, &_start_code, &_end_code, &_start_stack, &_kstk_esp, &_kstk_eip - - /* - &_wchan, - &_exit_signal, &_processor, - &_rtprio, &_sched - */ - ); - if ( found == 0 ){ + "%d %s %c " + "%d %d %d %d %d " + "%lu %lu %lu %lu %lu " + "%lu %lu %ld %ld " /* utime stime cutime cstime */ + "%ld %ld " + "%ld " + "%ld " + "%lu " /* start_time */ + "%lu " + "%ld " // rss + "%lu %"KLF"u %"KLF"u %"KLF"u %"KLF"u %"KLF"u " + /* + "%*s %*s %*s %*s " + "%"KLF"u %*lu %*lu " + "%d %d " + "%lu %lu" + */ + + , + + &_pid, + _comm, + &_state, + &_ppid, &_pgrp, &_session, &_tty, &_tpgid, + &_flags, &_min_flt, &_cmin_flt, &_maj_flt, &_cmaj_flt, + &_utime, &_stime, &_cutime, &_cstime, + &_priority, &_nice, + &_alarm, + &_nlwp, + &_start_time, + &_vsize, + &_rss, + &_rss_rlim, &_start_code, &_end_code, &_start_stack, &_kstk_esp, &_kstk_eip + + /* + &_wchan, + &_exit_signal, &_processor, + &_rtprio, &_sched + */ + ); + if ( found == 0 ) { cout << "system error: reading proc info" << endl; } fclose( f ); } - - unsigned long getVirtualMemorySize(){ + + unsigned long getVirtualMemorySize() { return _vsize; } - - unsigned long getResidentSize(){ + + unsigned long getResidentSize() { return (unsigned long)_rss * 4 * 1024; } - - int _pid; + + int _pid; // The process ID. - - char _comm[128]; + + char _comm[128]; // The filename of the executable, in parentheses. This is visible whether or not the executable is swapped out. - + char _state; //One character from the string "RSDZTW" where R is running, S is sleeping in an interruptible wait, D is waiting in uninterruptible // disk sleep, Z is zombie, T is traced or stopped (on a signal), and W is paging. - + int _ppid; // The PID of the parent. - + int _pgrp; // The process group ID of the process. - + int _session; // The session ID of the process. - + int _tty; // The tty the process uses. - + int _tpgid; // The process group ID of the process which currently owns the tty that the process is connected to. - + unsigned long _flags; // %lu // The kernel flags word of the process. For bit meanings, see the PF_* defines in <linux/sched.h>. Details depend on the kernel version. - + unsigned long _min_flt; // %lu // The number of minor faults the process has made which have not required loading a memory page from disk. - + unsigned long _cmin_flt; // %lu // The number of minor faults that the process - + unsigned long _maj_flt; // %lu // The number of major faults the process has made which have required loading a memory page from disk. - + unsigned long _cmaj_flt; // %lu // The number of major faults that the process - + unsigned long _utime; // %lu // The number of jiffies that this process has been scheduled in user mode. - + unsigned long _stime; // %lu // The number of jiffies that this process has been scheduled in kernel mode. - + long _cutime; // %ld // The number of jiffies that this removed field. - + long _cstime; // %ld - + long _priority; long _nice; - + long _nlwp; // %ld // The time in jiffies before the next SIGALRM is sent to the process due to an interval timer. - + unsigned long _alarm; - + unsigned long _start_time; // %lu // The time in jiffies the process started after system boot. - + unsigned long _vsize; // %lu // Virtual memory size in bytes. - + long _rss; // %ld // Resident Set Size: number of pages the process has in real memory, minus 3 for administrative purposes. This is just the pages which // count towards text, data, or stack space. This does not include pages which have not been demand-loaded in, or which are swapped out - + unsigned long _rss_rlim; // %lu // Current limit in bytes on the rss of the process (usually 4294967295 on i386). - + unsigned long _start_code; // %lu // The address above which program text can run. - + unsigned long _end_code; // %lu // The address below which program text can run. - + unsigned long _start_stack; // %lu // The address of the start of the stack. - + unsigned long _kstk_esp; // %lu // The current value of esp (stack pointer), as found in the kernel stack page for the process. - + unsigned long _kstk_eip; // %lu // The current EIP (instruction pointer). - - - + + + }; - ProcessInfo::ProcessInfo( pid_t pid ) : _pid( pid ){ + ProcessInfo::ProcessInfo( pid_t pid ) : _pid( pid ) { } - ProcessInfo::~ProcessInfo(){ + ProcessInfo::~ProcessInfo() { } - bool ProcessInfo::supported(){ + bool ProcessInfo::supported() { return true; } - - int ProcessInfo::getVirtualMemorySize(){ + + int ProcessInfo::getVirtualMemorySize() { LinuxProc p(_pid); return (int)( p.getVirtualMemorySize() / ( 1024.0 * 1024 ) ); } - - int ProcessInfo::getResidentSize(){ + + int ProcessInfo::getResidentSize() { LinuxProc p(_pid); return (int)( p.getResidentSize() / ( 1024.0 * 1024 ) ); } - void ProcessInfo::getExtraInfo(BSONObjBuilder& info){ + void ProcessInfo::getExtraInfo(BSONObjBuilder& info) { struct mallinfo malloc_info = mallinfo(); // structure has same name as function that returns it. (see malloc.h) info.append("heap_usage_bytes", malloc_info.uordblks); @@ -219,18 +220,18 @@ namespace mongo { info.append("page_faults", (int)p._maj_flt); } - bool ProcessInfo::blockCheckSupported(){ + bool ProcessInfo::blockCheckSupported() { return true; } - - bool ProcessInfo::blockInMemory( char * start ){ + + bool ProcessInfo::blockInMemory( char * start ) { static long pageSize = 0; - if ( pageSize == 0 ){ + if ( pageSize == 0 ) { pageSize = sysconf( _SC_PAGESIZE ); } start = start - ( (unsigned long long)start % pageSize ); unsigned char x = 0; - if ( mincore( start , 128 , &x ) ){ + if ( mincore( start , 128 , &x ) ) { log() << "mincore failed: " << errnoWithDescription() << endl; return 1; } diff --git a/util/processinfo_none.cpp b/util/processinfo_none.cpp index b54cb13..7d1e84d 100644 --- a/util/processinfo_none.cpp +++ b/util/processinfo_none.cpp @@ -22,32 +22,32 @@ using namespace std; namespace mongo { - - ProcessInfo::ProcessInfo( pid_t pid ){ + + ProcessInfo::ProcessInfo( pid_t pid ) { } - ProcessInfo::~ProcessInfo(){ + ProcessInfo::~ProcessInfo() { } - bool ProcessInfo::supported(){ + bool ProcessInfo::supported() { return false; } - - int ProcessInfo::getVirtualMemorySize(){ + + int ProcessInfo::getVirtualMemorySize() { return -1; } - - int ProcessInfo::getResidentSize(){ + + int ProcessInfo::getResidentSize() { return -1; } void ProcessInfo::getExtraInfo(BSONObjBuilder& info) {} - - bool ProcessInfo::blockCheckSupported(){ + + bool ProcessInfo::blockCheckSupported() { return false; } - - bool ProcessInfo::blockInMemory( char * start ){ + + bool ProcessInfo::blockInMemory( char * start ) { assert(0); return true; } diff --git a/util/processinfo_win32.cpp b/util/processinfo_win32.cpp index 5fc6ab5..d62b21b 100644 --- a/util/processinfo_win32.cpp +++ b/util/processinfo_win32.cpp @@ -25,27 +25,27 @@ using namespace std; -int getpid(){ +int getpid() { return GetCurrentProcessId(); } namespace mongo { - - int _wconvertmtos( SIZE_T s ){ + + int _wconvertmtos( SIZE_T s ) { return (int)( s / ( 1024 * 1024 ) ); } - - ProcessInfo::ProcessInfo( pid_t pid ){ + + ProcessInfo::ProcessInfo( pid_t pid ) { } - ProcessInfo::~ProcessInfo(){ + ProcessInfo::~ProcessInfo() { } - bool ProcessInfo::supported(){ + bool ProcessInfo::supported() { return true; } - - int ProcessInfo::getVirtualMemorySize(){ + + int ProcessInfo::getVirtualMemorySize() { MEMORYSTATUSEX mse; mse.dwLength = sizeof(mse); assert( GlobalMemoryStatusEx( &mse ) ); @@ -53,8 +53,8 @@ namespace mongo { assert( x <= 0x7fffffff ); return (int) x; } - - int ProcessInfo::getResidentSize(){ + + int ProcessInfo::getResidentSize() { PROCESS_MEMORY_COUNTERS pmc; assert( GetProcessMemoryInfo( GetCurrentProcess() , &pmc, sizeof(pmc) ) ); return _wconvertmtos( pmc.WorkingSetSize ); @@ -62,11 +62,11 @@ namespace mongo { void ProcessInfo::getExtraInfo(BSONObjBuilder& info) {} - bool ProcessInfo::blockCheckSupported(){ + bool ProcessInfo::blockCheckSupported() { return false; } - - bool ProcessInfo::blockInMemory( char * start ){ + + bool ProcessInfo::blockInMemory( char * start ) { assert(0); return true; } diff --git a/util/queue.h b/util/queue.h index 35e02a8..6a1e33a 100644 --- a/util/queue.h +++ b/util/queue.h @@ -18,12 +18,13 @@ #pragma once #include "../pch.h" -#include "../util/goodies.h" #include <queue> +#include "../util/timer.h" + namespace mongo { - + /** * simple blocking queue */ @@ -31,42 +32,67 @@ namespace mongo { public: BlockingQueue() : _lock("BlockingQueue") { } - void push(T const& t){ + void push(T const& t) { scoped_lock l( _lock ); _queue.push( t ); _condition.notify_one(); } - + bool empty() const { scoped_lock l( _lock ); return _queue.empty(); } - - bool tryPop( T & t ){ + + bool tryPop( T & t ) { scoped_lock l( _lock ); if ( _queue.empty() ) return false; - + t = _queue.front(); _queue.pop(); - + return true; } - - T blockingPop(){ + + T blockingPop() { scoped_lock l( _lock ); while( _queue.empty() ) _condition.wait( l.boost() ); - + T t = _queue.front(); _queue.pop(); - return t; + return t; } - + + + /** + * blocks waiting for an object until maxSecondsToWait passes + * if got one, return true and set in t + * otherwise return false and t won't be changed + */ + bool blockingPop( T& t , int maxSecondsToWait ) { + + Timer timer; + + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + xt.sec += maxSecondsToWait; + + scoped_lock l( _lock ); + while( _queue.empty() ) { + if ( ! _condition.timed_wait( l.boost() , xt ) ) + return false; + } + + t = _queue.front(); + _queue.pop(); + return true; + } + private: std::queue<T> _queue; - + mutable mongo::mutex _lock; boost::condition _condition; }; diff --git a/util/ramlog.h b/util/ramlog.h index 393527d..fc588e6 100644 --- a/util/ramlog.h +++ b/util/ramlog.h @@ -23,7 +23,7 @@ namespace mongo { class RamLog : public Tee { - enum { + enum { N = 128, C = 256 }; @@ -31,7 +31,7 @@ namespace mongo { unsigned h, n; public: - RamLog() { + RamLog() { h = 0; n = 0; for( int i = 0; i < N; i++ ) lines[i][C-1] = 0; @@ -52,7 +52,7 @@ namespace mongo { v.push_back(lines[i]); } - static int repeats(const vector<const char *>& v, int i) { + static int repeats(const vector<const char *>& v, int i) { for( int j = i-1; j >= 0 && j+8 > i; j-- ) { if( strcmp(v[i]+20,v[j]+20) == 0 ) { for( int x = 1; ; x++ ) { @@ -67,14 +67,14 @@ namespace mongo { } - static string clean(const vector<const char *>& v, int i, string line="") { + static string clean(const vector<const char *>& v, int i, string line="") { if( line.empty() ) line = v[i]; if( i > 0 && strncmp(v[i], v[i-1], 11) == 0 ) return string(" ") + line.substr(11); return v[i]; } - static string color(string line) { + static string color(string line) { string s = str::after(line, "replSet "); if( str::startsWith(s, "warning") || startsWith(s, "error") ) return html::red(line); @@ -85,16 +85,16 @@ namespace mongo { return html::yellow(line); return line; //html::blue(line); } - + return line; } /* turn http:... into an anchor */ - string linkify(const char *s) { + string linkify(const char *s) { const char *p = s; const char *h = strstr(p, "http://"); if( h == 0 ) return s; - + const char *sp = h + 7; while( *sp && *sp != ' ' ) sp++; @@ -115,15 +115,15 @@ namespace mongo { int r = repeats(v, i); if( r < 0 ) { s << color( linkify( clean(v,i).c_str() ) ); - } + } else { stringstream x; x << string(v[i], 0, 20); int nr = (i-r); int last = i+nr-1; for( ; r < i ; r++ ) x << '.'; - if( 1 ) { - stringstream r; + if( 1 ) { + stringstream r; if( nr == 1 ) r << "repeat last line"; else r << "repeats last " << nr << " lines; ends " << string(v[last]+4,0,15); first = false; s << html::a("", r.str(), clean(v,i,x.str())); @@ -135,7 +135,7 @@ namespace mongo { } s << "</pre>\n"; } - + }; diff --git a/util/ramstore.cpp b/util/ramstore.cpp deleted file mode 100644 index 0bdf2e2..0000000 --- a/util/ramstore.cpp +++ /dev/null @@ -1,93 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc.info -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -#include "pch.h" -#include "mmap.h" - -namespace mongo { - - //extern bool checkNsFilesOnLoad; - -static set<RamStoreFile*> files; - -void RamStoreFile::grow(int offset, int len) { - cout << "GROW ofs:" << offset << " len:" << len; - assert( len > 0 ); - Node& n = _m[offset]; - cout << " oldlen:" << n.len << endl; - assert( n.len > 0 ); - if( len > n.len ) { - n.p = (char *) realloc(n.p, len); - memset(((char *)n.p) + n.len, 0, len - n.len); - n.len = len; - } -} - -/* maxLen can be -1 for existing data */ -void* RamStoreFile::at(int offset, int maxLen) { - if( offset != _last ) { - if( _m.count(_last) ) { - _m[_last].check(); - if( !(offset < _last || offset >= _last + _m[_last].len) ) { - cout << offset << ' ' << _last << ' ' << _m[_last].len << endl; - assert(false); - } - } - } - _last = offset; - - Node& n = _m[offset]; - if( n.len == 0 ) { - // create - if( strstr(name, ".ns") == 0 ) - cout << "CREATE " << name << " ofs:" << offset << " len:" << maxLen << endl; - assert( maxLen >= 0 ); - n.p = (char *) calloc(maxLen+1, 1); - n.len = maxLen; - } - assert( n.len >= maxLen ); - n.check(); - return n.p; - } - -void RamStoreFile::Node::check() { - assert( p[len] == 0 ); -} - -void RamStoreFile::check() { - for( std::map<int,Node>::iterator i = _m.begin(); i != _m.end(); i++ ) { - i->second.check(); - } -} - -void RamStoreFile::validate() { - for( set<RamStoreFile*>::iterator i = files.begin(); i != files.end(); i++ ) { - (*i)->check(); - } -} - -RamStoreFile::~RamStoreFile() { - check(); - files.erase(this); -} - -RamStoreFile::RamStoreFile() : _len(0) { - // checkNsFilesOnLoad = false; - files.insert(this); -} - -} - diff --git a/util/ramstore.h b/util/ramstore.h deleted file mode 100644 index f75a57a..0000000 --- a/util/ramstore.h +++ /dev/null @@ -1,86 +0,0 @@ -// ramstore.h - -// mmap.h - -/* Copyright 2009 10gen Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -extern bool checkNsFilesOnLoad; - -class RamStoreFile : public MongoFile { - char name[256]; - struct Node { - char *p; - int len; - Node() : len(0) { } - void check(); - }; - std::map<int,Node> _m; - long _len; - - static void validate(); - void check(); - - int _last; - - void grow(int offset, int len); - - /* maxLen can be -1 for existing data */ - void* at(int offset, int maxLen); - -protected: - virtual void close() { - cout << "ramstore dealloc not yet implemented" << endl; - if( _len ) { - _len = 0; - } - } - virtual void flush(bool sync) { } - -public: - ~RamStoreFile(); - RamStoreFile(); - - virtual long length() { return _len; } - - class Pointer { - RamStoreFile* _f; - friend class RamStoreFile; - public: - void* at(int offset, int len) { - assert( len <= /*MaxBSONObjectSize*/4*1024*1024 + 128 ); - return _f->at(offset,len); - } - void grow(int offset, int len) { - assert( len <= /*MaxBSONObjectSize*/4*1024*1024 + 128 ); - _f->grow(offset,len); - } - bool isNull() const { return _f == 0; } - }; - - Pointer map( const char *filename ) { - assert(false); return Pointer(); - } - Pointer map(const char *_filename, long &length, int options=0) { - strncpy(name, _filename, sizeof(name)-1); - Pointer p; - p._f = this; - return p; - } - - static bool exists(boost::filesystem::path p) { - return false; - } -}; diff --git a/util/signal_handlers.cpp b/util/signal_handlers.cpp new file mode 100644 index 0000000..0e9ec7a --- /dev/null +++ b/util/signal_handlers.cpp @@ -0,0 +1,122 @@ +// signal_handlers.cpp + +/** +* Copyright (C) 2010 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#include "pch.h" + +#include <cstdarg> +#include <cstdio> +#include <cstdlib> + +#if !defined(_WIN32) // TODO: windows support +#include <unistd.h> +#endif + +#if !defined(_WIN32) && !defined(NOEXECINFO) +#include <execinfo.h> +#endif + +#include "log.h" +#include "signal_handlers.h" + +namespace mongo { + + /* + * WARNING: PLEASE READ BEFORE CHANGING THIS MODULE + * + * All code in this module should be singal-friendly. Before adding any system + * call or other dependency, please make sure the latter still holds. + * + */ + + static int rawWrite( int fd , char* c , int size ) { +#if !defined(_WIN32) + + int toWrite = size; + int writePos = 0; + int wrote; + while ( toWrite > 0 ) { + wrote = write( fd , &c[writePos] , toWrite ); + if ( wrote < 1 ) break; + toWrite -= wrote; + writePos += wrote; + } + return writePos; + +#else + + return -1; + +#endif + } + + static int formattedWrite( int fd , const char* format, ... ) { + const int MAX_ENTRY = 256; + static char entryBuf[MAX_ENTRY]; + + va_list ap; + va_start( ap , format ); + int entrySize = vsnprintf( entryBuf , MAX_ENTRY-1 , format , ap ); + if ( entrySize < 0 ) { + return -1; + } + + if ( rawWrite( fd , entryBuf , entrySize ) < 0 ) { + return -1; + } + + return 0; + } + + static void formattedBacktrace( int fd ) { + +#if !defined(_WIN32) && !defined(NOEXECINFO) + + int numFrames; + const int MAX_DEPTH = 20; + void* stackFrames[MAX_DEPTH]; + + numFrames = backtrace( stackFrames , 20 ); + for ( int i = 0; i < numFrames; i++ ) { + formattedWrite( fd , "%p " , stackFrames[i] ); + } + formattedWrite( fd , "\n" ); + + backtrace_symbols_fd( stackFrames , numFrames , fd ); + +#else + + formattedWrite( fd, "backtracing not implemented for this platform yet\n" ); + +#endif + + } + + void printStackAndExit( int signalNum ) { + int fd = Logstream::getLogDesc(); + + if ( fd >= 0 ) { + formattedWrite( fd , "Received signal %d\n" , signalNum ); + formattedWrite( fd , "Backtrace: " ); + formattedBacktrace( fd ); + formattedWrite( fd , "===\n" ); + } + + ::exit( EXIT_ABRUPT ); + } + +} // namespace mongo diff --git a/util/signal_handlers.h b/util/signal_handlers.h new file mode 100644 index 0000000..9d3a735 --- /dev/null +++ b/util/signal_handlers.h @@ -0,0 +1,34 @@ +// signal_handlers.h + +/** +* Copyright (C) 2010 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#pragma once + +#include "../pch.h" + +namespace mongo { + + /** + * Obtains the log file handler and writes the current thread's stack trace to + * it. This call issues an exit(). The function can safely be called from within a + * signal handler. + * + * @param signal that this hadler is called for + */ + void printStackAndExit( int signalNum ); + +} // namespace mongo diff --git a/util/sock.cpp b/util/sock.cpp index 3b97c4b..ef3ed0e 100644 --- a/util/sock.cpp +++ b/util/sock.cpp @@ -1,4 +1,4 @@ -// sock.cpp +// @file sock.cpp /* Copyright 2009 10gen Inc. * @@ -26,6 +26,15 @@ namespace mongo { void enableIPv6(bool state) { ipv6 = state; } bool IPv6Enabled() { return ipv6; } + string getAddrInfoStrError(int code) { +#if !defined(_WIN32) + return gai_strerror(code); +#else + /* gai_strerrorA is not threadsafe on windows. don't use it. */ + return errnoWithDescription(code); +#endif + } + SockAddr::SockAddr(int sourcePort) { memset(as<sockaddr_in>().sin_zero, 0, sizeof(as<sockaddr_in>().sin_zero)); as<sockaddr_in>().sin_family = AF_INET; @@ -38,7 +47,7 @@ namespace mongo { if (!strcmp(iporhost, "localhost")) iporhost = "127.0.0.1"; - if (strchr(iporhost, '/')){ + if (strchr(iporhost, '/')) { #ifdef _WIN32 uassert(13080, "no unix socket support on windows", false); #endif @@ -46,21 +55,37 @@ namespace mongo { as<sockaddr_un>().sun_family = AF_UNIX; strcpy(as<sockaddr_un>().sun_path, iporhost); addressSize = sizeof(sockaddr_un); - }else{ + } + else { addrinfo* addrs = NULL; addrinfo hints; memset(&hints, 0, sizeof(addrinfo)); hints.ai_socktype = SOCK_STREAM; //hints.ai_flags = AI_ADDRCONFIG; // This is often recommended but don't do it. SERVER-1579 + hints.ai_flags |= AI_NUMERICHOST; // first pass tries w/o DNS lookup hints.ai_family = (IPv6Enabled() ? AF_UNSPEC : AF_INET); stringstream ss; ss << port; int ret = getaddrinfo(iporhost, ss.str().c_str(), &hints, &addrs); - if (ret){ + + // old C compilers on IPv6-capable hosts return EAI_NODATA error +#ifdef EAI_NODATA + int nodata = (ret == EAI_NODATA); +#else + int nodata = false; +#endif + if (ret == EAI_NONAME || nodata) { + // iporhost isn't an IP address, allow DNS lookup + hints.ai_flags &= ~AI_NUMERICHOST; + ret = getaddrinfo(iporhost, ss.str().c_str(), &hints, &addrs); + } + + if (ret) { log() << "getaddrinfo(\"" << iporhost << "\") failed: " << gai_strerror(ret) << endl; - *this = SockAddr(port); - }else{ + *this = SockAddr(port); + } + else { //TODO: handle other addresses in linked list; assert(addrs->ai_addrlen <= sizeof(sa)); memcpy(&sa, addrs->ai_addr, addrs->ai_addrlen); @@ -69,13 +94,13 @@ namespace mongo { } } } - + bool SockAddr::isLocalHost() const { - switch (getType()){ - case AF_INET: return getAddr() == "127.0.0.1"; - case AF_INET6: return getAddr() == "::1"; - case AF_UNIX: return true; - default: return false; + switch (getType()) { + case AF_INET: return getAddr() == "127.0.0.1"; + case AF_INET6: return getAddr() == "::1"; + case AF_UNIX: return true; + default: return false; } assert(false); return false; @@ -191,19 +216,20 @@ namespace mongo { SockAddr unknownAddress( "0.0.0.0", 0 ); ListeningSockets* ListeningSockets::_instance = new ListeningSockets(); - - ListeningSockets* ListeningSockets::get(){ + + ListeningSockets* ListeningSockets::get() { return _instance; } - - string getHostNameCached(){ - static string host; - if ( host.empty() ){ - string s = getHostName(); - host = s; - } - return host; + string _hostNameCached; + static void _hostNameCachedInit() { + _hostNameCached = getHostName(); + } + boost::once_flag _hostNameCachedInitFlags = BOOST_ONCE_INIT; + + string getHostNameCached() { + boost::call_once( _hostNameCachedInit , _hostNameCachedInitFlags ); + return _hostNameCached; } } // namespace mongo diff --git a/util/sock.h b/util/sock.h index 897be8a..84690fe 100644 --- a/util/sock.h +++ b/util/sock.h @@ -1,4 +1,4 @@ -// sock.h +// @file sock.h /* Copyright 2009 10gen Inc. * @@ -23,21 +23,18 @@ #include <sstream> #include "goodies.h" #include "../db/jsobj.h" +#include "../db/cmdline.h" namespace mongo { const int SOCK_FAMILY_UNKNOWN_ERROR=13078; + string getAddrInfoStrError(int code); #if defined(_WIN32) typedef short sa_family_t; typedef int socklen_t; - inline int getLastError() { - return WSAGetLastError(); - } - inline const char* gai_strerror(int code) { - return ::gai_strerrorA(code); - } + inline int getLastError() { return WSAGetLastError(); } inline void disableNagle(int sock) { int x = 1; if ( setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, (char *) &x, sizeof(x)) ) @@ -45,8 +42,7 @@ namespace mongo { if ( setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, (char *) &x, sizeof(x)) ) out() << "ERROR: SO_KEEPALIVE failed" << endl; } - inline void prebindOptions( int sock ) { - } + inline void prebindOptions( int sock ) { } // This won't actually be used on windows struct sockaddr_un { @@ -56,6 +52,8 @@ namespace mongo { #else + extern CmdLine cmdLine; + } // namespace mongo #include <sys/socket.h> @@ -111,8 +109,8 @@ namespace mongo { #endif - inline string makeUnixSockPath(int port){ - return "/tmp/mongodb-" + BSONObjBuilder::numStr(port) + ".sock"; + inline string makeUnixSockPath(int port) { + return cmdLine.socket + "/mongodb-" + BSONObjBuilder::numStr(port) + ".sock"; } inline void setSockTimeouts(int sock, double secs) { @@ -148,7 +146,7 @@ namespace mongo { template <typename T> const T& as() const { return *(const T*)(&sa); } - string toString(bool includePort=true) const{ + string toString(bool includePort=true) const { string out = getAddr(); if (includePort && getType() != AF_UNIX && getType() != AF_UNSPEC) out += ':' + BSONObjBuilder::numStr(getPort()); @@ -161,34 +159,34 @@ namespace mongo { } unsigned getPort() const { - switch (getType()){ - case AF_INET: return ntohs(as<sockaddr_in>().sin_port); - case AF_INET6: return ntohs(as<sockaddr_in6>().sin6_port); - case AF_UNIX: return 0; - case AF_UNSPEC: return 0; - default: massert(SOCK_FAMILY_UNKNOWN_ERROR, "unsupported address family", false); return 0; + switch (getType()) { + case AF_INET: return ntohs(as<sockaddr_in>().sin_port); + case AF_INET6: return ntohs(as<sockaddr_in6>().sin6_port); + case AF_UNIX: return 0; + case AF_UNSPEC: return 0; + default: massert(SOCK_FAMILY_UNKNOWN_ERROR, "unsupported address family", false); return 0; } } string getAddr() const { - switch (getType()){ - case AF_INET: - case AF_INET6: { - const int buflen=128; - char buffer[buflen]; - int ret = getnameinfo(raw(), addressSize, buffer, buflen, NULL, 0, NI_NUMERICHOST); - massert(13082, gai_strerror(ret), ret == 0); - return buffer; - } - - case AF_UNIX: return (addressSize > 2 ? as<sockaddr_un>().sun_path : "anonymous unix socket"); - case AF_UNSPEC: return "(NONE)"; - default: massert(SOCK_FAMILY_UNKNOWN_ERROR, "unsupported address family", false); return ""; + switch (getType()) { + case AF_INET: + case AF_INET6: { + const int buflen=128; + char buffer[buflen]; + int ret = getnameinfo(raw(), addressSize, buffer, buflen, NULL, 0, NI_NUMERICHOST); + massert(13082, getAddrInfoStrError(ret), ret == 0); + return buffer; + } + + case AF_UNIX: return (addressSize > 2 ? as<sockaddr_un>().sun_path : "anonymous unix socket"); + case AF_UNSPEC: return "(NONE)"; + default: massert(SOCK_FAMILY_UNKNOWN_ERROR, "unsupported address family", false); return ""; } } bool isLocalHost() const; - + bool operator==(const SockAddr& r) const { if (getType() != r.getType()) return false; @@ -196,12 +194,12 @@ namespace mongo { if (getPort() != r.getPort()) return false; - switch (getType()){ - case AF_INET: return as<sockaddr_in>().sin_addr.s_addr == r.as<sockaddr_in>().sin_addr.s_addr; - case AF_INET6: return memcmp(as<sockaddr_in6>().sin6_addr.s6_addr, r.as<sockaddr_in6>().sin6_addr.s6_addr, sizeof(in6_addr)) == 0; - case AF_UNIX: return strcmp(as<sockaddr_un>().sun_path, r.as<sockaddr_un>().sun_path) == 0; - case AF_UNSPEC: return true; // assume all unspecified addresses are the same - default: massert(SOCK_FAMILY_UNKNOWN_ERROR, "unsupported address family", false); + switch (getType()) { + case AF_INET: return as<sockaddr_in>().sin_addr.s_addr == r.as<sockaddr_in>().sin_addr.s_addr; + case AF_INET6: return memcmp(as<sockaddr_in6>().sin6_addr.s6_addr, r.as<sockaddr_in6>().sin6_addr.s6_addr, sizeof(in6_addr)) == 0; + case AF_UNIX: return strcmp(as<sockaddr_un>().sun_path, r.as<sockaddr_un>().sun_path) == 0; + case AF_UNSPEC: return true; // assume all unspecified addresses are the same + default: massert(SOCK_FAMILY_UNKNOWN_ERROR, "unsupported address family", false); } } bool operator!=(const SockAddr& r) const { @@ -218,12 +216,12 @@ namespace mongo { else if (getPort() > r.getPort()) return false; - switch (getType()){ - case AF_INET: return as<sockaddr_in>().sin_addr.s_addr < r.as<sockaddr_in>().sin_addr.s_addr; - case AF_INET6: return memcmp(as<sockaddr_in6>().sin6_addr.s6_addr, r.as<sockaddr_in6>().sin6_addr.s6_addr, sizeof(in6_addr)) < 0; - case AF_UNIX: return strcmp(as<sockaddr_un>().sun_path, r.as<sockaddr_un>().sun_path) < 0; - case AF_UNSPEC: return false; - default: massert(SOCK_FAMILY_UNKNOWN_ERROR, "unsupported address family", false); + switch (getType()) { + case AF_INET: return as<sockaddr_in>().sin_addr.s_addr < r.as<sockaddr_in>().sin_addr.s_addr; + case AF_INET6: return memcmp(as<sockaddr_in6>().sin6_addr.s6_addr, r.as<sockaddr_in6>().sin6_addr.s6_addr, sizeof(in6_addr)) < 0; + case AF_UNIX: return strcmp(as<sockaddr_un>().sun_path, r.as<sockaddr_un>().sun_path) < 0; + case AF_UNSPEC: return false; + default: massert(SOCK_FAMILY_UNKNOWN_ERROR, "unsupported address family", false); } } @@ -231,7 +229,7 @@ namespace mongo { sockaddr* raw() {return (sockaddr*)&sa;} socklen_t addressSize; - private: + private: struct sockaddr_storage sa; }; @@ -253,32 +251,52 @@ namespace mongo { class ListeningSockets { public: - ListeningSockets() : _mutex("ListeningSockets"), _sockets( new set<int>() ) { } - void add( int sock ){ + ListeningSockets() + : _mutex("ListeningSockets") + , _sockets( new set<int>() ) + , _socketPaths( new set<string>() ) + { } + void add( int sock ) { scoped_lock lk( _mutex ); _sockets->insert( sock ); } - void remove( int sock ){ + void addPath( string path ) { + scoped_lock lk( _mutex ); + _socketPaths->insert( path ); + } + void remove( int sock ) { scoped_lock lk( _mutex ); _sockets->erase( sock ); } - void closeAll(){ - set<int>* s; + void closeAll() { + set<int>* sockets; + set<string>* paths; + { scoped_lock lk( _mutex ); - s = _sockets; + sockets = _sockets; _sockets = new set<int>(); + paths = _socketPaths; + _socketPaths = new set<string>(); } - for ( set<int>::iterator i=s->begin(); i!=s->end(); i++ ) { + + for ( set<int>::iterator i=sockets->begin(); i!=sockets->end(); i++ ) { int sock = *i; log() << "closing listening socket: " << sock << endl; closesocket( sock ); - } + } + + for ( set<string>::iterator i=paths->begin(); i!=paths->end(); i++ ) { + string path = *i; + log() << "removing socket file: " << path << endl; + ::remove( path.c_str() ); + } } static ListeningSockets* get(); private: mongo::mutex _mutex; set<int>* _sockets; + set<string>* _socketPaths; // for unix domain sockets static ListeningSockets* _instance; }; diff --git a/util/stringutils.cpp b/util/stringutils.cpp index 3f989fd..229f57b 100644 --- a/util/stringutils.cpp +++ b/util/stringutils.cpp @@ -20,13 +20,13 @@ namespace mongo { - void splitStringDelim( const string& str , vector<string>* res , char delim ){ + void splitStringDelim( const string& str , vector<string>* res , char delim ) { if ( str.empty() ) return; size_t beg = 0; size_t pos = str.find( delim ); - while ( pos != string::npos ){ + while ( pos != string::npos ) { res->push_back( str.substr( beg, pos - beg) ); beg = ++pos; pos = str.find( delim, beg ); @@ -34,8 +34,8 @@ namespace mongo { res->push_back( str.substr( beg ) ); } - void joinStringDelim( const vector<string>& strs , string* res , char delim ){ - for ( vector<string>::const_iterator it = strs.begin(); it != strs.end(); ++it ){ + void joinStringDelim( const vector<string>& strs , string* res , char delim ) { + for ( vector<string>::const_iterator it = strs.begin(); it != strs.end(); ++it ) { if ( it !=strs.begin() ) res->push_back( delim ); res->append( *it ); } diff --git a/util/stringutils.h b/util/stringutils.h index 6b79c33..60571e6 100644 --- a/util/stringutils.h +++ b/util/stringutils.h @@ -20,24 +20,26 @@ namespace mongo { + // see also mongoutils/str.h - perhaps move these there? + void splitStringDelim( const string& str , vector<string>* res , char delim ); void joinStringDelim( const vector<string>& strs , string* res , char delim ); - inline string tolowerString( const string& input ){ + inline string tolowerString( const string& input ) { string::size_type sz = input.size(); - + boost::scoped_array<char> line(new char[sz+1]); char * copy = line.get(); - - for ( string::size_type i=0; i<sz; i++ ){ + + for ( string::size_type i=0; i<sz; i++ ) { char c = input[i]; copy[i] = (char)tolower( (int)c ); } copy[sz] = 0; return string(copy); } - + } // namespace mongo #endif // UTIL_STRING_UTILS_HEADER diff --git a/util/text.cpp b/util/text.cpp index f381e01..51a2556 100644 --- a/util/text.cpp +++ b/util/text.cpp @@ -19,9 +19,9 @@ #include "text.h" #include "unittest.h" -namespace mongo{ +namespace mongo { - inline int leadingOnes(unsigned char c){ + inline int leadingOnes(unsigned char c) { if (c < 0x80) return 0; static const char _leadingOnes[128] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0x80 - 0x8F @@ -32,24 +32,25 @@ namespace mongo{ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // 0xD0 - 0xD9 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, // 0xE0 - 0xE9 4, 4, 4, 4, 4, 4, 4, 4, // 0xF0 - 0xF7 - 5, 5, 5, 5, // 0xF8 - 0xFB - 6, 6, // 0xFC - 0xFD - 7, // 0xFE - 8, // 0xFF + 5, 5, 5, 5, // 0xF8 - 0xFB + 6, 6, // 0xFC - 0xFD + 7, // 0xFE + 8, // 0xFF }; return _leadingOnes[c & 0x7f]; } - bool isValidUTF8(const char *s){ + bool isValidUTF8(const char *s) { int left = 0; // how many bytes are left in the current codepoint - while (*s){ + while (*s) { const unsigned char c = (unsigned char) *(s++); const int ones = leadingOnes(c); - if (left){ + if (left) { if (ones != 1) return false; // should be a continuation byte left--; - }else{ + } + else { if (ones == 0) continue; // ASCII byte if (ones == 1) return false; // unexpected continuation byte if (c > 0xF4) return false; // codepoint too large (< 0x10FFFF) @@ -61,53 +62,50 @@ namespace mongo{ } if (left!=0) return false; // string ended mid-codepoint return true; - } - - #if defined(_WIN32) - - std::string toUtf8String(const std::wstring& wide) - { - if (wide.size() > boost::integer_traits<int>::const_max) - throw std::length_error( - "Wide string cannot be more than INT_MAX characters long."); - if (wide.size() == 0) - return ""; - - // Calculate necessary buffer size - int len = ::WideCharToMultiByte( - CP_UTF8, 0, wide.c_str(), static_cast<int>(wide.size()), - NULL, 0, NULL, NULL); - - // Perform actual conversion - if (len > 0) - { - std::vector<char> buffer(len); - len = ::WideCharToMultiByte( - CP_UTF8, 0, wide.c_str(), static_cast<int>(wide.size()), - &buffer[0], static_cast<int>(buffer.size()), NULL, NULL); - if (len > 0) - { - assert(len == static_cast<int>(buffer.size())); - return std::string(&buffer[0], buffer.size()); - } - } - - throw boost::system::system_error( - ::GetLastError(), boost::system::system_category); - } + } + +#if defined(_WIN32) + + std::string toUtf8String(const std::wstring& wide) { + if (wide.size() > boost::integer_traits<int>::const_max) + throw std::length_error( + "Wide string cannot be more than INT_MAX characters long."); + if (wide.size() == 0) + return ""; + + // Calculate necessary buffer size + int len = ::WideCharToMultiByte( + CP_UTF8, 0, wide.c_str(), static_cast<int>(wide.size()), + NULL, 0, NULL, NULL); + + // Perform actual conversion + if (len > 0) { + std::vector<char> buffer(len); + len = ::WideCharToMultiByte( + CP_UTF8, 0, wide.c_str(), static_cast<int>(wide.size()), + &buffer[0], static_cast<int>(buffer.size()), NULL, NULL); + if (len > 0) { + assert(len == static_cast<int>(buffer.size())); + return std::string(&buffer[0], buffer.size()); + } + } + + throw boost::system::system_error( + ::GetLastError(), boost::system::system_category); + } #if defined(_UNICODE) - std::wstring toWideString(const char *s) { + std::wstring toWideString(const char *s) { std::basic_ostringstream<TCHAR> buf; buf << s; return buf.str(); } #endif - #endif +#endif struct TextUnitTest : public UnitTest { - void run() { + void run() { assert( parseLL("123") == 123 ); assert( parseLL("-123000000000") == -123000000000LL ); } diff --git a/util/text.h b/util/text.h index 4ba622f..fc2da21 100644 --- a/util/text.h +++ b/util/text.h @@ -33,52 +33,57 @@ #pragma once namespace mongo { - + class StringSplitter { public: + /** @param big the string to be split + @param splitter the delimiter + */ StringSplitter( const char * big , const char * splitter ) - : _big( big ) , _splitter( splitter ){ + : _big( big ) , _splitter( splitter ) { } - bool more(){ + /** @return true if more to be taken via next() */ + bool more() { return _big[0]; } - string next(){ + /** get next split string fragment */ + string next() { const char * foo = strstr( _big , _splitter ); - if ( foo ){ + if ( foo ) { string s( _big , foo - _big ); _big = foo + 1; while ( *_big && strstr( _big , _splitter ) == _big ) _big++; return s; } - + string s = _big; _big += strlen( _big ); return s; } - - void split( vector<string>& l ){ - while ( more() ){ + + void split( vector<string>& l ) { + while ( more() ) { l.push_back( next() ); } } - - vector<string> split(){ + + vector<string> split() { vector<string> l; split( l ); return l; } - static vector<string> split( const string& big , const string& splitter ){ + static vector<string> split( const string& big , const string& splitter ) { StringSplitter ss( big.c_str() , splitter.c_str() ); return ss.split(); } - static string join( vector<string>& l , const string& split ){ + static string join( vector<string>& l , const string& split ) { stringstream ss; - for ( unsigned i=0; i<l.size(); i++ ){ + for ( unsigned i=0; i<l.size(); i++ ) { if ( i > 0 ) ss << split; ss << l[i]; @@ -90,20 +95,20 @@ namespace mongo { const char * _big; const char * _splitter; }; - + /* This doesn't defend against ALL bad UTF8, but it will guarantee that the * string can be converted to sequence of codepoints. However, it doesn't * guarantee that the codepoints are valid. */ bool isValidUTF8(const char *s); - inline bool isValidUTF8(string s) { return isValidUTF8(s.c_str()); } + inline bool isValidUTF8(string s) { return isValidUTF8(s.c_str()); } #if defined(_WIN32) std::string toUtf8String(const std::wstring& wide); std::wstring toWideString(const char *s); - + /* like toWideString but UNICODE macro sensitive */ # if !defined(_UNICODE) #error temp error @@ -111,9 +116,9 @@ namespace mongo { # else inline std::wstring toNativeString(const char *s) { return toWideString(s); } # endif - + #endif - + // expect that n contains a base ten number and nothing else after it // NOTE win version hasn't been tested directly inline long long parseLL( const char *n ) { @@ -124,11 +129,12 @@ namespace mongo { errno = 0; ret = strtoll( n, &endPtr, 10 ); uassert( 13305, "could not convert string to long long", *endPtr == 0 && errno == 0 ); -#elif _MSC_VER>=1600 // 1600 is VS2k10 1500 is VS2k8 +#elif _MSC_VER>=1600 // 1600 is VS2k10 1500 is VS2k8 size_t endLen = 0; try { ret = stoll( n, &endLen, 10 ); - } catch ( ... ) { + } + catch ( ... ) { endLen = 0; } uassert( 13306, "could not convert string to long long", endLen != 0 && n[ endLen ] == 0 ); diff --git a/util/time_support.h b/util/time_support.h new file mode 100644 index 0000000..5dedec9 --- /dev/null +++ b/util/time_support.h @@ -0,0 +1,201 @@ +// @file time_support.h + +/* Copyright 2010 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <cstdio> // sscanf +#include <ctime> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/xtime.hpp> +#undef assert +#define assert MONGO_assert + +namespace mongo { + + inline void time_t_to_Struct(time_t t, struct tm * buf , bool local = false ) { +#if defined(_WIN32) + if ( local ) + localtime_s( buf , &t ); + else + gmtime_s(buf, &t); +#else + if ( local ) + localtime_r(&t, buf); + else + gmtime_r(&t, buf); +#endif + } + + // uses ISO 8601 dates without trailing Z + // colonsOk should be false when creating filenames + inline string terseCurrentTime(bool colonsOk=true) { + struct tm t; + time_t_to_Struct( time(0) , &t ); + + const char* fmt = (colonsOk ? "%Y-%m-%dT%H:%M:%S" : "%Y-%m-%dT%H-%M-%S"); + char buf[32]; + assert(strftime(buf, sizeof(buf), fmt, &t) == 19); + return buf; + } + + inline boost::gregorian::date currentDate() { + boost::posix_time::ptime now = boost::posix_time::second_clock::local_time(); + return now.date(); + } + + // parses time of day in "hh:mm" format assuming 'hh' is 00-23 + inline bool toPointInTime( const string& str , boost::posix_time::ptime* timeOfDay ) { + int hh = 0; + int mm = 0; + if ( 2 != sscanf( str.c_str() , "%d:%d" , &hh , &mm ) ) { + return false; + } + + // verify that time is well formed + if ( ( hh / 24 ) || ( mm / 60 ) ) { + return false; + } + + boost::posix_time::ptime res( currentDate() , boost::posix_time::hours( hh ) + boost::posix_time::minutes( mm ) ); + *timeOfDay = res; + return true; + } + +#define MONGO_asctime _asctime_not_threadsafe_ +#define asctime MONGO_asctime +#define MONGO_gmtime _gmtime_not_threadsafe_ +#define gmtime MONGO_gmtime +#define MONGO_localtime _localtime_not_threadsafe_ +#define localtime MONGO_localtime +#define MONGO_ctime _ctime_is_not_threadsafe_ +#define ctime MONGO_ctime + +#if defined(_WIN32) + inline void sleepsecs(int s) { + Sleep(s*1000); + } + inline void sleepmillis(long long s) { + assert( s <= 0xffffffff ); + Sleep((DWORD) s); + } + inline void sleepmicros(long long s) { + if ( s <= 0 ) + return; + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + xt.sec += (int)( s / 1000000 ); + xt.nsec += (int)(( s % 1000000 ) * 1000); + if ( xt.nsec >= 1000000000 ) { + xt.nsec -= 1000000000; + xt.sec++; + } + boost::thread::sleep(xt); + } +#elif defined(__sunos__) + inline void sleepsecs(int s) { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + xt.sec += s; + boost::thread::sleep(xt); + } + inline void sleepmillis(long long s) { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + xt.sec += (int)( s / 1000 ); + xt.nsec += (int)(( s % 1000 ) * 1000000); + if ( xt.nsec >= 1000000000 ) { + xt.nsec -= 1000000000; + xt.sec++; + } + boost::thread::sleep(xt); + } + inline void sleepmicros(long long s) { + if ( s <= 0 ) + return; + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + xt.sec += (int)( s / 1000000 ); + xt.nsec += (int)(( s % 1000000 ) * 1000); + if ( xt.nsec >= 1000000000 ) { + xt.nsec -= 1000000000; + xt.sec++; + } + boost::thread::sleep(xt); + } +#else + inline void sleepsecs(int s) { + struct timespec t; + t.tv_sec = s; + t.tv_nsec = 0; + if ( nanosleep( &t , 0 ) ) { + cout << "nanosleep failed" << endl; + } + } + inline void sleepmicros(long long s) { + if ( s <= 0 ) + return; + struct timespec t; + t.tv_sec = (int)(s / 1000000); + t.tv_nsec = 1000 * ( s % 1000000 ); + struct timespec out; + if ( nanosleep( &t , &out ) ) { + cout << "nanosleep failed" << endl; + } + } + inline void sleepmillis(long long s) { + sleepmicros( s * 1000 ); + } +#endif + + // note this wraps + inline int tdiff(unsigned told, unsigned tnew) { + return WrappingInt::diff(tnew, told); + } + + /** curTimeMillis will overflow - use curTimeMicros64 instead if you care about that. */ + inline unsigned curTimeMillis() { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + unsigned t = xt.nsec / 1000000; + return (xt.sec & 0xfffff) * 1000 + t; + } + + /** Date_t is milliseconds since epoch */ + inline Date_t jsTime() { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + unsigned long long t = xt.nsec / 1000000; + return ((unsigned long long) xt.sec * 1000) + t; + } + + inline unsigned long long curTimeMicros64() { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + unsigned long long t = xt.nsec / 1000; + return (((unsigned long long) xt.sec) * 1000000) + t; + } + + // measures up to 1024 seconds. or, 512 seconds with tdiff that is... + inline unsigned curTimeMicros() { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + unsigned t = xt.nsec / 1000; + unsigned secs = xt.sec % 1024; + return secs*1000000 + t; + } + +} // namespace mongo diff --git a/util/timer.h b/util/timer.h new file mode 100644 index 0000000..f5a21f8 --- /dev/null +++ b/util/timer.h @@ -0,0 +1,67 @@ +// @file timer.h + +/* Copyright 2010 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "time_support.h" + +namespace mongo { + + /** + * simple scoped timer + */ + class Timer { + public: + Timer() { + reset(); + } + + Timer( unsigned long long start ) { + old = start; + } + + int seconds() const { + return (int)(micros() / 1000000); + } + + int millis() const { + return (long)(micros() / 1000); + } + + unsigned long long micros() const { + unsigned long long n = curTimeMicros64(); + return n - old; + } + + unsigned long long micros(unsigned long long & n) const { // returns cur time in addition to timer result + n = curTimeMicros64(); + return n - old; + } + + unsigned long long startTime() { + return old; + } + + void reset() { + old = curTimeMicros64(); + } + + private: + unsigned long long old; + }; + +} // namespace mongo diff --git a/util/unittest.h b/util/unittest.h index caf8cb3..94be444 100644 --- a/util/unittest.h +++ b/util/unittest.h @@ -25,6 +25,9 @@ namespace mongo { To define a unit test, inherit from this and implement run. instantiate one object for the new class as a global. + + These tests are ran on *every* startup of mongod, so they have to be very lightweight. But it is a + good quick check for a bad build. */ struct UnitTest { UnitTest() { diff --git a/util/util.cpp b/util/util.cpp index b4b1053..216683a 100644 --- a/util/util.cpp +++ b/util/util.cpp @@ -20,28 +20,47 @@ #include "unittest.h" #include "file_allocator.h" #include "optime.h" +#include "time_support.h" namespace mongo { + string hexdump(const char *data, unsigned len) { + assert( len < 1000000 ); + const unsigned char *p = (const unsigned char *) data; + stringstream ss; + for( unsigned i = 0; i < 4 && i < len; i++ ) { + ss << std::hex << setw(2) << setfill('0'); + unsigned n = p[i]; + ss << n; + ss << ' '; + } + string s = ss.str(); + return s; + } + boost::thread_specific_ptr<string> _threadName; - - void _setThreadName( const char * name ){ - static int N = 0; - if ( strcmp( name , "conn" ) == 0 ){ + + unsigned _setThreadName( const char * name ) { + if ( ! name ) name = "NONE"; + + static unsigned N = 0; + + if ( strcmp( name , "conn" ) == 0 ) { + unsigned n = ++N; stringstream ss; - ss << name << ++N; + ss << name << n; _threadName.reset( new string( ss.str() ) ); + return n; } - else { - _threadName.reset( new string(name) ); - } + + _threadName.reset( new string(name) ); + return 0; } #if defined(_WIN32) #define MS_VC_EXCEPTION 0x406D1388 #pragma pack(push,8) - typedef struct tagTHREADNAME_INFO - { + typedef struct tagTHREADNAME_INFO { DWORD dwType; // Must be 0x1000. LPCSTR szName; // Pointer to name (in user addr space). DWORD dwThreadID; // Thread ID (-1=caller thread). @@ -49,30 +68,42 @@ namespace mongo { } THREADNAME_INFO; #pragma pack(pop) - void setThreadName(const char *name) - { - _setThreadName( name ); - Sleep(10); + void setWinThreadName(const char *name) { + /* is the sleep here necessary??? + Sleep(10); + */ THREADNAME_INFO info; info.dwType = 0x1000; info.szName = name; info.dwThreadID = -1; info.dwFlags = 0; - __try - { - RaiseException( MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(ULONG_PTR), (ULONG_PTR*)&info ); - } - __except(EXCEPTION_EXECUTE_HANDLER) - { + __try { + RaiseException( MS_VC_EXCEPTION, 0, sizeof(info)/sizeof(ULONG_PTR), (ULONG_PTR*)&info ); + } + __except(EXCEPTION_EXECUTE_HANDLER) { } } + + unsigned setThreadName(const char *name) { + unsigned n = _setThreadName( name ); +#if !defined(_DEBUG) + // naming might be expensive so don't do "conn*" over and over + if( string("conn") == name ) + return n; +#endif + setWinThreadName(name); + return n; + } + #else - void setThreadName(const char * name ) { - _setThreadName( name ); + + unsigned setThreadName(const char * name ) { + return _setThreadName( name ); } + #endif - string getThreadName(){ + string getThreadName() { string * s = _threadName.get(); if ( s ) return *s; @@ -89,8 +120,6 @@ namespace mongo { int tlogLevel = 0; mongo::mutex Logstream::mutex("Logstream"); int Logstream::doneSetup = Logstream::magicNumber(); - - bool goingAway = false; bool isPrime(int n) { int z = 2; @@ -140,13 +169,9 @@ namespace mongo { } } utilTest; - - // The mutex contained in this object may be held on shutdown. - FileAllocator &theFileAllocator_ = *(new FileAllocator()); - FileAllocator &theFileAllocator() { return theFileAllocator_; } - + OpTime OpTime::last(0, 0); - + /* this is a good place to set a breakpoint when debugging, as lots of warning things (assert, wassert) call it. */ @@ -174,11 +199,11 @@ namespace mongo { Logstream::logLockless("\n"); } - ostream& operator<<( ostream &s, const ThreadSafeString &o ){ + ostream& operator<<( ostream &s, const ThreadSafeString &o ) { s << o.toString(); return s; } - bool __destroyingStatics = false; - + bool StaticObserver::_destroyingStatics = false; + } // namespace mongo diff --git a/util/version.cpp b/util/version.cpp index c5ca8d4..1755a92 100644 --- a/util/version.cpp +++ b/util/version.cpp @@ -1,20 +1,32 @@ -#include "pch.h" +// @file version.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pch.h" #include <cstdlib> #include <iostream> #include <iomanip> #include <sstream> #include <string> - +#include "unittest.h" #include "version.h" namespace mongo { - // - // mongo processes version support - // - - const char versionString[] = "1.6.6-pre-"; + const char versionString[] = "1.8.0"; string mongodVersion() { stringstream ss; @@ -22,24 +34,16 @@ namespace mongo { return ss.str(); } - // - // git version support - // - #ifndef _SCONS // only works in scons - const char * gitVersion(){ return "not-scons"; } + const char * gitVersion() { return "not-scons"; } #endif void printGitVersion() { log() << "git version: " << gitVersion() << endl; } - // - // sys info support - // - #ifndef _SCONS #if defined(_WIN32) - string sysInfo(){ + string sysInfo() { stringstream ss; ss << "not-scons win"; ss << " mscver:" << _MSC_FULL_VER << " built:" << __DATE__; @@ -51,17 +55,18 @@ namespace mongo { return ss.str(); } #else - string sysInfo(){ return ""; } + string sysInfo() { return ""; } #endif #endif - void printSysInfo() { log() << "sys info: " << sysInfo() << endl; } + void printSysInfo() { + log() << "build sys info: " << sysInfo() << endl; + } // // 32 bit systems warning // - - void show_warnings(){ + void show_warnings() { // each message adds a leading but not a trailing newline bool warned = false; @@ -79,11 +84,12 @@ namespace mongo { cout << endl; cout << "** NOTE: when using MongoDB 32 bit, you are limited to about 2 gigabytes of data" << endl; cout << "** see http://blog.mongodb.org/post/137788967/32-bit-limitations" << endl; + cout << "** with --dur, the limit is lower" << endl; warned = true; } #ifdef __linux__ - if (boost::filesystem::exists("/proc/vz") && !boost::filesystem::exists("/proc/bc")){ + if (boost::filesystem::exists("/proc/vz") && !boost::filesystem::exists("/proc/bc")) { cout << endl; cout << "** WARNING: You are running in OpenVZ. This is known to be broken!!!" << endl; warned = true; @@ -94,4 +100,38 @@ namespace mongo { cout << endl; } + int versionCmp(StringData rhs, StringData lhs) { + if (strcmp(rhs.data(),lhs.data()) == 0) + return 0; + + // handle "1.2.3-" and "1.2.3-pre" + if (rhs.size() < lhs.size()) { + if (strncmp(rhs.data(), lhs.data(), rhs.size()) == 0 && lhs.data()[rhs.size()] == '-') + return +1; + } + else if (rhs.size() > lhs.size()) { + if (strncmp(rhs.data(), lhs.data(), lhs.size()) == 0 && rhs.data()[lhs.size()] == '-') + return -1; + } + + return lexNumCmp(rhs.data(), lhs.data()); + } + + class VersionCmpTest : public UnitTest { + public: + void run() { + assert( versionCmp("1.2.3", "1.2.3") == 0 ); + assert( versionCmp("1.2.3", "1.2.4") < 0 ); + assert( versionCmp("1.2.3", "1.2.20") < 0 ); + assert( versionCmp("1.2.3", "1.20.3") < 0 ); + assert( versionCmp("2.2.3", "10.2.3") < 0 ); + assert( versionCmp("1.2.3", "1.2.3-") > 0 ); + assert( versionCmp("1.2.3", "1.2.3-pre") > 0 ); + assert( versionCmp("1.2.3", "1.2.4-") < 0 ); + assert( versionCmp("1.2.3-", "1.2.3") < 0 ); + assert( versionCmp("1.2.3-pre", "1.2.3") < 0 ); + + log(1) << "versionCmpTest passed" << endl; + } + } versionCmpTest; } diff --git a/util/version.h b/util/version.h index ea22a35..779fbdc 100644 --- a/util/version.h +++ b/util/version.h @@ -9,7 +9,8 @@ namespace mongo { // mongo version extern const char versionString[]; - string mongodVersion(); + string mongodVersion(); + int versionCmp(StringData rhs, StringData lhs); // like strcmp const char * gitVersion(); void printGitVersion(); |