diff options
Diffstat (limited to 'util')
53 files changed, 7134 insertions, 0 deletions
diff --git a/util/allocator.h b/util/allocator.h new file mode 100644 index 0000000..af8032c --- /dev/null +++ b/util/allocator.h @@ -0,0 +1,49 @@ +// allocator.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace mongo { + + inline void * ourmalloc(size_t size) { + void *x = malloc(size); + if ( x == 0 ) dbexit( EXIT_OOM_MALLOC , "malloc fails"); + return x; + } + + inline void * ourrealloc(void *ptr, size_t size) { + void *x = realloc(ptr, size); + if ( x == 0 ) dbexit( EXIT_OOM_REALLOC , "realloc fails"); + return x; + } + +#define malloc mongo::ourmalloc +#define realloc mongo::ourrealloc + +#if defined(_WIN32) + inline void our_debug_free(void *p) { +#if 0 + // this is not safe if you malloc < 4 bytes so we don't use anymore + unsigned *u = (unsigned *) p; + u[0] = 0xEEEEEEEE; +#endif + free(p); + } +#define free our_debug_free +#endif + +} // namespace mongo diff --git a/util/assert_util.cpp b/util/assert_util.cpp new file mode 100644 index 0000000..d1d85b2 --- /dev/null +++ b/util/assert_util.cpp @@ -0,0 +1,170 @@ +// assert_util.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "assert_util.h" +#include "assert.h" +#include "file.h" + +namespace mongo { + + string getDbContext(); + + Assertion lastAssert[4]; + + /* "warning" assert -- safe to continue, so we don't throw exception. */ + void wasserted(const char *msg, const char *file, unsigned line) { + problem() << "Assertion failure " << msg << ' ' << file << ' ' << dec << line << endl; + sayDbContext(); + raiseError(0,msg && *msg ? msg : "wassertion failure"); + lastAssert[1].set(msg, getDbContext().c_str(), file, line); + } + + void asserted(const char *msg, const char *file, unsigned line) { + problem() << "Assertion failure " << msg << ' ' << file << ' ' << dec << line << endl; + sayDbContext(); + raiseError(0,msg && *msg ? msg : "assertion failure"); + lastAssert[0].set(msg, getDbContext().c_str(), file, line); + stringstream temp; + temp << "assertion " << file << ":" << line; + AssertionException e; + e.msg = temp.str(); + breakpoint(); + throw e; + } + + void uassert_nothrow(const char *msg) { + lastAssert[3].set(msg, getDbContext().c_str(), "", 0); + raiseError(0,msg); + } + + int uacount = 0; + void uasserted(int msgid, const char *msg) { + if ( ++uacount < 100 ) + log() << "User Exception " << msgid << ":" << msg << endl; + else + RARELY log() << "User Exception " << msg << endl; + lastAssert[3].set(msg, getDbContext().c_str(), "", 0); + raiseError(msgid,msg); + throw UserException(msgid, msg); + } + + void msgasserted(int msgid, const char *msg) { + log() << "Assertion: " << msgid << ":" << msg << endl; + lastAssert[2].set(msg, getDbContext().c_str(), "", 0); + raiseError(msgid,msg && *msg ? msg : "massert failure"); + breakpoint(); + printStackTrace(); // TEMP?? should we get rid of this? TODO + throw MsgAssertionException(msgid, msg); + } + + boost::mutex *Assertion::_mutex = new boost::mutex(); + + string Assertion::toString() { + if( _mutex == 0 ) + return ""; + + boostlock lk(*_mutex); + + if ( !isSet() ) + return ""; + + stringstream ss; + ss << msg << '\n'; + if ( *context ) + ss << context << '\n'; + if ( *file ) + ss << file << ' ' << line << '\n'; + return ss.str(); + } + + + class LoggingManager { + public: + LoggingManager() + : _enabled(0) , _file(0) { + } + + void start( const string& lp , bool append ){ + uassert( 10268 , "LoggingManager already started" , ! _enabled ); + _append = append; + + // test path + FILE * test = fopen( lp.c_str() , _append ? "a" : "w" ); + if ( ! test ){ + cout << "can't open [" << lp << "] for log file" << endl; + dbexit( EXIT_BADOPTIONS ); + assert( 0 ); + } + + _path = lp; + _enabled = 1; + rotate(); + } + + void rotate(){ + if ( ! _enabled ){ + cout << "LoggingManager not enabled" << endl; + return; + } + + if ( _file ){ +#ifdef _WIN32 + cout << "log rotation doesn't work on windows" << endl; + return; +#else + struct tm t; + localtime_r( &_opened , &t ); + + stringstream ss; + ss << _path << "." << ( 1900 + t.tm_year ) << "-" << t.tm_mon << "-" << t.tm_mday + << "_" << t.tm_hour << "-" << t.tm_min << "-" << t.tm_sec; + string s = ss.str(); + rename( _path.c_str() , s.c_str() ); +#endif + } + + _file = freopen( _path.c_str() , _append ? "a" : "w" , stdout ); + if ( ! _file ){ + cerr << "can't open: " << _path.c_str() << " for log file" << endl; + dbexit( EXIT_BADOPTIONS ); + assert(0); + } + _opened = time(0); + } + + private: + + bool _enabled; + string _path; + bool _append; + + FILE * _file; + time_t _opened; + + } loggingManager; + + void initLogging( const string& lp , bool append ){ + cout << "all output going to: " << lp << endl; + loggingManager.start( lp , append ); + } + + void rotateLogs( int signal ){ + loggingManager.rotate(); + } +} + diff --git a/util/assert_util.h b/util/assert_util.h new file mode 100644 index 0000000..ccb60a0 --- /dev/null +++ b/util/assert_util.h @@ -0,0 +1,186 @@ +// assert_util.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#pragma once + +#include "../db/lasterror.h" + +namespace mongo { + + /* these are manipulated outside of mutexes, so be careful */ + struct Assertion { + Assertion() { + msg[0] = msg[127] = 0; + context[0] = context[127] = 0; + file = ""; + line = 0; + when = 0; + } + private: + static boost::mutex *_mutex; + char msg[128]; + char context[128]; + const char *file; + unsigned line; + time_t when; + public: + void set(const char *m, const char *ctxt, const char *f, unsigned l) { + if( _mutex == 0 ) { + /* asserted during global variable initialization */ + return; + } + boostlock lk(*_mutex); + strncpy(msg, m, 127); + strncpy(context, ctxt, 127); + file = f; + line = l; + when = time(0); + } + std::string toString(); + bool isSet() { + return when != 0; + } + }; + + enum { + AssertRegular = 0, + AssertW = 1, + AssertMsg = 2, + AssertUser = 3 + }; + + /* last assert of diff types: regular, wassert, msgassert, uassert: */ + extern Assertion lastAssert[4]; + + class DBException : public std::exception { + public: + virtual const char* what() const throw() = 0; + virtual string toString() const { + return what(); + } + virtual int getCode() = 0; + operator string() const { return toString(); } + }; + + class AssertionException : public DBException { + public: + int code; + string msg; + AssertionException() { code = 0; } + virtual ~AssertionException() throw() { } + virtual bool severe() { + return true; + } + virtual bool isUserAssertion() { + return false; + } + virtual int getCode(){ return code; } + virtual const char* what() const throw() { return msg.c_str(); } + }; + + /* UserExceptions are valid errors that a user can cause, like out of disk space or duplicate key */ + class UserException : public AssertionException { + public: + UserException(int c , const string& m) { + code = c; + msg = m; + } + virtual bool severe() { + return false; + } + virtual bool isUserAssertion() { + return true; + } + virtual string toString() const { + return "userassert:" + msg; + } + }; + + class MsgAssertionException : public AssertionException { + public: + MsgAssertionException(int c, const char *m) { + code = c; + msg = m; + } + virtual bool severe() { + return false; + } + virtual string toString() const { + return "massert:" + msg; + } + }; + + void asserted(const char *msg, const char *file, unsigned line); + void wasserted(const char *msg, const char *file, unsigned line); + void uasserted(int msgid, const char *msg); + inline void uasserted(int msgid , string msg) { uasserted(msgid, msg.c_str()); } + void uassert_nothrow(const char *msg); // reported via lasterror, but don't throw exception + void msgasserted(int msgid, const char *msg); + inline void msgasserted(int msgid, string msg) { msgasserted(msgid, msg.c_str()); } + +#ifdef assert +#undef assert +#endif + +#define assert(_Expression) (void)( (!!(_Expression)) || (mongo::asserted(#_Expression, __FILE__, __LINE__), 0) ) + + /* "user assert". if asserts, user did something wrong, not our code */ +//#define uassert( 10269 , _Expression) (void)( (!!(_Expression)) || (uasserted(#_Expression, __FILE__, __LINE__), 0) ) +#define uassert(msgid, msg,_Expression) (void)( (!!(_Expression)) || (mongo::uasserted(msgid, msg), 0) ) + +#define xassert(_Expression) (void)( (!!(_Expression)) || (mongo::asserted(#_Expression, __FILE__, __LINE__), 0) ) + +#define yassert 1 + + /* warning only - keeps going */ +#define wassert(_Expression) (void)( (!!(_Expression)) || (mongo::wasserted(#_Expression, __FILE__, __LINE__), 0) ) + + /* display a message, no context, and throw assertionexception + + easy way to throw an exception and log something without our stack trace + display happening. + */ +#define massert(msgid, msg,_Expression) (void)( (!!(_Expression)) || (mongo::msgasserted(msgid, msg), 0) ) + + /* dassert is 'debug assert' -- might want to turn off for production as these + could be slow. + */ +#if defined(_DEBUG) +#define dassert assert +#else +#define dassert(x) +#endif + + // some special ids that we want to duplicate + + // > 10000 asserts + // < 10000 UserException + +#define ASSERT_ID_DUPKEY 11000 + +} // namespace mongo + +#define BOOST_CHECK_EXCEPTION( expression ) \ + try { \ + expression; \ + } catch ( const std::exception &e ) { \ + problem() << "caught boost exception: " << e.what() << endl; \ + assert( false ); \ + } catch ( ... ) { \ + massert( 10437 , "unknown boost failed" , false ); \ + } diff --git a/util/background.cpp b/util/background.cpp new file mode 100644 index 0000000..ac3a48c --- /dev/null +++ b/util/background.cpp @@ -0,0 +1,64 @@ +//background.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "goodies.h" +#include "background.h" + +namespace mongo { + + BackgroundJob *BackgroundJob::grab = 0; + boost::mutex &BackgroundJob::mutex = *( new boost::mutex ); + + /* static */ + void BackgroundJob::thr() { + assert( grab ); + BackgroundJob *us = grab; + assert( us->state == NotStarted ); + us->state = Running; + grab = 0; + us->run(); + us->state = Done; + if ( us->deleteSelf ) + delete us; + } + + BackgroundJob& BackgroundJob::go() { + boostlock bl(mutex); + assert( grab == 0 ); + grab = this; + boost::thread t(thr); + while ( grab ) + sleepmillis(2); + return *this; + } + + bool BackgroundJob::wait(int msMax) { + assert( state != NotStarted ); + int ms = 1; + Date_t start = jsTime(); + while ( state != Done ) { + sleepmillis(ms); + if ( ms < 1000 ) + ms = ms * 2; + if ( msMax && ( int( jsTime() - start ) > msMax) ) + return false; + } + return true; + } + +} // namespace mongo diff --git a/util/background.h b/util/background.h new file mode 100644 index 0000000..ff044cb --- /dev/null +++ b/util/background.h @@ -0,0 +1,73 @@ +// background.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace mongo { + + /* object-orienty background thread dispatching. + + subclass and define run() + + It is ok to call go() more than once -- if the previous invocation + has finished. Thus one pattern of use is to embed a backgroundjob + in your object and reuse it (or same thing with inheritance). + */ + + class BackgroundJob { + protected: + /* define this to do your work! */ + virtual void run() = 0; + + public: + enum State { + NotStarted, + Running, + Done + }; + State getState() const { + return state; + } + bool running() const { + return state == Running; + } + + bool deleteSelf; // delete self when Done? + + BackgroundJob() { + deleteSelf = false; + state = NotStarted; + } + virtual ~BackgroundJob() { } + + // start job. returns before it's finished. + BackgroundJob& go(); + + // wait for completion. this spins with sleep() so not terribly efficient. + // returns true if did not time out. + // + // note you can call wait() more than once if the first call times out. + bool wait(int msMax = 0); + + private: + static BackgroundJob *grab; + static boost::mutex &mutex; + static void thr(); + volatile State state; + }; + +} // namespace mongo diff --git a/util/base64.cpp b/util/base64.cpp new file mode 100644 index 0000000..cf2f485 --- /dev/null +++ b/util/base64.cpp @@ -0,0 +1,144 @@ +// util/base64.cpp + + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" + +namespace mongo { + namespace base64 { + + class Alphabet { + public: + Alphabet(){ + encode = (unsigned char*) + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789" + "+/"; + + decode = (unsigned char*)malloc(257); + memset( decode , 0 , 256 ); + for ( int i=0; i<64; i++ ){ + decode[ encode[i] ] = i; + } + + test(); + } + ~Alphabet(){ + free( decode ); + } + + void test(){ + assert( strlen( (char*)encode ) == 64 ); + for ( int i=0; i<26; i++ ) + assert( encode[i] == toupper( encode[i+26] ) ); + } + + char e( int x ){ + return encode[x&0x3f]; + } + + private: + const unsigned char * encode; + public: + unsigned char * decode; + } alphabet; + + + void encode( stringstream& ss , const char * data , int size ){ + for ( int i=0; i<size; i+=3 ){ + int left = size - i; + const unsigned char * start = (const unsigned char*)data + i; + + // byte 0 + ss << alphabet.e(start[0]>>2); + + // byte 1 + unsigned char temp = ( start[0] << 4 ); + if ( left == 1 ){ + ss << alphabet.e(temp); + break; + } + temp |= ( ( start[1] >> 4 ) & 0xF ); + ss << alphabet.e(temp); + + // byte 2 + temp = ( start[1] & 0xF ) << 2; + if ( left == 2 ){ + ss << alphabet.e(temp); + break; + } + temp |= ( ( start[2] >> 6 ) & 0x3 ); + ss << alphabet.e(temp); + + // byte 3 + ss << alphabet.e(start[2] & 0x3f); + } + + int mod = size % 3; + if ( mod == 1 ){ + ss << "=="; + } + else if ( mod == 2 ){ + ss << "="; + } + } + + + string encode( const char * data , int size ){ + stringstream ss; + encode( ss , data ,size ); + return ss.str(); + } + + string encode( const string& s ){ + return encode( s.c_str() , s.size() ); + } + + + void decode( stringstream& ss , const string& s ){ + uassert( 10270 , "invalid base64" , s.size() % 4 == 0 ); + const unsigned char * data = (const unsigned char*)s.c_str(); + int size = s.size(); + + unsigned char buf[3]; + for ( int i=0; i<size; i+=4){ + const unsigned char * start = data + i; + buf[0] = ( ( alphabet.decode[start[0]] << 2 ) & 0xFC ) | ( ( alphabet.decode[start[1]] >> 4 ) & 0x3 ); + buf[1] = ( ( alphabet.decode[start[1]] << 4 ) & 0xF0 ) | ( ( alphabet.decode[start[2]] >> 2 ) & 0xF ); + buf[2] = ( ( alphabet.decode[start[2]] << 6 ) & 0xC0 ) | ( ( alphabet.decode[start[3]] & 0x3F ) ); + + int len = 3; + if ( start[3] == '=' ){ + len = 2; + if ( start[2] == '=' ){ + len = 1; + } + } + ss.write( (const char*)buf , len ); + } + } + + string decode( const string& s ){ + stringstream ss; + decode( ss , s ); + return ss.str(); + } + + } +} + diff --git a/util/base64.h b/util/base64.h new file mode 100644 index 0000000..62caceb --- /dev/null +++ b/util/base64.h @@ -0,0 +1,32 @@ +// util/base64.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +namespace mongo { + namespace base64 { + + void encode( stringstream& ss , const char * data , int size ); + string encode( const char * data , int size ); + string encode( const string& s ); + + void decode( stringstream& ss , const string& s ); + string decode( const string& s ); + + + void testAlphabet(); + } +} diff --git a/util/builder.h b/util/builder.h new file mode 100644 index 0000000..5046b72 --- /dev/null +++ b/util/builder.h @@ -0,0 +1,207 @@ +/* builder.h */ + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../stdafx.h" +#include <string.h> + +namespace mongo { + + class StringBuilder; + + class BufBuilder { + public: + BufBuilder(int initsize = 512) : size(initsize) { + if ( size > 0 ) { + data = (char *) malloc(size); + assert(data); + } else { + data = 0; + } + l = 0; + } + ~BufBuilder() { + kill(); + } + + void kill() { + if ( data ) { + free(data); + data = 0; + } + } + + void reset( int maxSize = 0 ){ + l = 0; + if ( maxSize && size > maxSize ){ + free(data); + data = (char*)malloc(maxSize); + size = maxSize; + } + + } + + /* leave room for some stuff later */ + void skip(int n) { + grow(n); + } + + /* note this may be deallocated (realloced) if you keep writing. */ + char* buf() { + return data; + } + + /* assume ownership of the buffer - you must then free it */ + void decouple() { + data = 0; + } + + template<class T> void append(T j) { + *((T*)grow(sizeof(T))) = j; + } + void append(short j) { + append<short>(j); + } + void append(int j) { + append<int>(j); + } + void append(unsigned j) { + append<unsigned>(j); + } + void append(bool j) { + append<bool>(j); + } + void append(double j) { + append<double>(j); + } + + void append(const void *src, int len) { + memcpy(grow(len), src, len); + } + + void append(const char *str) { + append((void*) str, strlen(str)+1); + } + + void append(const string &str) { + append( (void *)str.c_str(), str.length() + 1 ); + } + + int len() const { + return l; + } + + void setlen( int newLen ){ + l = newLen; + } + + private: + /* returns the pre-grow write position */ + char* grow(int by) { + int oldlen = l; + l += by; + if ( l > size ) { + int a = size * 2; + if ( a == 0 ) + a = 512; + if ( l > a ) + a = l + 16 * 1024; + assert( a < 64 * 1024 * 1024 ); + data = (char *) realloc(data, a); + size= a; + } + return data + oldlen; + } + + char *data; + int l; + int size; + + friend class StringBuilder; + }; + + class StringBuilder { + public: + StringBuilder( int initsize=256 ) + : _buf( initsize ){ + } + +#define SBNUM(val,maxSize,macro) \ + int prev = _buf.l; \ + int z = sprintf( _buf.grow(maxSize) , macro , (val) ); \ + _buf.l = prev + z; \ + return *this; + + + StringBuilder& operator<<( double x ){ + SBNUM( x , 25 , "%g" ); + } + StringBuilder& operator<<( int x ){ + SBNUM( x , 11 , "%d" ); + } + StringBuilder& operator<<( unsigned x ){ + SBNUM( x , 11 , "%u" ); + } + StringBuilder& operator<<( long x ){ + SBNUM( x , 22 , "%ld" ); + } + StringBuilder& operator<<( unsigned long x ){ + SBNUM( x , 22 , "%lu" ); + } + StringBuilder& operator<<( long long x ){ + SBNUM( x , 22 , "%lld" ); + } + StringBuilder& operator<<( unsigned long long x ){ + SBNUM( x , 22 , "%llu" ); + } + StringBuilder& operator<<( short x ){ + SBNUM( x , 8 , "%hd" ); + } + StringBuilder& operator<<( char c ){ + _buf.grow( 1 )[0] = c; + return *this; + } + + void append( const char * str ){ + int x = strlen( str ); + memcpy( _buf.grow( x ) , str , x ); + } + StringBuilder& operator<<( const char * str ){ + append( str ); + return *this; + } + StringBuilder& operator<<( const string& s ){ + append( s.c_str() ); + return *this; + } + + // access + + void reset( int maxSize = 0 ){ + _buf.reset( maxSize ); + } + + string str(){ + return string(_buf.data,0,_buf.l); + } + + private: + BufBuilder _buf; + }; + +} // namespace mongo diff --git a/util/debug_util.cpp b/util/debug_util.cpp new file mode 100644 index 0000000..283053f --- /dev/null +++ b/util/debug_util.cpp @@ -0,0 +1,59 @@ +// debug_util.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "../db/cmdline.h" +#include "../db/jsobj.h" + +namespace mongo { + +#if defined(_DEBUG) && !defined(_WIN32) + /* Magic gdb trampoline + * Do not call directly! call setupSIGTRAPforGDB() + * Assumptions: + * 1) gdbserver is on your path + * 2) You have run "handle SIGSTOP noprint" in gdb + * 3) cmdLine.port + 2000 is free + */ + void launchGDB(int){ + // Don't come back here + signal(SIGTRAP, SIG_IGN); + + int newPort = cmdLine.port + 2000; + string newPortStr = "localhost:" + BSONObjBuilder::numStr(newPort); + string pidToDebug = BSONObjBuilder::numStr(getpid()); + + cout << "\n\n\t**** Launching gdbserver on " << newPortStr << " ****" << endl << endl; + if (fork() == 0){ + //child + execlp("gdbserver", "gdbserver", "--attach", newPortStr.c_str(), pidToDebug.c_str(), NULL); + perror(NULL); + }else{ + //parent + raise(SIGSTOP); // pause all threads until gdb connects and continues + raise(SIGTRAP); // break inside gdbserver + } + } + + void setupSIGTRAPforGDB(){ + assert( signal(SIGTRAP , launchGDB ) != SIG_ERR ); + } +#else + void setupSIGTRAPforGDB() { + } +#endif +} diff --git a/util/debug_util.h b/util/debug_util.h new file mode 100644 index 0000000..6d633c5 --- /dev/null +++ b/util/debug_util.h @@ -0,0 +1,97 @@ +// debug_util.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifndef _WIN32 +#include <signal.h> +#endif // ndef _WIN32 + +namespace mongo { + +// for debugging + typedef struct _Ints { + int i[100]; + } *Ints; + typedef struct _Chars { + char c[200]; + } *Chars; + + typedef char CHARS[400]; + + typedef struct _OWS { + int size; + char type; + char string[400]; + } *OWS; + +// for now, running on win32 means development not production -- +// use this to log things just there. +#if defined(_WIN32) +#define WIN if( 1 ) +#else +#define WIN if( 0 ) +#endif + +#if defined(_DEBUG) +#define DEV if( 1 ) +#else +#define DEV if( 0 ) +#endif + +#define DEBUGGING if( 0 ) + +// The following declare one unique counter per enclosing function. +// NOTE The implementation double-increments on a match, but we don't really care. +#define SOMETIMES( occasion, howOften ) for( static unsigned occasion = 0; ++occasion % howOften == 0; ) +#define OCCASIONALLY SOMETIMES( occasionally, 16 ) +#define RARELY SOMETIMES( rarely, 128 ) +#define ONCE for( static bool undone = true; undone; undone = false ) + +#if defined(_WIN32) +#define strcasecmp _stricmp +#endif + + // Sets SIGTRAP handler to launch GDB + // Noop unless on *NIX and compiled with _DEBUG + void setupSIGTRAPforGDB(); + +#if defined(_WIN32) + inline void breakpoint() {} //noop +#else // defined(_WIN32) + // code to raise a breakpoint in GDB + inline void breakpoint(){ + ONCE { + //prevent SIGTRAP from crashing the program if default action is specified and we are not in gdb + struct sigaction current; + sigaction(SIGTRAP, NULL, ¤t); + if (current.sa_handler == SIG_DFL){ + signal(SIGTRAP, SIG_IGN); + } + } + + raise(SIGTRAP); + } +#endif // defined(_WIN32) + + // conditional breakpoint + inline void breakif(bool test){ + if (test) + breakpoint(); + } + +} // namespace mongo diff --git a/util/embedded_builder.h b/util/embedded_builder.h new file mode 100644 index 0000000..d945bfb --- /dev/null +++ b/util/embedded_builder.h @@ -0,0 +1,91 @@ +// embedded_builder.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace mongo { + // utility class for assembling hierarchical objects + class EmbeddedBuilder { + public: + EmbeddedBuilder( BSONObjBuilder *b ) { + _builders.push_back( make_pair( "", b ) ); + } + // It is assumed that the calls to prepareContext will be made with the 'name' + // parameter in lex ascending order. + void prepareContext( string &name ) { + int i = 1, n = _builders.size(); + while( i < n && + name.substr( 0, _builders[ i ].first.length() ) == _builders[ i ].first && + ( name[ _builders[i].first.length() ] == '.' || name[ _builders[i].first.length() ] == 0 ) + ){ + name = name.substr( _builders[ i ].first.length() + 1 ); + ++i; + } + for( int j = n - 1; j >= i; --j ) { + popBuilder(); + } + for( string next = splitDot( name ); !next.empty(); next = splitDot( name ) ) { + addBuilder( next ); + } + } + void appendAs( const BSONElement &e, string name ) { + if ( e.type() == Object && e.valuesize() == 5 ) { // empty object -- this way we can add to it later + string dummyName = name + ".foo"; + prepareContext( dummyName ); + return; + } + prepareContext( name ); + back()->appendAs( e, name.c_str() ); + } + BufBuilder &subarrayStartAs( string name ) { + prepareContext( name ); + return back()->subarrayStart( name.c_str() ); + } + void done() { + while( ! _builderStorage.empty() ) + popBuilder(); + } + + static string splitDot( string & str ) { + size_t pos = str.find( '.' ); + if ( pos == string::npos ) + return ""; + string ret = str.substr( 0, pos ); + str = str.substr( pos + 1 ); + return ret; + } + + private: + void addBuilder( const string &name ) { + shared_ptr< BSONObjBuilder > newBuilder( new BSONObjBuilder( back()->subobjStart( name.c_str() ) ) ); + _builders.push_back( make_pair( name, newBuilder.get() ) ); + _builderStorage.push_back( newBuilder ); + } + void popBuilder() { + back()->done(); + _builders.pop_back(); + _builderStorage.pop_back(); + } + + BSONObjBuilder *back() { return _builders.back().second; } + + vector< pair< string, BSONObjBuilder * > > _builders; + vector< shared_ptr< BSONObjBuilder > > _builderStorage; + + }; + +} //namespace mongo diff --git a/util/file.h b/util/file.h new file mode 100644 index 0000000..347e2d6 --- /dev/null +++ b/util/file.h @@ -0,0 +1,168 @@ +// file.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if !defined(_WIN32) +#include "errno.h" +#include <sys/mman.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#else +#include <windows.h> +#endif + +namespace mongo { + +#ifndef __sunos__ +typedef uint64_t fileofs; +#else +typedef boost::uint64_t fileofs; +#endif + +class FileInterface { +public: + void open(const char *fn) {} + void write(fileofs o, const char *data, unsigned len) {} + void read(fileofs o, char *data, unsigned len) {} + bool bad() {return false;} + bool is_open() {return false;} + fileofs len() { return 0; } +}; + +#if defined(_WIN32) +#include <io.h> +std::wstring toWideString(const char *s); + +class File : public FileInterface { + HANDLE fd; + bool _bad; + void err(BOOL b=false) { /* false = error happened */ + if( !b && !_bad ) { + _bad = true; + log() << "File I/O error " << GetLastError() << '\n'; + } + } +public: + File() { + fd = INVALID_HANDLE_VALUE; + _bad = true; + } + ~File() { + if( is_open() ) CloseHandle(fd); + fd = INVALID_HANDLE_VALUE; + } + void open(const char *filename, bool readOnly=false ) { + std::wstring filenamew = toWideString(filename); + fd = CreateFile( + filenamew.c_str(), ( readOnly ? 0 : GENERIC_WRITE ) | GENERIC_READ, FILE_SHARE_READ, + NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if( !is_open() ) { + out() << "CreateFile failed " << filename << endl; + } + else + _bad = false; + } + void write(fileofs o, const char *data, unsigned len) { + LARGE_INTEGER li; + li.QuadPart = o; + SetFilePointerEx(fd, li, NULL, FILE_BEGIN); + DWORD written; + err( WriteFile(fd, data, len, &written, NULL) ); + } + void read(fileofs o, char *data, unsigned len) { + DWORD read; + LARGE_INTEGER li; + li.QuadPart = o; + SetFilePointerEx(fd, li, NULL, FILE_BEGIN); + int ok = ReadFile(fd, data, len, &read, 0); + if( !ok ) + err(ok); + else + massert( 10438 , "ReadFile error - truncated file?", read == len); + } + bool bad() { return _bad; } + bool is_open() { return fd != INVALID_HANDLE_VALUE; } + fileofs len() { + LARGE_INTEGER li; + li.LowPart = GetFileSize(fd, (DWORD *) &li.HighPart); + if( li.HighPart == 0 && li.LowPart == INVALID_FILE_SIZE ) { + err( false ); + return 0; + } + return li.QuadPart; + } + void fsync() { FlushFileBuffers(fd); } +}; + +#else + +class File : public FileInterface { + int fd; + bool _bad; + void err(bool ok) { + if( !ok && !_bad ) { + _bad = true; + log() << "File I/O " << OUTPUT_ERRNO << '\n'; + } + } +public: + File() { + fd = -1; + _bad = true; + } + ~File() { + if( is_open() ) ::close(fd); + fd = -1; + } + +#ifndef O_NOATIME +#define O_NOATIME 0 +#define lseek64 lseek +#endif + + void open(const char *filename, bool readOnly=false ) { + fd = ::open(filename, O_CREAT | ( readOnly ? 0 : O_RDWR ) | O_NOATIME, S_IRUSR | S_IWUSR); + if ( fd <= 0 ) { + out() << "couldn't open " << filename << ' ' << OUTPUT_ERRNO << endl; + return; + } + _bad = false; + } + void write(fileofs o, const char *data, unsigned len) { + lseek64(fd, o, SEEK_SET); + err( ::write(fd, data, len) == (int) len ); + } + void read(fileofs o, char *data, unsigned len) { + lseek(fd, o, SEEK_SET); + err( ::read(fd, data, len) == (int) len ); + } + bool bad() { return _bad; } + bool is_open() { return fd > 0; } + fileofs len() { + return lseek(fd, 0, SEEK_END); + } + void fsync() { ::fsync(fd); } +}; + + +#endif + + +} + diff --git a/util/file_allocator.h b/util/file_allocator.h new file mode 100644 index 0000000..73159d3 --- /dev/null +++ b/util/file_allocator.h @@ -0,0 +1,229 @@ +//file_allocator.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../stdafx.h" +#include <fcntl.h> +#include <errno.h> +#if defined(__freebsd__) +#include <sys/stat.h> +#endif + +#ifndef O_NOATIME +#define O_NOATIME 0 +#endif + +namespace mongo { + + /* Handles allocation of contiguous files on disk. Allocation may be + requested asynchronously or synchronously. + */ + class FileAllocator { + /* The public functions may not be called concurrently. The allocation + functions may be called multiple times per file, but only the first + size specified per file will be used. + */ + public: +#if !defined(_WIN32) + FileAllocator() : failed_() {} +#endif + void start() { +#if !defined(_WIN32) + Runner r( *this ); + boost::thread t( r ); +#endif + } + // May be called if file exists. If file exists, or its allocation has + // been requested, size is updated to match existing file size. + void requestAllocation( const string &name, long &size ) { + /* Some of the system calls in the file allocator don't work in win, + so no win support - 32 or 64 bit. Plus we don't seem to need preallocation + on windows anyway as we don't have to pre-zero the file there. + */ +#if !defined(_WIN32) + boostlock lk( pendingMutex_ ); + if ( failed_ ) + return; + long oldSize = prevSize( name ); + if ( oldSize != -1 ) { + size = oldSize; + return; + } + pending_.push_back( name ); + pendingSize_[ name ] = size; + pendingUpdated_.notify_all(); +#endif + } + // Returns when file has been allocated. If file exists, size is + // updated to match existing file size. + void allocateAsap( const string &name, long &size ) { +#if !defined(_WIN32) + boostlock lk( pendingMutex_ ); + long oldSize = prevSize( name ); + if ( oldSize != -1 ) { + size = oldSize; + if ( !inProgress( name ) ) + return; + } + checkFailure(); + pendingSize_[ name ] = size; + if ( pending_.size() == 0 ) + pending_.push_back( name ); + else if ( pending_.front() != name ) { + pending_.remove( name ); + list< string >::iterator i = pending_.begin(); + ++i; + pending_.insert( i, name ); + } + pendingUpdated_.notify_all(); + while( inProgress( name ) ) { + checkFailure(); + pendingUpdated_.wait( lk ); + } +#endif + } + + void waitUntilFinished() const { +#if !defined(_WIN32) + if ( failed_ ) + return; + boostlock lk( pendingMutex_ ); + while( pending_.size() != 0 ) + pendingUpdated_.wait( lk ); +#endif + } + + private: +#if !defined(_WIN32) + void checkFailure() { + massert( 12520, "file allocation failure", !failed_ ); + } + + // caller must hold pendingMutex_ lock. Returns size if allocated or + // allocation requested, -1 otherwise. + long prevSize( const string &name ) const { + if ( pendingSize_.count( name ) > 0 ) + return pendingSize_[ name ]; + if ( boost::filesystem::exists( name ) ) + return boost::filesystem::file_size( name ); + return -1; + } + + // caller must hold pendingMutex_ lock. + bool inProgress( const string &name ) const { + for( list< string >::const_iterator i = pending_.begin(); i != pending_.end(); ++i ) + if ( *i == name ) + return true; + return false; + } + + mutable boost::mutex pendingMutex_; + mutable boost::condition pendingUpdated_; + list< string > pending_; + mutable map< string, long > pendingSize_; + bool failed_; + + struct Runner { + Runner( FileAllocator &allocator ) : a_( allocator ) {} + FileAllocator &a_; + void operator()() { + while( 1 ) { + { + boostlock lk( a_.pendingMutex_ ); + if ( a_.pending_.size() == 0 ) + a_.pendingUpdated_.wait( lk ); + } + while( 1 ) { + string name; + long size; + { + boostlock lk( a_.pendingMutex_ ); + if ( a_.pending_.size() == 0 ) + break; + name = a_.pending_.front(); + size = a_.pendingSize_[ name ]; + } + try { + long fd = open(name.c_str(), O_CREAT | O_RDWR | O_NOATIME, S_IRUSR | S_IWUSR); + if ( fd <= 0 ) { + stringstream ss; + ss << "couldn't open " << name << ' ' << OUTPUT_ERRNO; + massert( 10439 , ss.str(), fd <= 0 ); + } + +#if defined(POSIX_FADV_DONTNEED) + if( posix_fadvise(fd, 0, size, POSIX_FADV_DONTNEED) ) { + log() << "warning: posix_fadvise fails " << name << ' ' << OUTPUT_ERRNO << endl; + } +#endif + + /* make sure the file is the full desired length */ + off_t filelen = lseek(fd, 0, SEEK_END); + if ( filelen < size ) { + massert( 10440 , "failure creating new datafile", filelen == 0 ); + // Check for end of disk. + massert( 10441 , "Unable to allocate file of desired size", + size - 1 == lseek(fd, size - 1, SEEK_SET) ); + massert( 10442 , "Unable to allocate file of desired size", + 1 == write(fd, "", 1) ); + lseek(fd, 0, SEEK_SET); + log() << "allocating new datafile " << name << ", filling with zeroes..." << endl; + Timer t; + long z = 256 * 1024; + char buf[z]; + memset(buf, 0, z); + long left = size; + while ( 1 ) { + if ( left <= z ) { + massert( 10443 , "write failed", left == write(fd, buf, left) ); + break; + } + massert( 10444 , "write failed", z == write(fd, buf, z) ); + left -= z; + } + log() << "done allocating datafile " << name << ", size: " << size/1024/1024 << "MB, took " << ((double)t.millis())/1000.0 << " secs" << endl; + } + close( fd ); + + } catch ( ... ) { + problem() << "Failed to allocate new file: " << name + << ", size: " << size << ", aborting." << endl; + try { + BOOST_CHECK_EXCEPTION( boost::filesystem::remove( name ) ); + } catch ( ... ) { + } + boostlock lk( a_.pendingMutex_ ); + a_.failed_ = true; + // not erasing from pending + a_.pendingUpdated_.notify_all(); + return; // no more allocation + } + + { + boostlock lk( a_.pendingMutex_ ); + a_.pendingSize_.erase( name ); + a_.pending_.pop_front(); + a_.pendingUpdated_.notify_all(); + } + } + } + } + }; +#endif + }; + + FileAllocator &theFileAllocator(); +} // namespace mongo diff --git a/util/goodies.h b/util/goodies.h new file mode 100644 index 0000000..7eebc0a --- /dev/null +++ b/util/goodies.h @@ -0,0 +1,526 @@ +// goodies.h +// miscellaneous junk + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if defined(_WIN32) +# include <windows.h> +#endif + +namespace mongo { + +#if !defined(_WIN32) && !defined(NOEXECINFO) + +} // namespace mongo + +#include <pthread.h> +#include <execinfo.h> + +namespace mongo { + + inline pthread_t GetCurrentThreadId() { + return pthread_self(); + } + + /* use "addr2line -CFe <exe>" to parse. */ + inline void printStackTrace( ostream &o = cout ) { + void *b[20]; + size_t size; + char **strings; + size_t i; + + size = backtrace(b, 20); + strings = backtrace_symbols(b, size); + + for (i = 0; i < size; i++) + o << hex << b[i] << dec << ' '; + o << '\n'; + for (i = 0; i < size; i++) + o << ' ' << strings[i] << '\n'; + + free (strings); + } +#else + inline void printStackTrace( ostream &o = cout ) { } +#endif + + /* set to TRUE if we are exiting */ + extern bool goingAway; + + /* find the multimap member which matches a particular key and value. + + note this can be slow if there are a lot with the same key. + */ + template<class C,class K,class V> inline typename C::iterator kv_find(C& c, const K& k,const V& v) { + pair<typename C::iterator,typename C::iterator> p = c.equal_range(k); + + for ( typename C::iterator it=p.first; it!=p.second; ++it) + if ( it->second == v ) + return it; + + return c.end(); + } + + bool isPrime(int n); + int nextPrime(int n); + + inline void dumpmemory(const char *data, int len) { + if ( len > 1024 ) + len = 1024; + try { + const char *q = data; + const char *p = q; + while ( len > 0 ) { + for ( int i = 0; i < 16; i++ ) { + if ( *p >= 32 && *p <= 126 ) + cout << *p; + else + cout << '.'; + p++; + } + cout << " "; + p -= 16; + for ( int i = 0; i < 16; i++ ) + cout << (unsigned) ((unsigned char)*p++) << ' '; + cout << endl; + len -= 16; + } + } catch (...) { + } + } + +// PRINT(2+2); prints "2+2: 4" +#define PRINT(x) cout << #x ": " << (x) << endl +// PRINTFL; prints file:line +#define PRINTFL cout << __FILE__ ":" << __LINE__ << endl + +#undef yassert + +#undef assert +#define assert xassert +#define yassert 1 + + struct WrappingInt { + WrappingInt() { + x = 0; + } + WrappingInt(unsigned z) : x(z) { } + volatile unsigned x; + operator unsigned() const { + return x; + } + + // returns original value (like x++) + WrappingInt atomicIncrement(){ +#if defined(_WIN32) + // InterlockedIncrement returns the new value + return InterlockedIncrement((volatile long*)&x)-1; //long is 32bits in Win64 +#elif defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) + // this is in GCC >= 4.1 + return __sync_fetch_and_add(&x, 1); +#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) + // from boost 1.39 interprocess/detail/atomic.hpp + int r; + int val = 1; + asm volatile + ( + "lock\n\t" + "xadd %1, %0": + "+m"( x ), "=r"( r ): // outputs (%0, %1) + "1"( val ): // inputs (%2 == %1) + "memory", "cc" // clobbers + ); + return r; +#else +# error "unsupported compiler or platform" +#endif + } + + static int diff(unsigned a, unsigned b) { + return a-b; + } + bool operator<=(WrappingInt r) { + // platform dependent + int df = (r.x - x); + return df >= 0; + } + bool operator>(WrappingInt r) { + return !(r<=*this); + } + }; + +} // namespace mongo + +#include <ctime> + +namespace mongo { + + inline void time_t_to_String(time_t t, char *buf) { +#if defined(_WIN32) + ctime_s(buf, 64, &t); +#else + ctime_r(&t, buf); +#endif + buf[24] = 0; // don't want the \n + } + +#define asctime _asctime_not_threadsafe_ +#define gmtime _gmtime_not_threadsafe_ +#define localtime _localtime_not_threadsafe_ +#define ctime _ctime_is_not_threadsafe_ + +#if defined(_WIN32) || defined(__sunos__) + inline void sleepsecs(int s) { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + xt.sec += s; + boost::thread::sleep(xt); + } + inline void sleepmillis(int s) { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + xt.sec += ( s / 1000 ); + xt.nsec += ( s % 1000 ) * 1000000; + if ( xt.nsec >= 1000000000 ) { + xt.nsec -= 1000000000; + xt.sec++; + } + boost::thread::sleep(xt); + } + inline void sleepmicros(int s) { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + xt.sec += ( s / 1000000 ); + xt.nsec += ( s % 1000000 ) * 1000; + if ( xt.nsec >= 1000000000 ) { + xt.nsec -= 1000000000; + xt.sec++; + } + boost::thread::sleep(xt); + } +#else + inline void sleepsecs(int s) { + struct timespec t; + t.tv_sec = s; + t.tv_nsec = 0; + if ( nanosleep( &t , 0 ) ){ + cout << "nanosleep failed" << endl; + } + } + inline void sleepmicros(int s) { + struct timespec t; + t.tv_sec = (int)(s / 1000000); + t.tv_nsec = s % 1000000; + if ( nanosleep( &t , 0 ) ){ + cout << "nanosleep failed" << endl; + } + } + inline void sleepmillis(int s) { + sleepmicros( s * 1000 ); + } +#endif + +// note this wraps + inline int tdiff(unsigned told, unsigned tnew) { + return WrappingInt::diff(tnew, told); + } + inline unsigned curTimeMillis() { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + unsigned t = xt.nsec / 1000000; + return (xt.sec & 0xfffff) * 1000 + t; + } + + struct Date_t { + // TODO: make signed (and look for related TODO's) + unsigned long long millis; + Date_t(): millis(0) {} + Date_t(unsigned long long m): millis(m) {} + operator unsigned long long&() { return millis; } + operator const unsigned long long&() const { return millis; } + }; + + inline Date_t jsTime() { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + unsigned long long t = xt.nsec / 1000000; + return ((unsigned long long) xt.sec * 1000) + t; + } + + inline unsigned long long curTimeMicros64() { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + unsigned long long t = xt.nsec / 1000; + return (((unsigned long long) xt.sec) * 1000000) + t; + } + +// measures up to 1024 seconds. or, 512 seconds with tdiff that is... + inline unsigned curTimeMicros() { + boost::xtime xt; + boost::xtime_get(&xt, boost::TIME_UTC); + unsigned t = xt.nsec / 1000; + unsigned secs = xt.sec % 1024; + return secs*1000000 + t; + } + using namespace boost; + typedef boost::mutex::scoped_lock boostlock; + typedef boost::recursive_mutex::scoped_lock recursive_boostlock; + +// simple scoped timer + class Timer { + public: + Timer() { + reset(); + } + Timer( unsigned long long start ) { + old = start; + } + int seconds(){ + return (int)(micros() / 1000000); + } + int millis() { + return (long)(micros() / 1000); + } + unsigned long long micros() { + unsigned long long n = curTimeMicros64(); + return n - old; + } + unsigned long long micros(unsigned long long & n) { // returns cur time in addition to timer result + n = curTimeMicros64(); + return n - old; + } + unsigned long long startTime(){ + return old; + } + void reset() { + old = curTimeMicros64(); + } + private: + unsigned long long old; + }; + + /* + + class DebugMutex : boost::noncopyable { + friend class lock; + boost::mutex m; + int locked; + public: + DebugMutex() : locked(0); { } + bool isLocked() { return locked; } + }; + + */ + +//typedef boostlock lock; + + inline bool startsWith(const char *str, const char *prefix) { + unsigned l = strlen(prefix); + if ( strlen(str) < l ) return false; + return strncmp(str, prefix, l) == 0; + } + + inline bool endsWith(const char *p, const char *suffix) { + int a = strlen(p); + int b = strlen(suffix); + if ( b > a ) return false; + return strcmp(p + a - b, suffix) == 0; + } + +} // namespace mongo + +#include "boost/detail/endian.hpp" + +namespace mongo { + + inline unsigned long swapEndian(unsigned long x) { + return + ((x & 0xff) << 24) | + ((x & 0xff00) << 8) | + ((x & 0xff0000) >> 8) | + ((x & 0xff000000) >> 24); + } + +#if defined(BOOST_LITTLE_ENDIAN) + inline unsigned long fixEndian(unsigned long x) { + return x; + } +#else + inline unsigned long fixEndian(unsigned long x) { + return swapEndian(x); + } +#endif + + // Like strlen, but only scans up to n bytes. + // Returns -1 if no '0' found. + inline int strnlen( const char *s, int n ) { + for( int i = 0; i < n; ++i ) + if ( !s[ i ] ) + return i; + return -1; + } + +#if !defined(_WIN32) + typedef int HANDLE; + inline void strcpy_s(char *dst, unsigned len, const char *src) { + strcpy(dst, src); + } +#else + typedef void *HANDLE; +#endif + + /* thread local "value" rather than a pointer + good for things which have copy constructors (and the copy constructor is fast enough) + e.g. + ThreadLocalValue<int> myint; + */ + template<class T> + class ThreadLocalValue { + public: + ThreadLocalValue( T def = 0 ) : _default( def ) { } + + T get() { + T * val = _val.get(); + if ( val ) + return *val; + return _default; + } + + void set( const T& i ) { + T *v = _val.get(); + if( v ) { + *v = i; + return; + } + v = new T(i); + _val.reset( v ); + } + + private: + T _default; + boost::thread_specific_ptr<T> _val; + }; + + class ProgressMeter { + public: + ProgressMeter( long long total , int secondsBetween = 3 , int checkInterval = 100 ) + : _total( total ) , _secondsBetween( secondsBetween ) , _checkInterval( checkInterval ) , + _done(0) , _hits(0) , _lastTime( (int) time(0) ){ + } + + bool hit( int n = 1 ){ + _done += n; + _hits++; + if ( _hits % _checkInterval ) + return false; + + int t = (int) time(0); + if ( t - _lastTime < _secondsBetween ) + return false; + + if ( _total > 0 ){ + int per = (int)( ( (double)_done * 100.0 ) / (double)_total ); + cout << "\t\t" << _done << "/" << _total << "\t" << per << "%" << endl; + } + _lastTime = t; + return true; + } + + long long done(){ + return _done; + } + + long long hits(){ + return _hits; + } + + private: + + long long _total; + int _secondsBetween; + int _checkInterval; + + long long _done; + long long _hits; + int _lastTime; + }; + + class TicketHolder { + public: + TicketHolder( int num ){ + _outof = num; + _num = num; + } + + bool tryAcquire(){ + boostlock lk( _mutex ); + if ( _num <= 0 ){ + if ( _num < 0 ){ + cerr << "DISASTER! in TicketHolder" << endl; + } + return false; + } + _num--; + return true; + } + + void release(){ + boostlock lk( _mutex ); + _num++; + } + + void resize( int newSize ){ + boostlock lk( _mutex ); + int used = _outof - _num; + if ( used > newSize ){ + cout << "ERROR: can't resize since we're using (" << used << ") more than newSize(" << newSize << ")" << endl; + return; + } + + _outof = newSize; + _num = _outof - used; + } + + int available(){ + return _num; + } + + int used(){ + return _outof - _num; + } + + private: + int _outof; + int _num; + boost::mutex _mutex; + }; + + class TicketHolderReleaser { + public: + TicketHolderReleaser( TicketHolder * holder ){ + _holder = holder; + } + + ~TicketHolderReleaser(){ + _holder->release(); + } + private: + TicketHolder * _holder; + }; + +} // namespace mongo diff --git a/util/hashtab.h b/util/hashtab.h new file mode 100644 index 0000000..214c0ae --- /dev/null +++ b/util/hashtab.h @@ -0,0 +1,164 @@ +/* hashtab.h + + Simple, fixed size hash table. Darn simple. + + Uses a contiguous block of memory, so you can put it in a memory mapped file very easily. +*/ + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../stdafx.h" +#include <map> + +namespace mongo { + +#pragma pack(1) + + /* you should define: + + int Key::hash() return > 0 always. + */ + + template < + class Key, + class Type + > + class HashTable : boost::noncopyable { + public: + const char *name; + struct Node { + int hash; + Key k; + Type value; + bool inUse() { + return hash != 0; + } + void setUnused() { + hash = 0; + } + } *nodes; + int n; + int maxChain; + + int _find(const Key& k, bool& found) { + found = false; + int h = k.hash(); + int i = h % n; + int start = i; + int chain = 0; + int firstNonUsed = -1; + while ( 1 ) { + if ( !nodes[i].inUse() ) { + if ( firstNonUsed < 0 ) + firstNonUsed = i; + } + + if ( nodes[i].hash == h && nodes[i].k == k ) { + if ( chain >= 200 ) + out() << "warning: hashtable " << name << " long chain " << endl; + found = true; + return i; + } + chain++; + i = (i+1) % n; + if ( i == start ) { + // shouldn't get here / defensive for infinite loops + out() << "error: hashtable " << name << " is full n:" << n << endl; + return -1; + } + if( chain >= maxChain ) { + if ( firstNonUsed >= 0 ) + return firstNonUsed; + out() << "error: hashtable " << name << " max chain n:" << n << endl; + return -1; + } + } + } + + public: + /* buf must be all zeroes on initialization. */ + HashTable(void *buf, int buflen, const char *_name) : name(_name) { + int m = sizeof(Node); + // out() << "hashtab init, buflen:" << buflen << " m:" << m << endl; + n = buflen / m; + if ( (n & 1) == 0 ) + n--; + maxChain = (int) (n * 0.05); + nodes = (Node *) buf; + + assert( sizeof(Node) == 628 ); + //out() << "HashTable() " << _name << " sizeof(node):" << sizeof(Node) << " n:" << n << endl; + } + + Type* get(const Key& k) { + bool found; + int i = _find(k, found); + if ( found ) + return &nodes[i].value; + return 0; + } + + void kill(const Key& k) { + bool found; + int i = _find(k, found); + if ( i >= 0 && found ) { + nodes[i].k.kill(); + nodes[i].setUnused(); + } + } +/* + void drop(const Key& k) { + bool found; + int i = _find(k, found); + if ( i >= 0 && found ) { + nodes[i].setUnused(); + } + } +*/ + /** returns false if too full */ + bool put(const Key& k, const Type& value) { + bool found; + int i = _find(k, found); + if ( i < 0 ) + return false; + if ( !found ) { + nodes[i].k = k; + nodes[i].hash = k.hash(); + } + else { + assert( nodes[i].hash == k.hash() ); + } + nodes[i].value = value; + return true; + } + + typedef void (*IteratorCallback)( const Key& k , Type& v ); + + void iterall( IteratorCallback callback ){ + for ( int i=0; i<n; i++ ){ + if ( ! nodes[i].inUse() ) + continue; + callback( nodes[i].k , nodes[i].value ); + } + } + + }; + +#pragma pack() + +} // namespace mongo diff --git a/util/httpclient.cpp b/util/httpclient.cpp new file mode 100644 index 0000000..284bb63 --- /dev/null +++ b/util/httpclient.cpp @@ -0,0 +1,61 @@ +// httpclient.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "httpclient.h" + +namespace mongo { + + int HttpClient::get( string url , map<string,string>& headers, stringstream& data ){ + uassert( 10271 , "invalid url" , url.find( "http://" ) == 0 ); + url = url.substr( 7 ); + + string host , path; + if ( url.find( "/" ) == string::npos ){ + host = url; + path = "/"; + } + else { + host = url.substr( 0 , url.find( "/" ) ); + path = url.substr( url.find( "/" ) ); + } + + int port = 80; + uassert( 10272 , "non standard port not supported yet" , host.find( ":" ) == string::npos ); + + cout << "host [" << host << "]" << endl; + cout << "path [" << path << "]" << endl; + cout << "port: " << port << endl; + + string req; + { + stringstream ss; + ss << "GET " << path << " HTTP/1.1\r\n"; + ss << "Host: " << host << "\r\n"; + ss << "Connection: Close\r\n"; + ss << "User-Agent: mongodb http client\r\n"; + ss << "\r\n"; + + req = ss.str(); + } + + cout << req << endl; + + return -1; + } + +} diff --git a/util/httpclient.h b/util/httpclient.h new file mode 100644 index 0000000..14f0d87 --- /dev/null +++ b/util/httpclient.h @@ -0,0 +1,29 @@ +// httpclient.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../stdafx.h" + +namespace mongo { + + class HttpClient { + public: + int get( string url , map<string,string>& headers, stringstream& data ); + }; +} + diff --git a/util/log.h b/util/log.h new file mode 100644 index 0000000..a9f43c8 --- /dev/null +++ b/util/log.h @@ -0,0 +1,247 @@ +// log.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <string.h> + +namespace mongo { + + using boost::shared_ptr; + + // Utility interface for stringifying object only when val() called. + class LazyString { + public: + virtual ~LazyString() {} + virtual string val() const = 0; + }; + + // Utility class for stringifying object only when val() called. + template< class T > + class LazyStringImpl : public LazyString { + public: + LazyStringImpl( const T &t ) : t_( t ) {} + virtual string val() const { return (string)t_; } + private: + const T& t_; + }; + + class Nullstream { + public: + virtual ~Nullstream() {} + virtual Nullstream& operator<<(const char *) { + return *this; + } + virtual Nullstream& operator<<(char *) { + return *this; + } + virtual Nullstream& operator<<(char) { + return *this; + } + virtual Nullstream& operator<<(int) { + return *this; + } + virtual Nullstream& operator<<(ExitCode) { + return *this; + } + virtual Nullstream& operator<<(unsigned long) { + return *this; + } + virtual Nullstream& operator<<(long) { + return *this; + } + virtual Nullstream& operator<<(unsigned) { + return *this; + } + virtual Nullstream& operator<<(double) { + return *this; + } + virtual Nullstream& operator<<(void *) { + return *this; + } + virtual Nullstream& operator<<(const void *) { + return *this; + } + virtual Nullstream& operator<<(long long) { + return *this; + } + virtual Nullstream& operator<<(unsigned long long) { + return *this; + } + virtual Nullstream& operator<<(bool) { + return *this; + } + virtual Nullstream& operator<<(const LazyString&) { + return *this; + } + template< class T > + Nullstream& operator<<(T *t) { + return operator<<( static_cast<void*>( t ) ); + } + template< class T > + Nullstream& operator<<(const T *t) { + return operator<<( static_cast<const void*>( t ) ); + } + template< class T > + Nullstream& operator<<(const shared_ptr<T> p ){ + return *this; + } + template< class T > + Nullstream& operator<<(const T &t) { + return operator<<( static_cast<const LazyString&>( LazyStringImpl< T >( t ) ) ); + } + virtual Nullstream& operator<< (ostream& ( *endl )(ostream&)) { + return *this; + } + virtual Nullstream& operator<< (ios_base& (*hex)(ios_base&)) { + return *this; + } + virtual void flush(){} + }; + extern Nullstream nullstream; + +#define LOGIT { ss << x; return *this; } + + class Logstream : public Nullstream { + static boost::mutex &mutex; + static int doneSetup; + stringstream ss; + public: + static int magicNumber(){ + return 1717; + } + void flush() { + // this ensures things are sane + if ( doneSetup == 1717 ){ + boostlock lk(mutex); + cout << ss.str(); + cout.flush(); + } + ss.str(""); + } + Logstream& operator<<(const char *x) LOGIT + Logstream& operator<<(char *x) LOGIT + Logstream& operator<<(char x) LOGIT + Logstream& operator<<(int x) LOGIT + Logstream& operator<<(ExitCode x) LOGIT + Logstream& operator<<(long x) LOGIT + Logstream& operator<<(unsigned long x) LOGIT + Logstream& operator<<(unsigned x) LOGIT + Logstream& operator<<(double x) LOGIT + Logstream& operator<<(void *x) LOGIT + Logstream& operator<<(const void *x) LOGIT + Logstream& operator<<(long long x) LOGIT + Logstream& operator<<(unsigned long long x) LOGIT + Logstream& operator<<(bool x) LOGIT + Logstream& operator<<(const LazyString& x) { + ss << x.val(); + return *this; + } + Logstream& operator<< (ostream& ( *_endl )(ostream&)) { + ss << '\n'; + flush(); + return *this; + } + Logstream& operator<< (ios_base& (*_hex)(ios_base&)) { + ss << _hex; + return *this; + } + + template< class T > + Nullstream& operator<<(const shared_ptr<T> p ){ + T * t = p.get(); + if ( ! t ) + *this << "null"; + else + *this << t; + return *this; + } + + Logstream& prolog() { + char now[64]; + time_t_to_String(time(0), now); + now[20] = 0; + ss << now; + return *this; + } + + private: + static thread_specific_ptr<Logstream> tsp; + public: + static Logstream& get() { + Logstream *p = tsp.get(); + if( p == 0 ) + tsp.reset( p = new Logstream() ); + return *p; + } + }; + + extern int logLevel; + + inline Nullstream& out( int level = 0 ) { + if ( level > logLevel ) + return nullstream; + return Logstream::get(); + } + + /* flush the log stream if the log level is + at the specified level or higher. */ + inline void logflush(int level = 0) { + if( level > logLevel ) + Logstream::get().flush(); + } + + /* without prolog */ + inline Nullstream& _log( int level = 0 ){ + if ( level > logLevel ) + return nullstream; + return Logstream::get(); + } + + inline Nullstream& log( int level = 0 ) { + if ( level > logLevel ) + return nullstream; + return Logstream::get().prolog(); + } + + /* TODOCONCURRENCY */ + inline ostream& stdcout() { + return cout; + } + + /* default impl returns "" -- mongod overrides */ + extern const char * (*getcurns)(); + + inline Nullstream& problem( int level = 0 ) { + if ( level > logLevel ) + return nullstream; + Logstream& l = Logstream::get().prolog(); + l << ' ' << getcurns() << ' '; + return l; + } + + /** + log to a file rather than stdout + defined in assert_util.cpp + */ + void initLogging( const string& logpath , bool append ); + void rotateLogs( int signal = 0 ); + +#define OUTPUT_ERRNOX(x) "errno:" << x << " " << strerror(x) +#define OUTPUT_ERRNO OUTPUT_ERRNOX(errno) + +} // namespace mongo diff --git a/util/lruishmap.h b/util/lruishmap.h new file mode 100644 index 0000000..c390cb2 --- /dev/null +++ b/util/lruishmap.h @@ -0,0 +1,78 @@ +// lru-ish map.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../stdafx.h" +#include "../util/goodies.h" + +namespace mongo { + + /* Your K object must define: + int hash() - must always return > 0. + operator== + */ + + template <class K, class V, int MaxChain> + class LRUishMap { + public: + LRUishMap(int _n) { + n = nextPrime(_n); + keys = new K[n]; + hashes = new int[n]; + for ( int i = 0; i < n; i++ ) hashes[i] = 0; + } + ~LRUishMap() { + delete[] keys; + delete[] hashes; + } + + int _find(const K& k, bool& found) { + int h = k.hash(); + assert( h > 0 ); + int j = h % n; + int first = j; + for ( int i = 0; i < MaxChain; i++ ) { + if ( hashes[j] == h ) { + if ( keys[j] == k ) { + found = true; + return j; + } + } + else if ( hashes[j] == 0 ) { + found = false; + return j; + } + } + found = false; + return first; + } + + V* find(const K& k) { + bool found; + int j = _find(k, found); + return found ? &values[j] : 0; + } + + private: + int n; + K *keys; + int *hashes; + V *values; + }; + +} // namespace mongo diff --git a/util/md5.c b/util/md5.c new file mode 100644 index 0000000..c35d96c --- /dev/null +++ b/util/md5.c @@ -0,0 +1,381 @@ +/* + Copyright (C) 1999, 2000, 2002 Aladdin Enterprises. All rights reserved. + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + L. Peter Deutsch + ghost@aladdin.com + + */ +/* $Id: md5.c,v 1.6 2002/04/13 19:20:28 lpd Exp $ */ +/* + Independent implementation of MD5 (RFC 1321). + + This code implements the MD5 Algorithm defined in RFC 1321, whose + text is available at + http://www.ietf.org/rfc/rfc1321.txt + The code is derived from the text of the RFC, including the test suite + (section A.5) but excluding the rest of Appendix A. It does not include + any code or documentation that is identified in the RFC as being + copyrighted. + + The original and principal author of md5.c is L. Peter Deutsch + <ghost@aladdin.com>. Other authors are noted in the change history + that follows (in reverse chronological order): + + 2002-04-13 lpd Clarified derivation from RFC 1321; now handles byte order + either statically or dynamically; added missing #include <string.h> + in library. + 2002-03-11 lpd Corrected argument list for main(), and added int return + type, in test program and T value program. + 2002-02-21 lpd Added missing #include <stdio.h> in test program. + 2000-07-03 lpd Patched to eliminate warnings about "constant is + unsigned in ANSI C, signed in traditional"; made test program + self-checking. + 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. + 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5). + 1999-05-03 lpd Original version. + */ + +#include "md5.h" +#include <string.h> + +#undef BYTE_ORDER /* 1 = big-endian, -1 = little-endian, 0 = unknown */ +#ifdef ARCH_IS_BIG_ENDIAN +# define BYTE_ORDER (ARCH_IS_BIG_ENDIAN ? 1 : -1) +#else +# define BYTE_ORDER 0 +#endif + +#define T_MASK ((md5_word_t)~0) +#define T1 /* 0xd76aa478 */ (T_MASK ^ 0x28955b87) +#define T2 /* 0xe8c7b756 */ (T_MASK ^ 0x173848a9) +#define T3 0x242070db +#define T4 /* 0xc1bdceee */ (T_MASK ^ 0x3e423111) +#define T5 /* 0xf57c0faf */ (T_MASK ^ 0x0a83f050) +#define T6 0x4787c62a +#define T7 /* 0xa8304613 */ (T_MASK ^ 0x57cfb9ec) +#define T8 /* 0xfd469501 */ (T_MASK ^ 0x02b96afe) +#define T9 0x698098d8 +#define T10 /* 0x8b44f7af */ (T_MASK ^ 0x74bb0850) +#define T11 /* 0xffff5bb1 */ (T_MASK ^ 0x0000a44e) +#define T12 /* 0x895cd7be */ (T_MASK ^ 0x76a32841) +#define T13 0x6b901122 +#define T14 /* 0xfd987193 */ (T_MASK ^ 0x02678e6c) +#define T15 /* 0xa679438e */ (T_MASK ^ 0x5986bc71) +#define T16 0x49b40821 +#define T17 /* 0xf61e2562 */ (T_MASK ^ 0x09e1da9d) +#define T18 /* 0xc040b340 */ (T_MASK ^ 0x3fbf4cbf) +#define T19 0x265e5a51 +#define T20 /* 0xe9b6c7aa */ (T_MASK ^ 0x16493855) +#define T21 /* 0xd62f105d */ (T_MASK ^ 0x29d0efa2) +#define T22 0x02441453 +#define T23 /* 0xd8a1e681 */ (T_MASK ^ 0x275e197e) +#define T24 /* 0xe7d3fbc8 */ (T_MASK ^ 0x182c0437) +#define T25 0x21e1cde6 +#define T26 /* 0xc33707d6 */ (T_MASK ^ 0x3cc8f829) +#define T27 /* 0xf4d50d87 */ (T_MASK ^ 0x0b2af278) +#define T28 0x455a14ed +#define T29 /* 0xa9e3e905 */ (T_MASK ^ 0x561c16fa) +#define T30 /* 0xfcefa3f8 */ (T_MASK ^ 0x03105c07) +#define T31 0x676f02d9 +#define T32 /* 0x8d2a4c8a */ (T_MASK ^ 0x72d5b375) +#define T33 /* 0xfffa3942 */ (T_MASK ^ 0x0005c6bd) +#define T34 /* 0x8771f681 */ (T_MASK ^ 0x788e097e) +#define T35 0x6d9d6122 +#define T36 /* 0xfde5380c */ (T_MASK ^ 0x021ac7f3) +#define T37 /* 0xa4beea44 */ (T_MASK ^ 0x5b4115bb) +#define T38 0x4bdecfa9 +#define T39 /* 0xf6bb4b60 */ (T_MASK ^ 0x0944b49f) +#define T40 /* 0xbebfbc70 */ (T_MASK ^ 0x4140438f) +#define T41 0x289b7ec6 +#define T42 /* 0xeaa127fa */ (T_MASK ^ 0x155ed805) +#define T43 /* 0xd4ef3085 */ (T_MASK ^ 0x2b10cf7a) +#define T44 0x04881d05 +#define T45 /* 0xd9d4d039 */ (T_MASK ^ 0x262b2fc6) +#define T46 /* 0xe6db99e5 */ (T_MASK ^ 0x1924661a) +#define T47 0x1fa27cf8 +#define T48 /* 0xc4ac5665 */ (T_MASK ^ 0x3b53a99a) +#define T49 /* 0xf4292244 */ (T_MASK ^ 0x0bd6ddbb) +#define T50 0x432aff97 +#define T51 /* 0xab9423a7 */ (T_MASK ^ 0x546bdc58) +#define T52 /* 0xfc93a039 */ (T_MASK ^ 0x036c5fc6) +#define T53 0x655b59c3 +#define T54 /* 0x8f0ccc92 */ (T_MASK ^ 0x70f3336d) +#define T55 /* 0xffeff47d */ (T_MASK ^ 0x00100b82) +#define T56 /* 0x85845dd1 */ (T_MASK ^ 0x7a7ba22e) +#define T57 0x6fa87e4f +#define T58 /* 0xfe2ce6e0 */ (T_MASK ^ 0x01d3191f) +#define T59 /* 0xa3014314 */ (T_MASK ^ 0x5cfebceb) +#define T60 0x4e0811a1 +#define T61 /* 0xf7537e82 */ (T_MASK ^ 0x08ac817d) +#define T62 /* 0xbd3af235 */ (T_MASK ^ 0x42c50dca) +#define T63 0x2ad7d2bb +#define T64 /* 0xeb86d391 */ (T_MASK ^ 0x14792c6e) + + +static void +md5_process(md5_state_t *pms, const md5_byte_t *data /*[64]*/) +{ + md5_word_t + a = pms->abcd[0], b = pms->abcd[1], + c = pms->abcd[2], d = pms->abcd[3]; + md5_word_t t; +#if BYTE_ORDER > 0 + /* Define storage only for big-endian CPUs. */ + md5_word_t X[16]; +#else + /* Define storage for little-endian or both types of CPUs. */ + md5_word_t xbuf[16]; + const md5_word_t *X; +#endif + + { +#if BYTE_ORDER == 0 + /* + * Determine dynamically whether this is a big-endian or + * little-endian machine, since we can use a more efficient + * algorithm on the latter. + */ + static const int w = 1; + + if (*((const md5_byte_t *)&w)) /* dynamic little-endian */ +#endif +#if BYTE_ORDER <= 0 /* little-endian */ + { + /* + * On little-endian machines, we can process properly aligned + * data without copying it. + */ + if (!((data - (const md5_byte_t *)0) & 3)) { + /* data are properly aligned */ + X = (const md5_word_t *)data; + } else { + /* not aligned */ + memcpy(xbuf, data, 64); + X = xbuf; + } + } +#endif +#if BYTE_ORDER == 0 + else /* dynamic big-endian */ +#endif +#if BYTE_ORDER >= 0 /* big-endian */ + { + /* + * On big-endian machines, we must arrange the bytes in the + * right order. + */ + const md5_byte_t *xp = data; + int i; + +# if BYTE_ORDER == 0 + X = xbuf; /* (dynamic only) */ +# else +# define xbuf X /* (static only) */ +# endif + for (i = 0; i < 16; ++i, xp += 4) + xbuf[i] = xp[0] + (xp[1] << 8) + (xp[2] << 16) + (xp[3] << 24); + } +#endif + } + +#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32 - (n)))) + + /* Round 1. */ + /* Let [abcd k s i] denote the operation + a = b + ((a + F(b,c,d) + X[k] + T[i]) <<< s). */ +#define F(x, y, z) (((x) & (y)) | (~(x) & (z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + F(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 0, 7, T1); + SET(d, a, b, c, 1, 12, T2); + SET(c, d, a, b, 2, 17, T3); + SET(b, c, d, a, 3, 22, T4); + SET(a, b, c, d, 4, 7, T5); + SET(d, a, b, c, 5, 12, T6); + SET(c, d, a, b, 6, 17, T7); + SET(b, c, d, a, 7, 22, T8); + SET(a, b, c, d, 8, 7, T9); + SET(d, a, b, c, 9, 12, T10); + SET(c, d, a, b, 10, 17, T11); + SET(b, c, d, a, 11, 22, T12); + SET(a, b, c, d, 12, 7, T13); + SET(d, a, b, c, 13, 12, T14); + SET(c, d, a, b, 14, 17, T15); + SET(b, c, d, a, 15, 22, T16); +#undef SET + + /* Round 2. */ + /* Let [abcd k s i] denote the operation + a = b + ((a + G(b,c,d) + X[k] + T[i]) <<< s). */ +#define G(x, y, z) (((x) & (z)) | ((y) & ~(z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + G(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 1, 5, T17); + SET(d, a, b, c, 6, 9, T18); + SET(c, d, a, b, 11, 14, T19); + SET(b, c, d, a, 0, 20, T20); + SET(a, b, c, d, 5, 5, T21); + SET(d, a, b, c, 10, 9, T22); + SET(c, d, a, b, 15, 14, T23); + SET(b, c, d, a, 4, 20, T24); + SET(a, b, c, d, 9, 5, T25); + SET(d, a, b, c, 14, 9, T26); + SET(c, d, a, b, 3, 14, T27); + SET(b, c, d, a, 8, 20, T28); + SET(a, b, c, d, 13, 5, T29); + SET(d, a, b, c, 2, 9, T30); + SET(c, d, a, b, 7, 14, T31); + SET(b, c, d, a, 12, 20, T32); +#undef SET + + /* Round 3. */ + /* Let [abcd k s t] denote the operation + a = b + ((a + H(b,c,d) + X[k] + T[i]) <<< s). */ +#define H(x, y, z) ((x) ^ (y) ^ (z)) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + H(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 5, 4, T33); + SET(d, a, b, c, 8, 11, T34); + SET(c, d, a, b, 11, 16, T35); + SET(b, c, d, a, 14, 23, T36); + SET(a, b, c, d, 1, 4, T37); + SET(d, a, b, c, 4, 11, T38); + SET(c, d, a, b, 7, 16, T39); + SET(b, c, d, a, 10, 23, T40); + SET(a, b, c, d, 13, 4, T41); + SET(d, a, b, c, 0, 11, T42); + SET(c, d, a, b, 3, 16, T43); + SET(b, c, d, a, 6, 23, T44); + SET(a, b, c, d, 9, 4, T45); + SET(d, a, b, c, 12, 11, T46); + SET(c, d, a, b, 15, 16, T47); + SET(b, c, d, a, 2, 23, T48); +#undef SET + + /* Round 4. */ + /* Let [abcd k s t] denote the operation + a = b + ((a + I(b,c,d) + X[k] + T[i]) <<< s). */ +#define I(x, y, z) ((y) ^ ((x) | ~(z))) +#define SET(a, b, c, d, k, s, Ti)\ + t = a + I(b,c,d) + X[k] + Ti;\ + a = ROTATE_LEFT(t, s) + b + /* Do the following 16 operations. */ + SET(a, b, c, d, 0, 6, T49); + SET(d, a, b, c, 7, 10, T50); + SET(c, d, a, b, 14, 15, T51); + SET(b, c, d, a, 5, 21, T52); + SET(a, b, c, d, 12, 6, T53); + SET(d, a, b, c, 3, 10, T54); + SET(c, d, a, b, 10, 15, T55); + SET(b, c, d, a, 1, 21, T56); + SET(a, b, c, d, 8, 6, T57); + SET(d, a, b, c, 15, 10, T58); + SET(c, d, a, b, 6, 15, T59); + SET(b, c, d, a, 13, 21, T60); + SET(a, b, c, d, 4, 6, T61); + SET(d, a, b, c, 11, 10, T62); + SET(c, d, a, b, 2, 15, T63); + SET(b, c, d, a, 9, 21, T64); +#undef SET + + /* Then perform the following additions. (That is increment each + of the four registers by the value it had before this block + was started.) */ + pms->abcd[0] += a; + pms->abcd[1] += b; + pms->abcd[2] += c; + pms->abcd[3] += d; +} + +void +md5_init(md5_state_t *pms) +{ + pms->count[0] = pms->count[1] = 0; + pms->abcd[0] = 0x67452301; + pms->abcd[1] = /*0xefcdab89*/ T_MASK ^ 0x10325476; + pms->abcd[2] = /*0x98badcfe*/ T_MASK ^ 0x67452301; + pms->abcd[3] = 0x10325476; +} + +void +md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes) +{ + const md5_byte_t *p = data; + int left = nbytes; + int offset = (pms->count[0] >> 3) & 63; + md5_word_t nbits = (md5_word_t)(nbytes << 3); + + if (nbytes <= 0) + return; + + /* Update the message length. */ + pms->count[1] += nbytes >> 29; + pms->count[0] += nbits; + if (pms->count[0] < nbits) + pms->count[1]++; + + /* Process an initial partial block. */ + if (offset) { + int copy = (offset + nbytes > 64 ? 64 - offset : nbytes); + + memcpy(pms->buf + offset, p, copy); + if (offset + copy < 64) + return; + p += copy; + left -= copy; + md5_process(pms, pms->buf); + } + + /* Process full blocks. */ + for (; left >= 64; p += 64, left -= 64) + md5_process(pms, p); + + /* Process a final partial block. */ + if (left) + memcpy(pms->buf, p, left); +} + +void +md5_finish(md5_state_t *pms, md5_byte_t digest[16]) +{ + static const md5_byte_t pad[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + md5_byte_t data[8]; + int i; + + /* Save the length before padding. */ + for (i = 0; i < 8; ++i) + data[i] = (md5_byte_t)(pms->count[i >> 2] >> ((i & 3) << 3)); + /* Pad to 56 bytes mod 64. */ + md5_append(pms, pad, ((55 - (pms->count[0] >> 3)) & 63) + 1); + /* Append the length. */ + md5_append(pms, data, 8); + for (i = 0; i < 16; ++i) + digest[i] = (md5_byte_t)(pms->abcd[i >> 2] >> ((i & 3) << 3)); +} diff --git a/util/md5.h b/util/md5.h new file mode 100644 index 0000000..d001234 --- /dev/null +++ b/util/md5.h @@ -0,0 +1,91 @@ +/* + Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved. + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + L. Peter Deutsch + ghost@aladdin.com + + */ +/* $Id: md5.h,v 1.4 2002/04/13 19:20:28 lpd Exp $ */ +/* + Independent implementation of MD5 (RFC 1321). + + This code implements the MD5 Algorithm defined in RFC 1321, whose + text is available at + http://www.ietf.org/rfc/rfc1321.txt + The code is derived from the text of the RFC, including the test suite + (section A.5) but excluding the rest of Appendix A. It does not include + any code or documentation that is identified in the RFC as being + copyrighted. + + The original and principal author of md5.h is L. Peter Deutsch + <ghost@aladdin.com>. Other authors are noted in the change history + that follows (in reverse chronological order): + + 2002-04-13 lpd Removed support for non-ANSI compilers; removed + references to Ghostscript; clarified derivation from RFC 1321; + now handles byte order either statically or dynamically. + 1999-11-04 lpd Edited comments slightly for automatic TOC extraction. + 1999-10-18 lpd Fixed typo in header comment (ansi2knr rather than md5); + added conditionalization for C++ compilation from Martin + Purschke <purschke@bnl.gov>. + 1999-05-03 lpd Original version. + */ + +#ifndef md5_INCLUDED +# define md5_INCLUDED + +/* + * This package supports both compile-time and run-time determination of CPU + * byte order. If ARCH_IS_BIG_ENDIAN is defined as 0, the code will be + * compiled to run only on little-endian CPUs; if ARCH_IS_BIG_ENDIAN is + * defined as non-zero, the code will be compiled to run only on big-endian + * CPUs; if ARCH_IS_BIG_ENDIAN is not defined, the code will be compiled to + * run on either big- or little-endian CPUs, but will run slightly less + * efficiently on either one than if ARCH_IS_BIG_ENDIAN is defined. + */ + +typedef unsigned char md5_byte_t; /* 8-bit byte */ +typedef unsigned int md5_word_t; /* 32-bit word */ + +/* Define the state of the MD5 Algorithm. */ +typedef struct md5_state_s { + md5_word_t count[2]; /* message length in bits, lsw first */ + md5_word_t abcd[4]; /* digest buffer */ + md5_byte_t buf[64]; /* accumulate block */ +} md5_state_t; + +#ifdef __cplusplus +extern "C" +{ +#endif + + /* Initialize the algorithm. */ + void md5_init(md5_state_t *pms); + + /* Append a string to the message. */ + void md5_append(md5_state_t *pms, const md5_byte_t *data, int nbytes); + + /* Finish the message and return the digest. */ + void md5_finish(md5_state_t *pms, md5_byte_t digest[16]); + +#ifdef __cplusplus +} /* end extern "C" */ +#endif + +#endif /* md5_INCLUDED */ diff --git a/util/md5.hpp b/util/md5.hpp new file mode 100644 index 0000000..d955910 --- /dev/null +++ b/util/md5.hpp @@ -0,0 +1,53 @@ +// md5.hpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "md5.h" + +namespace mongo { + + typedef unsigned char md5digest[16]; + + inline void md5(const void *buf, int nbytes, md5digest digest) { + md5_state_t st; + md5_init(&st); + md5_append(&st, (const md5_byte_t *) buf, nbytes); + md5_finish(&st, digest); + } + + inline void md5(const char *str, md5digest digest) { + md5(str, strlen(str), digest); + } + + inline std::string digestToString( md5digest digest ){ + static const char * letters = "0123456789abcdef"; + stringstream ss; + for ( int i=0; i<16; i++){ + unsigned char c = digest[i]; + ss << letters[ ( c >> 4 ) & 0xf ] << letters[ c & 0xf ]; + } + return ss.str(); + } + + inline std::string md5simpledigest( string s ){ + md5digest d; + md5( s.c_str() , d ); + return digestToString( d ); + } + +} // namespace mongo diff --git a/util/md5main.cpp b/util/md5main.cpp new file mode 100644 index 0000000..d5b4982 --- /dev/null +++ b/util/md5main.cpp @@ -0,0 +1,144 @@ +/* + Copyright (C) 2002 Aladdin Enterprises. All rights reserved. + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + L. Peter Deutsch + ghost@aladdin.com + + */ +/* $Id: md5main.c,v 1.1 2002/04/13 19:20:28 lpd Exp $ */ +/* + Independent implementation of MD5 (RFC 1321). + + This code implements the MD5 Algorithm defined in RFC 1321, whose + text is available at + http://www.ietf.org/rfc/rfc1321.txt + The code is derived from the text of the RFC, including the test suite + (section A.5) but excluding the rest of Appendix A. It does not include + any code or documentation that is identified in the RFC as being + copyrighted. + + The original and principal author of md5.c is L. Peter Deutsch + <ghost@aladdin.com>. Other authors are noted in the change history + that follows (in reverse chronological order): + + 2002-04-13 lpd Splits off main program into a separate file, md5main.c. + */ + +#include "stdafx.h" +#include "md5.h" +#include <math.h> +#include <stdio.h> +#include <string.h> + +/* + * This file builds an executable that performs various functions related + * to the MD5 library. Typical compilation: + * gcc -o md5main -lm md5main.c md5.c + */ +static const char *const usage = "\ +Usage:\n\ + md5main --test # run the self-test (A.5 of RFC 1321)\n\ + md5main --t-values # print the T values for the library\n\ + md5main --version # print the version of the package\n\ +"; +static const char *const version = "2002-04-13"; + +/* modified: not static, renamed */ +/* Run the self-test. */ +/*static*/ int +//do_test(void) +do_md5_test(void) +{ + static const char *const test[7*2] = { + "", "d41d8cd98f00b204e9800998ecf8427e", + "a", "0cc175b9c0f1b6a831c399e269772661", + "abc", "900150983cd24fb0d6963f7d28e17f72", + "message digest", "f96b697d7cb7938d525a2f31aaf161d0", + "abcdefghijklmnopqrstuvwxyz", "c3fcd3d76192e4007dfb496cca67e13b", + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + "d174ab98d277d9f5a5611c2c9f419d9f", + "12345678901234567890123456789012345678901234567890123456789012345678901234567890", "57edf4a22be3c955ac49da2e2107b67a" + }; + int i; + int status = 0; + + for (i = 0; i < 7*2; i += 2) { + md5_state_t state; + md5_byte_t digest[16]; + char hex_output[16*2 + 1]; + int di; + + md5_init(&state); + md5_append(&state, (const md5_byte_t *)test[i], strlen(test[i])); + md5_finish(&state, digest); + for (di = 0; di < 16; ++di) + sprintf(hex_output + di * 2, "%02x", digest[di]); + if (strcmp(hex_output, test[i + 1])) { + printf("MD5 (\"%s\") = ", test[i]); + puts(hex_output); + printf("**** ERROR, should be: %s\n", test[i + 1]); + status = 1; + } + } +// if (status == 0) +/*modified commented out: puts("md5 self-test completed successfully."); */ + return status; +} + +/* Print the T values. */ +static int +do_t_values(void) +{ + int i; + for (i = 1; i <= 64; ++i) { + unsigned long v = (unsigned long)(4294967296.0 * fabs(sin((double)i))); + + /* + * The following nonsense is only to avoid compiler warnings about + * "integer constant is unsigned in ANSI C, signed with -traditional". + */ + if (v >> 31) { + printf("#define T%d /* 0x%08lx */ (T_MASK ^ 0x%08lx)\n", i, + v, (unsigned long)(unsigned int)(~v)); + } else { + printf("#define T%d 0x%08lx\n", i, v); + } + } + return 0; +} + +/* modified from original code changed function name main->md5main */ +/* Main program */ +int +md5main(int argc, char *argv[]) +{ + if (argc == 2) { + if (!strcmp(argv[1], "--test")) + return do_md5_test(); + if (!strcmp(argv[1], "--t-values")) + return do_t_values(); + if (!strcmp(argv[1], "--version")) { + puts(version); + return 0; + } + } + puts(usage); + return 0; +} + diff --git a/util/message.cpp b/util/message.cpp new file mode 100644 index 0000000..0fbc2d2 --- /dev/null +++ b/util/message.cpp @@ -0,0 +1,466 @@ +/* message + + todo: authenticate; encrypt? +*/ + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "message.h" +#include <time.h> +#include "../util/goodies.h" +#include "../util/background.h" +#include <fcntl.h> +#include <errno.h> +#include "../db/cmdline.h" + +namespace mongo { + + bool objcheck = false; + +// if you want trace output: +#define mmm(x) + +#ifdef MSG_NOSIGNAL + const int portSendFlags = MSG_NOSIGNAL; +#else + const int portSendFlags = 0; +#endif + + /* listener ------------------------------------------------------------------- */ + + bool Listener::init() { + SockAddr me; + if ( ip.empty() ) + me = SockAddr( port ); + else + me = SockAddr( ip.c_str(), port ); + sock = ::socket(AF_INET, SOCK_STREAM, 0); + if ( sock == INVALID_SOCKET ) { + log() << "ERROR: listen(): invalid socket? " << OUTPUT_ERRNO << endl; + return false; + } + prebindOptions( sock ); + if ( ::bind(sock, (sockaddr *) &me.sa, me.addressSize) != 0 ) { + log() << "listen(): bind() failed " << OUTPUT_ERRNO << " for port: " << port << endl; + closesocket(sock); + return false; + } + + if ( ::listen(sock, 128) != 0 ) { + log() << "listen(): listen() failed " << OUTPUT_ERRNO << endl; + closesocket(sock); + return false; + } + + return true; + } + + void Listener::listen() { + static long connNumber = 0; + SockAddr from; + while ( 1 ) { + int s = accept(sock, (sockaddr *) &from.sa, &from.addressSize); + if ( s < 0 ) { + if ( errno == ECONNABORTED || errno == EBADF ) { + log() << "Listener on port " << port << " aborted" << endl; + return; + } + log() << "Listener: accept() returns " << s << " " << OUTPUT_ERRNO << endl; + continue; + } + disableNagle(s); + if ( ! cmdLine.quiet ) log() << "connection accepted from " << from.toString() << " #" << ++connNumber << endl; + accepted( new MessagingPort(s, from) ); + } + } + + /* messagingport -------------------------------------------------------------- */ + + class PiggyBackData { + public: + PiggyBackData( MessagingPort * port ) { + _port = port; + _buf = new char[1300]; + _cur = _buf; + } + + ~PiggyBackData() { + flush(); + delete( _cur ); + } + + void append( Message& m ) { + assert( m.data->len <= 1300 ); + + if ( len() + m.data->len > 1300 ) + flush(); + + memcpy( _cur , m.data , m.data->len ); + _cur += m.data->len; + } + + int flush() { + if ( _buf == _cur ) + return 0; + + int x = ::send( _port->sock , _buf , len() , 0 ); + _cur = _buf; + return x; + } + + int len() { + return _cur - _buf; + } + + private: + + MessagingPort* _port; + + char * _buf; + char * _cur; + }; + + class Ports { + set<MessagingPort*>& ports; + boost::mutex& m; + public: + // we "new" this so it is still be around when other automatic global vars + // are being destructed during termination. + Ports() : ports( *(new set<MessagingPort*>()) ), + m( *(new boost::mutex()) ) { } + void closeAll() { \ + boostlock bl(m); + for ( set<MessagingPort*>::iterator i = ports.begin(); i != ports.end(); i++ ) + (*i)->shutdown(); + } + void insert(MessagingPort* p) { + boostlock bl(m); + ports.insert(p); + } + void erase(MessagingPort* p) { + boostlock bl(m); + ports.erase(p); + } + } ports; + + + + void closeAllSockets() { + ports.closeAll(); + } + + MessagingPort::MessagingPort(int _sock, SockAddr& _far) : sock(_sock), piggyBackData(0), farEnd(_far) { + ports.insert(this); + } + + MessagingPort::MessagingPort() { + ports.insert(this); + sock = -1; + piggyBackData = 0; + } + + void MessagingPort::shutdown() { + if ( sock >= 0 ) { + closesocket(sock); + sock = -1; + } + } + + MessagingPort::~MessagingPort() { + if ( piggyBackData ) + delete( piggyBackData ); + shutdown(); + ports.erase(this); + } + + class ConnectBG : public BackgroundJob { + public: + int sock; + int res; + SockAddr farEnd; + void run() { + res = ::connect(sock, (sockaddr *) &farEnd.sa, farEnd.addressSize); + } + }; + + bool MessagingPort::connect(SockAddr& _far) + { + farEnd = _far; + + sock = socket(AF_INET, SOCK_STREAM, 0); + if ( sock == INVALID_SOCKET ) { + log() << "ERROR: connect(): invalid socket? " << OUTPUT_ERRNO << endl; + return false; + } + +#if 0 + long fl = fcntl(sock, F_GETFL, 0); + assert( fl >= 0 ); + fl |= O_NONBLOCK; + fcntl(sock, F_SETFL, fl); + + int res = ::connect(sock, (sockaddr *) &farEnd.sa, farEnd.addressSize); + if ( res ) { + if ( errno == EINPROGRESS ) + closesocket(sock); + sock = -1; + return false; + } + +#endif + + ConnectBG bg; + bg.sock = sock; + bg.farEnd = farEnd; + bg.go(); + + // int res = ::connect(sock, (sockaddr *) &farEnd.sa, farEnd.addressSize); + if ( bg.wait(5000) ) { + if ( bg.res ) { + closesocket(sock); + sock = -1; + return false; + } + } + else { + // time out the connect + closesocket(sock); + sock = -1; + bg.wait(); // so bg stays in scope until bg thread terminates + return false; + } + + disableNagle(sock); + +#ifdef SO_NOSIGPIPE + // osx + const int one = 1; + setsockopt(sock, SOL_SOCKET, SO_NOSIGPIPE, &one, sizeof(int)); +#endif + + return true; + } + + bool MessagingPort::recv(Message& m) { +again: + mmm( out() << "* recv() sock:" << this->sock << endl; ) + int len = -1; + + char *lenbuf = (char *) &len; + int lft = 4; + while ( 1 ) { + int x = ::recv(sock, lenbuf, lft, 0); + if ( x == 0 ) { + DEV out() << "MessagingPort recv() conn closed? " << farEnd.toString() << endl; + m.reset(); + return false; + } + if ( x < 0 ) { + log() << "MessagingPort recv() " << OUTPUT_ERRNO << " " << farEnd.toString()<<endl; + m.reset(); + return false; + } + lft -= x; + if ( lft == 0 ) + break; + lenbuf += x; + log() << "MessagingPort recv() got " << x << " bytes wanted 4, lft=" << lft << endl; + assert( lft > 0 ); + } + + if ( len < 0 || len > 16000000 ) { + if ( len == -1 ) { + // Endian check from the database, after connecting, to see what mode server is running in. + unsigned foo = 0x10203040; + int x = ::send(sock, (char *) &foo, 4, portSendFlags ); + if ( x <= 0 ) { + log() << "MessagingPort endian send() " << OUTPUT_ERRNO << ' ' << farEnd.toString() << endl; + return false; + } + goto again; + } + + if ( len == 542393671 ){ + // an http GET + log() << "looks like you're trying to access db over http on native driver port. please add 1000 for webserver" << endl; + string msg = "You are trying to access MongoDB on the native driver port. For http diagnostic access, add 1000 to the port number\n"; + stringstream ss; + ss << "HTTP/1.0 200 OK\r\nConnection: close\r\nContent-Type: text/plain\r\nContent-Length: " << msg.size() << "\r\n\r\n" << msg; + string s = ss.str(); + ::send( sock , s.c_str(), s.size(), 0 ); + return false; + } + log() << "bad recv() len: " << len << '\n'; + return false; + } + + int z = (len+1023)&0xfffffc00; + assert(z>=len); + MsgData *md = (MsgData *) malloc(z); + md->len = len; + + if ( len <= 0 ) { + out() << "got a length of " << len << ", something is wrong" << endl; + return false; + } + + char *p = (char *) &md->id; + int left = len -4; + while ( 1 ) { + int x = ::recv(sock, p, left, 0); + if ( x == 0 ) { + DEV out() << "MessagingPort::recv(): conn closed? " << farEnd.toString() << endl; + m.reset(); + return false; + } + if ( x < 0 ) { + log() << "MessagingPort recv() " << OUTPUT_ERRNO << ' ' << farEnd.toString() << endl; + m.reset(); + return false; + } + left -= x; + p += x; + if ( left <= 0 ) + break; + } + + m.setData(md, true); + return true; + } + + void MessagingPort::reply(Message& received, Message& response) { + say(/*received.from, */response, received.data->id); + } + + void MessagingPort::reply(Message& received, Message& response, MSGID responseTo) { + say(/*received.from, */response, responseTo); + } + + bool MessagingPort::call(Message& toSend, Message& response) { + mmm( out() << "*call()" << endl; ) + MSGID old = toSend.data->id; + say(/*to,*/ toSend); + while ( 1 ) { + bool ok = recv(response); + if ( !ok ) + return false; + //out() << "got response: " << response.data->responseTo << endl; + if ( response.data->responseTo == toSend.data->id ) + break; + out() << "********************" << endl; + out() << "ERROR: MessagingPort::call() wrong id got:" << (unsigned)response.data->responseTo << " expect:" << (unsigned)toSend.data->id << endl; + out() << " toSend op: " << toSend.data->operation() << " old id:" << (unsigned)old << endl; + out() << " response msgid:" << (unsigned)response.data->id << endl; + out() << " response len: " << (unsigned)response.data->len << endl; + out() << " response op: " << response.data->operation() << endl; + out() << " farEnd: " << farEnd << endl; + assert(false); + response.reset(); + } + mmm( out() << "*call() end" << endl; ) + return true; + } + + void MessagingPort::say(Message& toSend, int responseTo) { + mmm( out() << "* say() sock:" << this->sock << " thr:" << GetCurrentThreadId() << endl; ) + toSend.data->id = nextMessageId(); + toSend.data->responseTo = responseTo; + + int x = -100; + + if ( piggyBackData && piggyBackData->len() ) { + mmm( out() << "* have piggy back" << endl; ) + if ( ( piggyBackData->len() + toSend.data->len ) > 1300 ) { + // won't fit in a packet - so just send it off + piggyBackData->flush(); + } + else { + piggyBackData->append( toSend ); + x = piggyBackData->flush(); + } + } + + if ( x == -100 ) + x = ::send(sock, (char*)toSend.data, toSend.data->len , portSendFlags ); + + if ( x <= 0 ) { + log() << "MessagingPort say send() " << OUTPUT_ERRNO << ' ' << farEnd.toString() << endl; + throw SocketException(); + } + + } + + void MessagingPort::piggyBack( Message& toSend , int responseTo ) { + + if ( toSend.data->len > 1300 ) { + // not worth saving because its almost an entire packet + say( toSend ); + return; + } + + // we're going to be storing this, so need to set it up + toSend.data->id = nextMessageId(); + toSend.data->responseTo = responseTo; + + if ( ! piggyBackData ) + piggyBackData = new PiggyBackData( this ); + + piggyBackData->append( toSend ); + } + + unsigned MessagingPort::remotePort(){ + return farEnd.getPort(); + } + + MSGID NextMsgId; + bool usingClientIds = 0; + ThreadLocalValue<int> clientId; + + struct MsgStart { + MsgStart() { + NextMsgId = (((unsigned) time(0)) << 16) ^ curTimeMillis(); + assert(MsgDataHeaderSize == 16); + } + } msgstart; + + MSGID nextMessageId(){ + MSGID msgid = NextMsgId.atomicIncrement(); + + if ( usingClientIds ){ + msgid = msgid & 0xFFFF; + msgid = msgid | clientId.get(); + } + + return msgid; + } + + bool doesOpGetAResponse( int op ){ + return op == dbQuery || op == dbGetMore; + } + + void setClientId( int id ){ + usingClientIds = true; + id = id & 0xFFFF0000; + massert( 10445 , "invalid id" , id ); + clientId.set( id ); + } + + int getClientId(){ + return clientId.get(); + } + +} // namespace mongo diff --git a/util/message.h b/util/message.h new file mode 100644 index 0000000..8d6a46e --- /dev/null +++ b/util/message.h @@ -0,0 +1,207 @@ +// message.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../util/sock.h" + +namespace mongo { + + class Message; + class MessagingPort; + class PiggyBackData; + typedef WrappingInt MSGID; + + class Listener { + public: + Listener(const string &_ip, int p) : ip(_ip), port(p) { } + virtual ~Listener() {} + bool init(); // set up socket + int socket() const { return sock; } + void listen(); // never returns (start a thread) + + /* spawn a thread, etc., then return */ + virtual void accepted(MessagingPort *mp) = 0; + private: + string ip; + int port; + int sock; + }; + + class AbstractMessagingPort { + public: + virtual ~AbstractMessagingPort() { } + virtual void reply(Message& received, Message& response, MSGID responseTo) = 0; // like the reply below, but doesn't rely on received.data still being available + virtual void reply(Message& received, Message& response) = 0; + + virtual unsigned remotePort() = 0 ; + }; + + class MessagingPort : public AbstractMessagingPort { + public: + MessagingPort(int sock, SockAddr& farEnd); + MessagingPort(); + virtual ~MessagingPort(); + + void shutdown(); + + bool connect(SockAddr& farEnd); + + /* it's assumed if you reuse a message object, that it doesn't cross MessagingPort's. + also, the Message data will go out of scope on the subsequent recv call. + */ + bool recv(Message& m); + void reply(Message& received, Message& response, MSGID responseTo); + void reply(Message& received, Message& response); + bool call(Message& toSend, Message& response); + void say(Message& toSend, int responseTo = -1); + + void piggyBack( Message& toSend , int responseTo = -1 ); + + virtual unsigned remotePort(); + private: + int sock; + PiggyBackData * piggyBackData; + public: + SockAddr farEnd; + + friend class PiggyBackData; + }; + + //#pragma pack() +#pragma pack(1) + + enum Operations { + opReply = 1, /* reply. responseTo is set. */ + dbMsg = 1000, /* generic msg command followed by a string */ + dbUpdate = 2001, /* update object */ + dbInsert = 2002, + //dbGetByOID = 2003, + dbQuery = 2004, + dbGetMore = 2005, + dbDelete = 2006, + dbKillCursors = 2007 + }; + + bool doesOpGetAResponse( int op ); + + struct MsgData { + int len; /* len of the msg, including this field */ + MSGID id; /* request/reply id's match... */ + MSGID responseTo; /* id of the message we are responding to */ + int _operation; + int operation() const { + return _operation; + } + void setOperation(int o) { + _operation = o; + } + char _data[4]; + + int& dataAsInt() { + return *((int *) _data); + } + + bool valid(){ + if ( len <= 0 || len > ( 1024 * 1024 * 10 ) ) + return false; + if ( _operation < 0 || _operation > 100000 ) + return false; + return true; + } + + int dataLen(); // len without header + }; + const int MsgDataHeaderSize = sizeof(MsgData) - 4; + inline int MsgData::dataLen() { + return len - MsgDataHeaderSize; + } + +#pragma pack() + + class Message { + public: + Message() { + data = 0; + freeIt = false; + } + Message( void * _data , bool _freeIt ) { + data = (MsgData*)_data; + freeIt = _freeIt; + }; + ~Message() { + reset(); + } + + SockAddr from; + MsgData *data; + + Message& operator=(Message& r) { + assert( data == 0 ); + data = r.data; + assert( r.freeIt ); + r.freeIt = false; + r.data = 0; + freeIt = true; + return *this; + } + + void reset() { + if ( freeIt && data ) + free(data); + data = 0; + freeIt = false; + } + + void setData(MsgData *d, bool _freeIt) { + assert( data == 0 ); + freeIt = _freeIt; + data = d; + } + void setData(int operation, const char *msgtxt) { + setData(operation, msgtxt, strlen(msgtxt)+1); + } + void setData(int operation, const char *msgdata, int len) { + assert(data == 0); + int dataLen = len + sizeof(MsgData) - 4; + MsgData *d = (MsgData *) malloc(dataLen); + memcpy(d->_data, msgdata, len); + d->len = fixEndian(dataLen); + d->setOperation(operation); + freeIt= true; + data = d; + } + + bool doIFreeIt() { + return freeIt; + } + + private: + bool freeIt; + }; + + class SocketException : public DBException { + public: + virtual const char* what() const throw() { return "socket exception"; } + virtual int getCode(){ return 9001; } + }; + + MSGID nextMessageId(); + + void setClientId( int id ); + int getClientId(); +} // namespace mongo diff --git a/util/message_server.h b/util/message_server.h new file mode 100644 index 0000000..cc40b76 --- /dev/null +++ b/util/message_server.h @@ -0,0 +1,49 @@ +// message_server.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + abstract database server + async io core, worker thread system + */ + +#pragma once + +#include "../stdafx.h" + +namespace mongo { + + class MessageHandler { + public: + virtual ~MessageHandler(){} + virtual void process( Message& m , AbstractMessagingPort* p ) = 0; + }; + + class MessageServer { + public: + MessageServer( int port , MessageHandler * handler ) : _port( port ) , _handler( handler ){} + virtual ~MessageServer(){} + + virtual void run() = 0; + + protected: + + int _port; + MessageHandler* _handler; + }; + + MessageServer * createServer( int port , MessageHandler * handler ); +} diff --git a/util/message_server_asio.cpp b/util/message_server_asio.cpp new file mode 100644 index 0000000..4d5fab0 --- /dev/null +++ b/util/message_server_asio.cpp @@ -0,0 +1,191 @@ +// message_server_asio.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef USE_ASIO + +#include <boost/asio.hpp> +#include <boost/bind.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/shared_ptr.hpp> + +#include <iostream> +#include <vector> + +#include "message.h" +#include "message_server.h" +#include "../util/thread_pool.h" + +using namespace boost; +using namespace boost::asio; +using namespace boost::asio::ip; +//using namespace std; + +namespace mongo { + namespace { + ThreadPool tp; + } + + class MessageServerSession : public boost::enable_shared_from_this<MessageServerSession> , public AbstractMessagingPort { + public: + MessageServerSession( MessageHandler * handler , io_service& ioservice ) : _handler( handler ) , _socket( ioservice ){ + + } + ~MessageServerSession(){ + cout << "disconnect from: " << _socket.remote_endpoint() << endl; + } + + tcp::socket& socket(){ + return _socket; + } + + void start(){ + cout << "MessageServerSession start from:" << _socket.remote_endpoint() << endl; + _startHeaderRead(); + } + + void handleReadHeader( const boost::system::error_code& error ){ + if ( _inHeader.len == 0 ) + return; + + if ( ! _inHeader.valid() ){ + cout << " got invalid header from: " << _socket.remote_endpoint() << " closing connected" << endl; + return; + } + + char * raw = (char*)malloc( _inHeader.len ); + + MsgData * data = (MsgData*)raw; + memcpy( data , &_inHeader , sizeof( _inHeader ) ); + assert( data->len == _inHeader.len ); + + uassert( 10273 , "_cur not empty! pipelining requests not supported" , ! _cur.data ); + + _cur.setData( data , true ); + async_read( _socket , + buffer( raw + sizeof( _inHeader ) , _inHeader.len - sizeof( _inHeader ) ) , + boost::bind( &MessageServerSession::handleReadBody , shared_from_this() , boost::asio::placeholders::error ) ); + } + + void handleReadBody( const boost::system::error_code& error ){ + tp.schedule(&MessageServerSession::process, shared_from_this()); + } + + void process(){ + _handler->process( _cur , this ); + + if (_reply.data){ + async_write( _socket , + buffer( (char*)_reply.data , _reply.data->len ) , + boost::bind( &MessageServerSession::handleWriteDone , shared_from_this() , boost::asio::placeholders::error ) ); + } else { + _cur.reset(); + _startHeaderRead(); + } + } + + void handleWriteDone( const boost::system::error_code& error ){ + _cur.reset(); + _reply.reset(); + _startHeaderRead(); + } + + virtual void reply( Message& received, Message& response ){ + reply( received , response , received.data->id ); + } + + virtual void reply( Message& query , Message& toSend, MSGID responseTo ){ + _reply = toSend; + + _reply.data->id = nextMessageId(); + _reply.data->responseTo = responseTo; + uassert( 10274 , "pipelining requests doesn't work yet" , query.data->id == _cur.data->id ); + } + + + virtual unsigned remotePort(){ + return _socket.remote_endpoint().port(); + } + + private: + + void _startHeaderRead(){ + _inHeader.len = 0; + async_read( _socket , + buffer( &_inHeader , sizeof( _inHeader ) ) , + boost::bind( &MessageServerSession::handleReadHeader , shared_from_this() , boost::asio::placeholders::error ) ); + } + + MessageHandler * _handler; + tcp::socket _socket; + MsgData _inHeader; + Message _cur; + Message _reply; + }; + + + class AsyncMessageServer : public MessageServer { + public: + AsyncMessageServer( int port , MessageHandler * handler ) + : MessageServer( port , handler ) + , _endpoint( tcp::v4() , port ) + , _acceptor( _ioservice , _endpoint ) + { + _accept(); + } + virtual ~AsyncMessageServer(){ + + } + + void run(){ + cout << "AsyncMessageServer starting to listen on: " << _port << endl; + _ioservice.run(); + cout << "AsyncMessageServer done listening on: " << _port << endl; + } + + void handleAccept( shared_ptr<MessageServerSession> session , + const boost::system::error_code& error ){ + if ( error ){ + cout << "handleAccept error!" << endl; + return; + } + session->start(); + _accept(); + } + + void _accept(){ + shared_ptr<MessageServerSession> session( new MessageServerSession( _handler , _ioservice ) ); + _acceptor.async_accept( session->socket() , + boost::bind( &AsyncMessageServer::handleAccept, + this, + session, + boost::asio::placeholders::error ) + ); + } + + private: + io_service _ioservice; + tcp::endpoint _endpoint; + tcp::acceptor _acceptor; + }; + + MessageServer * createServer( int port , MessageHandler * handler ){ + return new AsyncMessageServer( port , handler ); + } + +} + +#endif diff --git a/util/message_server_port.cpp b/util/message_server_port.cpp new file mode 100644 index 0000000..e5becc9 --- /dev/null +++ b/util/message_server_port.cpp @@ -0,0 +1,90 @@ +// message_server_port.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef USE_ASIO + +#include "message.h" +#include "message_server.h" + +namespace mongo { + + namespace pms { + + MessagingPort * grab = 0; + MessageHandler * handler; + + void threadRun(){ + assert( grab ); + MessagingPort * p = grab; + grab = 0; + + Message m; + try { + while ( 1 ){ + m.reset(); + + if ( ! p->recv(m) ) { + log() << "end connection " << p->farEnd.toString() << endl; + p->shutdown(); + break; + } + + handler->process( m , p ); + } + } + catch ( ... ){ + problem() << "uncaught exception in PortMessageServer::threadRun, closing connection" << endl; + delete p; + } + + } + + } + + class PortMessageServer : public MessageServer , public Listener { + public: + PortMessageServer( int port , MessageHandler * handler ) : + MessageServer( port , handler ) , + Listener( "", port ){ + + uassert( 10275 , "multiple PortMessageServer not supported" , ! pms::handler ); + pms::handler = handler; + } + + virtual void accepted(MessagingPort * p) { + assert( ! pms::grab ); + pms::grab = p; + boost::thread thr( pms::threadRun ); + while ( pms::grab ) + sleepmillis(1); + } + + void run(){ + assert( init() ); + listen(); + } + + }; + + + MessageServer * createServer( int port , MessageHandler * handler ){ + return new PortMessageServer( port , handler ); + } + +} + +#endif diff --git a/util/miniwebserver.cpp b/util/miniwebserver.cpp new file mode 100644 index 0000000..b492153 --- /dev/null +++ b/util/miniwebserver.cpp @@ -0,0 +1,224 @@ +// miniwebserver.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "miniwebserver.h" + +#include <pcrecpp.h> + +namespace mongo { + + MiniWebServer::MiniWebServer() { + sock = 0; + } + + bool MiniWebServer::init(const string &ip, int _port) { + port = _port; + SockAddr me; + if ( ip.empty() ) + me = SockAddr( port ); + else + me = SockAddr( ip.c_str(), port ); + sock = ::socket(AF_INET, SOCK_STREAM, 0); + if ( sock == INVALID_SOCKET ) { + log() << "ERROR: MiniWebServer listen(): invalid socket? " << OUTPUT_ERRNO << endl; + return false; + } + prebindOptions( sock ); + if ( ::bind(sock, (sockaddr *) &me.sa, me.addressSize) != 0 ) { + log() << "MiniWebServer: bind() failed port:" << port << " " << OUTPUT_ERRNO << endl; + if ( errno == EADDRINUSE ) + log() << " addr already in use" << endl; + closesocket(sock); + return false; + } + + if ( ::listen(sock, 16) != 0 ) { + log() << "MiniWebServer: listen() failed " << OUTPUT_ERRNO << endl; + closesocket(sock); + return false; + } + + return true; + } + + string MiniWebServer::parseURL( const char * buf ) { + const char * urlStart = strstr( buf , " " ); + if ( ! urlStart ) + return "/"; + + urlStart++; + + const char * end = strstr( urlStart , " " ); + if ( ! end ) { + end = strstr( urlStart , "\r" ); + if ( ! end ) { + end = strstr( urlStart , "\n" ); + } + } + + if ( ! end ) + return "/"; + + int diff = (int)(end-urlStart); + if ( diff < 0 || diff > 255 ) + return "/"; + + return string( urlStart , (int)(end-urlStart) ); + } + + void MiniWebServer::parseParams( map<string,string> & params , string query ) { + if ( query.size() == 0 ) + return; + + while ( query.size() ) { + + string::size_type amp = query.find( "&" ); + + string cur; + if ( amp == string::npos ) { + cur = query; + query = ""; + } + else { + cur = query.substr( 0 , amp ); + query = query.substr( amp + 1 ); + } + + string::size_type eq = cur.find( "=" ); + if ( eq == string::npos ) + continue; + + params[cur.substr(0,eq)] = cur.substr(eq+1); + } + return; + } + + string MiniWebServer::parseMethod( const char * headers ) { + const char * end = strstr( headers , " " ); + if ( ! end ) + return "GET"; + return string( headers , (int)(end-headers) ); + } + + const char *MiniWebServer::body( const char *buf ) { + const char *ret = strstr( buf, "\r\n\r\n" ); + return ret ? ret + 4 : ret; + } + + bool MiniWebServer::fullReceive( const char *buf ) { + const char *bod = body( buf ); + if ( !bod ) + return false; + const char *lenString = "Content-Length:"; + const char *lengthLoc = strstr( buf, lenString ); + if ( !lengthLoc ) + return true; + lengthLoc += strlen( lenString ); + long len = strtol( lengthLoc, 0, 10 ); + if ( long( strlen( bod ) ) == len ) + return true; + return false; + } + + void MiniWebServer::accepted(int s, const SockAddr &from) { + char buf[4096]; + int len = 0; + while ( 1 ) { + int x = ::recv(s, buf + len, sizeof(buf) - 1 - len, 0); + if ( x <= 0 ) { + return; + } + len += x; + buf[ len ] = 0; + if ( fullReceive( buf ) ) + break; + } + buf[len] = 0; + + string responseMsg; + int responseCode = 599; + vector<string> headers; + + try { + doRequest(buf, parseURL( buf ), responseMsg, responseCode, headers, from); + } + catch ( std::exception& e ){ + responseCode = 500; + responseMsg = "error loading page: "; + responseMsg += e.what(); + } + catch ( ... ){ + responseCode = 500; + responseMsg = "unknown error loading page"; + } + + stringstream ss; + ss << "HTTP/1.0 " << responseCode; + if ( responseCode == 200 ) ss << " OK"; + ss << "\r\n"; + if ( headers.empty() ) { + ss << "Content-Type: text/html\r\n"; + } + else { + for ( vector<string>::iterator i = headers.begin(); i != headers.end(); i++ ) + ss << *i << "\r\n"; + } + ss << "\r\n"; + ss << responseMsg; + string response = ss.str(); + + ::send(s, response.c_str(), response.size(), 0); + } + + string MiniWebServer::getHeader( const char * req , string wanted ){ + const char * headers = strstr( req , "\n" ); + if ( ! headers ) + return ""; + pcrecpp::StringPiece input( headers + 1 ); + + string name; + string val; + pcrecpp::RE re("([\\w\\-]+): (.*?)\r?\n"); + while ( re.Consume( &input, &name, &val) ){ + if ( name == wanted ) + return val; + } + return ""; + } + + void MiniWebServer::run() { + SockAddr from; + while ( 1 ) { + int s = accept(sock, (sockaddr *) &from.sa, &from.addressSize); + if ( s < 0 ) { + if ( errno == ECONNABORTED ) { + log() << "Listener on port " << port << " aborted." << endl; + return; + } + log() << "MiniWebServer: accept() returns " << s << " " << OUTPUT_ERRNO << endl; + sleepmillis(200); + continue; + } + disableNagle(s); + RARELY log() << "MiniWebServer: connection accepted from " << from.toString() << endl; + accepted( s, from ); + closesocket(s); + } + } + +} // namespace mongo diff --git a/util/miniwebserver.h b/util/miniwebserver.h new file mode 100644 index 0000000..27476d6 --- /dev/null +++ b/util/miniwebserver.h @@ -0,0 +1,59 @@ +// miniwebserver.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "message.h" + +namespace mongo { + + class MiniWebServer { + public: + MiniWebServer(); + virtual ~MiniWebServer() {} + + bool init(const string &ip, int _port); + void run(); + + virtual void doRequest( + const char *rq, // the full request + string url, + // set these and return them: + string& responseMsg, + int& responseCode, + vector<string>& headers, // if completely empty, content-type: text/html will be added + const SockAddr &from + ) = 0; + + int socket() const { return sock; } + + protected: + string parseURL( const char * buf ); + string parseMethod( const char * headers ); + string getHeader( const char * headers , string name ); + void parseParams( map<string,string> & params , string query ); + static const char *body( const char *buf ); + + private: + void accepted(int s, const SockAddr &from); + static bool fullReceive( const char *buf ); + + int port; + int sock; + }; + +} // namespace mongo diff --git a/util/mmap.cpp b/util/mmap.cpp new file mode 100644 index 0000000..f3103d0 --- /dev/null +++ b/util/mmap.cpp @@ -0,0 +1,95 @@ +// mmap.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "mmap.h" + +namespace mongo { + + set<MemoryMappedFile*> mmfiles; + boost::mutex mmmutex; + + MemoryMappedFile::~MemoryMappedFile() { + close(); + boostlock lk( mmmutex ); + mmfiles.erase(this); + } + + void MemoryMappedFile::created(){ + boostlock lk( mmmutex ); + mmfiles.insert(this); + } + + /*static*/ + int closingAllFiles = 0; + void MemoryMappedFile::closeAllFiles( stringstream &message ) { + if ( closingAllFiles ) { + message << "warning closingAllFiles=" << closingAllFiles << endl; + return; + } + ++closingAllFiles; + ProgressMeter pm( mmfiles.size() , 2 , 1 ); + for ( set<MemoryMappedFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ){ + (*i)->close(); + pm.hit(); + } + message << " closeAllFiles() finished" << endl; + --closingAllFiles; + } + + long long MemoryMappedFile::totalMappedLength(){ + unsigned long long total = 0; + + boostlock lk( mmmutex ); + for ( set<MemoryMappedFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ) + total += (*i)->length(); + + return total; + } + + int MemoryMappedFile::flushAll( bool sync ){ + int num = 0; + + boostlock lk( mmmutex ); + for ( set<MemoryMappedFile*>::iterator i = mmfiles.begin(); i != mmfiles.end(); i++ ){ + num++; + MemoryMappedFile * mmf = *i; + if ( ! mmf ) + continue; + mmf->flush( sync ); + } + return num; + } + + + void MemoryMappedFile::updateLength( const char *filename, long &length ) { + if ( !boost::filesystem::exists( filename ) ) + return; + // make sure we map full length if preexisting file. + boost::uintmax_t l = boost::filesystem::file_size( filename ); + assert( l <= 0x7fffffff ); + length = (long) l; + } + + void* MemoryMappedFile::map(const char *filename) { + boost::uintmax_t l = boost::filesystem::file_size( filename ); + assert( l <= 0x7fffffff ); + long i = (long)l; + return map( filename , i ); + } + +} // namespace mongo diff --git a/util/mmap.h b/util/mmap.h new file mode 100644 index 0000000..ed4ca99 --- /dev/null +++ b/util/mmap.h @@ -0,0 +1,63 @@ +// mmap.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace mongo { + + class MemoryMappedFile { + public: + + MemoryMappedFile(); + ~MemoryMappedFile(); /* closes the file if open */ + void close(); + + // Throws exception if file doesn't exist. + void* map( const char *filename ); + + /* Creates with length if DNE, otherwise uses existing file length, + passed length. + */ + void* map(const char *filename, long &length); + + void flush(bool sync); + + void* viewOfs() { + return view; + } + + long length() { + return len; + } + + static void updateLength( const char *filename, long &length ); + + static long long totalMappedLength(); + static void closeAllFiles( stringstream &message ); + static int flushAll( bool sync ); + + private: + void created(); + + HANDLE fd; + HANDLE maphandle; + void *view; + long len; + }; + + +} // namespace mongo diff --git a/util/mmap_mm.cpp b/util/mmap_mm.cpp new file mode 100644 index 0000000..aa9b275 --- /dev/null +++ b/util/mmap_mm.cpp @@ -0,0 +1,51 @@ +// mmap_mm.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "mmap.h" + +/* in memory (no file) version */ + +namespace mongo { + + MemoryMappedFile::MemoryMappedFile() { + fd = 0; + maphandle = 0; + view = 0; + len = 0; + } + + void MemoryMappedFile::close() { + if ( view ) + delete( view ); + view = 0; + len = 0; + } + + void* MemoryMappedFile::map(const char *filename, size_t length) { + path p( filename ); + + view = malloc( length ); + return view; + } + + void MemoryMappedFile::flush(bool sync) { + } + + +} + diff --git a/util/mmap_posix.cpp b/util/mmap_posix.cpp new file mode 100644 index 0000000..1237220 --- /dev/null +++ b/util/mmap_posix.cpp @@ -0,0 +1,94 @@ +// mmap_posix.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "mmap.h" +#include "file_allocator.h" + +#include <errno.h> +#include <sys/mman.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +namespace mongo { + + MemoryMappedFile::MemoryMappedFile() { + fd = 0; + maphandle = 0; + view = 0; + len = 0; + created(); + } + + void MemoryMappedFile::close() { + if ( view ) + munmap(view, len); + view = 0; + + if ( fd ) + ::close(fd); + fd = 0; + } + +#ifndef O_NOATIME +#define O_NOATIME 0 +#endif + + void* MemoryMappedFile::map(const char *filename, long &length) { + // length may be updated by callee. + theFileAllocator().allocateAsap( filename, length ); + len = length; + + massert( 10446 , (string)"mmap() can't map area of size 0 [" + filename + "]" , length > 0 ); + + + fd = open(filename, O_RDWR | O_NOATIME); + if ( fd <= 0 ) { + out() << "couldn't open " << filename << ' ' << OUTPUT_ERRNO << endl; + return 0; + } + + off_t filelen = lseek(fd, 0, SEEK_END); + if ( filelen != length ){ + cout << "wanted length: " << length << " filelen: " << filelen << endl; + cout << sizeof(size_t) << endl; + massert( 10447 , "file size allocation failed", filelen == length ); + } + lseek( fd, 0, SEEK_SET ); + + view = mmap(NULL, length, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); + if ( view == MAP_FAILED ) { + out() << " mmap() failed for " << filename << " len:" << length << " " << OUTPUT_ERRNO << endl; + if ( errno == ENOMEM ){ + out() << " mmap failed with out of memory, if you're using 32-bits, then you probably need to upgrade to 64" << endl; + } + return 0; + } + return view; + } + + void MemoryMappedFile::flush(bool sync) { + if ( view == 0 || fd == 0 ) + return; + if ( msync(view, len, sync ? MS_SYNC : MS_ASYNC) ) + problem() << "msync " << OUTPUT_ERRNO << endl; + } + + +} // namespace mongo + diff --git a/util/mmap_win.cpp b/util/mmap_win.cpp new file mode 100644 index 0000000..8a0d306 --- /dev/null +++ b/util/mmap_win.cpp @@ -0,0 +1,101 @@ +// mmap_win.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "mmap.h" +#include <windows.h> + +namespace mongo { + + MemoryMappedFile::MemoryMappedFile() { + fd = 0; + maphandle = 0; + view = 0; + len = 0; + created(); + } + + void MemoryMappedFile::close() { + if ( view ) + UnmapViewOfFile(view); + view = 0; + if ( maphandle ) + CloseHandle(maphandle); + maphandle = 0; + if ( fd ) + CloseHandle(fd); + fd = 0; + } + + std::wstring toWideString(const char *s) { + std::basic_ostringstream<TCHAR> buf; + buf << s; + return buf.str(); + } + + unsigned mapped = 0; + + void* MemoryMappedFile::map(const char *_filename, long &length) { + /* big hack here: Babble uses db names with colons. doesn't seem to work on windows. temporary perhaps. */ + char filename[256]; + strncpy(filename, _filename, 255); + filename[255] = 0; + { + size_t len = strlen( filename ); + for ( size_t i=len-1; i>=0; i-- ){ + if ( filename[i] == '/' || + filename[i] == '\\' ) + break; + + if ( filename[i] == ':' ) + filename[i] = '_'; + } + } + + updateLength( filename, length ); + std::wstring filenamew = toWideString(filename); + + fd = CreateFile( + filenamew.c_str(), GENERIC_WRITE | GENERIC_READ, FILE_SHARE_READ, + NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if ( fd == INVALID_HANDLE_VALUE ) { + out() << "Create/OpenFile failed " << filename << ' ' << GetLastError() << endl; + return 0; + } + + mapped += length; + + maphandle = CreateFileMapping(fd, NULL, PAGE_READWRITE, 0, length, NULL); + if ( maphandle == NULL ) { + out() << "CreateFileMapping failed " << filename << ' ' << GetLastError() << endl; + return 0; + } + + view = MapViewOfFile(maphandle, FILE_MAP_ALL_ACCESS, 0, 0, 0); + if ( view == 0 ) { + out() << "MapViewOfFile failed " << filename << " " << OUTPUT_ERRNO << " "; + out() << GetLastError(); + out() << endl; + } + len = length; + return view; + } + + void MemoryMappedFile::flush(bool) { + } + +} diff --git a/util/mvar.h b/util/mvar.h new file mode 100644 index 0000000..7d17051 --- /dev/null +++ b/util/mvar.h @@ -0,0 +1,116 @@ +// mvar.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace mongo { + + /* This is based on haskell's MVar synchronization primitive: + * http://www.haskell.org/ghc/docs/latest/html/libraries/base-4.2.0.0/Control-Concurrent-MVar.html + * + * It is a thread-safe queue that can hold at most one object. + * You can also think of it as a box that can be either full or empty. + */ + + template <typename T> + class MVar { + public: + enum State {EMPTY=0, FULL}; + + // create an empty MVar + MVar() + : _state(EMPTY) + {} + + // creates a full MVar + MVar(const T& val) + : _state(FULL) + , _value(val) + {} + + // puts val into the MVar and returns true or returns false if full + // never blocks + bool tryPut(const T& val){ + // intentionally repeat test before and after lock + if (_state == FULL) return false; + Mutex::scoped_lock lock(_mutex); + if (_state == FULL) return false; + + _state = FULL; + _value = val; + + // unblock threads waiting to 'take' + _condition.notify_all(); + + return true; + } + + // puts val into the MVar + // will block if the MVar is already full + void put(const T& val){ + Mutex::scoped_lock lock(_mutex); + while (!tryPut(val)){ + // unlocks lock while waiting and relocks before returning + _condition.wait(lock); + } + } + + // takes val out of the MVar and returns true or returns false if empty + // never blocks + bool tryTake(T& out){ + // intentionally repeat test before and after lock + if (_state == EMPTY) return false; + Mutex::scoped_lock lock(_mutex); + if (_state == EMPTY) return false; + + _state = EMPTY; + out = _value; + + // unblock threads waiting to 'put' + _condition.notify_all(); + + return true; + } + + // takes val out of the MVar + // will block if the MVar is empty + T take(){ + T ret = T(); + + Mutex::scoped_lock lock(_mutex); + while (!tryTake(ret)){ + // unlocks lock while waiting and relocks before returning + _condition.wait(lock); + } + + return ret; + } + + + // Note: this is fast because there is no locking, but state could + // change before you get a chance to act on it. + // Mainly useful for sanity checks / asserts. + State getState(){ return _state; } + + + private: + State _state; + T _value; + typedef boost::recursive_mutex Mutex; + Mutex _mutex; + boost::condition _condition; + }; + +} diff --git a/util/ntservice.cpp b/util/ntservice.cpp new file mode 100644 index 0000000..602d98a --- /dev/null +++ b/util/ntservice.cpp @@ -0,0 +1,175 @@ +// ntservice.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "ntservice.h" + +#if defined(_WIN32) + +namespace mongo { + + void shutdown(); + + SERVICE_STATUS_HANDLE ServiceController::_statusHandle = null; + std::wstring ServiceController::_serviceName; + ServiceCallback ServiceController::_serviceCallback = null; + + ServiceController::ServiceController() { + } + + bool ServiceController::installService( const std::wstring& serviceName, const std::wstring& displayName, const std::wstring& serviceDesc, int argc, char* argv[] ) { + + std::string commandLine; + + for ( int i = 0; i < argc; i++ ) { + std::string arg( argv[ i ] ); + + // replace install command to indicate process is being started as a service + if ( arg == "--install" ) + arg = "--service"; + + commandLine += arg + " "; + } + + SC_HANDLE schSCManager = ::OpenSCManager( null, null, SC_MANAGER_ALL_ACCESS ); + if ( schSCManager == null ) + return false; + + std::basic_ostringstream< TCHAR > commandLineWide; + commandLineWide << commandLine.c_str(); + + // create new service + SC_HANDLE schService = ::CreateService( schSCManager, serviceName.c_str(), displayName.c_str(), + SERVICE_ALL_ACCESS, SERVICE_WIN32_OWN_PROCESS, + SERVICE_AUTO_START, SERVICE_ERROR_NORMAL, + commandLineWide.str().c_str(), null, null, L"\0\0", null, null ); + + if ( schService == null ) { + ::CloseServiceHandle( schSCManager ); + return false; + } + + SERVICE_DESCRIPTION serviceDescription; + serviceDescription.lpDescription = (LPTSTR)serviceDesc.c_str(); + + // set new service description + bool serviceInstalled = ::ChangeServiceConfig2( schService, SERVICE_CONFIG_DESCRIPTION, &serviceDescription ); + + if ( serviceInstalled ) { + SC_ACTION aActions[ 3 ] = { { SC_ACTION_RESTART, 0 }, { SC_ACTION_RESTART, 0 }, { SC_ACTION_RESTART, 0 } }; + + SERVICE_FAILURE_ACTIONS serviceFailure; + ZeroMemory( &serviceFailure, sizeof( SERVICE_FAILURE_ACTIONS ) ); + serviceFailure.cActions = 3; + serviceFailure.lpsaActions = aActions; + + // set service recovery options + serviceInstalled = ::ChangeServiceConfig2( schService, SERVICE_CONFIG_FAILURE_ACTIONS, &serviceFailure ); + } + + ::CloseServiceHandle( schService ); + ::CloseServiceHandle( schSCManager ); + + return serviceInstalled; + } + + bool ServiceController::removeService( const std::wstring& serviceName ) { + SC_HANDLE schSCManager = ::OpenSCManager( null, null, SC_MANAGER_ALL_ACCESS ); + if ( schSCManager == null ) + return false; + + SC_HANDLE schService = ::OpenService( schSCManager, serviceName.c_str(), SERVICE_ALL_ACCESS ); + + if ( schService == null ) { + ::CloseServiceHandle( schSCManager ); + return false; + } + + SERVICE_STATUS serviceStatus; + + // stop service if running + if ( ::ControlService( schService, SERVICE_CONTROL_STOP, &serviceStatus ) ) { + while ( ::QueryServiceStatus( schService, &serviceStatus ) ) { + if ( serviceStatus.dwCurrentState == SERVICE_STOP_PENDING ) + Sleep( 1000 ); + } + } + + bool serviceRemoved = ::DeleteService( schService ); + + ::CloseServiceHandle( schService ); + ::CloseServiceHandle( schSCManager ); + + return serviceRemoved; + } + + bool ServiceController::startService( const std::wstring& serviceName, ServiceCallback startService ) { + _serviceName = serviceName; + _serviceCallback = startService; + + SERVICE_TABLE_ENTRY dispTable[] = { + { (LPTSTR)serviceName.c_str(), (LPSERVICE_MAIN_FUNCTION)ServiceController::initService }, + { null, null } + }; + + return StartServiceCtrlDispatcher( dispTable ); + } + + bool ServiceController::reportStatus( DWORD reportState, DWORD waitHint ) { + if ( _statusHandle == null ) + return false; + + static DWORD checkPoint = 1; + + SERVICE_STATUS ssStatus; + + ssStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS; + ssStatus.dwServiceSpecificExitCode = 0; + ssStatus.dwControlsAccepted = reportState == SERVICE_START_PENDING ? 0 : SERVICE_ACCEPT_STOP; + ssStatus.dwCurrentState = reportState; + ssStatus.dwWin32ExitCode = NO_ERROR; + ssStatus.dwWaitHint = waitHint; + ssStatus.dwCheckPoint = ( reportState == SERVICE_RUNNING || reportState == SERVICE_STOPPED ) ? 0 : checkPoint++; + + return SetServiceStatus( _statusHandle, &ssStatus ); + } + + void WINAPI ServiceController::initService( DWORD argc, LPTSTR *argv ) { + _statusHandle = RegisterServiceCtrlHandler( _serviceName.c_str(), serviceCtrl ); + if ( !_statusHandle ) + return; + + reportStatus( SERVICE_START_PENDING, 1000 ); + + _serviceCallback(); + + reportStatus( SERVICE_STOPPED ); + } + + void WINAPI ServiceController::serviceCtrl( DWORD ctrlCode ) { + switch ( ctrlCode ) { + case SERVICE_CONTROL_STOP: + case SERVICE_CONTROL_SHUTDOWN: + shutdown(); + reportStatus( SERVICE_STOPPED ); + return; + } + } + +} // namespace mongo + +#endif diff --git a/util/ntservice.h b/util/ntservice.h new file mode 100644 index 0000000..00b8a0a --- /dev/null +++ b/util/ntservice.h @@ -0,0 +1,48 @@ +// ntservice.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if defined(_WIN32) +#include <windows.h> + +namespace mongo { + + typedef bool ( *ServiceCallback )( void ); + + class ServiceController { + public: + ServiceController(); + virtual ~ServiceController() {} + + static bool installService( const std::wstring& serviceName, const std::wstring& displayName, const std::wstring& serviceDesc, int argc, char* argv[] ); + static bool removeService( const std::wstring& serviceName ); + static bool startService( const std::wstring& serviceName, ServiceCallback startService ); + static bool reportStatus( DWORD reportState, DWORD waitHint = 0 ); + + static void WINAPI initService( DWORD argc, LPTSTR *argv ); + static void WINAPI serviceCtrl( DWORD ctrlCode ); + + protected: + static std::wstring _serviceName; + static SERVICE_STATUS_HANDLE _statusHandle; + static ServiceCallback _serviceCallback; + }; + +} // namespace mongo + +#endif diff --git a/util/optime.h b/util/optime.h new file mode 100644 index 0000000..b7d4f61 --- /dev/null +++ b/util/optime.h @@ -0,0 +1,104 @@ +// optime.h - OpTime class + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../db/concurrency.h" + +namespace mongo { + + /* Operation sequence #. A combination of current second plus an ordinal value. + */ +#pragma pack(4) + class OpTime { + unsigned i; + unsigned secs; + static OpTime last; + public: + unsigned getSecs() const { + return secs; + } + OpTime(Date_t date) { + reinterpret_cast<unsigned long long&>(*this) = date.millis; + } + OpTime(unsigned long long date) { + reinterpret_cast<unsigned long long&>(*this) = date; + } + OpTime(unsigned a, unsigned b) { + secs = a; + i = b; + } + OpTime() { + secs = 0; + i = 0; + } + static OpTime now() { + unsigned t = (unsigned) time(0); +// DEV assertInWriteLock(); + if ( last.secs == t ) { + last.i++; + return last; + } + last = OpTime(t, 1); + return last; + } + + /* We store OpTime's in the database as BSON Date datatype -- we needed some sort of + 64 bit "container" for these values. While these are not really "Dates", that seems a + better choice for now than say, Number, which is floating point. Note the BinData type + is perhaps the cleanest choice, lacking a true unsigned64 datatype, but BinData has 5 + bytes of overhead. + */ + unsigned long long asDate() const { + return *((unsigned long long *) &i); + } + // unsigned long long& asDate() { return *((unsigned long long *) &i); } + + bool isNull() { + return secs == 0; + } + + string toStringLong() const { + char buf[64]; + time_t_to_String(secs, buf); + stringstream ss; + ss << buf << ' '; + ss << hex << secs << ':' << i; + return ss.str(); + } + + string toString() const { + stringstream ss; + ss << hex << secs << ':' << i; + return ss.str(); + } + operator string() const { return toString(); } + bool operator==(const OpTime& r) const { + return i == r.i && secs == r.secs; + } + bool operator!=(const OpTime& r) const { + return !(*this == r); + } + bool operator<(const OpTime& r) const { + if ( secs != r.secs ) + return secs < r.secs; + return i < r.i; + } + }; +#pragma pack() + +} // namespace mongo diff --git a/util/processinfo.h b/util/processinfo.h new file mode 100644 index 0000000..83c3bcf --- /dev/null +++ b/util/processinfo.h @@ -0,0 +1,59 @@ +// processinfo.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <sys/types.h> + +#ifndef _WIN32 +#include <unistd.h> +#else +typedef int pid_t; +int getpid(); +#endif + +namespace mongo { + + class BSONObjBuilder; + + class ProcessInfo { + public: + ProcessInfo( pid_t pid = getpid() ); + ~ProcessInfo(); + + /** + * @return mbytes + */ + int getVirtualMemorySize(); + + /** + * @return mbytes + */ + int getResidentSize(); + + /** + * Append platform-specific data to obj + */ + void getExtraInfo(BSONObjBuilder& info); + + bool supported(); + + private: + pid_t _pid; + }; + +} diff --git a/util/processinfo_darwin.cpp b/util/processinfo_darwin.cpp new file mode 100644 index 0000000..904f967 --- /dev/null +++ b/util/processinfo_darwin.cpp @@ -0,0 +1,95 @@ +// processinfo_darwin.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "processinfo.h" + + + +#include <mach/task_info.h> + +#include <mach/mach_init.h> +#include <mach/mach_host.h> +#include <mach/mach_traps.h> +#include <mach/task.h> +#include <mach/vm_map.h> +#include <mach/shared_memory_server.h> +#include <iostream> + +using namespace std; + +namespace mongo { + + ProcessInfo::ProcessInfo( pid_t pid ) : _pid( pid ){ + } + + ProcessInfo::~ProcessInfo(){ + } + + bool ProcessInfo::supported(){ + return true; + } + + int ProcessInfo::getVirtualMemorySize(){ + task_t result; + + mach_port_t task; + + if ( ( result = task_for_pid( mach_task_self() , _pid , &task) ) != KERN_SUCCESS ){ + cout << "error getting task\n"; + return 0; + } + +#if !defined(__LP64__) + task_basic_info_32 ti; +#else + task_basic_info_64 ti; +#endif + mach_msg_type_number_t count = TASK_BASIC_INFO_COUNT; + if ( ( result = task_info( task , TASK_BASIC_INFO , (task_info_t)&ti, &count ) ) != KERN_SUCCESS ){ + cout << "error getting task_info: " << result << endl; + return 0; + } + return (int)((double)ti.virtual_size / (1024.0 * 1024 * 2 ) ); + } + + int ProcessInfo::getResidentSize(){ + task_t result; + + mach_port_t task; + + if ( ( result = task_for_pid( mach_task_self() , _pid , &task) ) != KERN_SUCCESS ){ + cout << "error getting task\n"; + return 0; + } + + +#if !defined(__LP64__) + task_basic_info_32 ti; +#else + task_basic_info_64 ti; +#endif + mach_msg_type_number_t count = TASK_BASIC_INFO_COUNT; + if ( ( result = task_info( task , TASK_BASIC_INFO , (task_info_t)&ti, &count ) ) != KERN_SUCCESS ){ + cout << "error getting task_info: " << result << endl; + return 0; + } + return (int)( ti.resident_size / (1024 * 1024 ) ); + } + + void ProcessInfo::getExtraInfo(BSONObjBuilder& info) {} + +} diff --git a/util/processinfo_linux2.cpp b/util/processinfo_linux2.cpp new file mode 100644 index 0000000..3e00c06 --- /dev/null +++ b/util/processinfo_linux2.cpp @@ -0,0 +1,215 @@ +// processinfo_linux2.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "processinfo.h" + +#include <iostream> +#include <stdio.h> +#include <malloc.h> +#include <db/jsobj.h> + +using namespace std; + +#define KLONG long +#define KLF "l" + +namespace mongo { + + class LinuxProc { + public: + LinuxProc( pid_t pid = getpid() ){ + char name[128]; + sprintf( name , "/proc/%d/stat" , pid ); + + FILE * f = fopen( name , "r"); + + int found = fscanf(f, + "%d %s %c " + "%d %d %d %d %d " + "%lu %lu %lu %lu %lu " + "%lu %lu %ld %ld " /* utime stime cutime cstime */ + "%ld %ld " + "%ld " + "%ld " + "%lu " /* start_time */ + "%lu " + "%ld " // rss + "%lu %"KLF"u %"KLF"u %"KLF"u %"KLF"u %"KLF"u " + /* + "%*s %*s %*s %*s " + "%"KLF"u %*lu %*lu " + "%d %d " + "%lu %lu" + */ + + , + + &_pid, + _comm, + &_state, + &_ppid, &_pgrp, &_session, &_tty, &_tpgid, + &_flags, &_min_flt, &_cmin_flt, &_maj_flt, &_cmaj_flt, + &_utime, &_stime, &_cutime, &_cstime, + &_priority, &_nice, + &_alarm, + &_nlwp, + &_start_time, + &_vsize, + &_rss, + &_rss_rlim, &_start_code, &_end_code, &_start_stack, &_kstk_esp, &_kstk_eip + + /* + &_wchan, + &_exit_signal, &_processor, + &_rtprio, &_sched + */ + ); + if ( found == 0 ){ + cout << "system error: reading proc info" << endl; + } + fclose( f ); + } + + unsigned long getVirtualMemorySize(){ + return _vsize; + } + + unsigned long getResidentSize(){ + return (unsigned long)_rss * 4 * 1024; + } + + int _pid; + // The process ID. + + char _comm[128]; + // The filename of the executable, in parentheses. This is visible whether or not the executable is swapped out. + + char _state; + //One character from the string "RSDZTW" where R is running, S is sleeping in an interruptible wait, D is waiting in uninterruptible + // disk sleep, Z is zombie, T is traced or stopped (on a signal), and W is paging. + + int _ppid; + // The PID of the parent. + + int _pgrp; + // The process group ID of the process. + + int _session; + // The session ID of the process. + + int _tty; + // The tty the process uses. + + int _tpgid; + // The process group ID of the process which currently owns the tty that the process is connected to. + + unsigned long _flags; // %lu + // The kernel flags word of the process. For bit meanings, see the PF_* defines in <linux/sched.h>. Details depend on the kernel version. + + unsigned long _min_flt; // %lu + // The number of minor faults the process has made which have not required loading a memory page from disk. + + unsigned long _cmin_flt; // %lu + // The number of minor faults that the process + + unsigned long _maj_flt; // %lu + // The number of major faults the process has made which have required loading a memory page from disk. + + unsigned long _cmaj_flt; // %lu + // The number of major faults that the process + + unsigned long _utime; // %lu + // The number of jiffies that this process has been scheduled in user mode. + + unsigned long _stime; // %lu + // The number of jiffies that this process has been scheduled in kernel mode. + + long _cutime; // %ld + // The number of jiffies that this removed field. + + long _cstime; // %ld + + long _priority; + long _nice; + + long _nlwp; // %ld + // The time in jiffies before the next SIGALRM is sent to the process due to an interval timer. + + unsigned long _alarm; + + unsigned long _start_time; // %lu + // The time in jiffies the process started after system boot. + + unsigned long _vsize; // %lu + // Virtual memory size in bytes. + + long _rss; // %ld + // Resident Set Size: number of pages the process has in real memory, minus 3 for administrative purposes. This is just the pages which + // count towards text, data, or stack space. This does not include pages which have not been demand-loaded in, or which are swapped out + + unsigned long _rss_rlim; // %lu + // Current limit in bytes on the rss of the process (usually 4294967295 on i386). + + unsigned long _start_code; // %lu + // The address above which program text can run. + + unsigned long _end_code; // %lu + // The address below which program text can run. + + unsigned long _start_stack; // %lu + // The address of the start of the stack. + + unsigned long _kstk_esp; // %lu + // The current value of esp (stack pointer), as found in the kernel stack page for the process. + + unsigned long _kstk_eip; // %lu + // The current EIP (instruction pointer). + + + + }; + + + ProcessInfo::ProcessInfo( pid_t pid ) : _pid( pid ){ + } + + ProcessInfo::~ProcessInfo(){ + } + + bool ProcessInfo::supported(){ + return true; + } + + int ProcessInfo::getVirtualMemorySize(){ + LinuxProc p(_pid); + return (int)( p.getVirtualMemorySize() / ( 1024.0 * 1024 ) ); + } + + int ProcessInfo::getResidentSize(){ + LinuxProc p(_pid); + return (int)( p.getResidentSize() / ( 1024.0 * 1024 ) ); + } + + void ProcessInfo::getExtraInfo(BSONObjBuilder& info){ + struct mallinfo malloc_info = mallinfo(); // structure has same name as function that returns it. (see malloc.h) + info.append("heap_usage_bytes", malloc_info.uordblks); + + LinuxProc p(_pid); + info.append("page_faults", (int)p._maj_flt); + } + +} diff --git a/util/processinfo_none.cpp b/util/processinfo_none.cpp new file mode 100644 index 0000000..57f4ca3 --- /dev/null +++ b/util/processinfo_none.cpp @@ -0,0 +1,46 @@ +// processinfo_none.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "processinfo.h" + +#include <iostream> +using namespace std; + +namespace mongo { + + ProcessInfo::ProcessInfo( pid_t pid ){ + } + + ProcessInfo::~ProcessInfo(){ + } + + bool ProcessInfo::supported(){ + return false; + } + + int ProcessInfo::getVirtualMemorySize(){ + return -1; + } + + int ProcessInfo::getResidentSize(){ + return -1; + } + + void ProcessInfo::getExtraInfo(BSONObjBuilder& info) {} + +} diff --git a/util/processinfo_win32.cpp b/util/processinfo_win32.cpp new file mode 100644 index 0000000..0f0bf2e --- /dev/null +++ b/util/processinfo_win32.cpp @@ -0,0 +1,64 @@ +// processinfo_win32.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "processinfo.h" + +#include <iostream> + +#include <windows.h> +#include <psapi.h> + +using namespace std; + +int getpid(){ + return GetCurrentProcessId(); +} + +namespace mongo { + + int _wconvertmtos( SIZE_T s ){ + return (int)( s / ( 1024 * 1024 ) ); + } + + ProcessInfo::ProcessInfo( pid_t pid ){ + } + + ProcessInfo::~ProcessInfo(){ + } + + bool ProcessInfo::supported(){ + return true; + } + + int ProcessInfo::getVirtualMemorySize(){ + MEMORYSTATUSEX mse; + mse.dwLength = sizeof(mse); + assert( GlobalMemoryStatusEx( &mse ) ); + DWORDLONG x = (mse.ullTotalVirtual - mse.ullAvailVirtual) / (1024 * 1024) ; + assert( x <= 0x7fffffff ); + return (int) x; + } + + int ProcessInfo::getResidentSize(){ + PROCESS_MEMORY_COUNTERS pmc; + assert( GetProcessMemoryInfo( GetCurrentProcess() , &pmc, sizeof(pmc) ) ); + return _wconvertmtos( pmc.WorkingSetSize ); + } + + void ProcessInfo::getExtraInfo(BSONObjBuilder& info) {} +} diff --git a/util/queue.h b/util/queue.h new file mode 100644 index 0000000..8f4fbaf --- /dev/null +++ b/util/queue.h @@ -0,0 +1,72 @@ +// queue.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../stdafx.h" +#include "../util/goodies.h" + +#include <queue> + +namespace mongo { + + /** + * simple blocking queue + */ + template<typename T> class BlockingQueue : boost::noncopyable { + public: + void push(T const& t){ + boostlock l( _lock ); + _queue.push( t ); + _condition.notify_one(); + } + + bool empty() const { + boostlock l( _lock ); + return _queue.empty(); + } + + bool tryPop( T & t ){ + boostlock l( _lock ); + if ( _queue.empty() ) + return false; + + t = _queue.front(); + _queue.pop(); + + return true; + } + + T blockingPop(){ + + boostlock l( _lock ); + while( _queue.empty() ) + _condition.wait( l ); + + T t = _queue.front(); + _queue.pop(); + return t; + } + + private: + std::queue<T> _queue; + + mutable boost::mutex _lock; + boost::condition _condition; + }; + +} diff --git a/util/sock.cpp b/util/sock.cpp new file mode 100644 index 0000000..5172692 --- /dev/null +++ b/util/sock.cpp @@ -0,0 +1,209 @@ +// sock.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "sock.h" + +namespace mongo { + + static boost::mutex sock_mutex; + + string hostbyname(const char *hostname) { + static string unknown = "0.0.0.0"; + if ( unknown == hostname ) + return unknown; + + boostlock lk(sock_mutex); +#if defined(_WIN32) + if( inet_addr(hostname) != INADDR_NONE ) + return hostname; +#else + struct in_addr temp; + if ( inet_aton( hostname, &temp ) ) + return hostname; +#endif + struct hostent *h; + h = gethostbyname(hostname); + if ( h == 0 ) return ""; + return inet_ntoa( *((struct in_addr *)(h->h_addr)) ); + } + + void sendtest() { + out() << "sendtest\n"; + SockAddr me(27016); + SockAddr dest("127.0.0.1", 27015); + UDPConnection c; + if ( c.init(me) ) { + char buf[256]; + out() << "sendto: "; + out() << c.sendto(buf, sizeof(buf), dest) << " " << OUTPUT_ERRNO << endl; + } + out() << "end\n"; + } + + void listentest() { + out() << "listentest\n"; + SockAddr me(27015); + SockAddr sender; + UDPConnection c; + if ( c.init(me) ) { + char buf[256]; + out() << "recvfrom: "; + out() << c.recvfrom(buf, sizeof(buf), sender) << " " << OUTPUT_ERRNO << endl; + } + out() << "end listentest\n"; + } + + void xmain(); + struct SockStartupTests { + SockStartupTests() { +#if defined(_WIN32) + WSADATA d; + if ( WSAStartup(MAKEWORD(2,2), &d) != 0 ) { + out() << "ERROR: wsastartup failed " << OUTPUT_ERRNO << endl; + problem() << "ERROR: wsastartup failed " << OUTPUT_ERRNO << endl; + dbexit( EXIT_NTSERVICE_ERROR ); + } +#endif + //out() << "ntohl:" << ntohl(256) << endl; + //sendtest(); + //listentest(); + } + } sstests; + +#if 0 + void smain() { + + WSADATA wsaData; + SOCKET RecvSocket; + sockaddr_in RecvAddr; + int Port = 27015; + char RecvBuf[1024]; + int BufLen = 1024; + sockaddr_in SenderAddr; + int SenderAddrSize = sizeof(SenderAddr); + + //----------------------------------------------- + // Initialize Winsock + WSAStartup(MAKEWORD(2,2), &wsaData); + + //----------------------------------------------- + // Create a receiver socket to receive datagrams + RecvSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + prebindOptions( RecvSocket ); + + //----------------------------------------------- + // Bind the socket to any address and the specified port. + RecvAddr.sin_family = AF_INET; + RecvAddr.sin_port = htons(Port); + RecvAddr.sin_addr.s_addr = htonl(INADDR_ANY); + + ::bind(RecvSocket, (SOCKADDR *) &RecvAddr, sizeof(RecvAddr)); + + //----------------------------------------------- + // Call the recvfrom function to receive datagrams + // on the bound socket. + printf("Receiving datagrams...\n"); + recvfrom(RecvSocket, + RecvBuf, + BufLen, + 0, + (SOCKADDR *)&SenderAddr, + &SenderAddrSize); + + //----------------------------------------------- + // Close the socket when finished receiving datagrams + printf("Finished receiving. Closing socket.\n"); + closesocket(RecvSocket); + + //----------------------------------------------- + // Clean up and exit. + printf("Exiting.\n"); + WSACleanup(); + return; + } + + + + + void xmain() { + + WSADATA wsaData; + SOCKET RecvSocket; + sockaddr_in RecvAddr; + int Port = 27015; + char RecvBuf[1024]; + int BufLen = 1024; + sockaddr_in SenderAddr; + int SenderAddrSize = sizeof(SenderAddr); + + //----------------------------------------------- + // Initialize Winsock + WSAStartup(MAKEWORD(2,2), &wsaData); + + //----------------------------------------------- + // Create a receiver socket to receive datagrams + + RecvSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + prebindOptions( RecvSocket ); + + //----------------------------------------------- + // Bind the socket to any address and the specified port. + RecvAddr.sin_family = AF_INET; + RecvAddr.sin_port = htons(Port); + RecvAddr.sin_addr.s_addr = htonl(INADDR_ANY); + + SockAddr a(Port); + ::bind(RecvSocket, (SOCKADDR *) &a.sa, a.addressSize); +// bind(RecvSocket, (SOCKADDR *) &RecvAddr, sizeof(RecvAddr)); + + SockAddr b; + + //----------------------------------------------- + // Call the recvfrom function to receive datagrams + // on the bound socket. + printf("Receiving datagrams...\n"); + recvfrom(RecvSocket, + RecvBuf, + BufLen, + 0, + (SOCKADDR *) &b.sa, &b.addressSize); +// (SOCKADDR *)&SenderAddr, +// &SenderAddrSize); + + //----------------------------------------------- + // Close the socket when finished receiving datagrams + printf("Finished receiving. Closing socket.\n"); + closesocket(RecvSocket); + + //----------------------------------------------- + // Clean up and exit. + printf("Exiting.\n"); + WSACleanup(); + return; + } + +#endif + + ListeningSockets* ListeningSockets::_instance = new ListeningSockets(); + + ListeningSockets* ListeningSockets::get(){ + return _instance; + } + + +} // namespace mongo diff --git a/util/sock.h b/util/sock.h new file mode 100644 index 0000000..5798a71 --- /dev/null +++ b/util/sock.h @@ -0,0 +1,280 @@ +// sock.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../stdafx.h" + +#include <stdio.h> +#include <sstream> +#include "goodies.h" + +#ifdef _WIN32 +#include <windows.h> +#include <winsock.h> +#endif + +namespace mongo { + +#if defined(_WIN32) + + typedef int socklen_t; + inline int getLastError() { + return WSAGetLastError(); + } + inline void disableNagle(int sock) { + int x = 1; + if ( setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, (char *) &x, sizeof(x)) ) + out() << "ERROR: disableNagle failed" << endl; + } + inline void prebindOptions( int sock ) { + } +#else + +} // namespace mongo + +#include <sys/socket.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <netinet/tcp.h> +#include <arpa/inet.h> +#include <errno.h> +#include <netdb.h> + +namespace mongo { + + inline void closesocket(int s) { + close(s); + } + const int INVALID_SOCKET = -1; + typedef int SOCKET; + + inline void disableNagle(int sock) { + int x = 1; + +#ifdef SOL_TCP + int level = SOL_TCP; +#else + int level = SOL_SOCKET; +#endif + + if ( setsockopt(sock, level, TCP_NODELAY, (char *) &x, sizeof(x)) ) + log() << "ERROR: disableNagle failed" << endl; + + } + inline void prebindOptions( int sock ) { + DEV log() << "doing prebind option" << endl; + int x = 1; + if ( setsockopt( sock , SOL_SOCKET, SO_REUSEADDR, &x, sizeof(x)) < 0 ) + out() << "Failed to set socket opt, SO_REUSEADDR" << endl; + } + + +#endif + + inline void setSockReceiveTimeout(int sock, int secs) { +// todo - finish - works? + struct timeval tv; + tv.tv_sec = 0;//secs; + tv.tv_usec = 1000; + int rc = setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *) &tv, sizeof(tv)); + if ( rc ) { + out() << "ERROR: setsockopt RCVTIMEO failed rc:" << rc << " " << OUTPUT_ERRNO << " secs:" << secs << " sock:" << sock << endl; + } + } + + // If an ip address is passed in, just return that. If a hostname is passed + // in, look up its ip and return that. Returns "" on failure. + string hostbyname(const char *hostname); + + struct SockAddr { + SockAddr() { + addressSize = sizeof(sockaddr_in); + memset(&sa, 0, sizeof(sa)); + } + SockAddr(int sourcePort); /* listener side */ + SockAddr(const char *ip, int port); /* EndPoint (remote) side, or if you want to specify which interface locally */ + + struct sockaddr_in sa; + socklen_t addressSize; + + bool isLocalHost() const { +#if defined(_WIN32) + return sa.sin_addr.S_un.S_addr == 0x100007f; +#else + return sa.sin_addr.s_addr == 0x100007f; +#endif + } + + string toString() const{ + stringstream out; + out << inet_ntoa(sa.sin_addr) << ':' + << ntohs(sa.sin_port); + return out.str(); + } + + operator string() const{ + return toString(); + } + + unsigned getPort() { + return sa.sin_port; + } + + bool localhost() const { return inet_addr( "127.0.0.1" ) == sa.sin_addr.s_addr; } + + bool operator==(const SockAddr& r) const { + return sa.sin_addr.s_addr == r.sa.sin_addr.s_addr && + sa.sin_port == r.sa.sin_port; + } + bool operator!=(const SockAddr& r) const { + return !(*this == r); + } + bool operator<(const SockAddr& r) const { + if ( sa.sin_port >= r.sa.sin_port ) + return false; + return sa.sin_addr.s_addr < r.sa.sin_addr.s_addr; + } + }; + + const int MaxMTU = 16384; + + class UDPConnection { + public: + UDPConnection() { + sock = 0; + } + ~UDPConnection() { + if ( sock ) { + closesocket(sock); + sock = 0; + } + } + bool init(const SockAddr& myAddr); + int recvfrom(char *buf, int len, SockAddr& sender); + int sendto(char *buf, int len, const SockAddr& EndPoint); + int mtu(const SockAddr& sa) { + return sa.isLocalHost() ? 16384 : 1480; + } + + SOCKET sock; + }; + + inline int UDPConnection::recvfrom(char *buf, int len, SockAddr& sender) { + return ::recvfrom(sock, buf, len, 0, (sockaddr *) &sender.sa, &sender.addressSize); + } + + inline int UDPConnection::sendto(char *buf, int len, const SockAddr& EndPoint) { + if ( 0 && rand() < (RAND_MAX>>4) ) { + out() << " NOTSENT "; + // out() << curTimeMillis() << " .TEST: NOT SENDING PACKET" << endl; + return 0; + } + return ::sendto(sock, buf, len, 0, (sockaddr *) &EndPoint.sa, EndPoint.addressSize); + } + + inline bool UDPConnection::init(const SockAddr& myAddr) { + sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + if ( sock == INVALID_SOCKET ) { + out() << "invalid socket? " << OUTPUT_ERRNO << endl; + return false; + } + //out() << sizeof(sockaddr_in) << ' ' << myAddr.addressSize << endl; + if ( ::bind(sock, (sockaddr *) &myAddr.sa, myAddr.addressSize) != 0 ) { + out() << "udp init failed" << endl; + closesocket(sock); + sock = 0; + return false; + } + socklen_t optLen; + int rcvbuf; + if (getsockopt(sock, + SOL_SOCKET, + SO_RCVBUF, + (char*)&rcvbuf, + &optLen) != -1) + out() << "SO_RCVBUF:" << rcvbuf << endl; + return true; + } + + inline SockAddr::SockAddr(int sourcePort) { + memset(sa.sin_zero, 0, sizeof(sa.sin_zero)); + sa.sin_family = AF_INET; + sa.sin_port = htons(sourcePort); + sa.sin_addr.s_addr = htonl(INADDR_ANY); + addressSize = sizeof(sa); + } + + inline SockAddr::SockAddr(const char * iporhost , int port) { + string ip = hostbyname( iporhost ); + memset(sa.sin_zero, 0, sizeof(sa.sin_zero)); + sa.sin_family = AF_INET; + sa.sin_port = htons(port); + sa.sin_addr.s_addr = inet_addr(ip.c_str()); + addressSize = sizeof(sa); + } + + inline string getHostName() { + char buf[256]; + int ec = gethostname(buf, 127); + if ( ec || *buf == 0 ) { + log() << "can't get this server's hostname " << OUTPUT_ERRNO << endl; + return ""; + } + return buf; + } + + class ListeningSockets { + public: + ListeningSockets() : _sockets( new set<int>() ){ + } + + void add( int sock ){ + boostlock lk( _mutex ); + _sockets->insert( sock ); + } + void remove( int sock ){ + boostlock lk( _mutex ); + _sockets->erase( sock ); + } + + void closeAll(){ + set<int>* s; + { + boostlock lk( _mutex ); + s = _sockets; + _sockets = new set<int>(); + } + + for ( set<int>::iterator i=s->begin(); i!=s->end(); i++ ){ + int sock = *i; + log() << "going to close listening socket: " << sock << endl; + closesocket( sock ); + } + + } + + static ListeningSockets* get(); + + private: + boost::mutex _mutex; + set<int>* _sockets; + static ListeningSockets* _instance; + }; + +} // namespace mongo diff --git a/util/thread_pool.cpp b/util/thread_pool.cpp new file mode 100644 index 0000000..b95bc1d --- /dev/null +++ b/util/thread_pool.cpp @@ -0,0 +1,139 @@ +/* threadpool.cpp +*/ + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "thread_pool.h" +#include "mvar.h" + + +namespace mongo{ +namespace threadpool{ + +// Worker thread +class Worker : boost::noncopyable { +public: + explicit Worker(ThreadPool& owner) + : _owner(owner) + , _is_done(true) + , _thread(boost::bind(&Worker::loop, this)) + {} + + // destructor will block until current operation is completed + // Acts as a "join" on this thread + ~Worker(){ + _task.put(Task()); + _thread.join(); + } + + void set_task(Task& func){ + assert(!func.empty()); + assert(_is_done); + _is_done = false; + + _task.put(func); + } + + private: + ThreadPool& _owner; + MVar<Task> _task; + bool _is_done; // only used for error detection + boost::thread _thread; + + void loop(){ + while (true) { + Task task = _task.take(); + if (task.empty()) + break; // ends the thread + + try { + task(); + } catch (std::exception e){ + log() << "Unhandled exception in worker thread: " << e.what() << endl;; + } catch (...){ + log() << "Unhandled non-exception in worker thread" << endl; + } + _is_done = true; + _owner.task_done(this); + } + } +}; + +ThreadPool::ThreadPool(int nThreads) + : _tasksRemaining(0) + , _nThreads(nThreads) +{ + boostlock lock(_mutex); + while (nThreads-- > 0){ + Worker* worker = new Worker(*this); + _freeWorkers.push_front(worker); + } +} + +ThreadPool::~ThreadPool(){ + join(); + + assert(_tasks.empty()); + + // O(n) but n should be small + assert(_freeWorkers.size() == (unsigned)_nThreads); + + while(!_freeWorkers.empty()){ + delete _freeWorkers.front(); + _freeWorkers.pop_front(); + } +} + +void ThreadPool::join(){ + boostlock lock(_mutex); + while(_tasksRemaining){ + _condition.wait(lock); + } +} + +void ThreadPool::schedule(Task task){ + boostlock lock(_mutex); + + _tasksRemaining++; + + if (!_freeWorkers.empty()){ + _freeWorkers.front()->set_task(task); + _freeWorkers.pop_front(); + }else{ + _tasks.push_back(task); + } +} + +// should only be called by a worker from the worker thread +void ThreadPool::task_done(Worker* worker){ + boostlock lock(_mutex); + + if (!_tasks.empty()){ + worker->set_task(_tasks.front()); + _tasks.pop_front(); + }else{ + _freeWorkers.push_front(worker); + } + + _tasksRemaining--; + + if(_tasksRemaining == 0) + _condition.notify_all(); +} + +} //namespace threadpool +} //namespace mongo diff --git a/util/thread_pool.h b/util/thread_pool.h new file mode 100644 index 0000000..91c2969 --- /dev/null +++ b/util/thread_pool.h @@ -0,0 +1,82 @@ +// thread_pool.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <boost/function.hpp> +#include <boost/bind.hpp> +#undef assert +#define assert xassert + +namespace mongo { + +namespace threadpool { + class Worker; + + typedef boost::function<void(void)> Task; //nullary function or functor + + // exported to the mongo namespace + class ThreadPool : boost::noncopyable{ + public: + explicit ThreadPool(int nThreads=8); + + // blocks until all tasks are complete (tasks_remaining() == 0) + // You should not call schedule while in the destructor + ~ThreadPool(); + + // blocks until all tasks are complete (tasks_remaining() == 0) + // does not prevent new tasks from being scheduled so could wait forever. + // Also, new tasks could be scheduled after this returns. + void join(); + + + // task will be copied a few times so make sure it's relatively cheap + void schedule(Task task); + + // Helpers that wrap schedule and boost::bind. + // Functor and args will be copied a few times so make sure it's relatively cheap + template<typename F, typename A> + void schedule(F f, A a){ schedule(boost::bind(f,a)); } + template<typename F, typename A, typename B> + void schedule(F f, A a, B b){ schedule(boost::bind(f,a,b)); } + template<typename F, typename A, typename B, typename C> + void schedule(F f, A a, B b, C c){ schedule(boost::bind(f,a,b,c)); } + template<typename F, typename A, typename B, typename C, typename D> + void schedule(F f, A a, B b, C c, D d){ schedule(boost::bind(f,a,b,c,d)); } + template<typename F, typename A, typename B, typename C, typename D, typename E> + void schedule(F f, A a, B b, C c, D d, E e){ schedule(boost::bind(f,a,b,c,d,e)); } + + + int tasks_remaining() { return _tasksRemaining; } + + private: + boost::mutex _mutex; + boost::condition _condition; + + list<Worker*> _freeWorkers; //used as LIFO stack (always front) + list<Task> _tasks; //used as FIFO queue (push_back, pop_front) + int _tasksRemaining; // in queue + currently processing + int _nThreads; // only used for sanity checking. could be removed in the future. + + // should only be called by a worker from the worker's thread + void task_done(Worker* worker); + friend class Worker; + }; + +} //namespace threadpool + +using threadpool::ThreadPool; + +} //namespace mongo diff --git a/util/top.cpp b/util/top.cpp new file mode 100644 index 0000000..98d9598 --- /dev/null +++ b/util/top.cpp @@ -0,0 +1,18 @@ +// top.cpp + +#include "stdafx.h" +#include "top.h" + +namespace mongo { + + Top::T Top::_snapshotStart = Top::currentTime(); + Top::D Top::_snapshotDuration; + Top::UsageMap Top::_totalUsage; + Top::UsageMap Top::_snapshotA; + Top::UsageMap Top::_snapshotB; + Top::UsageMap &Top::_snapshot = Top::_snapshotA; + Top::UsageMap &Top::_nextSnapshot = Top::_snapshotB; + boost::mutex Top::topMutex; + + +} diff --git a/util/top.h b/util/top.h new file mode 100644 index 0000000..aaf7c3f --- /dev/null +++ b/util/top.h @@ -0,0 +1,183 @@ +// top.h : DB usage monitor. + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <boost/date_time/posix_time/posix_time.hpp> +#undef assert +#define assert xassert + +namespace mongo { + + /* Records per namespace utilization of the mongod process. + No two functions of this class may be called concurrently. + */ + class Top { + typedef boost::posix_time::ptime T; + typedef boost::posix_time::time_duration D; + typedef boost::tuple< D, int, int, int > UsageData; + public: + Top() : _read(false), _write(false) { } + + /* these are used to record activity: */ + + void clientStart( const char *client ) { + clientStop(); + _currentStart = currentTime(); + _current = client; + } + + /* indicate current request is a read operation. */ + void setRead() { _read = true; } + + void setWrite() { _write = true; } + + void clientStop() { + if ( _currentStart == T() ) + return; + D d = currentTime() - _currentStart; + + { + boostlock L(topMutex); + recordUsage( _current, d ); + } + + _currentStart = T(); + _read = false; + _write = false; + } + + /* these are used to fetch the stats: */ + + struct Usage { + string ns; + D time; + double pct; + int reads, writes, calls; + }; + + static void usage( vector< Usage > &res ) { + boostlock L(topMutex); + + // Populate parent namespaces + UsageMap snapshot; + UsageMap totalUsage; + fillParentNamespaces( snapshot, _snapshot ); + fillParentNamespaces( totalUsage, _totalUsage ); + + multimap< D, string, more > sorted; + for( UsageMap::iterator i = snapshot.begin(); i != snapshot.end(); ++i ) + sorted.insert( make_pair( i->second.get<0>(), i->first ) ); + for( multimap< D, string, more >::iterator i = sorted.begin(); i != sorted.end(); ++i ) { + if ( trivialNs( i->second.c_str() ) ) + continue; + Usage u; + u.ns = i->second; + u.time = totalUsage[ u.ns ].get<0>(); + u.pct = _snapshotDuration != D() ? 100.0 * i->first.ticks() / _snapshotDuration.ticks() : 0; + u.reads = snapshot[ u.ns ].get<1>(); + u.writes = snapshot[ u.ns ].get<2>(); + u.calls = snapshot[ u.ns ].get<3>(); + res.push_back( u ); + } + for( UsageMap::iterator i = totalUsage.begin(); i != totalUsage.end(); ++i ) { + if ( snapshot.count( i->first ) != 0 || trivialNs( i->first.c_str() ) ) + continue; + Usage u; + u.ns = i->first; + u.time = i->second.get<0>(); + u.pct = 0; + u.reads = 0; + u.writes = 0; + u.calls = 0; + res.push_back( u ); + } + } + + static void completeSnapshot() { + boostlock L(topMutex); + + if ( &_snapshot == &_snapshotA ) { + _snapshot = _snapshotB; + _nextSnapshot = _snapshotA; + } else { + _snapshot = _snapshotA; + _nextSnapshot = _snapshotB; + } + _snapshotDuration = currentTime() - _snapshotStart; + _snapshotStart = currentTime(); + _nextSnapshot.clear(); + } + + private: + static boost::mutex topMutex; + static bool trivialNs( const char *ns ) { + const char *ret = strrchr( ns, '.' ); + return ret && ret[ 1 ] == '\0'; + } + typedef map<string,UsageData> UsageMap; // duration, # reads, # writes, # total calls + static T currentTime() { + return boost::posix_time::microsec_clock::universal_time(); + } + void recordUsage( const string &client, D duration ) { + recordUsageForMap( _totalUsage, client, duration ); + recordUsageForMap( _nextSnapshot, client, duration ); + } + void recordUsageForMap( UsageMap &map, const string &client, D duration ) { + UsageData& g = map[client]; + g.get< 0 >() += duration; + if ( _read && !_write ) + g.get< 1 >()++; + else if ( !_read && _write ) + g.get< 2 >()++; + g.get< 3 >()++; + } + static void fillParentNamespaces( UsageMap &to, const UsageMap &from ) { + for( UsageMap::const_iterator i = from.begin(); i != from.end(); ++i ) { + string current = i->first; + size_t dot = current.rfind( "." ); + if ( dot == string::npos || dot != current.length() - 1 ) { + inc( to[ current ], i->second ); + } + while( dot != string::npos ) { + current = current.substr( 0, dot ); + inc( to[ current ], i->second ); + dot = current.rfind( "." ); + } + } + } + static void inc( UsageData &to, const UsageData &from ) { + to.get<0>() += from.get<0>(); + to.get<1>() += from.get<1>(); + to.get<2>() += from.get<2>(); + to.get<3>() += from.get<3>(); + } + struct more { bool operator()( const D &a, const D &b ) { return a > b; } }; + string _current; + T _currentStart; + static T _snapshotStart; + static D _snapshotDuration; + static UsageMap _totalUsage; + static UsageMap _snapshotA; + static UsageMap _snapshotB; + static UsageMap &_snapshot; + static UsageMap &_nextSnapshot; + bool _read; + bool _write; + }; + +} // namespace mongo diff --git a/util/unittest.h b/util/unittest.h new file mode 100644 index 0000000..caf8cb3 --- /dev/null +++ b/util/unittest.h @@ -0,0 +1,59 @@ +// unittest.h + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +namespace mongo { + + /* The idea here is to let all initialization of global variables (classes inheriting from UnitTest) + complete before we run the tests -- otherwise order of initilization being arbitrary may mess + us up. The app's main() function should call runTests(). + + To define a unit test, inherit from this and implement run. instantiate one object for the new class + as a global. + */ + struct UnitTest { + UnitTest() { + registerTest(this); + } + virtual ~UnitTest() {} + + // assert if fails + virtual void run() = 0; + + static bool testsInProgress() { return running; } + private: + static vector<UnitTest*> *tests; + static bool running; + public: + static void registerTest(UnitTest *t) { + if ( tests == 0 ) + tests = new vector<UnitTest*>(); + tests->push_back(t); + } + + static void runTests() { + running = true; + for ( vector<UnitTest*>::iterator i = tests->begin(); i != tests->end(); i++ ) { + (*i)->run(); + } + running = false; + } + }; + + +} // namespace mongo diff --git a/util/util.cpp b/util/util.cpp new file mode 100644 index 0000000..78d8d52 --- /dev/null +++ b/util/util.cpp @@ -0,0 +1,137 @@ +// util.cpp + +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "stdafx.h" +#include "goodies.h" +#include "unittest.h" +#include "top.h" +#include "file_allocator.h" +#include "optime.h" + +namespace mongo { + + vector<UnitTest*> *UnitTest::tests = 0; + bool UnitTest::running = false; + + Nullstream nullstream; + + thread_specific_ptr<Logstream> Logstream::tsp; + + const char *default_getcurns() { return ""; } + const char * (*getcurns)() = default_getcurns; + + int logLevel = 0; + boost::mutex &Logstream::mutex = *( new boost::mutex ); + int Logstream::doneSetup = Logstream::magicNumber(); + + bool goingAway = false; + + bool isPrime(int n) { + int z = 2; + while ( 1 ) { + if ( z*z > n ) + break; + if ( n % z == 0 ) + return false; + z++; + } + return true; + } + + int nextPrime(int n) { + n |= 1; // 2 goes to 3...don't care... + while ( !isPrime(n) ) + n += 2; + return n; + } + + struct UtilTest : public UnitTest { + void run() { + assert( WrappingInt(0) <= WrappingInt(0) ); + assert( WrappingInt(0) <= WrappingInt(1) ); + assert( !(WrappingInt(1) <= WrappingInt(0)) ); + assert( (WrappingInt(0xf0000000) <= WrappingInt(0)) ); + assert( (WrappingInt(0xf0000000) <= WrappingInt(9000)) ); + assert( !(WrappingInt(300) <= WrappingInt(0xe0000000)) ); + + assert( tdiff(3, 4) == 1 ); + assert( tdiff(4, 3) == -1 ); + assert( tdiff(0xffffffff, 0) == 1 ); + + assert( isPrime(3) ); + assert( isPrime(2) ); + assert( isPrime(13) ); + assert( isPrime(17) ); + assert( !isPrime(9) ); + assert( !isPrime(6) ); + assert( nextPrime(4) == 5 ); + assert( nextPrime(8) == 11 ); + + assert( endsWith("abcde", "de") ); + assert( !endsWith("abcde", "dasdfasdfashkfde") ); + + assert( swapEndian(0x01020304) == 0x04030201 ); + + } + } utilTest; + + // The mutex contained in this object may be held on shutdown. + FileAllocator &theFileAllocator_ = *(new FileAllocator()); + FileAllocator &theFileAllocator() { return theFileAllocator_; } + + OpTime OpTime::last(0, 0); + + /* this is a good place to set a breakpoint when debugging, as lots of warning things + (assert, wassert) call it. + */ + void sayDbContext(const char *errmsg) { + if ( errmsg ) { + problem() << errmsg << endl; + } + printStackTrace(); + } + + void rawOut( const string &s ) { + if( s.empty() ) return; + char now[64]; + time_t_to_String(time(0), now); + now[20] = 0; +#if defined(_WIN32) + (std::cout << now << " " << s).flush(); +#else + assert( write( STDOUT_FILENO, now, 20 ) > 0 ); + assert( write( STDOUT_FILENO, " ", 1 ) > 0 ); + assert( write( STDOUT_FILENO, s.c_str(), s.length() ) > 0 ); + fsync( STDOUT_FILENO ); +#endif + } + +#ifndef _SCONS + // only works in scons + const char * gitVersion(){ return ""; } + const char * sysInfo(){ return ""; } +#endif + + void printGitVersion() { log() << "git version: " << gitVersion() << endl; } + void printSysInfo() { log() << "sys info: " << sysInfo() << endl; } + string mongodVersion() { + stringstream ss; + ss << "db version v" << versionString << ", pdfile version " << VERSION << "." << VERSION_MINOR; + return ss.str(); + } + +} // namespace mongo |