Imported Upstream version 1.4.0

10 files changed, 611 insertions, 233 deletions

diff --git a/tools/bridge.cpp b/tools/bridge.cpp
index 42c3287..5535719 100644
--- a/tools/bridge.cpp
+++ b/tools/bridge.cpp
@@ -106,6 +106,8 @@ void check( bool b ) {
 }
 
 int main( int argc, char **argv ) {
+    static StaticObserver staticObserver;
+    
     setupSignals();
 
     check( argc == 5 );
diff --git a/tools/dump.cpp b/tools/dump.cpp
index 4cbd2e1..52e95ce 100644
--- a/tools/dump.cpp
+++ b/tools/dump.cpp
@@ -28,7 +28,7 @@ namespace po = boost::program_options;
 
 class Dump : public Tool {
 public:
-    Dump() : Tool( "dump" , "*" ){
+    Dump() : Tool( "dump" , true , "*" ){
         add_options()
             ("out,o", po::value<string>()->default_value("dump"), "output directory")
             ;
@@ -39,7 +39,7 @@ public:
         
         ofstream out;
         out.open( outputFile.string().c_str() , ios_base::out | ios_base::binary  );
-        uassert( 10262 ,  "couldn't open file" , out.good() );
+        ASSERT_STREAM_GOOD( 10262 ,  "couldn't open file" , out );
 
         ProgressMeter m( conn( true ).count( coll.c_str() , BSONObj() , QueryOption_SlaveOk ) );
 
diff --git a/tools/export.cpp b/tools/export.cpp
index ad27f27..aabebf3 100644
--- a/tools/export.cpp
+++ b/tools/export.cpp
@@ -54,7 +54,7 @@ public:
                 string dir = outfile.substr( 0 , idx + 1 );
                 create_directories( dir );
             }
-            ofstream * s = new ofstream( outfile.c_str() , ios_base::out | ios_base::binary );
+            ofstream * s = new ofstream( outfile.c_str() , ios_base::out );
             fileStream.reset( s );
             outPtr = s;
             if ( ! s->good() ){
@@ -120,7 +120,7 @@ public:
         }
 
         
-        cout << "exported " << num << " records" << endl;
+        cerr << "exported " << num << " records" << endl;
 
         return 0;
     }
diff --git a/tools/files.cpp b/tools/files.cpp
index e69a070..2cbda12 100644
--- a/tools/files.cpp
+++ b/tools/files.cpp
@@ -139,12 +139,14 @@ public:
 
             }
 
+            conn().getLastError();
             cout << "done!";
             return 0;
         }
 
         if ( cmd == "delete" ){
             g.removeFile(filename);
+            conn().getLastError();
             cout << "done!";
             return 0;
         }
diff --git a/tools/import.cpp b/tools/import.cpp
index 47cbe32..e34e73d 100644
--- a/tools/import.cpp
+++ b/tools/import.cpp
@@ -76,28 +76,44 @@ class Import : public Tool {
             }
             pos++;
             
-            int skip = 1;
+            bool done = false;
+            string data;
             char * end;
             if ( _type == CSV && line[0] == '"' ){
-                line++;
-                end = strstr( line , "\"" );
-                skip = 2;
-            }
-            else {
+                line++; //skip first '"'
+
+                while (true) {
+                    end = strchr( line , '"' );
+                    if (!end){
+                        data += line;
+                        done = true;
+                        break;
+                    } else if (end[1] == '"') {
+                        // two '"'s get appended as one
+                        data.append(line, end-line+1); //include '"'
+                        line = end+2; //skip both '"'s
+                    } else if (end[-1] == '\\') {
+                        // "\\\"" gets appended as '"'
+                        data.append(line, end-line-1); //exclude '\\'
+                        data.append("\"");
+                        line = end+1; //skip the '"'
+                    } else {
+                        data.append(line, end-line);
+                        line = end+2; //skip '"' and ','
+                        break;
+                    }
+                }
+            } else {
                 end = strstr( line , _sep );
+                if ( ! end ){
+                    done = true;
+                    data = string( line );
+                } else {
+                    data = string( line , end - line );
+                    line = end+1;
+                }
             }
-            
-            bool done = false;
-            string data;
 
-            if ( ! end ){
-                done = true;
-                data = string( line );
-            }
-            else {
-                data = string( line , end - line );
-            }
-            
             if ( _headerLine ){
                 while ( isspace( data[0] ) )
                     data = data.substr( 1 );
@@ -108,7 +124,6 @@ class Import : public Tool {
             
             if ( done )
                 break;
-            line = end + skip;
         }
         return b.obj();
     }
@@ -135,7 +150,7 @@ public:
 
         istream * in = &cin;
 
-        ifstream file( filename.c_str() , ios_base::in | ios_base::binary);
+        ifstream file( filename.c_str() , ios_base::in);
 
         if ( filename.size() > 0 && filename != "-" ){
             if ( ! exists( filename ) ){
@@ -201,7 +216,7 @@ public:
         log(1) << "filesize: " << fileSize << endl;
         ProgressMeter pm( fileSize );
         const int BUF_SIZE = 1024 * 1024 * 4;
-        boost::scoped_array<char> line(new char[BUF_SIZE]);
+        boost::scoped_array<char> line(new char[BUF_SIZE+2]);
         while ( *in ){
             char * buf = line.get();
             in->getline( buf , BUF_SIZE );
@@ -214,6 +229,8 @@ public:
             if ( ! len )
                 continue;
             
+            buf[len+1] = 0;
+
             if ( in->rdstate() == ios_base::eofbit )
                 break;
             assert( in->rdstate() == 0 );
@@ -238,6 +255,8 @@ public:
         }
 
         cout << "imported " << num << " objects" << endl;
+
+        conn().getLastError();
         
         if ( errors == 0 )
             return 0;
diff --git a/tools/restore.cpp b/tools/restore.cpp
index 19e3a26..6fcf2d3 100644
--- a/tools/restore.cpp
+++ b/tools/restore.cpp
@@ -31,7 +31,15 @@ namespace po = boost::program_options;
 
 class Restore : public Tool {
 public:
-    Restore() : Tool( "restore" , "" , "" ){
+    
+    bool _drop;
+    bool _objcheck;
+    
+    Restore() : Tool( "restore" , true , "" , "" ) , _drop(false),_objcheck(false){
+        add_options()
+            ("drop" , "drop each collection before import" )
+            ("objcheck" , "validate object before inserting" )
+            ;
         add_hidden_options()
             ("dir", po::value<string>()->default_value("dump"), "directory to restore from")
             ;
@@ -45,6 +53,8 @@ public:
     int run(){
         auth();
         path root = getParam("dir");
+        _drop = hasParam( "drop" );
+        _objcheck = hasParam( "objcheck" );
 
         /* If _db is not "" then the user specified a db name to restore as.
          *
@@ -56,6 +66,7 @@ public:
          * .bson file, or a single .bson file itself (a collection).
          */
         drillDown(root, _db != "", _coll != "");
+        conn().getLastError();
         return EXIT_CLEAN;
     }
 
@@ -128,6 +139,11 @@ public:
 
         out() << "\t going into namespace [" << ns << "]" << endl;
 
+        if ( _drop ){
+            out() << "\t dropping" << endl;
+            conn().dropCollection( ns );
+        }
+
         string fileString = root.string();
         ifstream file( fileString.c_str() , ios_base::in | ios_base::binary);
         if ( ! file.is_open() ){
@@ -141,7 +157,8 @@ public:
         long long num = 0;
 
         const int BUF_SIZE = 1024 * 1024 * 5;
-        char * buf = (char*)malloc( BUF_SIZE );
+        boost::scoped_array<char> buf_holder(new char[BUF_SIZE]);
+        char * buf = buf_holder.get();
 
         ProgressMeter m( fileLength );
 
@@ -156,6 +173,22 @@ public:
             file.read( buf + 4 , size - 4 );
 
             BSONObj o( buf );
+            if ( _objcheck && ! o.valid() ){
+                cerr << "INVALID OBJECT - going try and pring out " << endl;
+                cerr << "size: " << size << endl;
+                BSONObjIterator i(o);
+                while ( i.more() ){
+                    BSONElement e = i.next();
+                    try {
+                        e.validate();
+                    }
+                    catch ( ... ){
+                        cerr << "\t\t NEXT ONE IS INVALID" << endl;
+                    }
+                    cerr << "\t name : " << e.fieldName() << " " << e.type() << endl;
+                    cerr << "\t " << e << endl;
+                }
+            }
             conn().insert( ns.c_str() , o );
 
             read += o.objsize();
@@ -164,8 +197,6 @@ public:
             m.hit( o.objsize() );
         }
 
-        free( buf );
-
         uassert( 10265 ,  "counts don't match" , m.done() == fileLength );
         out() << "\t "  << m.hits() << " objects" << endl;
     }
diff --git a/tools/sniffer.cpp b/tools/sniffer.cpp
index 9590d8f..14d32bd 100644
--- a/tools/sniffer.cpp
+++ b/tools/sniffer.cpp
@@ -1,4 +1,20 @@
 // sniffer.cpp
+/*
+ *    Copyright (C) 2010 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 
 /*
   TODO:
@@ -20,6 +36,7 @@
 
 #include "../util/builder.h"
 #include "../util/message.h"
+#include "../util/mmap.h"
 #include "../db/dbmessage.h"
 #include "../client/dbclient.h"
 
@@ -50,6 +67,7 @@ using mongo::BSONObj;
 using mongo::BufBuilder;
 using mongo::DBClientConnection;
 using mongo::QueryResult;
+using mongo::MemoryMappedFile;
 
 #define SNAP_LEN 65535
 
@@ -129,10 +147,12 @@ map< Connection, bool > seen;
 map< Connection, int > bytesRemainingInMessage;
 map< Connection, boost::shared_ptr< BufBuilder > > messageBuilder;
 map< Connection, unsigned > expectedSeq;
-map< Connection, DBClientConnection* > forwarder;
+map< Connection, boost::shared_ptr<DBClientConnection> > forwarder;
 map< Connection, long long > lastCursor;
 map< Connection, map< long long, long long > > mapCursor;
 
+void processMessage( Connection& c , Message& d );
+
 void got_packet(u_char *args, const struct pcap_pkthdr *header, const u_char *packet){
 
     const struct sniff_ip* ip = (struct sniff_ip*)(packet + captureHeaderSize);
@@ -224,6 +244,12 @@ void got_packet(u_char *args, const struct pcap_pkthdr *header, const u_char *pa
          << "  " << m.data->len << " bytes "
          << " id:" << hex << m.data->id << dec << "\t" << m.data->id;
 
+    processMessage( c , m );
+}
+
+void processMessage( Connection& c , Message& m ){
+    DbMessage d(m);
+
     if ( m.data->operation() == mongo::opReply )
         cout << " - " << m.data->responseTo;
     cout << endl;
@@ -281,11 +307,9 @@ void got_packet(u_char *args, const struct pcap_pkthdr *header, const u_char *pa
 
     if ( !forwardAddress.empty() ) {
         if ( m.data->operation() != mongo::opReply ) {
-            DBClientConnection *conn = forwarder[ c ];
+            boost::shared_ptr<DBClientConnection> conn = forwarder[ c ];
             if ( !conn ) {
-                // These won't get freed on error, oh well hopefully we'll just
-                // abort in that case anyway.
-                conn = new DBClientConnection( true );
+                conn.reset(new DBClientConnection( true ));
                 conn->connect( forwardAddress );
                 forwarder[ c ] = conn;
             }
@@ -328,13 +352,41 @@ void got_packet(u_char *args, const struct pcap_pkthdr *header, const u_char *pa
     }
 }
 
+void processDiagLog( const char * file ){
+    Connection c;
+    MemoryMappedFile f;
+    long length;
+    
+    char * root = (char*)f.map( file , length , MemoryMappedFile::SEQUENTIAL );
+    assert( root );
+    assert( length > 0 );
+    
+    char * pos = root;
+
+    long read = 0;
+    while ( read < length ){
+        Message m(pos,false);
+        int len = m.data->len;
+        DbMessage d(m);
+        cout << len << " " << d.getns() << endl;
+        
+        processMessage( c , m );
+
+        read += len;
+        pos += len;
+    }
+    
+    f.close();
+}
+
 void usage() {
     cout <<
-    "Usage: mongosniff [--help] [--forward host:port] [--source (NET <interface> | FILE <filename>)] [<port0> <port1> ... ]\n"
+    "Usage: mongosniff [--help] [--forward host:port] [--source (NET <interface> | (FILE | DIAGLOG) <filename>)] [<port0> <port1> ... ]\n"
     "--forward       Forward all parsed request messages to mongod instance at \n"
     "                specified host:port\n"
     "--source        Source of traffic to sniff, either a network interface or a\n"
-    "                file containing perviously captured packets, in pcap format.\n"
+    "                file containing previously captured packets in pcap format,\n"
+    "                or a file containing output from mongod's --diaglog option.\n"
     "                If no source is specified, mongosniff will attempt to sniff\n"
     "                from one of the machine's network interfaces.\n"
     "<port0>...      These parameters are used to filter sniffing.  By default, \n"
@@ -352,9 +404,10 @@ int main(int argc, char **argv){
     struct bpf_program fp;
     bpf_u_int32 mask;
     bpf_u_int32 net;
-
+    
     bool source = false;
     bool replay = false;
+    bool diaglog = false;
     const char *file = 0;
 
     vector< const char * > args;
@@ -367,18 +420,22 @@ int main(int argc, char **argv){
             if ( arg == string( "--help" ) ) {
                 usage();
                 return 0;
-            } else if ( arg == string( "--forward" ) ) {
+            } 
+            else if ( arg == string( "--forward" ) ) {
                 forwardAddress = args[ ++i ];
-            } else if ( arg == string( "--source" ) ) {
+            } 
+            else if ( arg == string( "--source" ) ) {
                 uassert( 10266 ,  "can't use --source twice" , source == false );
                 uassert( 10267 ,  "source needs more args" , args.size() > i + 2);
                 source = true;
                 replay = ( args[ ++i ] == string( "FILE" ) );
-                if ( replay )
+                diaglog = ( args[ i ] == string( "DIAGLOG" ) );
+                if ( replay || diaglog )
                     file = args[ ++i ];
                 else
                     dev = args[ ++i ];
-            } else {
+            }
+            else {
                 serverPorts.insert( atoi( args[ i ] ) );
             }
         }
@@ -389,8 +446,19 @@ int main(int argc, char **argv){
 
     if ( !serverPorts.size() )
         serverPorts.insert( 27017 );
-
-    if ( !replay ) {
+    
+    if ( diaglog ){
+        processDiagLog( file );
+        return 0;
+    }
+    else if ( replay ){
+        handle = pcap_open_offline(file, errbuf);
+        if ( ! handle ){
+            cerr << "error opening capture file!" << endl;
+            return -1;
+        }
+    }
+    else {
         if ( !dev ) {
             dev = pcap_lookupdev(errbuf);
             if ( ! dev ) {
@@ -408,13 +476,7 @@ int main(int argc, char **argv){
             cerr << "error opening device: " << errbuf << endl;
             return -1;
         }
-    } else {
-        handle = pcap_open_offline(file, errbuf);
-        if ( ! handle ){
-            cerr << "error opening capture file!" << endl;
-            return -1;
-        }
-    }
+    } 
 
     switch ( pcap_datalink( handle ) ){
     case DLT_EN10MB:
@@ -440,9 +502,6 @@ int main(int argc, char **argv){
     pcap_freecode(&fp);
     pcap_close(handle);
 
-    for( map< Connection, DBClientConnection* >::iterator i = forwarder.begin(); i != forwarder.end(); ++i )
-        free( i->second );
-
     return 0;
 }
 
diff --git a/tools/stat.cpp b/tools/stat.cpp
new file mode 100644
index 0000000..f66f3f1
--- /dev/null
+++ b/tools/stat.cpp
@@ -0,0 +1,194 @@
+// stat.cpp
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "stdafx.h"
+#include "client/dbclient.h"
+#include "db/json.h"
+
+#include "tool.h"
+
+#include <fstream>
+#include <iostream>
+
+#include <boost/program_options.hpp>
+
+namespace po = boost::program_options;
+
+namespace mongo {
+    
+    class Stat : public Tool {
+    public:
+
+        Stat() : Tool( "stat" , false , "admin" ){
+            _sleep = 1;
+            _rowNum = 0;
+            _showHeaders = true;
+
+            add_hidden_options()
+                ( "sleep" , po::value<int>() , "time to sleep between calls" )
+                ;
+            add_options()
+                ("noheaders", "don't output column names")
+                ("rowcount,n", po::value<int>()->default_value(0), "number of stats lines to print (0 for indefinite)")
+                ;
+
+            addPositionArg( "sleep" , 1 );
+        }
+
+        virtual void printExtraHelp( ostream & out ){
+            out << "usage: " << _name << " [options] [sleep time]" << endl;
+            out << "sleep time: time to wait (in seconds) between calls" << endl;
+        }
+
+        BSONObj stats(){
+            BSONObj out;
+            if ( ! conn().simpleCommand( _db , &out , "serverStatus" ) ){
+                cout << "error: " << out << endl;
+                return BSONObj();
+            }
+            return out.getOwned();
+        }
+
+        double diff( const string& name , const BSONObj& a , const BSONObj& b ){
+            BSONElement x = a.getFieldDotted( name.c_str() );
+            BSONElement y = b.getFieldDotted( name.c_str() );
+            if ( ! x.isNumber() || ! y.isNumber() )
+                return -1;
+            return ( y.number() - x.number() ) / _sleep;
+        }
+        
+        double percent( const char * outof , const char * val , const BSONObj& a , const BSONObj& b ){
+            double x = ( b.getFieldDotted( val ).number() - a.getFieldDotted( val ).number() );
+            double y = ( b.getFieldDotted( outof ).number() - a.getFieldDotted( outof ).number() );
+            if ( y == 0 )
+                return 0;
+            return x / y;
+        }
+
+        void cellstart( stringstream& ss , string name , unsigned& width ){
+            if ( ! _showHeaders ) {
+                return;
+            }
+            if ( name.size() > width )
+                width = name.size();
+            if ( _rowNum % 20 == 0 )
+                cout << setw(width) << name << " ";            
+        }
+
+        void cell( stringstream& ss , string name , unsigned width , double val ){
+            cellstart( ss , name , width );
+            ss << setw(width) << setprecision(3) << val << " ";
+        }
+
+        void cell( stringstream& ss , string name , unsigned width , int val ){
+            cellstart( ss , name , width );
+            ss << setw(width) << val << " ";
+        }
+
+        void cell( stringstream& ss , string name , unsigned width , const string& val ){
+            assert( val.size() <= width );
+            cellstart( ss , name , width );
+            ss << setw(width) << val << " ";
+        }
+
+
+        string doRow( const BSONObj& a , const BSONObj& b ){
+            stringstream ss;
+
+            if ( b["opcounters"].type() == Object ){
+                BSONObj ax = a["opcounters"].embeddedObject();
+                BSONObj bx = b["opcounters"].embeddedObject();
+                BSONObjIterator i( bx );
+                while ( i.more() ){
+                    BSONElement e = i.next();
+                    cell( ss , (string)(e.fieldName()) + "/s" , 6 , (int)diff( e.fieldName() , ax , bx ) );
+                }
+            }
+
+            if ( b.getFieldDotted("mem.supported").trueValue() ){
+                BSONObj bx = b["mem"].embeddedObject();
+                BSONObjIterator i( bx );
+                cell( ss , "mapped" , 6 , bx["mapped"].numberInt() );
+                cell( ss , "vsize" , 6 , bx["virtual"].numberInt() );
+                cell( ss , "res" , 6 , bx["resident"].numberInt() );
+            }
+            
+            cell( ss , "% locked" , 8 , percent( "globalLock.totalTime" , "globalLock.lockTime" , a , b ) );
+            cell( ss , "% idx miss" , 8 , percent( "indexCounters.btree.accesses" , "indexCounters.btree.misses" , a , b ) );
+
+            cell( ss , "conn" , 5 , b.getFieldDotted( "connections.current" ).numberInt() );
+
+            {
+                struct tm t;
+                time_t_to_Struct( time(0), &t , true );
+                stringstream temp;
+                temp << setfill('0') << setw(2) << t.tm_hour 
+                     << ":" 
+                     << setfill('0') << setw(2) << t.tm_min
+                     << ":" 
+                     << setfill('0') << setw(2) << t.tm_sec;
+                cell( ss , "time" , 8 , temp.str() );
+            }
+
+            if ( _showHeaders && _rowNum % 20 == 0 ){
+                // this is the newline after the header line
+                cout << endl;
+            }
+            _rowNum++;
+
+            return ss.str();
+        }
+        
+        int run(){ 
+            _sleep = getParam( "sleep" , _sleep );
+            if ( hasParam( "noheaders" ) ) {
+                _showHeaders = false;
+            }
+            _rowCount = getParam( "rowcount" , 0 );
+
+            BSONObj prev = stats();
+            if ( prev.isEmpty() )
+                return -1;
+
+            while ( _rowCount == 0 || _rowNum < _rowCount ){
+                sleepsecs(_sleep);
+                BSONObj now = stats();
+                if ( now.isEmpty() )
+                    return -2;
+                
+                cout << doRow( prev , now ) << endl;
+                
+                prev = now;
+            }
+            return 0;
+        }
+        
+
+        int _sleep;
+        int _rowNum;
+        int _rowCount;
+        bool _showHeaders;
+    };
+
+}
+
+int main( int argc , char ** argv ) {
+    mongo::Stat stat;
+    return stat.main( argc , argv );
+}
+
diff --git a/tools/tool.cpp b/tools/tool.cpp
index 8243a45..c92a0c4 100644
--- a/tools/tool.cpp
+++ b/tools/tool.cpp
@@ -1,3 +1,19 @@
+/*
+ *    Copyright (C) 2010 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 // Tool.cpp
 
 #include "tool.h"
@@ -14,225 +30,259 @@ using namespace mongo;
 
 namespace po = boost::program_options;
 
-mongo::Tool::Tool( string name , string defaultDB , string defaultCollection ) :
-    _name( name ) , _db( defaultDB ) , _coll( defaultCollection ) , _conn(0), _paired(false) {
-
-    _options = new po::options_description( "options" );
-    _options->add_options()
-        ("help","produce help message")
-        ("host,h",po::value<string>(), "mongo host to connect to" )
-        ("db,d",po::value<string>(), "database to use" )
-        ("collection,c",po::value<string>(), "collection to use (some commands)" )
-        ("username,u",po::value<string>(), "username" )
-        ("password,p",po::value<string>(), "password" )
-        ("dbpath",po::value<string>(), "directly access mongod data files in this path, instead of connecting to a mongod instance" )
-        ("verbose,v", "be more verbose (include multiple times for more verbosity e.g. -vvvvv)")
-        ;
-
-    _hidden_options = new po::options_description( name + " hidden options" );
-
-    /* support for -vv -vvvv etc. */
-    for (string s = "vv"; s.length() <= 10; s.append("v")) {
-        _hidden_options->add_options()(s.c_str(), "verbose");
-    }
-}
-
-mongo::Tool::~Tool(){
-    delete( _options );
-    delete( _hidden_options );
-    if ( _conn )
-        delete _conn;
-}
-
-void mongo::Tool::printExtraHelp( ostream & out ){
-}
+namespace mongo {
 
-void mongo::Tool::printHelp(ostream &out) {
-    printExtraHelp(out);
-    _options->print(out);
-}
+    Tool::Tool( string name , bool localDBAllowed , string defaultDB , string defaultCollection ) :
+        _name( name ) , _db( defaultDB ) , _coll( defaultCollection ) , _conn(0), _paired(false) {
+    
+        _options = new po::options_description( "options" );
+        _options->add_options()
+            ("help","produce help message")
+            ("verbose,v", "be more verbose (include multiple times for more verbosity e.g. -vvvvv)")
+            ("host,h",po::value<string>(), "mongo host to connect to (\"left,right\" for pairs)" )
+            ("db,d",po::value<string>(), "database to use" )
+            ("collection,c",po::value<string>(), "collection to use (some commands)" )
+            ("username,u",po::value<string>(), "username" )
+            ("password,p",po::value<string>(), "password" )
+            ;
+        if ( localDBAllowed )
+            _options->add_options()
+                ("dbpath",po::value<string>(), "directly access mongod data "
+                 "files in the given path, instead of connecting to a mongod "
+                 "instance - needs to lock the data directory, so cannot be "
+                 "used if a mongod is currently accessing the same path" )
+                ("directoryperdb", "if dbpath specified, each db is in a separate directory" )
+                ;
+
+        _hidden_options = new po::options_description( name + " hidden options" );
+
+        /* support for -vv -vvvv etc. */
+        for (string s = "vv"; s.length() <= 10; s.append("v")) {
+            _hidden_options->add_options()(s.c_str(), "verbose");
+        }
+    }
 
-int mongo::Tool::main( int argc , char ** argv ){
-    boost::filesystem::path::default_name_check( boost::filesystem::no_check );
-
-    _name = argv[0];
-
-    /* using the same style as db.cpp */
-    int command_line_style = (((po::command_line_style::unix_style ^
-                                po::command_line_style::allow_guessing) |
-                               po::command_line_style::allow_long_disguise) ^
-                              po::command_line_style::allow_sticky);
-    try {
-        po::options_description all_options("all options");
-        all_options.add(*_options).add(*_hidden_options);
-
-        po::store( po::command_line_parser( argc , argv ).
-                   options(all_options).
-                   positional( _positonalOptions ).
-                   style(command_line_style).run() , _params );
-
-        po::notify( _params );
-    } catch (po::error &e) {
-        cout << "ERROR: " << e.what() << endl << endl;
-        printHelp(cout);
-        return EXIT_BADOPTIONS;
+    Tool::~Tool(){
+        delete( _options );
+        delete( _hidden_options );
+        if ( _conn )
+            delete _conn;
     }
 
-    if ( _params.count( "help" ) ){
-        printHelp(cerr);
-        return 0;
+    void Tool::printExtraHelp( ostream & out ){
     }
 
-    if ( _params.count( "verbose" ) ) {
-        logLevel = 1;
+    void Tool::printHelp(ostream &out) {
+        printExtraHelp(out);
+        _options->print(out);
     }
 
-    for (string s = "vv"; s.length() <= 10; s.append("v")) {
-        if (_params.count(s)) {
-            logLevel = s.length();
+    int Tool::main( int argc , char ** argv ){
+        static StaticObserver staticObserver;
+        
+        cmdLine.prealloc = false;
+
+        boost::filesystem::path::default_name_check( boost::filesystem::no_check );
+
+        _name = argv[0];
+
+        /* using the same style as db.cpp */
+        int command_line_style = (((po::command_line_style::unix_style ^
+                                    po::command_line_style::allow_guessing) |
+                                   po::command_line_style::allow_long_disguise) ^
+                                  po::command_line_style::allow_sticky);
+        try {
+            po::options_description all_options("all options");
+            all_options.add(*_options).add(*_hidden_options);
+
+            po::store( po::command_line_parser( argc , argv ).
+                       options(all_options).
+                       positional( _positonalOptions ).
+                       style(command_line_style).run() , _params );
+
+            po::notify( _params );
+        } catch (po::error &e) {
+            cerr << "ERROR: " << e.what() << endl << endl;
+            printHelp(cerr);
+            return EXIT_BADOPTIONS;
         }
-    }
 
-    if ( ! hasParam( "dbpath" ) ) {
-        _host = "127.0.0.1";
-        if ( _params.count( "host" ) )
-            _host = _params["host"].as<string>();
+        if ( _params.count( "help" ) ){
+            printHelp(cerr);
+            return 0;
+        }
 
-        if ( _host.find( "," ) == string::npos ){
-            DBClientConnection * c = new DBClientConnection();
-            _conn = c;
+        if ( _params.count( "verbose" ) ) {
+            logLevel = 1;
+        }
 
-            string errmsg;
-            if ( ! c->connect( _host , errmsg ) ){
-                cerr << "couldn't connect to [" << _host << "] " << errmsg << endl;
-                return -1;
+        for (string s = "vv"; s.length() <= 10; s.append("v")) {
+            if (_params.count(s)) {
+                logLevel = s.length();
             }
         }
-        else {
-            log(1) << "using pairing" << endl;
-            DBClientPaired * c = new DBClientPaired();
-            _paired = true;
-            _conn = c;
 
-            if ( ! c->connect( _host ) ){
-                cerr << "couldn't connect to paired server: " << _host << endl;
+        bool useDirectClient = hasParam( "dbpath" );
+
+        if ( ! useDirectClient ) {
+            _host = "127.0.0.1";
+            if ( _params.count( "host" ) )
+                _host = _params["host"].as<string>();
+
+            if ( _host.find( "," ) == string::npos ){
+                DBClientConnection * c = new DBClientConnection();
+                _conn = c;
+
+                string errmsg;
+                if ( ! c->connect( _host , errmsg ) ){
+                    cerr << "couldn't connect to [" << _host << "] " << errmsg << endl;
+                    return -1;
+                }
+            }
+            else {
+                log(1) << "using pairing" << endl;
+                DBClientPaired * c = new DBClientPaired();
+                _paired = true;
+                _conn = c;
+
+                if ( ! c->connect( _host ) ){
+                    cerr << "couldn't connect to paired server: " << _host << endl;
+                    return -1;
+                }
+            }
+
+            cerr << "connected to: " << _host << endl;
+        }
+        else {
+            if ( _params.count( "directoryperdb" ) ) {
+                directoryperdb = true;
+            }
+            Client::initThread("tools");
+            _conn = new DBDirectClient();
+            _host = "DIRECT";
+            static string myDbpath = getParam( "dbpath" );
+            dbpath = myDbpath.c_str();
+            try {
+                acquirePathLock();
+            }
+            catch ( DBException& e ){
+                cerr << endl << "If you are running a mongod on the same "
+                    "path you should connect to that instead of direct data "
+                    "file access" << endl << endl;
+                dbexit( EXIT_CLEAN );
                 return -1;
             }
+
+            theFileAllocator().start();
         }
 
-        cerr << "connected to: " << _host << endl;
-    }
-    else {
-        Client::initThread("tools");
-        _conn = new DBDirectClient();
-        _host = "DIRECT";
-        static string myDbpath = getParam( "dbpath" );
-        mongo::dbpath = myDbpath.c_str();
-        mongo::acquirePathLock();
-        theFileAllocator().start();
-    }
+        if ( _params.count( "db" ) )
+            _db = _params["db"].as<string>();
 
-    if ( _params.count( "db" ) )
-        _db = _params["db"].as<string>();
+        if ( _params.count( "collection" ) )
+            _coll = _params["collection"].as<string>();
 
-    if ( _params.count( "collection" ) )
-        _coll = _params["collection"].as<string>();
+        if ( _params.count( "username" ) )
+            _username = _params["username"].as<string>();
 
-    if ( _params.count( "username" ) )
-        _username = _params["username"].as<string>();
+        if ( _params.count( "password" ) )
+            _password = _params["password"].as<string>();
 
-    if ( _params.count( "password" ) )
-        _password = _params["password"].as<string>();
+        int ret = -1;
+        try {
+            ret = run();
+        }
+        catch ( DBException& e ){
+            cerr << "assertion: " << e.toString() << endl;
+            ret = -1;
+        }
+    
+        if ( currentClient.get() )
+            currentClient->shutdown();
 
-    int ret = -1;
-    try {
-        ret = run();
-    }
-    catch ( DBException& e ){
-        cerr << "assertion: " << e.toString() << endl;
-        ret = -1;
+        if ( useDirectClient )
+            dbexit( EXIT_CLEAN );
+        return ret;
     }
-    
-    if ( currentClient.get() )
-        currentClient->shutdown();
-    
-    return ret;
-}
 
-mongo::DBClientBase& mongo::Tool::conn( bool slaveIfPaired ){
-    if ( _paired && slaveIfPaired )
-        return ((DBClientPaired*)_conn)->slaveConn();
-    return *_conn;
-}
+    DBClientBase& Tool::conn( bool slaveIfPaired ){
+        if ( _paired && slaveIfPaired )
+            return ((DBClientPaired*)_conn)->slaveConn();
+        return *_conn;
+    }
 
-void mongo::Tool::addFieldOptions(){
-    add_options()
-        ("fields,f" , po::value<string>() , "comma seperated list of field names e.g. -f name,age" )
-        ("fieldFile" , po::value<string>() , "file with fields names - 1 per line" )
-        ;
-}
+    void Tool::addFieldOptions(){
+        add_options()
+            ("fields,f" , po::value<string>() , "comma seperated list of field names e.g. -f name,age" )
+            ("fieldFile" , po::value<string>() , "file with fields names - 1 per line" )
+            ;
+    }
 
-void mongo::Tool::needFields(){
+    void Tool::needFields(){
 
-    if ( hasParam( "fields" ) ){
-        BSONObjBuilder b;
+        if ( hasParam( "fields" ) ){
+            BSONObjBuilder b;
         
-        string fields_arg = getParam("fields");
-        pcrecpp::StringPiece input(fields_arg);
+            string fields_arg = getParam("fields");
+            pcrecpp::StringPiece input(fields_arg);
         
-        string f;
-        pcrecpp::RE re("([\\w\\.]+),?" );
-        while ( re.Consume( &input, &f ) ){
-            _fields.push_back( f );
-            b.append( f.c_str() , 1 );
-        }
+            string f;
+            pcrecpp::RE re("([\\w\\.\\s]+),?" );
+            while ( re.Consume( &input, &f ) ){
+                _fields.push_back( f );
+                b.append( f.c_str() , 1 );
+            }
         
-        _fieldsObj = b.obj();
-        return;
-    }
+            _fieldsObj = b.obj();
+            return;
+        }
 
-    if ( hasParam( "fieldFile" ) ){
-        string fn = getParam( "fieldFile" );
-        if ( ! exists( fn ) )
-            throw UserException( 9999 , ((string)"file: " + fn ) + " doesn't exist" );
-
-        const int BUF_SIZE = 1024;
-        char line[ 1024 + 128];
-        ifstream file( fn.c_str() );
-
-        BSONObjBuilder b;
-        while ( file.rdstate() == ios_base::goodbit ){
-            file.getline( line , BUF_SIZE );
-            const char * cur = line;
-            while ( isspace( cur[0] ) ) cur++;
-            if ( strlen( cur ) == 0 )
-                continue;
-
-            _fields.push_back( cur );
-            b.append( cur , 1 );
+        if ( hasParam( "fieldFile" ) ){
+            string fn = getParam( "fieldFile" );
+            if ( ! exists( fn ) )
+                throw UserException( 9999 , ((string)"file: " + fn ) + " doesn't exist" );
+
+            const int BUF_SIZE = 1024;
+            char line[ 1024 + 128];
+            ifstream file( fn.c_str() );
+
+            BSONObjBuilder b;
+            while ( file.rdstate() == ios_base::goodbit ){
+                file.getline( line , BUF_SIZE );
+                const char * cur = line;
+                while ( isspace( cur[0] ) ) cur++;
+                if ( strlen( cur ) == 0 )
+                    continue;
+
+                _fields.push_back( cur );
+                b.append( cur , 1 );
+            }
+            _fieldsObj = b.obj();
+            return;
         }
-        _fieldsObj = b.obj();
-        return;
+
+        throw UserException( 9998 , "you need to specify fields" );
     }
 
-    throw UserException( 9998 , "you need to specify fields" );
-}
+    void Tool::auth( string dbname ){
+        if ( ! dbname.size() )
+            dbname = _db;
+
+        if ( ! ( _username.size() || _password.size() ) )
+            return;
 
-void mongo::Tool::auth( string dbname ){
-    if ( ! dbname.size() )
-        dbname = _db;
+        string errmsg;
+        if ( _conn->auth( dbname , _username , _password , errmsg ) )
+            return;
 
-    if ( ! ( _username.size() || _password.size() ) )
-        return;
+        // try against the admin db
+        string err2;
+        if ( _conn->auth( "admin" , _username , _password , errmsg ) )
+            return;
 
-    string errmsg;
-    if ( _conn->auth( dbname , _username , _password , errmsg ) )
-        return;
+        throw UserException( 9997 , (string)"auth failed: " + errmsg );
+    }
 
-    // try against the admin db
-    string err2;
-    if ( _conn->auth( "admin" , _username , _password , errmsg ) )
-        return;
 
-    throw mongo::UserException( 9997 , (string)"auth failed: " + errmsg );
+    void setupSignals(){}
 }
diff --git a/tools/tool.h b/tools/tool.h
index 18996ec..330fc2d 100644
--- a/tools/tool.h
+++ b/tools/tool.h
@@ -1,3 +1,19 @@
+/*
+ *    Copyright (C) 2010 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
 // Tool.h
 
 #pragma once
@@ -19,7 +35,7 @@ namespace mongo {
 
     class Tool {
     public:
-        Tool( string name , string defaultDB="test" , string defaultCollection="");
+        Tool( string name , bool localDBAllowed=true, string defaultDB="test" , string defaultCollection="");
         virtual ~Tool();
 
         int main( int argc , char ** argv );
@@ -39,6 +55,11 @@ namespace mongo {
                 return _params[name.c_str()].as<string>();
             return def;
         }
+        int getParam( string name , int def ){
+            if ( _params.count( name ) )
+                return _params[name.c_str()].as<int>();
+            return def;
+        }
         bool hasParam( string name ){
             return _params.count( name );
         }