Imported Upstream version 1.6.0

133 files changed, 19590 insertions, 9795 deletions

diff --git a/db/btree.cpp b/db/btree.cpp
index 0c8ca28..d646de8 100644
--- a/db/btree.cpp
+++ b/db/btree.cpp
@@ -16,7 +16,7 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "db.h"
 #include "btree.h"
 #include "pdfile.h"
@@ -55,8 +55,8 @@ namespace mongo {
     }
 
     int BucketBasics::Size() const {
-        assert( _Size == BucketSize );
-        return _Size;
+        assert( _wasSize == BucketSize );
+        return BucketSize;
     }
     inline void BucketBasics::setNotPacked() {
         flags &= ~Packed;
@@ -84,7 +84,7 @@ namespace mongo {
         bt_dmp=0;
     }
 
-    int BucketBasics::fullValidate(const DiskLoc& thisLoc, const BSONObj &order) {
+    int BucketBasics::fullValidate(const DiskLoc& thisLoc, const BSONObj &order, int *unusedCount) {
         {
             bool f = false;
             assert( f = true );
@@ -107,18 +107,24 @@ namespace mongo {
         for ( int i = 0; i < n; i++ ) {
             _KeyNode& kn = k(i);
 
-            if ( kn.isUsed() ) kc++;
+            if ( kn.isUsed() ) {
+                kc++;
+            } else {
+                if ( unusedCount ) {
+                    ++( *unusedCount );
+                }
+            }
             if ( !kn.prevChildBucket.isNull() ) {
                 DiskLoc left = kn.prevChildBucket;
                 BtreeBucket *b = left.btree();
                 wassert( b->parent == thisLoc );
-                kc += b->fullValidate(kn.prevChildBucket, order);
+                kc += b->fullValidate(kn.prevChildBucket, order, unusedCount);
             }
         }
         if ( !nextChild.isNull() ) {
             BtreeBucket *b = nextChild.btree();
             wassert( b->parent == thisLoc );
-            kc += b->fullValidate(nextChild, order);
+            kc += b->fullValidate(nextChild, order, unusedCount);
         }
 
         return kc;
@@ -126,7 +132,7 @@ namespace mongo {
 
     int nDumped = 0;
 
-    void BucketBasics::assertValid(const BSONObj &order, bool force) {
+    void BucketBasics::assertValid(const Ordering &order, bool force) {
         if ( !debug && !force )
             return;
         wassert( n >= 0 && n < Size() );
@@ -183,13 +189,14 @@ namespace mongo {
     }
 
     inline int BucketBasics::totalDataSize() const {
-        return Size() - (data-(char*)this);
+        return (int) (Size() - (data-(char*)this));
     }
 
     void BucketBasics::init() {
         parent.Null();
         nextChild.Null();
-        _Size = BucketSize;
+        _wasSize = BucketSize;
+        _reserved1 = 0;
         flags = Packed;
         n = 0;
         emptySize = totalDataSize();
@@ -247,7 +254,7 @@ namespace mongo {
     }
 
     /* add a key.  must be > all existing.  be careful to set next ptr right. */
-    bool BucketBasics::_pushBack(const DiskLoc& recordLoc, BSONObj& key, const BSONObj &order, DiskLoc prevChild) {
+    bool BucketBasics::_pushBack(const DiskLoc& recordLoc, BSONObj& key, const Ordering &order, DiskLoc prevChild) {
         int bytesNeeded = key.objsize() + sizeof(_KeyNode);
         if ( bytesNeeded > emptySize )
             return false;
@@ -268,12 +275,12 @@ namespace mongo {
     }*/
 
     /* insert a key in a bucket with no complexity -- no splits required */
-    bool BucketBasics::basicInsert(const DiskLoc& thisLoc, int keypos, const DiskLoc& recordLoc, const BSONObj& key, const BSONObj &order) {
+    bool BucketBasics::basicInsert(const DiskLoc& thisLoc, int &keypos, const DiskLoc& recordLoc, const BSONObj& key, const Ordering &order) {
         modified(thisLoc);
         assert( keypos >= 0 && keypos <= n );
         int bytesNeeded = key.objsize() + sizeof(_KeyNode);
         if ( bytesNeeded > emptySize ) {
-            pack( order );
+            pack( order, keypos );
             if ( bytesNeeded > emptySize )
                 return false;
         }
@@ -293,7 +300,7 @@ namespace mongo {
     /* when we delete things we just leave empty space until the node is
        full and then we repack it.
     */
-    void BucketBasics::pack( const BSONObj &order ) {
+    void BucketBasics::pack( const Ordering &order, int &refPos ) {
         if ( flags & Packed )
             return;
 
@@ -301,14 +308,29 @@ namespace mongo {
         char temp[BucketSize];
         int ofs = tdz;
         topSize = 0;
+        int i = 0;
         for ( int j = 0; j < n; j++ ) {
-            short ofsold = k(j).keyDataOfs();
-            int sz = keyNode(j).key.objsize();
+            if( j > 0 && ( j != refPos ) && k( j ).isUnused() && k( j ).prevChildBucket.isNull() ) {
+                continue; // key is unused and has no children - drop it
+            }
+            if( i != j ) {
+                if ( refPos == j ) {
+                    refPos = i; // i < j so j will never be refPos again
+                }
+                k( i ) = k( j );
+            }
+            short ofsold = k(i).keyDataOfs();
+            int sz = keyNode(i).key.objsize();
             ofs -= sz;
             topSize += sz;
             memcpy(temp+ofs, dataAt(ofsold), sz);
-            k(j).setKeyDataOfsSavingUse( ofs );
+            k(i).setKeyDataOfsSavingUse( ofs );
+            ++i;
         }
+        if ( refPos == n ) {
+            refPos = i;
+        }
+        n = i;
         int dataUsed = tdz - ofs;
         memcpy(data + ofs, temp + ofs, dataUsed);
         emptySize = tdz - dataUsed - n * sizeof(_KeyNode);
@@ -318,10 +340,10 @@ namespace mongo {
         assertValid( order );
     }
 
-    inline void BucketBasics::truncateTo(int N, const BSONObj &order) {
+    inline void BucketBasics::truncateTo(int N, const Ordering &order, int &refPos) {
         n = N;
         setNotPacked();
-        pack( order );
+        pack( order, refPos );
     }
 
     /* - BtreeBucket --------------------------------------------------- */
@@ -343,8 +365,37 @@ namespace mongo {
             break;
         }
     }
+    
+    int BtreeBucket::customBSONCmp( const BSONObj &l, const BSONObj &rBegin, int rBeginLen, const vector< const BSONElement * > &rEnd, const Ordering &o ) {
+        BSONObjIterator ll( l );
+        BSONObjIterator rr( rBegin );
+        vector< const BSONElement * >::const_iterator rr2 = rEnd.begin();
+        unsigned mask = 1;
+        for( int i = 0; i < rBeginLen; ++i, mask <<= 1 ) {
+            BSONElement lll = ll.next();
+            BSONElement rrr = rr.next();
+            ++rr2;
+            
+            int x = lll.woCompare( rrr, false );
+            if ( o.descending( mask ) )
+                x = -x;
+            if ( x != 0 )
+                return x;
+        }
+        for( ; ll.more(); mask <<= 1 ) {
+            BSONElement lll = ll.next();
+            BSONElement rrr = **rr2;
+            ++rr2;
+            int x = lll.woCompare( rrr, false );
+            if ( o.descending( mask ) )
+                x = -x;
+            if ( x != 0 )
+                return x;
+        }
+        return 0;
+    }
 
-    bool BtreeBucket::exists(const IndexDetails& idx, DiskLoc thisLoc, const BSONObj& key, BSONObj order) { 
+    bool BtreeBucket::exists(const IndexDetails& idx, DiskLoc thisLoc, const BSONObj& key, const Ordering& order) { 
         int pos;
         bool found;
         DiskLoc b = locate(idx, thisLoc, key, order, pos, found, minDiskLoc);
@@ -367,7 +418,7 @@ namespace mongo {
     */
     bool BtreeBucket::wouldCreateDup(
         const IndexDetails& idx, DiskLoc thisLoc, 
-        const BSONObj& key, BSONObj order,
+        const BSONObj& key, const Ordering& order,
         DiskLoc self) 
     { 
         int pos;
@@ -411,7 +462,7 @@ namespace mongo {
        note result might be an Unused location!
     */
 	char foo;
-    bool BtreeBucket::find(const IndexDetails& idx, const BSONObj& key, DiskLoc recordLoc, const BSONObj &order, int& pos, bool assertIfDup) {
+    bool BtreeBucket::find(const IndexDetails& idx, const BSONObj& key, DiskLoc recordLoc, const Ordering &order, int& pos, bool assertIfDup) {
 #if defined(_EXPERIMENT1)
 		{
 			char *z = (char *) this;
@@ -509,11 +560,11 @@ namespace mongo {
             assert(false);
         }
 found:
-        deallocBucket( thisLoc );
+        deallocBucket( thisLoc, id );
     }
     
-    void BtreeBucket::deallocBucket(const DiskLoc &thisLoc) {
-#if 1
+    void BtreeBucket::deallocBucket(const DiskLoc &thisLoc, IndexDetails &id) {
+#if 0
         /* as a temporary defensive measure, we zap the whole bucket, AND don't truly delete
            it (meaning it is ineligible for reuse).
            */
@@ -523,11 +574,8 @@ found:
         //defensive:
         n = -1;
         parent.Null();
-        massert( 10284 , "todo: use RecStoreInterface instead", false);
-        // TODO: this was broken anyway as deleteRecord does unindexRecord() call which assumes the data is a BSONObj, 
-        // and it isn't.
-        assert(false);
-//        theDataFileMgr.deleteRecord(id.indexNamespace().c_str(), thisLoc.rec(), thisLoc);
+        string ns = id.indexNamespace();
+        btreeStore->deleteRecord(ns.c_str(), thisLoc);
 #endif
     }
 
@@ -560,13 +608,13 @@ found:
     /* remove a key from the index */
     bool BtreeBucket::unindex(const DiskLoc& thisLoc, IndexDetails& id, BSONObj& key, const DiskLoc& recordLoc ) {
         if ( key.objsize() > KeyMax ) {
-            OCCASIONALLY problem() << "unindex: key too large to index, skipping " << id.indexNamespace() << /* ' ' << key.toString() << */ '\n';
+            OCCASIONALLY problem() << "unindex: key too large to index, skipping " << id.indexNamespace() << /* ' ' << key.toString() << */ endl;
             return false;
         }
 
         int pos;
         bool found;
-        DiskLoc loc = locate(id, thisLoc, key, id.keyPattern(), pos, found, recordLoc, 1);
+        DiskLoc loc = locate(id, thisLoc, key, Ordering::make(id.keyPattern()), pos, found, recordLoc, 1);
         if ( found ) {
             loc.btree()->delKeyAtPos(loc, id, pos);
             return true;
@@ -598,9 +646,11 @@ found:
 
     /* insert a key in this bucket, splitting if necessary.
        keypos - where to insert the key i3n range 0..n.  0=make leftmost, n=make rightmost.
+       NOTE this function may free some data, and as a result the value passed for keypos may
+       be invalid after calling insertHere()
     */
     void BtreeBucket::insertHere(DiskLoc thisLoc, int keypos,
-                                 DiskLoc recordLoc, const BSONObj& key, const BSONObj& order,
+                                 DiskLoc recordLoc, const BSONObj& key, const Ordering& order,
                                  DiskLoc lchild, DiskLoc rchild, IndexDetails& idx)
     {
         modified(thisLoc);
@@ -667,7 +717,7 @@ found:
 
         int split = n / 2;
         if ( keypos == n ) { // see SERVER-983
-            split = 0.9 * n;
+            split = (int) (0.9 * n);
             if ( split > n - 2 )
                 split = n - 2;
         }
@@ -718,7 +768,8 @@ found:
             }
         }
 
-        truncateTo(split, order);  // note this may trash splitkey.key.  thus we had to promote it before finishing up here.
+        int newpos = keypos;
+        truncateTo(split, order, newpos);  // note this may trash splitkey.key.  thus we had to promote it before finishing up here.
 
         // add our new key, there is room now
         {
@@ -726,7 +777,7 @@ found:
             if ( keypos <= split ) {
                 if ( split_debug )
                     out() << "  keypos<split, insertHere() the new key" << endl;
-                insertHere(thisLoc, keypos, recordLoc, key, order, lchild, rchild, idx);
+                insertHere(thisLoc, newpos, recordLoc, key, order, lchild, rchild, idx);
             } else {
                 int kp = keypos-split-1;
                 assert(kp>=0);
@@ -806,7 +857,7 @@ found:
         return DiskLoc();
     }
 
-    DiskLoc BtreeBucket::locate(const IndexDetails& idx, const DiskLoc& thisLoc, const BSONObj& key, const BSONObj &order, int& pos, bool& found, DiskLoc recordLoc, int direction) {
+    DiskLoc BtreeBucket::locate(const IndexDetails& idx, const DiskLoc& thisLoc, const BSONObj& key, const Ordering &order, int& pos, bool& found, DiskLoc recordLoc, int direction) {
         int p;
         found = find(idx, key, recordLoc, order, p, /*assertIfDup*/ false);
         if ( found ) {
@@ -829,10 +880,133 @@ found:
             return pos == n ? DiskLoc() /*theend*/ : thisLoc;
     }
 
+    bool BtreeBucket::customFind( int l, int h, const BSONObj &keyBegin, int keyBeginLen, const vector< const BSONElement * > &keyEnd, const Ordering &order, int direction, DiskLoc &thisLoc, int &keyOfs, pair< DiskLoc, int > &bestParent ) {
+        while( 1 ) {
+            if ( l + 1 == h ) {
+                keyOfs = ( direction > 0 ) ? h : l;
+                DiskLoc next = thisLoc.btree()->k( h ).prevChildBucket;
+                if ( !next.isNull() ) {
+                    bestParent = make_pair( thisLoc, keyOfs );
+                    thisLoc = next;
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+            int m = l + ( h - l ) / 2;
+            int cmp = customBSONCmp( thisLoc.btree()->keyNode( m ).key, keyBegin, keyBeginLen, keyEnd, order );
+            if ( cmp < 0 ) {
+                l = m;
+            } else if ( cmp > 0 ) {
+                h = m;
+            } else {
+                if ( direction < 0 ) {
+                    l = m;
+                } else {
+                    h = m;
+                }
+            }
+        }        
+    }
+    
+    // find smallest/biggest value greater-equal/less-equal than specified
+    // starting thisLoc + keyOfs will be strictly less than/strictly greater than keyBegin/keyBeginLen/keyEnd
+    // All the direction checks below allowed me to refactor the code, but possibly separate forward and reverse implementations would be more efficient
+    void BtreeBucket::advanceTo(const IndexDetails &id, DiskLoc &thisLoc, int &keyOfs, const BSONObj &keyBegin, int keyBeginLen, const vector< const BSONElement * > &keyEnd, const Ordering &order, int direction ) {
+        int l,h;
+        bool dontGoUp;
+        if ( direction > 0 ) {
+            l = keyOfs;
+            h = n - 1;
+            dontGoUp = ( customBSONCmp( keyNode( h ).key, keyBegin, keyBeginLen, keyEnd, order ) >= 0 );
+        } else {
+            l = 0;
+            h = keyOfs;
+            dontGoUp = ( customBSONCmp( keyNode( l ).key, keyBegin, keyBeginLen, keyEnd, order ) <= 0 );
+        }
+        pair< DiskLoc, int > bestParent;
+        if ( dontGoUp ) {
+            // this comparison result assures h > l
+            if ( !customFind( l, h, keyBegin, keyBeginLen, keyEnd, order, direction, thisLoc, keyOfs, bestParent ) ) {
+                return;
+            }
+        } else {
+            // go up parents until rightmost/leftmost node is >=/<= target or at top
+            while( !thisLoc.btree()->parent.isNull() ) {
+                thisLoc = thisLoc.btree()->parent;
+                if ( direction > 0 ) {
+                    if ( customBSONCmp( thisLoc.btree()->keyNode( thisLoc.btree()->n - 1 ).key, keyBegin, keyBeginLen, keyEnd, order ) >= 0 ) {
+                        break;
+                    }
+                } else {
+                    if ( customBSONCmp( thisLoc.btree()->keyNode( 0 ).key, keyBegin, keyBeginLen, keyEnd, order ) <= 0 ) {
+                        break;
+                    }                    
+                }
+            }
+        }
+        // go down until find smallest/biggest >=/<= target
+        while( 1 ) {
+            l = 0;
+            h = thisLoc.btree()->n - 1;
+            // leftmost/rightmost key may possibly be >=/<= search key
+            bool firstCheck;
+            if ( direction > 0 ) {
+                firstCheck = ( customBSONCmp( thisLoc.btree()->keyNode( 0 ).key, keyBegin, keyBeginLen, keyEnd, order ) >= 0 );
+            } else {
+                firstCheck = ( customBSONCmp( thisLoc.btree()->keyNode( h ).key, keyBegin, keyBeginLen, keyEnd, order ) <= 0 );
+            }
+            if ( firstCheck ) {
+                DiskLoc next;
+                if ( direction > 0 ) {
+                    next = thisLoc.btree()->k( 0 ).prevChildBucket;
+                    keyOfs = 0;
+                } else {
+                    next = thisLoc.btree()->nextChild;
+                    keyOfs = h;
+                }
+                if ( !next.isNull() ) {
+                    bestParent = make_pair( thisLoc, keyOfs );
+                    thisLoc = next;
+                    continue;
+                } else {
+                    return;
+                }
+            }
+            bool secondCheck;
+            if ( direction > 0 ) {
+                secondCheck = ( customBSONCmp( thisLoc.btree()->keyNode( h ).key, keyBegin, keyBeginLen, keyEnd, order ) < 0 );
+            } else {
+                secondCheck = ( customBSONCmp( thisLoc.btree()->keyNode( 0 ).key, keyBegin, keyBeginLen, keyEnd, order ) > 0 );
+            }
+            if ( secondCheck ) {
+                DiskLoc next;
+                if ( direction > 0 ) {
+                    next = thisLoc.btree()->nextChild;
+                } else {
+                    next = thisLoc.btree()->k( 0 ).prevChildBucket;
+                }
+                if ( next.isNull() ) {
+                    // if bestParent is null, we've hit the end and thisLoc gets set to DiskLoc()
+                    thisLoc = bestParent.first;
+                    keyOfs = bestParent.second;
+                    return;
+                } else {
+                    thisLoc = next;
+                    continue;
+                }
+            }
+            if ( !customFind( l, h, keyBegin, keyBeginLen, keyEnd, order, direction, thisLoc, keyOfs, bestParent ) ) {
+                return;
+            }
+        }
+    }
+
+    
     /* @thisLoc disk location of *this
     */
     int BtreeBucket::_insert(DiskLoc thisLoc, DiskLoc recordLoc,
-                             const BSONObj& key, const BSONObj &order, bool dupsAllowed,
+                             const BSONObj& key, const Ordering &order, bool dupsAllowed,
                              DiskLoc lChild, DiskLoc rChild, IndexDetails& idx) {
         if ( key.objsize() > KeyMax ) {
             problem() << "ERROR: key too large len:" << key.objsize() << " max:" << KeyMax << ' ' << key.objsize() << ' ' << idx.indexNamespace() << endl;
@@ -898,12 +1072,12 @@ found:
 
     /* todo: meaning of return code unclear clean up */
     int BtreeBucket::bt_insert(DiskLoc thisLoc, DiskLoc recordLoc,
-                            const BSONObj& key, const BSONObj &order, bool dupsAllowed,
+                            const BSONObj& key, const Ordering &order, bool dupsAllowed,
                             IndexDetails& idx, bool toplevel)
     {
         if ( toplevel ) {
             if ( key.objsize() > KeyMax ) {
-                problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace().c_str() << ' ' << key.objsize() << ' ' << key.toString() << '\n';
+                problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace().c_str() << ' ' << key.objsize() << ' ' << key.toString() << endl;
                 return 3;
             }
         }
@@ -921,7 +1095,9 @@ found:
     DiskLoc BtreeBucket::findSingle( const IndexDetails& indexdetails , const DiskLoc& thisLoc, const BSONObj& key ){
         int pos;
         bool found;
-        DiskLoc bucket = locate( indexdetails , indexdetails.head , key , BSONObj() , pos , found , minDiskLoc );
+        /* TODO: is it really ok here that the order is a default? */
+        Ordering o = Ordering::make(BSONObj());
+        DiskLoc bucket = locate( indexdetails , indexdetails.head , key , o , pos , found , minDiskLoc );
         if ( bucket.isNull() )
             return bucket;
 
@@ -958,7 +1134,8 @@ namespace mongo {
 
         DiskLoc rl;
         BSONObj key = fromjson("{x:9}");
-        BSONObj order = fromjson("{}");
+        BSONObj orderObj = fromjson("{}");
+        Ordering order = Ordering::make(orderObj);
 
         b->bt_insert(id.head, A, key, order, true, id);
         A.GETOFS() += 2;
@@ -974,7 +1151,7 @@ namespace mongo {
         b->k(2).setUnused();
         b->k(3).setUnused();
 
-        b->dumpTree(id.head, order);
+        b->dumpTree(id.head, orderObj);
 
         /*        b->bt_insert(id.head, B, key, order, false, id);
         b->k(1).setUnused();
@@ -989,17 +1166,20 @@ namespace mongo {
         // this should assert.  does it? (it might "accidentally" though, not asserting proves a problem, asserting proves nothing)
         b->bt_insert(id.head, C, key, order, false, id);
 
-        b->dumpTree(id.head, order);
+//        b->dumpTree(id.head, order);
     }
 
     /* --- BtreeBuilder --- */
 
     BtreeBuilder::BtreeBuilder(bool _dupsAllowed, IndexDetails& _idx) : 
-      dupsAllowed(_dupsAllowed), idx(_idx), n(0) 
+      dupsAllowed(_dupsAllowed), 
+      idx(_idx), 
+      n(0),
+      order( idx.keyPattern() ),
+      ordering( Ordering::make(idx.keyPattern()) )
     {
         first = cur = BtreeBucket::addBucket(idx);
         b = cur.btreemod();
-        order = idx.keyPattern();
         committed = false;
     }
 
@@ -1023,15 +1203,15 @@ namespace mongo {
             keyLast = key;
         }
 
-        if ( ! b->_pushBack(loc, key, order, DiskLoc()) ){
+        if ( ! b->_pushBack(loc, key, ordering, DiskLoc()) ){
             // no room
             if ( key.objsize() > KeyMax ) {
-                problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace().c_str() << ' ' << key.objsize() << ' ' << key.toString() << '\n';
+                problem() << "Btree::insert: key too large to index, skipping " << idx.indexNamespace().c_str() << ' ' << key.objsize() << ' ' << key.toString() << endl;
             }
             else { 
                 // bucket was full
                 newBucket();
-                b->pushBack(loc, key, order, DiskLoc());
+                b->pushBack(loc, key, ordering, DiskLoc());
             }
         }
         n++;
@@ -1060,13 +1240,13 @@ namespace mongo {
                 bool keepX = ( x->n != 0 );
                 DiskLoc keepLoc = keepX ? xloc : x->nextChild;
 
-                if ( ! up->_pushBack(r, k, order, keepLoc) ){
+                if ( ! up->_pushBack(r, k, ordering, keepLoc) ){
                     // current bucket full
                     DiskLoc n = BtreeBucket::addBucket(idx);
                     up->tempNext() = n;
                     upLoc = n; 
                     up = upLoc.btreemod();
-                    up->pushBack(r, k, order, keepLoc);
+                    up->pushBack(r, k, ordering, keepLoc);
                 }
 
                 DiskLoc nextLoc = x->tempNext(); /* get next in chain at current level */
@@ -1075,7 +1255,7 @@ namespace mongo {
                 } else {
                     if ( !x->nextChild.isNull() )
                         x->nextChild.btreemod()->parent = upLoc;
-                    x->deallocBucket( xloc );
+                    x->deallocBucket( xloc, idx );
                 }
                 xloc = nextLoc;
             }
diff --git a/db/btree.h b/db/btree.h
index b2e9ba9..233b4dc 100644
--- a/db/btree.h
+++ b/db/btree.h
@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "jsobj.h"
 #include "diskloc.h"
 #include "pdfile.h"
@@ -26,7 +26,6 @@
 namespace mongo {
 
 #pragma pack(1)
-
     struct _KeyNode {
         DiskLoc prevChildBucket; // the lchild
         DiskLoc recordLoc; // location of the record associated with the key
@@ -60,7 +59,6 @@ namespace mongo {
             return !isUnused();
         }
     };
-
 #pragma pack()
 
     class BucketBasics;
@@ -75,7 +73,6 @@ namespace mongo {
     };
 
 #pragma pack(1)
-
     /* this class is all about the storage management */
     class BucketBasics {
         friend class BtreeBuilder;
@@ -83,8 +80,11 @@ namespace mongo {
     public:
         void dumpTree(DiskLoc thisLoc, const BSONObj &order);
         bool isHead() { return parent.isNull(); }
-        void assertValid(const BSONObj &order, bool force = false);
-        int fullValidate(const DiskLoc& thisLoc, const BSONObj &order); /* traverses everything */
+        void assertValid(const Ordering &order, bool force = false);
+        void assertValid(const BSONObj &orderObj, bool force = false) { 
+            return assertValid(Ordering::make(orderObj),force); 
+        }
+        int fullValidate(const DiskLoc& thisLoc, const BSONObj &order, int *unusedCount = 0); /* traverses everything */
 
         KeyNode keyNode(int i) const {
             if ( i >= n ){
@@ -106,13 +106,13 @@ namespace mongo {
         /* returns false if node is full and must be split
            keypos is where to insert -- inserted after that key #.  so keypos=0 is the leftmost one.
         */
-        bool basicInsert(const DiskLoc& thisLoc, int keypos, const DiskLoc& recordLoc, const BSONObj& key, const BSONObj &order);
+        bool basicInsert(const DiskLoc& thisLoc, int &keypos, const DiskLoc& recordLoc, const BSONObj& key, const Ordering &order);
         
         /**
          * @return true if works, false if not enough space
          */
-        bool _pushBack(const DiskLoc& recordLoc, BSONObj& key, const BSONObj &order, DiskLoc prevChild);
-        void pushBack(const DiskLoc& recordLoc, BSONObj& key, const BSONObj &order, DiskLoc prevChild){
+        bool _pushBack(const DiskLoc& recordLoc, BSONObj& key, const Ordering &order, DiskLoc prevChild);
+        void pushBack(const DiskLoc& recordLoc, BSONObj& key, const Ordering &order, DiskLoc prevChild){
             bool ok = _pushBack( recordLoc , key , order , prevChild );
             assert(ok);
         }
@@ -130,12 +130,12 @@ namespace mongo {
         }
 
         int totalDataSize() const;
-        void pack( const BSONObj &order );
+        void pack( const Ordering &order, int &refPos);
         void setNotPacked();
         void setPacked();
         int _alloc(int bytes);
         void _unalloc(int bytes);
-        void truncateTo(int N, const BSONObj &order);
+        void truncateTo(int N, const Ordering &order, int &refPos);
         void markUnused(int keypos);
 
         /* BtreeBuilder uses the parent var as a temp place to maintain a linked list chain. 
@@ -152,7 +152,7 @@ namespace mongo {
             ss << "    n: " << n << endl;
             ss << "    parent: " << parent.toString() << endl;
             ss << "    nextChild: " << parent.toString() << endl;
-            ss << "    Size: " << _Size << " flags:" << flags << endl;
+            ss << "    flags:" << flags << endl;
             ss << "    emptySize: " << emptySize << " topSize: " << topSize << endl;
             return ss.str();
         }
@@ -164,7 +164,12 @@ namespace mongo {
     protected:
         void _shape(int level, stringstream&);
         DiskLoc nextChild; // child bucket off and to the right of the highest key.
-        int _Size; // total size of this btree node in bytes. constant.
+
+    private:
+        unsigned short _wasSize; // can be reused, value is 8192 in current pdfile version Apr2010
+        unsigned short _reserved1; // zero
+
+    protected:
         int Size() const;
         int flags;
         int emptySize; // size of the empty region
@@ -179,7 +184,9 @@ namespace mongo {
         }
         char data[4];
     };
+#pragma pack()
 
+#pragma pack(1)
     class BtreeBucket : public BucketBasics {
         friend class BtreeCursor;
     public:
@@ -191,20 +198,20 @@ namespace mongo {
              BSONObj order = ((IndexDetails&)idx).keyPattern();
            likewise below in bt_insert() etc.
         */
-        bool exists(const IndexDetails& idx, DiskLoc thisLoc, const BSONObj& key, BSONObj order);
+        bool exists(const IndexDetails& idx, DiskLoc thisLoc, const BSONObj& key, const Ordering& order);
 
         bool wouldCreateDup(
             const IndexDetails& idx, DiskLoc thisLoc, 
-            const BSONObj& key, BSONObj order,
+            const BSONObj& key, const Ordering& order,
             DiskLoc self); 
 
         static DiskLoc addBucket(IndexDetails&); /* start a new index off, empty */
-        void deallocBucket(const DiskLoc &thisLoc); // clear bucket memory, placeholder for deallocation
+        void deallocBucket(const DiskLoc &thisLoc, IndexDetails &id);
         
         static void renameIndexNamespace(const char *oldNs, const char *newNs);
 
         int bt_insert(DiskLoc thisLoc, DiskLoc recordLoc,
-                   const BSONObj& key, const BSONObj &order, bool dupsAllowed,
+                   const BSONObj& key, const Ordering &order, bool dupsAllowed,
                    IndexDetails& idx, bool toplevel = true);
 
         bool unindex(const DiskLoc& thisLoc, IndexDetails& id, BSONObj& key, const DiskLoc& recordLoc);
@@ -215,7 +222,7 @@ namespace mongo {
            found - returns true if exact match found.  note you can get back a position 
                    result even if found is false.
         */
-        DiskLoc locate(const IndexDetails& , const DiskLoc& thisLoc, const BSONObj& key, const BSONObj &order, 
+        DiskLoc locate(const IndexDetails& , const DiskLoc& thisLoc, const BSONObj& key, const Ordering &order, 
                        int& pos, bool& found, DiskLoc recordLoc, int direction=1);
         
         /**
@@ -227,6 +234,9 @@ namespace mongo {
 
         /* advance one key position in the index: */
         DiskLoc advance(const DiskLoc& thisLoc, int& keyOfs, int direction, const char *caller);
+        
+        void advanceTo(const IndexDetails &id, DiskLoc &thisLoc, int &keyOfs, const BSONObj &keyBegin, int keyBeginLen, const vector< const BSONElement * > &keyEnd, const Ordering &order, int direction );
+        
         DiskLoc getHead(const DiskLoc& thisLoc);
 
         /* get tree shape */
@@ -243,24 +253,28 @@ namespace mongo {
         }
         static BtreeBucket* allocTemp(); /* caller must release with free() */
         void insertHere(DiskLoc thisLoc, int keypos,
-                        DiskLoc recordLoc, const BSONObj& key, const BSONObj &order,
+                        DiskLoc recordLoc, const BSONObj& key, const Ordering &order,
                         DiskLoc lchild, DiskLoc rchild, IndexDetails&);
         int _insert(DiskLoc thisLoc, DiskLoc recordLoc,
-                    const BSONObj& key, const BSONObj &order, bool dupsAllowed,
+                    const BSONObj& key, const Ordering &order, bool dupsAllowed,
                     DiskLoc lChild, DiskLoc rChild, IndexDetails&);
-        bool find(const IndexDetails& idx, const BSONObj& key, DiskLoc recordLoc, const BSONObj &order, int& pos, bool assertIfDup);
+        bool find(const IndexDetails& idx, const BSONObj& key, DiskLoc recordLoc, const Ordering &order, int& pos, bool assertIfDup);
+        bool customFind( int l, int h, const BSONObj &keyBegin, int keyBeginLen, const vector< const BSONElement * > &keyEnd, const Ordering &order, int direction, DiskLoc &thisLoc, int &keyOfs, pair< DiskLoc, int > &bestParent );
         static void findLargestKey(const DiskLoc& thisLoc, DiskLoc& largestLoc, int& largestKey);
+        static int customBSONCmp( const BSONObj &l, const BSONObj &rBegin, int rBeginLen, const vector< const BSONElement * > &rEnd, const Ordering &o );
     public:
         // simply builds and returns a dup key error message string
         static string dupKeyError( const IndexDetails& idx , const BSONObj& key );
     };
+#pragma pack()
 
     class BtreeCursor : public Cursor {
     public:
         BtreeCursor( NamespaceDetails *_d, int _idxNo, const IndexDetails&, const BSONObj &startKey, const BSONObj &endKey, bool endKeyInclusive, int direction );
 
-        BtreeCursor( NamespaceDetails *_d, int _idxNo, const IndexDetails& _id, const BoundList &_bounds, int _direction );
-
+        BtreeCursor( NamespaceDetails *_d, int _idxNo, const IndexDetails& _id, const shared_ptr< FieldRangeVector > &_bounds, int _direction );
+        ~BtreeCursor(){
+        }
         virtual bool ok() {
             return !bucket.isNull();
         }
@@ -272,6 +286,7 @@ namespace mongo {
         virtual void noteLocation(); // updates keyAtKeyOfs...
         virtual void checkLocation();
         virtual bool supportGetMore() { return true; }
+        virtual bool supportYields() { return true; }
 
         /* used for multikey index traversal to avoid sending back dups. see Matcher::matches().
            if a multikey index traversal:
@@ -279,7 +294,6 @@ namespace mongo {
              otherwise, marks loc as sent.
              @return true if the loc has not been seen
         */
-        set<DiskLoc> dups;
         virtual bool getsetdup(DiskLoc loc) {
             if( multikey ) { 
                 pair<set<DiskLoc>::iterator, bool> p = dups.insert(loc);
@@ -326,7 +340,7 @@ namespace mongo {
         virtual string toString() {
             string s = string("BtreeCursor ") + indexDetails.indexName();
             if ( direction < 0 ) s += " reverse";
-            if ( bounds_.size() > 1 ) s += " multi";
+            if ( bounds_.get() && bounds_->size() > 1 ) s += " multi";
             return s;
         }
 
@@ -335,26 +349,31 @@ namespace mongo {
         }
 
         virtual BSONObj prettyIndexBounds() const {
-            BSONArrayBuilder ba;
-            if ( bounds_.size() == 0 ) {
-                ba << BSON_ARRAY( prettyKey( startKey ) << prettyKey( endKey ) );
+            if ( !_independentFieldRanges ) {
+                return BSON( "start" << prettyKey( startKey ) << "end" << prettyKey( endKey ) );
             } else {
-                for( BoundList::const_iterator i = bounds_.begin(); i != bounds_.end(); ++i ) {
-                    ba << BSON_ARRAY( prettyKey( i->first ) << prettyKey( i->second ) );
-                }
+                return bounds_->obj();
             }
-            return ba.arr();
         }
         
         void forgetEndKey() { endKey = BSONObj(); }
 
+        virtual CoveredIndexMatcher *matcher() const { return _matcher.get(); }
+        
+        virtual void setMatcher( shared_ptr< CoveredIndexMatcher > matcher ) {
+            _matcher = matcher;
+        }
+
+        // for debugging only
+        DiskLoc getBucket() const { return bucket; }
+        
     private:
         /* Our btrees may (rarely) have "unused" keys when items are deleted.
            Skip past them.
         */
-        void skipUnusedKeys();
-
-        /* Check if the current key is beyond endKey. */
+        bool skipUnusedKeys( bool mayJump );
+        bool skipOutOfRangeKeysAndCheckEnd();
+        void skipAndCheck();
         void checkEnd();
 
         // selective audits on construction
@@ -363,36 +382,40 @@ namespace mongo {
         // set initial bucket
         void init();
 
-        // init start / end keys with a new range
-        void initInterval();
-
+        void advanceTo( const BSONObj &keyBegin, int keyBeginLen, const vector< const BSONElement * > &keyEnd);
+        
         friend class BtreeBucket;
+        set<DiskLoc> dups;
         NamespaceDetails *d;
         int idxNo;
+        
         BSONObj startKey;
         BSONObj endKey;
         bool endKeyInclusive_;
+        
         bool multikey; // note this must be updated every getmore batch in case someone added a multikey...
 
         const IndexDetails& indexDetails;
         BSONObj order;
+        Ordering _ordering;
         DiskLoc bucket;
         int keyOfs;
         int direction; // 1=fwd,-1=reverse
         BSONObj keyAtKeyOfs; // so we can tell if things moved around on us between the query and the getMore call
         DiskLoc locAtKeyOfs;
-        BoundList bounds_;
-        unsigned boundIndex_;
+        shared_ptr< FieldRangeVector > bounds_;
+        auto_ptr< FieldRangeVector::Iterator > _boundsIterator;
         const IndexSpec& _spec;
+        shared_ptr< CoveredIndexMatcher > _matcher;
+        bool _independentFieldRanges;
     };
 
-#pragma pack()
 
     inline bool IndexDetails::hasKey(const BSONObj& key) { 
-        return head.btree()->exists(*this, head, key, keyPattern());
+        return head.btree()->exists(*this, head, key, Ordering::make(keyPattern()));
     }
     inline bool IndexDetails::wouldCreateDup(const BSONObj& key, DiskLoc self) { 
-        return head.btree()->wouldCreateDup(*this, head, key, keyPattern(), self);
+        return head.btree()->wouldCreateDup(*this, head, key, Ordering::make(keyPattern()), self);
     }
 
     /* build btree from the bottom up */
@@ -403,6 +426,7 @@ namespace mongo {
         unsigned long long n;
         BSONObj keyLast;
         BSONObj order;
+        Ordering ordering;
         bool committed;
 
         DiskLoc cur, first;
diff --git a/db/btreecursor.cpp b/db/btreecursor.cpp
index ab15c44..d6d0c09 100644
--- a/db/btreecursor.cpp
+++ b/db/btreecursor.cpp
@@ -16,7 +16,7 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "btree.h"
 #include "pdfile.h"
 #include "jsobj.h"
@@ -35,29 +35,39 @@ namespace mongo {
             multikey( d->isMultikey( idxNo ) ),
             indexDetails( _id ),
             order( _id.keyPattern() ),
+            _ordering( Ordering::make( order ) ),
             direction( _direction ),
-            boundIndex_(),
-            _spec( _id.getSpec() )
+            _spec( _id.getSpec() ),
+            _independentFieldRanges( false )
     {
         audit();
         init();
+        DEV assert( dups.size() == 0 );
     }
 
-    BtreeCursor::BtreeCursor( NamespaceDetails *_d, int _idxNo, const IndexDetails& _id, const vector< pair< BSONObj, BSONObj > > &_bounds, int _direction )
+    BtreeCursor::BtreeCursor( NamespaceDetails *_d, int _idxNo, const IndexDetails& _id, const shared_ptr< FieldRangeVector > &_bounds, int _direction )
         :
             d(_d), idxNo(_idxNo), 
             endKeyInclusive_( true ),
             multikey( d->isMultikey( idxNo ) ),
             indexDetails( _id ),
             order( _id.keyPattern() ),
+            _ordering( Ordering::make( order ) ),
             direction( _direction ),
-            bounds_( _bounds ),
-            boundIndex_(),
-            _spec( _id.getSpec() )
+            bounds_( ( assert( _bounds.get() ), _bounds ) ),
+            _boundsIterator( new FieldRangeVector::Iterator( *bounds_  ) ),
+            _spec( _id.getSpec() ),
+            _independentFieldRanges( true )
     {
-        assert( !bounds_.empty() );
+        massert( 13384, "BtreeCursor FieldRangeVector constructor doesn't accept special indexes", !_spec.getType() );
         audit();
-        initInterval();
+        startKey = bounds_->startKey();
+        bool found;
+        _boundsIterator->advance( startKey ); // handles initialization
+        bucket = indexDetails.head.btree()->
+        locate(indexDetails, indexDetails.head, startKey, _ordering, keyOfs, found, direction > 0 ? minDiskLoc : maxDiskLoc, direction);
+        skipAndCheck();
+        DEV assert( dups.size() == 0 );
     }
 
     void BtreeCursor::audit() {
@@ -82,21 +92,41 @@ namespace mongo {
         }
         bool found;
         bucket = indexDetails.head.btree()->
-        locate(indexDetails, indexDetails.head, startKey, order, keyOfs, found, direction > 0 ? minDiskLoc : maxDiskLoc, direction);
-        skipUnusedKeys();
-        checkEnd();        
+            locate(indexDetails, indexDetails.head, startKey, _ordering, keyOfs, found, direction > 0 ? minDiskLoc : maxDiskLoc, direction);
+        skipUnusedKeys( false );
+        checkEnd();
     }
     
-    void BtreeCursor::initInterval() {
-        do {
-            startKey = bounds_[ boundIndex_ ].first;
-            endKey = bounds_[ boundIndex_ ].second;
-            init();
-        } while ( !ok() && ++boundIndex_ < bounds_.size() );
+    void BtreeCursor::skipAndCheck() {
+        skipUnusedKeys( true );
+        while( 1 ) {
+            if ( !skipOutOfRangeKeysAndCheckEnd() ) {
+                break;
+            }
+            while( skipOutOfRangeKeysAndCheckEnd() );
+            if ( !skipUnusedKeys( true ) ) {
+                break;
+            }
+        }
     }
-
+    
+    bool BtreeCursor::skipOutOfRangeKeysAndCheckEnd() {
+        if ( !ok() ) {
+            return false;
+        }
+        int ret = _boundsIterator->advance( currKeyNode().key );
+        if ( ret == -2 ) {
+            bucket = DiskLoc();
+            return false;
+        } else if ( ret == -1 ) {
+            return false;
+        }
+        advanceTo( currKeyNode().key, ret, _boundsIterator->cmp() );
+        return true;
+    }
+    
     /* skip unused keys. */
-    void BtreeCursor::skipUnusedKeys() {
+    bool BtreeCursor::skipUnusedKeys( bool mayJump ) {
         int u = 0;
         while ( 1 ) {
             if ( !ok() )
@@ -107,12 +137,16 @@ namespace mongo {
                 break;
             bucket = b->advance(bucket, keyOfs, direction, "skipUnusedKeys");
             u++;
+            if ( mayJump && ( u % 10 == 0 ) ) {
+                skipOutOfRangeKeysAndCheckEnd();
+            }
         }
         if ( u > 10 )
             OCCASIONALLY log() << "btree unused skipped:" << u << '\n';
+        return u;
     }
 
-// Return a value in the set {-1, 0, 1} to represent the sign of parameter i.
+    // Return a value in the set {-1, 0, 1} to represent the sign of parameter i.
     int sgn( int i ) {
         if ( i == 0 )
             return 0;
@@ -130,17 +164,26 @@ namespace mongo {
                 bucket = DiskLoc();
         }
     }
-
+    
+    void BtreeCursor::advanceTo( const BSONObj &keyBegin, int keyBeginLen, const vector< const BSONElement * > &keyEnd) {
+        bucket.btree()->advanceTo( indexDetails, bucket, keyOfs, keyBegin, keyBeginLen, keyEnd, _ordering, direction );
+    }
+    
     bool BtreeCursor::advance() {
         killCurrentOp.checkForInterrupt();
         if ( bucket.isNull() )
             return false;
+
         bucket = bucket.btree()->advance(bucket, keyOfs, direction, "BtreeCursor::advance");
-        skipUnusedKeys();
-        checkEnd();
-        if( !ok() && ++boundIndex_ < bounds_.size() )
-            initInterval();
-        return !bucket.isNull();
+
+        if ( !_independentFieldRanges ) {
+            skipUnusedKeys( false );
+            checkEnd();
+            return ok();
+        }
+        
+        skipAndCheck();
+        return ok();
     }
 
     void BtreeCursor::noteLocation() {
@@ -173,15 +216,25 @@ namespace mongo {
 
             // Note keyAt() returns an empty BSONObj if keyOfs is now out of range,
             // which is possible as keys may have been deleted.
-            if ( b->keyAt(keyOfs).woEqual(keyAtKeyOfs) &&
+            int x = 0;
+            while( 1 ) {
+                if ( b->keyAt(keyOfs).woEqual(keyAtKeyOfs) &&
                     b->k(keyOfs).recordLoc == locAtKeyOfs ) {
-                if ( !b->k(keyOfs).isUsed() ) {
-                    /* we were deleted but still exist as an unused
-                       marker key. advance.
-                    */
-                    skipUnusedKeys();
+                        if ( !b->k(keyOfs).isUsed() ) {
+                            /* we were deleted but still exist as an unused
+                            marker key. advance.
+                            */
+                            skipUnusedKeys( false );
+                        }
+                        return;
                 }
-                return;
+
+                /* we check one key earlier too, in case a key was just deleted.  this is 
+                   important so that multi updates are reasonably fast.
+                   */
+                if( keyOfs == 0 || x++ )
+                    break;
+                keyOfs--;
             }
         }
 
@@ -192,10 +245,10 @@ namespace mongo {
         bool found;
 
         /* TODO: Switch to keep indexdetails and do idx.head! */
-        bucket = indexDetails.head.btree()->locate(indexDetails, indexDetails.head, keyAtKeyOfs, order, keyOfs, found, locAtKeyOfs, direction);
+        bucket = indexDetails.head.btree()->locate(indexDetails, indexDetails.head, keyAtKeyOfs, _ordering, keyOfs, found, locAtKeyOfs, direction);
         RARELY log() << "  key seems to have moved in the index, refinding. found:" << found << endl;
         if ( ! bucket.isNull() )
-            skipUnusedKeys();
+            skipUnusedKeys( false );
 
     }
 
diff --git a/db/cap.cpp b/db/cap.cpp
new file mode 100644
index 0000000..c676429
--- /dev/null
+++ b/db/cap.cpp
@@ -0,0 +1,393 @@
+// @file cap.cpp capped collection related 
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "pdfile.h"
+#include "db.h"
+#include "../util/mmap.h"
+#include "../util/hashtab.h"
+#include "../scripting/engine.h"
+#include "btree.h"
+#include <algorithm>
+#include <list>
+#include "query.h"
+#include "queryutil.h"
+#include "json.h"
+
+/*
+ capped collection layout
+
+ d's below won't exist if things align perfectly:
+
+ extent1             -> extent2                 -> extent3
+ -------------------    -----------------------    ---------------------
+ d r r r r r r r r d    d r r r r d r r r r r d    d r r r r r r r r r d
+                                ^   ^
+                           oldest   newest
+
+                        ^cappedFirstDeletedInCurExtent()
+                   ^cappedLastDelRecLastExtent()
+ ^cappedListOfAllDeletedRecords()
+*/
+
+
+namespace mongo {
+
+    /* combine adjacent deleted records *for the current extent* of the capped collection
+     
+       this is O(n^2) but we call it for capped tables where typically n==1 or 2!
+       (or 3...there will be a little unused sliver at the end of the extent.)
+    */
+    void NamespaceDetails::compact() {
+        assert(capped);
+
+        list<DiskLoc> drecs;
+
+        // Pull out capExtent's DRs from deletedList
+        DiskLoc i = cappedFirstDeletedInCurExtent();
+        for (; !i.isNull() && inCapExtent( i ); i = i.drec()->nextDeleted )
+            drecs.push_back( i );
+        cappedFirstDeletedInCurExtent() = i;
+
+        // This is the O(n^2) part.
+        drecs.sort();
+
+        list<DiskLoc>::iterator j = drecs.begin();
+        assert( j != drecs.end() );
+        DiskLoc a = *j;
+        while ( 1 ) {
+            j++;
+            if ( j == drecs.end() ) {
+                DEBUGGING out() << "TEMP: compact adddelrec\n";
+                addDeletedRec(a.drec(), a);
+                break;
+            }
+            DiskLoc b = *j;
+            while ( a.a() == b.a() && a.getOfs() + a.drec()->lengthWithHeaders == b.getOfs() ) {
+                // a & b are adjacent.  merge.
+                a.drec()->lengthWithHeaders += b.drec()->lengthWithHeaders;
+                j++;
+                if ( j == drecs.end() ) {
+                    DEBUGGING out() << "temp: compact adddelrec2\n";
+                    addDeletedRec(a.drec(), a);
+                    return;
+                }
+                b = *j;
+            }
+            DEBUGGING out() << "temp: compact adddelrec3\n";
+            addDeletedRec(a.drec(), a);
+            a = b;
+        }
+    }
+
+    DiskLoc &NamespaceDetails::cappedFirstDeletedInCurExtent() {
+        if ( cappedLastDelRecLastExtent().isNull() )
+            return cappedListOfAllDeletedRecords();
+        else
+            return cappedLastDelRecLastExtent().drec()->nextDeleted;
+    }
+
+    void NamespaceDetails::cappedCheckMigrate() {
+        // migrate old NamespaceDetails format
+        assert( capped );
+        if ( capExtent.a() == 0 && capExtent.getOfs() == 0 ) {
+            capFirstNewRecord = DiskLoc();
+            capFirstNewRecord.setInvalid();
+            // put all the DeletedRecords in cappedListOfAllDeletedRecords()
+            for ( int i = 1; i < Buckets; ++i ) {
+                DiskLoc first = deletedList[ i ];
+                if ( first.isNull() )
+                    continue;
+                DiskLoc last = first;
+                for (; !last.drec()->nextDeleted.isNull(); last = last.drec()->nextDeleted );
+                last.drec()->nextDeleted = cappedListOfAllDeletedRecords();
+                cappedListOfAllDeletedRecords() = first;
+                deletedList[ i ] = DiskLoc();
+            }
+            // NOTE cappedLastDelRecLastExtent() set to DiskLoc() in above
+
+            // Last, in case we're killed before getting here
+            capExtent = firstExtent;
+        }
+    }
+
+    bool NamespaceDetails::inCapExtent( const DiskLoc &dl ) const {
+        assert( !dl.isNull() );
+        // We could have a rec or drec, doesn't matter.
+        return dl.drec()->myExtent( dl ) == capExtent.ext();
+    }
+
+    bool NamespaceDetails::nextIsInCapExtent( const DiskLoc &dl ) const {
+        assert( !dl.isNull() );
+        DiskLoc next = dl.drec()->nextDeleted;
+        if ( next.isNull() )
+            return false;
+        return inCapExtent( next );
+    }
+
+    void NamespaceDetails::advanceCapExtent( const char *ns ) {
+        // We want cappedLastDelRecLastExtent() to be the last DeletedRecord of the prev cap extent
+        // (or DiskLoc() if new capExtent == firstExtent)
+        if ( capExtent == lastExtent )
+            cappedLastDelRecLastExtent() = DiskLoc();
+        else {
+            DiskLoc i = cappedFirstDeletedInCurExtent();
+            for (; !i.isNull() && nextIsInCapExtent( i ); i = i.drec()->nextDeleted );
+            cappedLastDelRecLastExtent() = i;
+        }
+
+        capExtent = theCapExtent()->xnext.isNull() ? firstExtent : theCapExtent()->xnext;
+
+        /* this isn't true if a collection has been renamed...that is ok just used for diagnostics */
+        //dassert( theCapExtent()->ns == ns );
+
+        theCapExtent()->assertOk();
+        capFirstNewRecord = DiskLoc();
+    }
+
+    DiskLoc NamespaceDetails::__capAlloc( int len ) {
+        DiskLoc prev = cappedLastDelRecLastExtent();
+        DiskLoc i = cappedFirstDeletedInCurExtent();
+        DiskLoc ret;
+        for (; !i.isNull() && inCapExtent( i ); prev = i, i = i.drec()->nextDeleted ) {
+            // We need to keep at least one DR per extent in cappedListOfAllDeletedRecords(),
+            // so make sure there's space to create a DR at the end.
+            if ( i.drec()->lengthWithHeaders >= len + 24 ) {
+                ret = i;
+                break;
+            }
+        }
+
+        /* unlink ourself from the deleted list */
+        if ( !ret.isNull() ) {
+            if ( prev.isNull() )
+                cappedListOfAllDeletedRecords() = ret.drec()->nextDeleted;
+            else
+                prev.drec()->nextDeleted = ret.drec()->nextDeleted;
+            ret.drec()->nextDeleted.setInvalid(); // defensive.
+            assert( ret.drec()->extentOfs < ret.getOfs() );
+        }
+
+        return ret;
+    }
+
+    DiskLoc NamespaceDetails::cappedAlloc(const char *ns, int len) { 
+        // signal done allocating new extents.
+        if ( !cappedLastDelRecLastExtent().isValid() )
+            cappedLastDelRecLastExtent() = DiskLoc();
+        
+        assert( len < 400000000 );
+        int passes = 0;
+        int maxPasses = ( len / 30 ) + 2; // 30 is about the smallest entry that could go in the oplog
+        if ( maxPasses < 5000 ){
+            // this is for bacwards safety since 5000 was the old value
+            maxPasses = 5000;
+        }
+        DiskLoc loc;
+
+        // delete records until we have room and the max # objects limit achieved.
+
+        /* this fails on a rename -- that is ok but must keep commented out */
+        //assert( theCapExtent()->ns == ns );
+
+        theCapExtent()->assertOk();
+        DiskLoc firstEmptyExtent;
+        while ( 1 ) {
+            if ( nrecords < max ) {
+                loc = __capAlloc( len );
+                if ( !loc.isNull() )
+                    break;
+            }
+
+            // If on first iteration through extents, don't delete anything.
+            if ( !capFirstNewRecord.isValid() ) {
+                advanceCapExtent( ns );
+                if ( capExtent != firstExtent )
+                    capFirstNewRecord.setInvalid();
+                // else signal done with first iteration through extents.
+                continue;
+            }
+
+            if ( !capFirstNewRecord.isNull() &&
+                    theCapExtent()->firstRecord == capFirstNewRecord ) {
+                // We've deleted all records that were allocated on the previous
+                // iteration through this extent.
+                advanceCapExtent( ns );
+                continue;
+            }
+
+            if ( theCapExtent()->firstRecord.isNull() ) {
+                if ( firstEmptyExtent.isNull() )
+                    firstEmptyExtent = capExtent;
+                advanceCapExtent( ns );
+                if ( firstEmptyExtent == capExtent ) {
+                    maybeComplain( ns, len );
+                    return DiskLoc();
+                }
+                continue;
+            }
+
+            DiskLoc fr = theCapExtent()->firstRecord;
+            theDataFileMgr.deleteRecord(ns, fr.rec(), fr, true); // ZZZZZZZZZZZZ
+            compact();
+            if( ++passes > maxPasses ) {
+                log() << "passes ns:" << ns << " len:" << len << " maxPasses: " << maxPasses << '\n';
+                log() << "passes max:" << max << " nrecords:" << nrecords << " datasize: " << datasize << endl;
+                massert( 10345 ,  "passes >= maxPasses in capped collection alloc", false );
+            }
+        }
+
+        // Remember first record allocated on this iteration through capExtent.
+        if ( capFirstNewRecord.isValid() && capFirstNewRecord.isNull() )
+            capFirstNewRecord = loc;
+
+        return loc;
+    }
+
+    void NamespaceDetails::dumpExtents() {
+        cout << "dumpExtents:" << endl;
+        for ( DiskLoc i = firstExtent; !i.isNull(); i = i.ext()->xnext ) {
+            Extent *e = i.ext();
+            stringstream ss;
+            e->dump(ss);
+            cout << ss.str() << endl;
+        }
+    }
+
+    void NamespaceDetails::cappedDumpDelInfo() { 
+        cout << "dl[0]: " << deletedList[0].toString() << endl;
+        for( DiskLoc z = deletedList[0]; !z.isNull(); z = z.drec()->nextDeleted ) { 
+            cout << "  drec:" << z.toString() << " dreclen:" << hex << z.drec()->lengthWithHeaders << 
+                " ext:" << z.drec()->myExtent(z)->myLoc.toString() << endl;
+        }
+        cout << "dl[1]: " << deletedList[1].toString() << endl;
+    }
+
+    /* everything from end on, eliminate from the capped collection.
+       @param inclusive if true, deletes end (i.e. closed or open range)
+    */
+    void NamespaceDetails::cappedTruncateAfter(const char *ns, DiskLoc end, bool inclusive) {
+        DEV assert( this == nsdetails(ns) );
+        assert( cappedLastDelRecLastExtent().isValid() );
+        
+        bool foundLast = false;
+        while( 1 ) {
+            if ( foundLast ) {
+                break;
+            }
+            DiskLoc curr = theCapExtent()->lastRecord;
+            assert( !curr.isNull() );
+            if ( curr == end ) {
+                if ( inclusive ) {
+                    foundLast = true;
+                } else {
+                    break;
+                }
+            }
+            
+            uassert( 13415, "emptying the collection is not allowed", nrecords > 1 );
+            
+            if ( !capLooped() ) {
+                theDataFileMgr.deleteRecord(ns, curr.rec(), curr, true);
+                compact();
+                if ( theCapExtent()->lastRecord.isNull() ) {
+                    assert( !theCapExtent()->xprev.isNull() );
+                    capExtent = theCapExtent()->xprev;
+                    theCapExtent()->assertOk();
+                    if ( capExtent == firstExtent ) {
+                        cappedLastDelRecLastExtent() = DiskLoc();
+                    } else {
+                        // slow - there's no prev ptr for deleted rec
+                        DiskLoc i = cappedListOfAllDeletedRecords();
+                        for( ;
+                            !i.drec()->nextDeleted.isNull() &&
+                            !inCapExtent( i.drec()->nextDeleted );
+                            i = i.drec()->nextDeleted );
+                        assert( !i.drec()->nextDeleted.isNull() ); // I believe there is always at least one drec per extent
+                        cappedLastDelRecLastExtent() = i;
+                    }
+                }
+                continue;
+            }
+
+            theDataFileMgr.deleteRecord(ns, curr.rec(), curr, true);
+            compact();
+            if ( curr == capFirstNewRecord ) { // invalid, but can compare locations
+                capExtent = ( capExtent == firstExtent ) ? lastExtent : theCapExtent()->xprev;
+                theCapExtent()->assertOk();
+                assert( !theCapExtent()->firstRecord.isNull() );
+                capFirstNewRecord = theCapExtent()->firstRecord;
+                if ( capExtent == firstExtent ) {
+                    cappedLastDelRecLastExtent() = DiskLoc();
+                } else {
+                    // slow - there's no prev ptr for deleted rec
+                    DiskLoc i = cappedListOfAllDeletedRecords();
+                    for( ;
+                        !i.drec()->nextDeleted.isNull() &&
+                        !inCapExtent( i.drec()->nextDeleted );
+                        i = i.drec()->nextDeleted );
+                    assert( !i.drec()->nextDeleted.isNull() ); // I believe there is always at least one drec per extent
+                    cappedLastDelRecLastExtent() = i;
+                }
+            }
+        }
+    }
+    
+    void NamespaceDetails::emptyCappedCollection( const char *ns ) {
+        DEV assert( this == nsdetails(ns) );
+        massert( 13424, "collection must be capped", capped );
+        massert( 13425, "background index build in progress", !backgroundIndexBuildInProgress );
+        massert( 13426, "indexes present", nIndexes == 0 );
+
+        ClientCursor::invalidate( ns );
+		NamespaceDetailsTransient::clearForPrefix( ns );
+
+        cappedLastDelRecLastExtent() = DiskLoc();
+        cappedListOfAllDeletedRecords() = DiskLoc();
+        
+        // preserve firstExtent/lastExtent
+        capExtent = firstExtent;
+        datasize = nrecords = 0;
+        // lastExtentSize preserve
+        // nIndexes preserve 0
+        // capped preserve true
+        // max preserve
+        paddingFactor = 1.0;
+        flags = 0;
+        capFirstNewRecord = DiskLoc();
+        capFirstNewRecord.setInvalid();
+        cappedLastDelRecLastExtent().setInvalid();
+        // dataFileVersion preserve
+        // indexFileVersion preserve
+        multiKeyIndexBits = 0;
+        reservedA = 0;
+        extraOffset = 0;
+        // backgroundIndexBuildInProgress preserve 0
+        memset(reserved, 0, sizeof(reserved));
+
+        for( DiskLoc ext = firstExtent; !ext.isNull(); ext = ext.ext()->xnext ) {
+            DiskLoc prev = ext.ext()->xprev;
+            DiskLoc next = ext.ext()->xnext;
+            DiskLoc empty = ext.ext()->reuse( ns );
+            ext.ext()->xprev = prev;
+            ext.ext()->xnext = next;
+            addDeletedRec( empty.drec(), empty );
+        }
+    }
+
+}
diff --git a/db/client.cpp b/db/client.cpp
index a2fe568..65c467a 100644
--- a/db/client.cpp
+++ b/db/client.cpp
@@ -20,16 +20,22 @@
    to an open socket (or logical connection if pooling on sockets) from a client.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "db.h"
 #include "client.h"
 #include "curop.h"
 #include "json.h"
 #include "security.h"
+#include "commands.h"
+#include "instance.h"
+#include "../s/d_logic.h"
+#include "dbwebserver.h"
+#include "../util/mongoutils/html.h"
+#include "../util/mongoutils/checksum.h"
 
 namespace mongo {
 
-    mongo::mutex Client::clientsMutex;
+    mongo::mutex Client::clientsMutex("clientsMutex");
     set<Client*> Client::clients; // always be in clientsMutex when manipulating this
     boost::thread_specific_ptr<Client> currentClient;
 
@@ -37,7 +43,8 @@ namespace mongo {
       _context(0),
       _shutdown(false),
       _desc(desc),
-      _god(0)
+      _god(0),
+      _lastOp(0)
     {
         _curOp = new CurOp( this );
         scoped_lock bl(clientsMutex);
@@ -54,6 +61,56 @@ namespace mongo {
             cout << "ERROR: Client::shutdown not called: " << _desc << endl;
     }
 
+    void Client::_dropns( const string& ns ){
+        Top::global.collectionDropped( ns );
+                    
+        dblock l;
+        Client::Context ctx( ns );
+        if ( ! nsdetails( ns.c_str() ) )
+            return;
+        
+        try {
+            string err;
+            BSONObjBuilder b;
+            dropCollection( ns , err , b );
+        }
+        catch ( ... ){
+            log() << "error dropping temp collection: " << ns << endl;
+        }
+
+    }
+    
+    void Client::_invalidateDB( const string& db ) {
+        assert( db.find( '.' ) == string::npos );
+
+        set<string>::iterator min = _tempCollections.lower_bound( db + "." );
+        set<string>::iterator max = _tempCollections.lower_bound( db + "|" );
+        
+        _tempCollections.erase( min , max );
+
+    }
+    
+    void Client::invalidateDB(const string& db) {
+        scoped_lock bl(clientsMutex);
+        for ( set<Client*>::iterator i = clients.begin(); i!=clients.end(); i++ ){
+            Client* cli = *i;
+            cli->_invalidateDB(db);
+        }
+    }
+
+    void Client::invalidateNS( const string& ns ){
+        scoped_lock bl(clientsMutex);
+        for ( set<Client*>::iterator i = clients.begin(); i!=clients.end(); i++ ){
+            Client* cli = *i;
+            cli->_tempCollections.erase( ns );
+        }
+    }
+
+
+    void Client::addTempCollection( const string& ns ) { 
+        _tempCollections.insert( ns ); 
+    }
+
     bool Client::shutdown(){
         _shutdown = true;
         if ( inShutdown() )
@@ -67,22 +124,8 @@ namespace mongo {
         
         if ( _tempCollections.size() ){
             didAnything = true;
-            for ( list<string>::iterator i = _tempCollections.begin(); i!=_tempCollections.end(); i++ ){
-                string ns = *i;
-                Top::global.collectionDropped( ns );
-                    
-                dblock l;
-                Client::Context ctx( ns );
-                if ( ! nsdetails( ns.c_str() ) )
-                    continue;
-                try {
-                    string err;
-                    BSONObjBuilder b;
-                    dropCollection( ns , err , b );
-                }
-                catch ( ... ){
-                    log() << "error dropping temp collection: " << ns << endl;
-                }
+            for ( set<string>::iterator i = _tempCollections.begin(); i!=_tempCollections.end(); i++ ){
+                _dropns( *i );
             }
             _tempCollections.clear();
         }
@@ -152,8 +195,15 @@ namespace mongo {
         _client->_curOp->enter( this );
         if ( doauth )
             _auth( lockState );
-    }
 
+        if ( _client->_curOp->getOp() != dbGetMore ){ // getMore's are special and should be handled else where
+            string errmsg;
+            if ( ! shardVersionOk( _ns , errmsg ) ){
+                msgasserted( StaleConfigInContextCode , (string)"[" + _ns + "] shard version not ok in Client::Context: " + errmsg );
+            }
+        }
+    }
+    
     void Client::Context::_auth( int lockState ){
         if ( _client->_ai.isAuthorizedForLock( _db->name , lockState ) )
             return;
@@ -162,8 +212,8 @@ namespace mongo {
         _client->_context = _oldContext; // note: _oldContext may be null
         
         stringstream ss;
-        ss << "unauthorized for db [" << _db->name << "] lock type: " << lockState << endl;
-        massert( 10057 , ss.str() , 0 );
+        ss << "unauthorized db:" << _db->name << " lock type:" << lockState << " client:" << _client->clientAddress();
+        uasserted( 10057 , ss.str() );
     }
 
     Client::Context::~Context() {
@@ -172,6 +222,12 @@ namespace mongo {
         _client->_context = _oldContext; // note: _oldContext may be null
     }
 
+    string Client::clientAddress() const {
+        if( _curOp )
+            return _curOp->getRemoteString(false);
+        return "";
+    }
+
     string Client::toString() const {
         stringstream ss;
         if ( _curOp )
@@ -203,7 +259,7 @@ namespace mongo {
         }
     }
 
-    BSONObj CurOp::infoNoauth() {
+    BSONObj CurOp::infoNoauth( int attempt ) {
         BSONObjBuilder b;
         b.append("opid", _opNum);
         bool a = _active && _start;
@@ -220,12 +276,35 @@ namespace mongo {
         
         b.append("ns", _ns);
         
-        if( haveQuery() ) {
-            b.append("query", query());
+        {
+            int size = querySize();
+            if ( size == 0 ){
+                // do nothing
+            }
+            else if ( size == 1 ){
+                b.append( "query" , _tooBig );
+            }
+            else if ( attempt > 2 ){
+                b.append( "query" , BSON( "err" << "can't get a clean object" ) );
+                log( LL_WARNING ) << "CurOp changing too much to get reading" << endl;
+                         
+            }
+            else {
+                int before = checksum( _queryBuf , size );
+                b.appendObject( "query" , _queryBuf , size );
+                int after = checksum( _queryBuf , size );
+                
+                if ( after != before ){
+                    // this means something changed
+                    // going to retry
+                    return infoNoauth( attempt + 1 );
+                }
+            }
         }
+
         // b.append("inLock",  ??
         stringstream clientStr;
-        clientStr << inet_ntoa( _remote.sin_addr ) << ":" << ntohs( _remote.sin_port );
+        clientStr << _remote.toString();
         b.append("client", clientStr.str());
 
         if ( _client )
@@ -234,32 +313,139 @@ namespace mongo {
         if ( ! _message.empty() ){
             if ( _progressMeter.isActive() ){
                 StringBuilder buf(128);
-                buf << _message << " " << _progressMeter.toString();
+                buf << _message.toString() << " " << _progressMeter.toString();
                 b.append( "msg" , buf.str() );
             }
             else {
-                b.append( "msg" , _message );
+                b.append( "msg" , _message.toString() );
             }
         }
 
         return b.obj();
     }
 
-    int Client::recommendedYieldMicros(){
+    void Client::gotHandshake( const BSONObj& o ){
+        BSONObjIterator i(o);
+
+        {
+            BSONElement id = i.next();
+            assert( id.type() );
+            _remoteId = id.wrap( "_id" );
+        }
+        
+        BSONObjBuilder b;
+        while ( i.more() )
+            b.append( i.next() );
+        _handshake = b.obj();
+    }
+
+    class HandshakeCmd : public Command {
+    public:
+        void help(stringstream& h) const { h << "internal"; }
+        HandshakeCmd() : Command( "handshake" ){}
+        virtual LockType locktype() const { return NONE; } 
+        virtual bool slaveOk() const { return true; }
+        virtual bool adminOnly() const { return false; }
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            Client& c = cc();
+            c.gotHandshake( cmdObj );
+            return 1;
+        }        
+
+    } handshakeCmd;
+
+    class ClientListPlugin : public WebStatusPlugin {
+    public:
+        ClientListPlugin() : WebStatusPlugin( "clients" , 20 ){}
+        virtual void init(){}
+        
+        virtual void run( stringstream& ss ){
+            using namespace mongoutils::html;
+
+            ss << "\n<table border=1 cellpadding=2 cellspacing=0>";
+            ss << "<tr align='left'>"
+               << th( a("", "Connections to the database, both internal and external.", "Client") )
+               << th( a("http://www.mongodb.org/display/DOCS/Viewing+and+Terminating+Current+Operation", "", "OpId") )
+               << "<th>Active</th>" 
+               << "<th>LockType</th>"
+               << "<th>Waiting</th>"
+               << "<th>SecsRunning</th>"
+               << "<th>Op</th>"
+               << th( a("http://www.mongodb.org/display/DOCS/Developer+FAQ#DeveloperFAQ-What%27sa%22namespace%22%3F", "", "Namespace") )
+               << "<th>Query</th>"
+               << "<th>client</th>"
+               << "<th>msg</th>"
+               << "<th>progress</th>"
+
+               << "</tr>\n";
+            {
+                scoped_lock bl(Client::clientsMutex);
+                for( set<Client*>::iterator i = Client::clients.begin(); i != Client::clients.end(); i++ ) { 
+                    Client *c = *i;
+                    CurOp& co = *(c->curop());
+                    ss << "<tr><td>" << c->desc() << "</td>";
+                    
+                    tablecell( ss , co.opNum() );
+                    tablecell( ss , co.active() );
+                    {
+                        int lt = co.getLockType();
+                        if( lt == -1 ) tablecell(ss, "R");
+                        else if( lt == 1 ) tablecell(ss, "W");
+                        else
+                            tablecell( ss ,  lt);
+                    }
+                    tablecell( ss , co.isWaitingForLock() );
+                    if ( co.active() )
+                        tablecell( ss , co.elapsedSeconds() );
+                    else
+                        tablecell( ss , "" );
+                    tablecell( ss , co.getOp() );
+                    tablecell( ss , co.getNS() );
+                    if ( co.haveQuery() )
+                        tablecell( ss , co.query() );
+                    else
+                        tablecell( ss , "" );
+                    tablecell( ss , co.getRemoteString() );
+
+                    tablecell( ss , co.getMessage() );
+                    tablecell( ss , co.getProgressMeter().toString() );
+
+
+                    ss << "</tr>\n";
+                }
+            }
+            ss << "</table>\n";
+
+        }
+        
+    } clientListPlugin;
+
+    int Client::recommendedYieldMicros( int * writers , int * readers ){
         int num = 0;
+        int w = 0;
+        int r = 0;
         {
             scoped_lock bl(clientsMutex);
-            num = clients.size();
+            for ( set<Client*>::iterator i=clients.begin(); i!=clients.end(); ++i ){
+                Client* c = *i;
+                if ( c->curop()->isWaitingForLock() ){
+                    num++;
+                    if ( c->curop()->getLockType() > 0 )
+                        w++;
+                    else
+                        r++;
+                }
+            }
         }
         
-        if ( --num <= 0 ) // -- is for myself
-            return 0;
-        
+        if ( writers )
+            *writers = w;
+        if ( readers )
+            *readers = r;
+
         if ( num > 50 )
             num = 50;
 
-        num *= 100;
-        return num;
+        return num * 100;
     }
-
 }
diff --git a/db/client.h b/db/client.h
index c484198..2456d7f 100644
--- a/db/client.h
+++ b/db/client.h
@@ -24,14 +24,17 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "security.h"
 #include "namespace.h"
 #include "lasterror.h"
 #include "stats/top.h"
+//#include "repl/rs.h"
 
 namespace mongo { 
 
+    extern class ReplSet *theReplSet;
+
     class AuthenticationInfo;
     class Database;
     class CurOp;
@@ -45,7 +48,7 @@ namespace mongo {
         static mongo::mutex clientsMutex;
         static set<Client*> clients; // always be in clientsMutex when manipulating this
 
-        static int recommendedYieldMicros();
+        static int recommendedYieldMicros( int * writers = 0 , int * readers = 0 );
 
         class GodScope {
             bool _prev;
@@ -80,16 +83,16 @@ namespace mongo {
         public:
             Context(const string& ns, string path=dbpath, mongolock * lock = 0 , bool doauth=true ) 
                 : _client( currentClient.get() ) , _oldContext( _client->_context ) , 
-                  _path( path ) , _lock( lock ) ,
-                  _ns( ns ){
+                  _path( path ) , _lock( lock ) , 
+                  _ns( ns ), _db(0){
                 _finishInit( doauth );
             }
             
             /* this version saves the context but doesn't yet set the new one: */
-
+            
             Context() 
                 : _client( currentClient.get() ) , _oldContext( _client->_context ), 
-                  _path( dbpath ) , _lock(0) , _justCreated(false){
+                  _path( dbpath ) , _lock(0) , _justCreated(false), _db(0){
                 _client->_context = this;
                 clear();
             }
@@ -101,20 +104,11 @@ namespace mongo {
             Context( string ns , Database * db, bool doauth=true );
             
             ~Context();
-            
-            Client* getClient() const { return _client; }
-            
-            Database* db() const {
-                return _db;
-            }
 
-            const char * ns() const {
-                return _ns.c_str();
-            }
-            
-            bool justCreated() const {
-                return _justCreated;
-            }
+            Client* getClient() const { return _client; }            
+            Database* db() const { return _db; }
+            const char * ns() const { return _ns.c_str(); }            
+            bool justCreated() const { return _justCreated; }
 
             bool equals( const string& ns , const string& path=dbpath ) const {
                 return _ns == ns && _path == path;
@@ -160,29 +154,53 @@ namespace mongo {
         CurOp * _curOp;
         Context * _context;
         bool _shutdown;
-        list<string> _tempCollections;
+        set<string> _tempCollections;
         const char *_desc;
         bool _god;
         AuthenticationInfo _ai;
+        ReplTime _lastOp;
+        BSONObj _handshake;
+        BSONObj _remoteId;
+
+        void _dropns( const string& ns );
 
     public:
-        
+        string clientAddress() const;
         AuthenticationInfo * getAuthenticationInfo(){ return &_ai; }
         bool isAdmin() { return _ai.isAuthorized( "admin" ); }
-
-        CurOp* curop() { return _curOp; }
-        
+        CurOp* curop() { return _curOp; }        
         Context* getContext(){ return _context; }
         Database* database() {  return _context ? _context->db() : 0; }
-        const char *ns() { return _context->ns(); }
+        const char *ns() const { return _context->ns(); }
+        const char *desc() const { return _desc; }
         
         Client(const char *desc);
         ~Client();
 
-        const char *desc() const { return _desc; }
+        void addTempCollection( const string& ns );
+        
+        void _invalidateDB(const string& db);
+        static void invalidateDB(const string& db);
+
+        static void invalidateNS( const string& ns );
 
-        void addTempCollection( const string& ns ){
-            _tempCollections.push_back( ns );
+        void setLastOp( ReplTime op ) {
+            _lastOp = op;
+        }
+
+        ReplTime getLastOp() const {
+            return _lastOp;
+        }
+
+        void appendLastOp( BSONObjBuilder& b ) {
+            if( theReplSet ) { 
+                b.append("lastOp" , (long long) _lastOp);
+            }
+            else {
+                OpTime lo(_lastOp);
+                if ( ! lo.isNull() )
+                    b.appendTimestamp( "lastOp" , lo.asDate() );
+            }
         }
 
         /* each thread which does db operations has a Client object in TLS.  
@@ -196,23 +214,34 @@ namespace mongo {
          */
         bool shutdown();
 
+        
+        /* this is for map/reduce writes */
         bool isGod() const { return _god; }
 
         friend class CurOp;
 
         string toString() const;
+
+        void gotHandshake( const BSONObj& o );
+
+        BSONObj getRemoteID() const { return _remoteId; }
+        BSONObj getHandshake() const { return _handshake; }
     };
     
     inline Client& cc() { 
-        return *currentClient.get();
+        Client * c = currentClient.get();
+        assert( c );
+        return *c;
     }
 
     /* each thread which does db operations has a Client object in TLS.  
        call this when your thread starts. 
     */
     inline void Client::initThread(const char *desc) {
+        setThreadName(desc);
         assert( currentClient.get() == 0 );
         currentClient.reset( new Client(desc) );
+        mongo::lastError.initThread();
     }
 
     inline Client::GodScope::GodScope(){
diff --git a/db/clientcursor.cpp b/db/clientcursor.cpp
index 1281fc3..ad7f9ce 100644
--- a/db/clientcursor.cpp
+++ b/db/clientcursor.cpp
@@ -22,12 +22,13 @@
    Cursor -- and its derived classes -- are our internal cursors.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "query.h"
 #include "introspect.h"
 #include <time.h>
 #include "db.h"
 #include "commands.h"
+#include "repl_block.h"
 
 namespace mongo {
 
@@ -128,7 +129,7 @@ namespace mongo {
 
         while ( 1 ) {
             toAdvance.push_back(j->second);
-            WIN assert( j->first == dl );
+            DEV assert( j->first == dl );
             ++j;
             if ( j == stop )
                 break;
@@ -155,8 +156,8 @@ namespace mongo {
             }
             c->advance();
             if ( c->eof() ) {
-                // advanced to end -- delete cursor
-                delete cc;
+                // advanced to end
+                // leave ClieneCursor in place so next getMore doesn't fail
             }
             else {
                 wassert( c->refLoc() != dl );
@@ -190,29 +191,67 @@ namespace mongo {
         DiskLoc cl = c->refLoc();
         if ( lastLoc() == cl ) {
             //log() << "info: lastloc==curloc " << ns << '\n';
-            return;
-        }
-        {
+        } else {
             recursive_scoped_lock lock(ccmutex);
             setLastLoc_inlock(cl);
-            c->noteLocation();
         }
+        // may be necessary for MultiCursor even when cl hasn't changed
+        c->noteLocation();
     }
     
-    bool ClientCursor::yield() {
-        // need to store on the stack in case this gets deleted
-        CursorId id = cursorid;
+    int ClientCursor::yieldSuggest() {
+        int writers = 0;
+        int readers = 0;
+        
+        int micros = Client::recommendedYieldMicros( &writers , &readers );
+        
+        if ( micros > 0 && writers == 0 && dbMutex.getState() <= 0 ){
+            // we have a read lock, and only reads are coming on, so why bother unlocking
+            micros = 0;
+        }
+        
+        return micros;
+    }
+    
+    bool ClientCursor::yieldSometimes(){
+        if ( ! _yieldSometimesTracker.ping() )
+            return true;
 
-        bool doingDeletes = _doingDeletes;
-        _doingDeletes = false;
+        int micros = yieldSuggest();
+        return ( micros > 0 ) ? yield( micros ) : true;
+    }
 
+    void ClientCursor::staticYield( int micros ) {
+        {
+            dbtempreleasecond unlock;
+            if ( unlock.unlocked() ){
+                if ( micros == -1 )
+                    micros = Client::recommendedYieldMicros();
+                if ( micros > 0 )
+                    sleepmicros( micros ); 
+            }
+            else {
+                log( LL_WARNING ) << "ClientCursor::yield can't unlock b/c of recursive lock" << endl;
+            }
+        }        
+    }
+    
+    bool ClientCursor::prepareToYield( YieldData &data ) {
+        if ( ! c->supportYields() )
+            return false;
+        // need to store in case 'this' gets deleted
+        data._id = cursorid;
+        
+        data._doingDeletes = _doingDeletes;
+        _doingDeletes = false;
+        
         updateLocation();
-
+        
         {
             /* a quick test that our temprelease is safe. 
-               todo: make a YieldingCursor class 
-               and then make the following code part of a unit test.
-            */
+             todo: make a YieldingCursor class 
+             and then make the following code part of a unit test.
+             */
             const int test = 0;
             static bool inEmpty = false;
             if( test && !inEmpty ) { 
@@ -228,20 +267,31 @@ namespace mongo {
                     dropDatabase(ns.c_str());
                 }
             }
-        }
-            
-        {
-            dbtempreleasecond unlock;
-            sleepmicros( Client::recommendedYieldMicros() );
-        }
-
-        if ( ClientCursor::find( id , false ) == 0 ){
-            // i was deleted
+        }        
+        return true;
+    }
+    
+    bool ClientCursor::recoverFromYield( const YieldData &data ) {
+        ClientCursor *cc = ClientCursor::find( data._id , false );
+        if ( cc == 0 ){
+            // id was deleted
             return false;
         }
+        
+        cc->_doingDeletes = data._doingDeletes;
+        cc->c->checkLocation();
+        return true;        
+    }
+    
+    bool ClientCursor::yield( int micros ) {
+        if ( ! c->supportYields() )
+            return true;
+        YieldData data; 
+        prepareToYield( data );
+        
+        staticYield( micros );
 
-        _doingDeletes = doingDeletes;
-        return true;
+        return ClientCursor::recoverFromYield( data );
     }
 
     int ctmLast = 0; // so we don't have to do find() which is a little slow very often.
@@ -255,26 +305,75 @@ namespace mongo {
                 break;
         }
         ctmLast = ctm;
-        DEV out() << "  alloccursorid " << x << endl;
+        //DEV tlog() << "  alloccursorid " << x << endl;
         return x;
     }
 
+    void ClientCursor::storeOpForSlave( DiskLoc last ){
+        if ( ! ( _queryOptions & QueryOption_OplogReplay ))
+            return;
+
+        if ( last.isNull() )
+            return;
+        
+        BSONElement e = last.obj()["ts"];
+        if ( e.type() == Date || e.type() == Timestamp )
+            _slaveReadTill = e._opTime();
+    }
+    
+    void ClientCursor::updateSlaveLocation( CurOp& curop ){
+        if ( _slaveReadTill.isNull() )
+            return;
+        mongo::updateSlaveLocation( curop , ns.c_str() , _slaveReadTill );
+    }
+
+
+
     // QUESTION: Restrict to the namespace from which this command was issued?
     // Alternatively, make this command admin-only?
     class CmdCursorInfo : public Command {
     public:
-        CmdCursorInfo() : Command( "cursorInfo" ) {}
-        virtual bool slaveOk() { return true; }
+        CmdCursorInfo() : Command( "cursorInfo", true ) {}
+        virtual bool slaveOk() const { return true; }
         virtual void help( stringstream& help ) const {
             help << " example: { cursorInfo : 1 }";
         }
-        virtual LockType locktype(){ return NONE; }
-        bool run(const char *dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
+        virtual LockType locktype() const { return NONE; }
+        bool run(const string&, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
             recursive_scoped_lock lock(ClientCursor::ccmutex);
+            result.append("totalOpen", unsigned( ClientCursor::clientCursorsById.size() ) );
             result.append("byLocation_size", unsigned( ClientCursor::byLoc.size() ) );
             result.append("clientCursors_size", unsigned( ClientCursor::clientCursorsById.size() ) );
             return true;
         }
     } cmdCursorInfo;
+    
+    void ClientCursorMonitor::run(){
+        Client::initThread("clientcursormon");
+        Client& client = cc();
+        
+        unsigned old = curTimeMillis();
+
+        while ( ! inShutdown() ){
+            unsigned now = curTimeMillis();
+            ClientCursor::idleTimeReport( now - old );
+            old = now;
+            sleepsecs(4);
+        }
+
+        client.shutdown();
+    }
+
+    void ClientCursor::find( const string& ns , set<CursorId>& all ){
+        recursive_scoped_lock lock(ccmutex);
+        
+        for ( CCById::iterator i=clientCursorsById.begin(); i!=clientCursorsById.end(); ++i ){
+            if ( i->second->ns == ns )
+                all.insert( i->first );
+        }
+    }
+
+
+    ClientCursorMonitor clientCursorMonitor;
 
 } // namespace mongo
diff --git a/db/clientcursor.h b/db/clientcursor.h
index 42919e3..32453fd 100644
--- a/db/clientcursor.h
+++ b/db/clientcursor.h
@@ -24,19 +24,22 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "cursor.h"
 #include "jsobj.h"
 #include "../util/message.h"
+#include "../util/background.h"
 #include "diskloc.h"
 #include "dbhelpers.h"
 #include "matcher.h"
+#include "../client/dbclient.h"
 
 namespace mongo {
 
     typedef long long CursorId; /* passed to the client so it can send back on getMore */
     class Cursor; /* internal server cursor base class */
     class ClientCursor;
+    class ParsedQuery;
 
     /* todo: make this map be per connection.  this will prevent cursor hijacking security attacks perhaps.
     */
@@ -58,6 +61,7 @@ namespace mongo {
         unsigned _pinValue;
 
         bool _doingDeletes;
+        ElapsedTracker _yieldSometimesTracker;
 
         static CCById clientCursorsById;
         static CCByLoc byLoc;
@@ -65,6 +69,8 @@ namespace mongo {
         
         static CursorId allocCursorId_inlock();
 
+
+        
     public:
         /* use this to assure we don't in the background time out cursor while it is under use.
            if you are using noTimeout() already, there is no risk anyway.
@@ -97,21 +103,56 @@ namespace mongo {
                 release();
             }
         }; 
+        
+        // This object assures safe and reliable cleanup of the ClientCursor.
+        // The implementation assumes that there will be no duplicate ids among cursors
+        // (which is assured if cursors must last longer than 1 second).
+        class CleanupPointer : boost::noncopyable {
+        public:
+            CleanupPointer() : _c( 0 ), _id( -1 ) {}
+            void reset( ClientCursor *c = 0 ) {
+                if ( c == _c ) {
+                    return;
+                }
+
+                if ( _c ) {
+                    // be careful in case cursor was deleted by someone else
+                    ClientCursor::erase( _id );
+                }
+                
+                if ( c ) {
+                    _c = c;
+                    _id = c->cursorid;
+                } else {
+                    _c = 0;
+                    _id = -1;
+                }
+            }
+            ~CleanupPointer() {
+                DESTRUCTOR_GUARD ( reset(); );
+            }
+            operator bool() { return _c; }
+            ClientCursor * operator-> () { return _c; }
+        private:
+            ClientCursor *_c;
+            CursorId _id;
+        };
 
         /*const*/ CursorId cursorid;
         string ns;
-        auto_ptr<CoveredIndexMatcher> matcher;
-        auto_ptr<Cursor> c;
-        int pos;                                 // # objects into the cursor so far 
+        shared_ptr<Cursor> c;
+        int pos;                  // # objects into the cursor so far 
         BSONObj query;
+        int _queryOptions;        // see enum QueryOptions dbclient.h
+        OpTime _slaveReadTill;
 
-        ClientCursor(auto_ptr<Cursor>& _c, const char *_ns, bool okToTimeout) : 
+        ClientCursor(int queryOptions, shared_ptr<Cursor>& _c, const string& _ns) :
             _idleAgeMillis(0), _pinValue(0), 
-            _doingDeletes(false), 
+            _doingDeletes(false), _yieldSometimesTracker(128,10),
             ns(_ns), c(_c), 
-            pos(0) 
+            pos(0), _queryOptions(queryOptions)
         {
-            if( !okToTimeout )
+            if( queryOptions & QueryOption_NoCursorTimeout )
                 noTimeout();
             recursive_scoped_lock lock(ccmutex);
             cursorid = allocCursorId_inlock();
@@ -123,6 +164,7 @@ namespace mongo {
             return _lastLoc;
         }
 
+        shared_ptr< ParsedQuery > pq;
         shared_ptr< FieldMatcher > fields; // which fields query wants returned
         Message originalMessage; // this is effectively an auto ptr for data the matcher points to
 
@@ -132,6 +174,8 @@ namespace mongo {
         static void invalidate(const char *nsPrefix);
 
         /**
+         * @param microsToSleep -1 : ask client 
+         *                     >=0 : sleep for that amount
          * do a dbtemprelease 
          * note: caller should check matcher.docMatcher().atomic() first and not yield if atomic - 
          *       we don't do herein as this->matcher (above) is only initialized for true queries/getmore.
@@ -140,7 +184,80 @@ namespace mongo {
          *         if false is returned, then this ClientCursor should be considered deleted - 
          *         in fact, the whole database could be gone.
          */
-        bool yield();
+        bool yield( int microsToSleep = -1 );
+
+        /**
+         * @return same as yield()
+         */
+        bool yieldSometimes();
+        
+        static int yieldSuggest();
+        static void staticYield( int micros );
+        
+        struct YieldData { CursorId _id; bool _doingDeletes; };
+        bool prepareToYield( YieldData &data );
+        static bool recoverFromYield( const YieldData &data );
+
+        struct YieldLock : boost::noncopyable {
+            explicit YieldLock( ptr<ClientCursor> cc )
+                : _canYield(cc->c->supportYields()) {
+                if ( _canYield ){
+                    cc->prepareToYield( _data );
+                    _unlock.reset(new dbtempreleasecond());
+                }
+            }
+            ~YieldLock(){
+                if ( _unlock ){
+                    log( LL_WARNING ) << "ClientCursor::YieldLock not closed properly" << endl;
+                    relock();
+                }
+            }
+
+            bool stillOk(){
+                if ( ! _canYield )
+                    return true;
+
+                relock();
+                
+                return ClientCursor::recoverFromYield( _data );
+            }
+
+            void relock(){
+                _unlock.reset();
+            }
+            
+        private:
+            bool _canYield;
+            YieldData _data;
+            
+            scoped_ptr<dbtempreleasecond> _unlock;
+
+        };
+
+        // --- some pass through helpers for Cursor ---
+
+        BSONObj indexKeyPattern() {
+            return c->indexKeyPattern();
+        }
+
+        bool ok(){
+            return c->ok();
+        }
+
+        bool advance(){
+            return c->advance();
+        }
+
+        bool currentMatches(){
+            if ( ! c->matcher() )
+                return true;
+            return c->matcher()->matchesCurrent( c.get() );
+        }
+
+        BSONObj current(){
+            return c->current();
+        }
+
     private:
         void setLastLoc_inlock(DiskLoc);
 
@@ -180,8 +297,6 @@ namespace mongo {
            */
         void updateLocation();
 
-        void cleanupByLocation(DiskLoc loc);
-        
         void mayUpgradeStorage() {
             /* if ( !ids_.get() )
                 return;
@@ -197,6 +312,9 @@ namespace mongo {
             _idleAgeMillis += millis;
             return _idleAgeMillis > 600000 && _pinValue == 0;
         }
+
+        void storeOpForSlave( DiskLoc last );
+        void updateSlaveLocation( CurOp& curop );
         
         unsigned idleTime(){
             return _idleAgeMillis;
@@ -218,7 +336,16 @@ public:
 
         static void informAboutToDeleteBucket(const DiskLoc& b);
         static void aboutToDelete(const DiskLoc& dl);
+
+        static void find( const string& ns , set<CursorId>& all );
+    };
+
+    class ClientCursorMonitor : public BackgroundJob {
+    public:
+        void run();
+        string name() { return "ClientCursorMonitor"; }
     };
 
+    extern ClientCursorMonitor clientCursorMonitor;
     
 } // namespace mongo
diff --git a/db/cloner.cpp b/db/cloner.cpp
index d300721..96890bf 100644
--- a/db/cloner.cpp
+++ b/db/cloner.cpp
@@ -16,10 +16,10 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "pdfile.h"
 #include "../client/dbclient.h"
-#include "../util/builder.h"
+#include "../bson/util/builder.h"
 #include "jsobj.h"
 #include "query.h"
 #include "commands.h"
@@ -37,10 +37,10 @@ namespace mongo {
         auto_ptr< DBClientWithCommands > conn;
         void copy(const char *from_ns, const char *to_ns, bool isindex, bool logForRepl,
                   bool masterSameProcess, bool slaveOk, Query q = Query());
-        void replayOpLog( DBClientCursor *c, const BSONObj &query );
+        struct Fun;
     public:
         Cloner() { }
-
+        
         /* slaveOk     - if true it is ok if the source of the data is !ismaster.
            useReplAuth - use the credentials we normally use as a replication slave for the cloning
            snapshot    - use $snapshot mode for copying collections.  note this should not be used when it isn't required, as it will be slower.
@@ -48,8 +48,8 @@ namespace mongo {
         */
         void setConnection( DBClientWithCommands *c ) { conn.reset( c ); }
         bool go(const char *masterHost, string& errmsg, const string& fromdb, bool logForRepl, bool slaveOk, bool useReplAuth, bool snapshot);
-        bool startCloneCollection( const char *fromhost, const char *ns, const BSONObj &query, string& errmsg, bool logForRepl, bool copyIndexes, int logSizeMb, long long &cursorId );
-        bool finishCloneCollection( const char *fromhost, const char *ns, const BSONObj &query, long long cursorId, string &errmsg );
+
+        bool copyCollection( const string& from , const string& ns , const BSONObj& query , string& errmsg , bool copyIndexes = true );
     };
 
     /* for index info object:
@@ -86,75 +86,108 @@ namespace mongo {
         return res;
     }
 
+    struct Cloner::Fun {
+        void operator()( DBClientCursorBatchIterator &i ) {
+            mongolock l( true );
+            if ( context ) {
+                context->relocked();
+            }
+            
+            while( i.moreInCurrentBatch() ) {
+                if ( n % 128 == 127 /*yield some*/ ) {
+                    dbtemprelease t;
+                }
+                
+                BSONObj tmp = i.nextSafe();
+            
+                /* assure object is valid.  note this will slow us down a little. */
+                if ( !tmp.valid() ) {
+                    stringstream ss;
+                    ss << "Cloner: skipping corrupt object from " << from_collection;
+                    BSONElement e = tmp.firstElement();
+                    try {
+                        e.validate();
+                        ss << " firstElement: " << e;
+                    }
+                    catch( ... ){
+                        ss << " firstElement corrupt";
+                    }
+                    out() << ss.str() << endl;
+                    continue;
+                }
+            
+                ++n;
+            
+                BSONObj js = tmp;
+                if ( isindex ) {
+                    assert( strstr(from_collection, "system.indexes") );
+                    js = fixindex(tmp);
+                    storedForLater->push_back( js.getOwned() );
+                    continue;
+                }
+            
+                try { 
+                    theDataFileMgr.insertWithObjMod(to_collection, js);
+                    if ( logForRepl )
+                        logOp("i", to_collection, js);
+                }
+                catch( UserException& e ) { 
+                    log() << "warning: exception cloning object in " << from_collection << ' ' << e.what() << " obj:" << js.toString() << '\n';
+                }
+            
+                RARELY if ( time( 0 ) - saveLast > 60 ) {
+                    log() << n << " objects cloned so far from collection " << from_collection << endl;
+                    saveLast = time( 0 );
+                }
+            }
+        }
+        int n;
+        bool isindex;
+        const char *from_collection;
+        const char *to_collection;
+        time_t saveLast;
+        list<BSONObj> *storedForLater;     
+        bool logForRepl;
+        Client::Context *context;
+    };
+    
     /* copy the specified collection
        isindex - if true, this is system.indexes collection, in which we do some transformation when copying.
     */
     void Cloner::copy(const char *from_collection, const char *to_collection, bool isindex, bool logForRepl, bool masterSameProcess, bool slaveOk, Query query) {
-        auto_ptr<DBClientCursor> c;
-        {
-            dbtemprelease r;
-            c = conn->query( from_collection, query, 0, 0, 0, QueryOption_NoCursorTimeout | ( slaveOk ? QueryOption_SlaveOk : 0 ) );
-        }
-        
         list<BSONObj> storedForLater;
         
-        massert( 13055 , "socket error in Cloner:copy" , c.get() );
-        long long n = 0;
-        time_t saveLast = time( 0 );
-        while ( 1 ) {
-            if( !c->moreInCurrentBatch() || n % 128 == 127 /*yield some*/ ) {
-                dbtemprelease r;
-                if ( !c->more() )
-                    break;
-            }
-            BSONObj tmp = c->next();
-
-            /* assure object is valid.  note this will slow us down a little. */
-            if ( !tmp.valid() ) {
-                stringstream ss;
-                ss << "Cloner: skipping corrupt object from " << from_collection;
-                BSONElement e = tmp.firstElement();
-                try {
-                    e.validate();
-                    ss << " firstElement: " << e;
-                }
-                catch( ... ){
-                    ss << " firstElement corrupt";
+        Fun f;
+        f.n = 0;
+        f.isindex = isindex;
+        f.from_collection = from_collection;
+        f.to_collection = to_collection;
+        f.saveLast = time( 0 );
+        f.storedForLater = &storedForLater;
+        f.logForRepl = logForRepl;
+        
+        int options = QueryOption_NoCursorTimeout | ( slaveOk ? QueryOption_SlaveOk : 0 );
+        {
+            dbtemprelease r;
+            f.context = r._context;
+            DBClientConnection *remote = dynamic_cast< DBClientConnection* >( conn.get() );
+            if ( remote ) {
+                remote->query( boost::function<void(DBClientCursorBatchIterator &)>( f ), from_collection, query, 0, options );
+            } else { // no exhaust mode for direct client, so we have this hack
+                auto_ptr<DBClientCursor> c = conn->query( from_collection, query, 0, 0, 0, options );
+                assert( c.get() );
+                while( c->more() ) {
+                    DBClientCursorBatchIterator i( *c );
+                    f( i );
                 }
-                out() << ss.str() << endl;
-                continue;
-            }
-
-            ++n;
-            
-            BSONObj js = tmp;
-            if ( isindex ) {
-                assert( strstr(from_collection, "system.indexes") );
-                js = fixindex(tmp);
-                storedForLater.push_back( js.getOwned() );
-                continue;
-            }
-
-            try { 
-                theDataFileMgr.insert(to_collection, js);
-                if ( logForRepl )
-                    logOp("i", to_collection, js);
-            }
-            catch( UserException& e ) { 
-                log() << "warning: exception cloning object in " << from_collection << ' ' << e.what() << " obj:" << js.toString() << '\n';
-            }
-            
-            RARELY if ( time( 0 ) - saveLast > 60 ) {
-                log() << n << " objects cloned so far from collection " << from_collection << endl;
-                saveLast = time( 0 );
             }
         }
-
+        
         if ( storedForLater.size() ){
             for ( list<BSONObj>::iterator i = storedForLater.begin(); i!=storedForLater.end(); i++ ){
                 BSONObj js = *i;
                 try { 
-                    theDataFileMgr.insert(to_collection, js);
+                    theDataFileMgr.insertWithObjMod(to_collection, js);
                     if ( logForRepl )
                         logOp("i", to_collection, js);
                 }
@@ -164,7 +197,45 @@ namespace mongo {
             }
         }
     }
+
+    bool copyCollectionFromRemote(const string& host, const string& ns, const BSONObj& query, string errmsg) {
+        Cloner c;
+        return c.copyCollection(host, ns, query, errmsg , /*copyIndexes*/ true);
+    }
+
+    bool Cloner::copyCollection( const string& from , const string& ns , const BSONObj& query , string& errmsg , bool copyIndexes ){
+        auto_ptr<DBClientConnection> myconn;
+        myconn.reset( new DBClientConnection() );
+        if ( ! myconn->connect( from , errmsg ) )
+            return false;
+
+        conn.reset( myconn.release() );
+        
+        writelock lk(ns); // TODO: make this lower down
+        Client::Context ctx(ns);
+
+        { // config
+            string temp = ctx.db()->name + ".system.namespaces";
+            BSONObj config = conn->findOne( temp , BSON( "name" << ns ) );
+            if ( config["options"].isABSONObj() )
+                if ( ! userCreateNS( ns.c_str() , config["options"].Obj() , errmsg, true , 0 ) )
+                    return false;
+        }
+        
+        { // main data
+            copy( ns.c_str() , ns.c_str() , false , true , false , true , Query(query).snapshot() );
+        }
+        
+        { // indexes
+            string temp = ctx.db()->name + ".system.indexes";
+            copy( temp.c_str() , temp.c_str() , true , true , false , true , BSON( "ns" << ns ) );
+        }
+        return true;
+    }
     
+    extern bool inDBRepair;
+    void ensureIdIndexForNewNs(const char *ns);
+
     bool Cloner::go(const char *masterHost, string& errmsg, const string& fromdb, bool logForRepl, bool slaveOk, bool useReplAuth, bool snapshot) {
 
 		massert( 10289 ,  "useReplAuth is not written to replication log", !useReplAuth || !logForRepl );
@@ -190,6 +261,7 @@ namespace mongo {
         {  
             dbtemprelease r;
 		
+            // just using exhaust for collection copying right now
             auto_ptr<DBClientCursor> c;
             {
                 if ( conn.get() ) {
@@ -228,14 +300,14 @@ namespace mongo {
                 const char *from_name = e.valuestr();
 
                 if( strstr(from_name, ".system.") ) { 
-                    /* system.users is cloned -- but nothing else from system. */
+                    /* system.users and s.js is cloned -- but nothing else from system.
+                     * system.indexes is handled specially at the end*/
                     if( legalClientSystemNS( from_name , true ) == 0 ){
                         log(2) << "\t\t not cloning because system collection" << endl;
                         continue;
                     }
                 }
-                if( strchr(from_name, '$') ) {
-                    // don't clone index namespaces -- we take care of those separately below.
+                if( ! nsDollarCheck( from_name ) ){
                     log(2) << "\t\t not cloning because has $ " << endl;
                     continue;
                 }            
@@ -257,19 +329,38 @@ namespace mongo {
             assert(p);
             string to_name = todb + p;
 
+            bool wantIdIndex = false;
             {
                 string err;
                 const char *toname = to_name.c_str();
-                userCreateNS(toname, options, err, logForRepl);
+                /* we defer building id index for performance - building it in batch is much faster */ 
+                userCreateNS(toname, options, err, logForRepl, &wantIdIndex);
             }
             log(1) << "\t\t cloning " << from_name << " -> " << to_name << endl;
             Query q;
             if( snapshot ) 
                 q.snapshot();
             copy(from_name, to_name.c_str(), false, logForRepl, masterSameProcess, slaveOk, q);
+
+            if( wantIdIndex ) {
+                /* we need dropDups to be true as we didn't do a true snapshot and this is before applying oplog operations 
+                   that occur during the initial sync.  inDBRepair makes dropDups be true.
+                   */
+                bool old = inDBRepair;
+                try {
+                    inDBRepair = true;
+                    ensureIdIndexForNewNs(to_name.c_str());
+                    inDBRepair = old;
+                }
+                catch(...) { 
+                    inDBRepair = old;
+                    throw;
+                }
+            }
         }
 
         // now build the indexes
+
         string system_indexes_from = fromdb + ".system.indexes";
         string system_indexes_to = todb + ".system.indexes";
         /* [dm]: is the ID index sometimes not called "_id_"?  There is other code in the system that looks for a "_id" prefix 
@@ -280,121 +371,6 @@ namespace mongo {
 
         return true;
     }
-
-    bool Cloner::startCloneCollection( const char *fromhost, const char *ns, const BSONObj &query, string &errmsg, bool logForRepl, bool copyIndexes, int logSizeMb, long long &cursorId ) {
-        char db[256];
-        nsToDatabase( ns, db );
-
-        NamespaceDetails *nsd = nsdetails( ns );
-        if ( nsd ){
-            /** note: its ok to clone into a collection, but only if the range you're copying 
-                doesn't exist on this server */
-            string err;
-            if ( runCount( ns , BSON( "query" << query ) , err ) > 0 ){
-                log() << "WARNING: data already exists for: " << ns << " in range : " << query << " deleting..." << endl;
-                deleteObjects( ns , query , false , logForRepl , false );
-            }
-        }
-
-        {
-            dbtemprelease r;
-            auto_ptr< DBClientConnection > c( new DBClientConnection() );
-            if ( !c->connect( fromhost, errmsg ) )
-                return false;
-            if( !replAuthenticate(c.get()) )
-                return false;
-            conn = c;
-
-            // Start temporary op log
-            BSONObjBuilder cmdSpec;
-            cmdSpec << "logCollection" << ns << "start" << 1;
-            if ( logSizeMb != INT_MIN )
-                cmdSpec << "logSizeMb" << logSizeMb;
-            BSONObj info;
-            if ( !conn->runCommand( db, cmdSpec.done(), info ) ) {
-                errmsg = "logCollection failed: " + (string)info;
-                return false;
-            }
-        }
-        
-        if ( ! nsd ) {
-            BSONObj spec = conn->findOne( string( db ) + ".system.namespaces", BSON( "name" << ns ) );
-            if ( !userCreateNS( ns, spec.getObjectField( "options" ), errmsg, true ) )
-                return false;
-        }
-
-        copy( ns, ns, false, logForRepl, false, false, query );
-
-        if ( copyIndexes ) {
-            string indexNs = string( db ) + ".system.indexes";
-            copy( indexNs.c_str(), indexNs.c_str(), true, logForRepl, false, false, BSON( "ns" << ns << "name" << NE << "_id_" ) );
-        }
-        
-        auto_ptr< DBClientCursor > c;
-        {
-            dbtemprelease r;
-            string logNS = "local.temp.oplog." + string( ns );
-            c = conn->query( logNS.c_str(), Query(), 0, 0, 0, QueryOption_CursorTailable );
-        }
-        if ( c->more() ) {
-            replayOpLog( c.get(), query );
-            cursorId = c->getCursorId();
-            massert( 10291 ,  "Expected valid tailing cursor", cursorId != 0 );
-        } else {
-            massert( 10292 ,  "Did not expect valid cursor for empty query result", c->getCursorId() == 0 );
-            cursorId = 0;
-        }
-        c->decouple();
-        return true;
-    }
-    
-    void Cloner::replayOpLog( DBClientCursor *c, const BSONObj &query ) {
-        Matcher matcher( query );
-        while( 1 ) {
-            BSONObj op;
-            {
-                dbtemprelease t;
-                if ( !c->more() )
-                    break;
-                op = c->next();
-            }
-            // For sharding v1.0, we don't allow shard key updates -- so just
-            // filter each insert by value.
-            if ( op.getStringField( "op" )[ 0 ] != 'i' || matcher.matches( op.getObjectField( "o" ) ) )
-                ReplSource::applyOperation( op );
-        }        
-    }
-    
-    bool Cloner::finishCloneCollection( const char *fromhost, const char *ns, const BSONObj &query, long long cursorId, string &errmsg ) {
-        char db[256];
-        nsToDatabase( ns, db );
-
-        auto_ptr< DBClientCursor > cur;
-        {
-            dbtemprelease r;
-            auto_ptr< DBClientConnection > c( new DBClientConnection() );
-            if ( !c->connect( fromhost, errmsg ) )
-                return false;
-            if( !replAuthenticate(c.get()) )
-                return false;
-            conn = c;            
-            string logNS = "local.temp.oplog." + string( ns );
-            if ( cursorId != 0 )
-                cur = conn->getMore( logNS.c_str(), cursorId );
-            else
-                cur = conn->query( logNS.c_str(), Query() );
-        }
-        replayOpLog( cur.get(), query );
-        {
-            dbtemprelease t;
-            BSONObj info;
-            if ( !conn->runCommand( db, BSON( "logCollection" << ns << "validateComplete" << 1 ), info ) ) {
-                errmsg = "logCollection failed: " + (string)info;
-                return false;
-            }
-        }
-        return true;
-    }
     
     /* slaveOk     - if true it is ok if the source of the data is !ismaster.
        useReplAuth - use the credentials we normally use as a replication slave for the cloning
@@ -413,169 +389,74 @@ namespace mongo {
     */
     class CmdClone : public Command {
     public:
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; }
+        virtual LockType locktype() const { return WRITE; }
         virtual void help( stringstream &help ) const {
             help << "clone this database from an instance of the db on another host\n";
-            help << "example: { clone : \"host13\" }";
+            help << "{ clone : \"host13\" }";
         }
         CmdClone() : Command("clone") { }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        virtual bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             string from = cmdObj.getStringField("clone");
             if ( from.empty() )
                 return false;
             /* replication note: we must logOp() not the command, but the cloned data -- if the slave
                were to clone it would get a different point-in-time and not match.
                */
-            return cloneFrom(from.c_str(), errmsg, cc().database()->name, 
+            return cloneFrom(from.c_str(), errmsg, dbname, 
                              /*logForReplication=*/!fromRepl, /*slaveok*/false, /*usereplauth*/false, /*snapshot*/true);
         }
     } cmdclone;
     
     class CmdCloneCollection : public Command {
     public:
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; }
+        virtual LockType locktype() const { return NONE; }
         CmdCloneCollection() : Command("cloneCollection") { }
         virtual void help( stringstream &help ) const {
-            help << " example: { cloneCollection: <collection ns>, from: <hostname>, query: <query> }";
+            help << "{ cloneCollection: <namespace>, from: <host> [,query: <query_filter>] [,copyIndexes:<bool>] }"
+                "\nCopies a collection from one server to another. Do not use on a single server as the destination "
+                "is placed at the same db.collection (namespace) as the source.\n"
+                "Warning: the local copy of 'ns' is emptied before the copying begins. Any existing data will be lost there."
+                ;
         }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        virtual bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             string fromhost = cmdObj.getStringField("from");
             if ( fromhost.empty() ) {
-                errmsg = "missing from spec";
+                errmsg = "missing 'from' parameter";
                 return false;
             }
+            {
+                HostAndPort h(fromhost);
+                if( h.isSelf() ) { 
+                    errmsg = "can't cloneCollection from self";
+                    return false;
+                }
+            }
             string collection = cmdObj.getStringField("cloneCollection");
             if ( collection.empty() ) {
-                errmsg = "missing cloneCollection spec";
+                errmsg = "bad 'cloneCollection' value";
                 return false;
             }
             BSONObj query = cmdObj.getObjectField("query");
             if ( query.isEmpty() )
                 query = BSONObj();
+            
             BSONElement copyIndexesSpec = cmdObj.getField("copyindexes");
             bool copyIndexes = copyIndexesSpec.isBoolean() ? copyIndexesSpec.boolean() : true;
-            // Will not be used if doesn't exist.
-            int logSizeMb = cmdObj.getIntField( "logSizeMb" );
-            
-            /* replication note: we must logOp() not the command, but the cloned data -- if the slave
-             were to clone it would get a different point-in-time and not match.
-             */
-            Client::Context ctx( collection );
             
-            log() << "cloneCollection.  db:" << ns << " collection:" << collection << " from: " << fromhost << " query: " << query << " logSizeMb: " << logSizeMb << ( copyIndexes ? "" : ", not copying indexes" ) << endl;
+            log() << "cloneCollection.  db:" << dbname << " collection:" << collection << " from: " << fromhost 
+                  << " query: " << query << " " << ( copyIndexes ? "" : ", not copying indexes" ) << endl;
             
             Cloner c;
-            long long cursorId;
-            if ( !c.startCloneCollection( fromhost.c_str(), collection.c_str(), query, errmsg, !fromRepl, copyIndexes, logSizeMb, cursorId ) )
-                return false;
-            return c.finishCloneCollection( fromhost.c_str(), collection.c_str(), query, cursorId, errmsg);
+            return c.copyCollection( fromhost , collection , query, errmsg , copyIndexes );
         }
     } cmdclonecollection;
 
-    class CmdStartCloneCollection : public Command {
-    public:
-        virtual bool slaveOk() {
-            return false;
-        }
-        virtual LockType locktype(){ return WRITE; }
-        CmdStartCloneCollection() : Command("startCloneCollection") { }
-        virtual void help( stringstream &help ) const {
-            help << " example: { startCloneCollection: <collection ns>, from: <hostname>, query: <query> }";
-            help << ", returned object includes a finishToken field, the value of which may be passed to the finishCloneCollection command";
-        }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            string fromhost = cmdObj.getStringField("from");
-            if ( fromhost.empty() ) {
-                errmsg = "missing from spec";
-                return false;
-            }
-            string collection = cmdObj.getStringField("startCloneCollection");
-            if ( collection.empty() ) {
-                errmsg = "missing startCloneCollection spec";
-                return false;
-            }
-            BSONObj query = cmdObj.getObjectField("query");
-            if ( query.isEmpty() )
-                query = BSONObj();
-            BSONElement copyIndexesSpec = cmdObj.getField("copyindexes");
-            bool copyIndexes = copyIndexesSpec.isBoolean() ? copyIndexesSpec.boolean() : true;
-            // Will not be used if doesn't exist.
-            int logSizeMb = cmdObj.getIntField( "logSizeMb" );
-            
-            /* replication note: we must logOp() not the command, but the cloned data -- if the slave
-             were to clone it would get a different point-in-time and not match.
-             */
-            Client::Context ctx(collection);
-            
-            log() << "startCloneCollection.  db:" << ns << " collection:" << collection << " from: " << fromhost << " query: " << query << endl;
-            
-            Cloner c;
-            long long cursorId;
-            bool res = c.startCloneCollection( fromhost.c_str(), collection.c_str(), query, errmsg, !fromRepl, copyIndexes, logSizeMb, cursorId );
-            
-            if ( res ) {
-                BSONObjBuilder b;
-                b << "fromhost" << fromhost;
-                b << "collection" << collection;
-                b << "query" << query;
-                b.appendDate( "cursorId", cursorId );
-                BSONObj token = b.done();
-                result << "finishToken" << token;
-            }
-            return res;
-        }
-    } cmdstartclonecollection;
-    
-    class CmdFinishCloneCollection : public Command {
-    public:
-        virtual bool slaveOk() {
-            return false;
-        }
-        virtual LockType locktype(){ return WRITE; }
-        CmdFinishCloneCollection() : Command("finishCloneCollection") { }
-        virtual void help( stringstream &help ) const {
-            help << " example: { finishCloneCollection: <finishToken> }";
-        }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            BSONObj fromToken = cmdObj.getObjectField("finishCloneCollection");
-            if ( fromToken.isEmpty() ) {
-                errmsg = "missing finishCloneCollection finishToken spec";
-                return false;
-            }
-            string fromhost = fromToken.getStringField( "fromhost" );
-            if ( fromhost.empty() ) {
-                errmsg = "missing fromhost spec";
-                return false;
-            }
-            string collection = fromToken.getStringField("collection");
-            if ( collection.empty() ) {
-                errmsg = "missing collection spec";
-                return false;
-            }
-            BSONObj query = fromToken.getObjectField("query");
-            if ( query.isEmpty() ) {
-                query = BSONObj();
-            }
-            long long cursorId = 0;
-            BSONElement cursorIdToken = fromToken.getField( "cursorId" );
-            if ( cursorIdToken.type() == Date ) {
-                cursorId = cursorIdToken._numberLong();
-            }
-            
-            Client::Context ctx( collection );
-            
-            log() << "finishCloneCollection.  db:" << ns << " collection:" << collection << " from: " << fromhost << " query: " << query << endl;
-            
-            Cloner c;
-            return c.finishCloneCollection( fromhost.c_str(), collection.c_str(), query, cursorId, errmsg );
-        }
-    } cmdfinishclonecollection;
 
     thread_specific_ptr< DBClientConnection > authConn_;
     /* Usage:
@@ -584,18 +465,18 @@ namespace mongo {
     class CmdCopyDbGetNonce : public Command {
     public:
         CmdCopyDbGetNonce() : Command("copydbgetnonce") { }
-        virtual bool adminOnly() {
+        virtual bool adminOnly() const {
             return true;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; }
+        virtual LockType locktype() const { return WRITE; }
         virtual void help( stringstream &help ) const {
             help << "get a nonce for subsequent copy db request from secure server\n";
             help << "usage: {copydbgetnonce: 1, fromhost: <hostname>}";
         }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             string fromhost = cmdObj.getStringField("fromhost");
             if ( fromhost.empty() ) {
                 /* copy from self */
@@ -610,7 +491,7 @@ namespace mongo {
                 if ( !authConn_->connect( fromhost, errmsg ) )
                     return false;
                 if( !authConn_->runCommand( "admin", BSON( "getnonce" << 1 ), ret ) ) {
-                    errmsg = "couldn't get nonce " + string( ret );
+                    errmsg = "couldn't get nonce " + ret.toString();
                     return false;
                 }
             }
@@ -625,18 +506,18 @@ namespace mongo {
     class CmdCopyDb : public Command {
     public:
         CmdCopyDb() : Command("copydb") { }
-        virtual bool adminOnly() {
+        virtual bool adminOnly() const {
             return true;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; }
+        virtual LockType locktype() const { return WRITE; }
         virtual void help( stringstream &help ) const {
             help << "copy a database from another host to this host\n";
             help << "usage: {copydb: 1, fromhost: <hostname>, fromdb: <db>, todb: <db>[, username: <username>, nonce: <nonce>, key: <key>]}";
         }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        virtual bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             string fromhost = cmdObj.getStringField("fromhost");
             if ( fromhost.empty() ) {
                 /* copy from self */
@@ -660,7 +541,7 @@ namespace mongo {
                 {
                     dbtemprelease t;
                     if ( !authConn_->runCommand( fromdb, BSON( "authenticate" << 1 << "user" << username << "nonce" << nonce << "key" << key ), ret ) ) {
-                        errmsg = "unable to login " + string( ret );
+                        errmsg = "unable to login " + ret.toString();
                         return false;
                     }
                 }
@@ -675,20 +556,20 @@ namespace mongo {
     class CmdRenameCollection : public Command {
     public:
         CmdRenameCollection() : Command( "renameCollection" ) {}
-        virtual bool adminOnly() {
+        virtual bool adminOnly() const {
             return true;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; }
+        virtual LockType locktype() const { return WRITE; }
         virtual bool logTheOp() {
             return true; // can't log steps when doing fast rename within a db, so always log the op rather than individual steps comprising it.
         }
         virtual void help( stringstream &help ) const {
             help << " example: { renameCollection: foo.a, to: bar.b }";
         }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        virtual bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             string source = cmdObj.getStringField( name.c_str() );
             string target = cmdObj.getStringField( "to" );
             if ( source.empty() || target.empty() ) {
@@ -750,7 +631,7 @@ namespace mongo {
                         break;
                 }
                 BSONObj o = c->next();
-                theDataFileMgr.insert( target.c_str(), o );
+                theDataFileMgr.insertWithObjMod( target.c_str(), o );
             }
             
             char cl[256];
@@ -780,7 +661,7 @@ namespace mongo {
                     }
                 }
                 BSONObj n = b.done();
-                theDataFileMgr.insert( targetIndexes.c_str(), n );
+                theDataFileMgr.insertWithObjMod( targetIndexes.c_str(), n );
             }
 
             {
diff --git a/db/cmdline.cpp b/db/cmdline.cpp
index 2d15279..d79cb87 100644
--- a/db/cmdline.cpp
+++ b/db/cmdline.cpp
@@ -16,9 +16,10 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "cmdline.h"
 #include "commands.h"
+#include "../util/processinfo.h"
 
 namespace po = boost::program_options;
 
@@ -41,8 +42,10 @@ namespace mongo {
             ("verbose,v", "be more verbose (include multiple times for more verbosity e.g. -vvvvv)")
             ("quiet", "quieter output")
             ("port", po::value<int>(&cmdLine.port), "specify port number")
+            ("bind_ip", po::value<string>(&cmdLine.bind_ip), "comma separated list of ip addresses to listen on - all local ips by default")
             ("logpath", po::value<string>() , "file to send all output to instead of stdout" )
             ("logappend" , "append to logpath instead of over-writing" )
+            ("pidfilepath", po::value<string>(), "directory for pidfile (if not set, no pidfile is created)")
 #ifndef _WIN32
             ("fork" , "fork server process" )
 #endif
@@ -114,25 +117,83 @@ namespace mongo {
             cmdLine.quiet = true;
         }
 
+        string logpath;
+
 #ifndef _WIN32
         if (params.count("fork")) {
             if ( ! params.count( "logpath" ) ){
                 cout << "--fork has to be used with --logpath" << endl;
                 ::exit(-1);
             }
+            
+            { // test logpath
+                logpath = params["logpath"].as<string>();
+                assert( logpath.size() );
+                if ( logpath[0] != '/' ){
+                    char temp[256];
+                    assert( getcwd( temp , 256 ) );
+                    logpath = (string)temp + "/" + logpath;
+                }
+                FILE * test = fopen( logpath.c_str() , "a" );
+                if ( ! test ){
+                    cout << "can't open [" << logpath << "] for log file: " << errnoWithDescription() << endl;
+                    ::exit(-1);
+                }
+                fclose( test );
+            }
+            
+            cout.flush();
+            cerr.flush();
+
             pid_t c = fork();
             if ( c ){
-                cout << "forked process: " << c << endl;
-                ::exit(0);
+                _exit(0);
+            }
+
+            if ( chdir("/") < 0 ){
+                cout << "Cant chdir() while forking server process: " << strerror(errno) << endl;
+                ::exit(-1);
             }
             setsid();
+            
+            pid_t c2 = fork();
+            if ( c2 ){
+                cout << "forked process: " << c2 << endl;
+                _exit(0);
+            }
+
+            // stdout handled in initLogging
+            //fclose(stdout);
+            //freopen("/dev/null", "w", stdout);
+
+            fclose(stderr);
+            fclose(stdin);
+
+            FILE* f = freopen("/dev/null", "w", stderr);
+            if ( f == NULL ){
+                cout << "Cant reassign stderr while forking server process: " << strerror(errno) << endl;
+                ::exit(-1);
+            }
+
+            f = freopen("/dev/null", "r", stdin);
+            if ( f == NULL ){
+                cout << "Cant reassign stdin while forking server process: " << strerror(errno) << endl;
+                ::exit(-1);
+            }
+
+            setupCoreSignals();
             setupSignals();
         }
 #endif
         if (params.count("logpath")) {
-            string lp = params["logpath"].as<string>();
-            uassert( 10033 ,  "logpath has to be non-zero" , lp.size() );
-            initLogging( lp , params.count( "logappend" ) );
+            if ( logpath.size() == 0 )
+                logpath = params["logpath"].as<string>();
+            uassert( 10033 ,  "logpath has to be non-zero" , logpath.size() );
+            initLogging( logpath , params.count( "logappend" ) );
+        }
+
+        if ( params.count("pidfilepath")) {
+            writePidFile( params["pidfilepath"].as<string>() );
         }
 
         {
@@ -144,15 +205,26 @@ namespace mongo {
 
         return true;
     }
+    
+    void ignoreSignal( int signal ){
+    }
+
+    void setupCoreSignals(){
+#if !defined(_WIN32)
+        assert( signal(SIGUSR1 , rotateLogs ) != SIG_ERR );
+        assert( signal(SIGHUP , ignoreSignal ) != SIG_ERR );
+#endif
+    }
 
     class CmdGetCmdLineOpts : Command{
         public:
         CmdGetCmdLineOpts(): Command("getCmdLineOpts") {}
-        virtual LockType locktype() { return NONE; }
-        virtual bool adminOnly() { return true; }
-        virtual bool slaveOk() { return true; }
+        void help(stringstream& h) const { h << "get argv"; }
+        virtual LockType locktype() const { return NONE; }
+        virtual bool adminOnly() const { return true; }
+        virtual bool slaveOk() const { return true; }
 
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl){
+        virtual bool run(const string&, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl){
             result.append("argv", argvArray);
             return true;
         }
diff --git a/db/cmdline.h b/db/cmdline.h
index 3e46c5e..ef1bd57 100644
--- a/db/cmdline.h
+++ b/db/cmdline.h
@@ -16,7 +16,7 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 
 namespace mongo {
     
@@ -25,8 +25,18 @@ namespace mongo {
     /* concurrency: OK/READ */
     struct CmdLine { 
         int port;              // --port
+        string bind_ip;        // --bind_ip
         bool rest;             // --rest
 
+        string _replSet;       // --replSet[/<seedlist>]
+        string ourSetName() const { 
+            string setname;
+            size_t sl = _replSet.find('/');
+            if( sl == string::npos )
+                return _replSet;
+            return _replSet.substr(0, sl);
+        }
+
         string source;         // --source
         string only;           // --only
         
@@ -43,6 +53,9 @@ namespace mongo {
         int defaultProfile;    // --profile
         int slowMS;            // --time in ms that is "slow"
 
+        int pretouch;          // --pretouch for replication application (experimental)
+        bool moveParanoia;     // for move chunk paranoia 
+
         enum { 
             DefaultDBPort = 27017,
 			ConfigServerPort = 27019,
@@ -51,7 +64,7 @@ namespace mongo {
 
         CmdLine() : 
             port(DefaultDBPort), rest(false), quiet(false), notablescan(false), prealloc(true), smallfiles(false),
-            quota(false), quotaFiles(8), cpu(false), oplogSize(0), defaultProfile(0), slowMS(100)
+            quota(false), quotaFiles(8), cpu(false), oplogSize(0), defaultProfile(0), slowMS(100), pretouch(0), moveParanoia( true )
         { } 
         
 
@@ -71,4 +84,5 @@ namespace mongo {
     
     extern CmdLine cmdLine;
     
+    void setupCoreSignals();
 }
diff --git a/db/commands.cpp b/db/commands.cpp
index 83d7219..ef219fe 100644
--- a/db/commands.cpp
+++ b/db/commands.cpp
@@ -17,21 +17,104 @@
  *    limitations under the License.
  */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "jsobj.h"
 #include "commands.h"
 #include "client.h"
-#include "replset.h"
+#include "replpair.h"
 
 namespace mongo {
 
+    map<string,Command*> * Command::_commandsByBestName;
+    map<string,Command*> * Command::_webCommands;
     map<string,Command*> * Command::_commands;
 
-    Command::Command(const char *_name) : name(_name) {
+    void Command::htmlHelp(stringstream& ss) const {
+        string helpStr;
+        {
+            stringstream h;
+            help(h);
+            helpStr = h.str();
+        }
+        ss << "\n<tr><td>";
+        bool web = _webCommands->count(name) != 0;
+        if( web ) ss << "<a href=\"/" << name << "?text\">";
+        ss << name;
+        if( web ) ss << "</a>";
+        ss << "</td>\n";
+        ss << "<td>";
+        int l = locktype();
+        //if( l == NONE ) ss << "N ";
+        if( l == READ ) ss << "R ";
+        else if( l == WRITE ) ss << "W ";
+        if( slaveOk() )
+            ss << "S ";
+        if( adminOnly() )
+            ss << "A";
+        ss << "</td>";
+        ss << "<td>";
+        if( helpStr != "no help defined" ) {
+            const char *p = helpStr.c_str();
+            while( *p ) { 
+                if( *p == '<' ) {
+                    ss << "&lt;";
+                    p++; continue;
+                }
+                else if( *p == '{' )
+                    ss << "<code>";
+                else if( *p == '}' ) {
+                    ss << "}</code>";
+                    p++;
+                    continue;
+                }
+                if( strncmp(p, "http:", 5) == 0 ) { 
+                    ss << "<a href=\"";
+                    const char *q = p;
+                    while( *q && *q != ' ' && *q != '\n' )
+                        ss << *q++;
+                    ss << "\">";
+                    q = p;
+                    if( startsWith(q, "http://www.mongodb.org/display/") )
+                        q += 31;
+                    while( *q && *q != ' ' && *q != '\n' ) {
+                        ss << (*q == '+' ? ' ' : *q);
+                        q++;
+                        if( *q == '#' ) 
+                            while( *q && *q != ' ' && *q != '\n' ) q++;
+                    }
+                    ss << "</a>";
+                    p = q;
+                    continue;
+                }
+                if( *p == '\n' ) ss << "<br>";
+                else ss << *p;
+                p++;
+            }
+        }
+        ss << "</td>";
+        ss << "</tr>\n";
+    }
+
+    Command::Command(const char *_name, bool web, const char *oldName) : name(_name) {
         // register ourself.
         if ( _commands == 0 )
             _commands = new map<string,Command*>;
-        (*_commands)[name] = this;
+        if( _commandsByBestName == 0 )
+            _commandsByBestName = new map<string,Command*>;
+        Command*& c = (*_commands)[name];
+        if ( c )
+            log() << "warning: 2 commands with name: " << _name << endl;
+        c = this;
+        (*_commandsByBestName)[name] = this;
+
+        if( web ) {
+            if( _webCommands == 0 )
+                _webCommands = new map<string,Command*>;
+            (*_webCommands)[name] = this;
+        }
+
+        if( oldName )
+            (*_commands)[oldName] = this;
     }
 
     void Command::help( stringstream& help ) const {
@@ -46,9 +129,7 @@ namespace mongo {
         bool ok = false;
         bool valid = false;
 
-        BSONElement e;
-        e = jsobj.firstElement();
-
+        BSONElement e = jsobj.firstElement();
         map<string,Command*>::iterator i;
 
         if ( e.eoo() )
@@ -60,9 +141,9 @@ namespace mongo {
             valid = true;
             string errmsg;
             Command *c = i->second;
-            if ( c->adminOnly() && strncmp(ns, "admin", 5) != 0 ) {
+            if ( c->adminOnly() && !startsWith(ns, "admin.") ) {
                 ok = false;
-                errmsg = "access denied";
+                errmsg = "access denied - use admin db";
             }
             else if ( jsobj.getBoolField( "help" ) ){
                 stringstream help;
@@ -71,7 +152,7 @@ namespace mongo {
                 anObjBuilder.append( "help" , help.str() );
             }
             else {
-                ok = c->run(ns, jsobj, errmsg, anObjBuilder, false);
+                ok = c->run( nsToDatabase( ns ) , jsobj, errmsg, anObjBuilder, false);
             }
 
             BSONObj tmp = anObjBuilder.asTempObj();
@@ -106,5 +187,12 @@ namespace mongo {
         return c->locktype();
     }
 
+    void Command::logIfSlow( const Timer& timer, const string& msg ) {
+        int ms = timer.millis();
+        if ( ms > cmdLine.slowMS ){
+            out() << msg << " took " << ms << " ms." << endl;
+        }
+    }
+    
     
 } // namespace mongo
diff --git a/db/commands.h b/db/commands.h
index 518dcb7..eea4a71 100644
--- a/db/commands.h
+++ b/db/commands.h
@@ -17,7 +17,7 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "jsobj.h"
 
 namespace mongo {
@@ -27,14 +27,15 @@ namespace mongo {
     class BufBuilder;
     class Client;
 
-// db "commands" (sent via db.$cmd.findOne(...))
-// subclass to make a command.
+    /** mongodb "commands" (sent via db.$cmd.findOne(...))
+        subclass to make a command.  define a singleton object for it.
+        */
     class Command {
     public:
         
         enum LockType { READ = -1 , NONE = 0 , WRITE = 1 };
 
-        string name;
+        const string name;
 
         /* run the given command
            implement this...
@@ -44,20 +45,22 @@ namespace mongo {
 
            return value is true if succeeded.  if false, set errmsg text.
         */
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) = 0;
+        virtual bool run(const string& db, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) = 0;
 
         /* 
 		   note: logTheTop() MUST be false if READ
            if NONE, can't use Client::Context setup
                     use with caution
 		 */
-        virtual LockType locktype() = 0;
+        virtual LockType locktype() const = 0;
 
         /* Return true if only the admin ns has privileges to run this command. */
-        virtual bool adminOnly() {
+        virtual bool adminOnly() const {
             return false;
         }
 
+        void htmlHelp(stringstream&) const;
+
         /* Like adminOnly, but even stricter: we must either be authenticated for admin db, 
            or, if running without auth, on the local interface.
 
@@ -68,7 +71,7 @@ namespace mongo {
         /* Return true if slaves of a replication pair are allowed to execute the command
            (the command directly from a client -- if fromRepl, always allowed).
         */
-        virtual bool slaveOk() = 0;
+        virtual bool slaveOk() const = 0;
         
         /* Return true if the client force a command to be run on a slave by
            turning on the 'slaveok' option in the command query.
@@ -93,7 +96,11 @@ namespace mongo {
         */
         virtual bool requiresAuth() { return true; }
 
-        Command(const char *_name);
+        /** @param webUI expose the command in the web ui as localhost:28017/<name> 
+            @param oldName an optional old, deprecated name for the command
+        */
+        Command(const char *_name, bool webUI = false, const char *oldName = 0);
+
         virtual ~Command() {}
 
     protected:
@@ -105,9 +112,16 @@ namespace mongo {
             return BSONObj();
         }
 
+        static void logIfSlow( const Timer& cmdTimer,  const string& msg);
+
         static map<string,Command*> * _commands;
+        static map<string,Command*> * _commandsByBestName;
+        static map<string,Command*> * _webCommands;
 
     public:
+        static const map<string,Command*>* commandsByBestName() { return _commandsByBestName; }
+        static const map<string,Command*>* webCommands() { return _webCommands; }
+        /** @return if command was found and executed */
         static bool runAgainstRegistered(const char *ns, BSONObj& jsobj, BSONObjBuilder& anObjBuilder);
         static LockType locktype( const string& name );
         static Command * findCommand( const string& name );
diff --git a/db/common.cpp b/db/common.cpp
index a199bd1..b7883f5 100644
--- a/db/common.cpp
+++ b/db/common.cpp
@@ -1,6 +1,21 @@
 // common.cpp
+/*
+ *    Copyright (C) 2010 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "concurrency.h"
 
 /**
@@ -9,6 +24,6 @@
 namespace mongo {
 
     /* we use new here so we don't have to worry about destructor orders at program shutdown */
-    MongoMutex &dbMutex( *(new MongoMutex) );
+    MongoMutex &dbMutex( *(new MongoMutex("rw:dbMutex")) );
 
 }
diff --git a/db/concurrency.h b/db/concurrency.h
index e841d61..9b91b0f 100644
--- a/db/concurrency.h
+++ b/db/concurrency.h
@@ -29,25 +29,11 @@
 
 #pragma once
 
-#if BOOST_VERSION >= 103500
-#include <boost/thread/shared_mutex.hpp>
-#undef assert
-#define assert xassert
-#define HAVE_READLOCK
-#else
-#warning built with boost version 1.34 or older - limited concurrency
-#endif
+#include "../util/concurrency/rwlock.h"
+#include "../util/mmap.h"
 
 namespace mongo {
 
-    inline bool readLockSupported(){
-#ifdef HAVE_READLOCK
-        return true;
-#else
-        return false;
-#endif
-    }
-
     string sayClientState();
     bool haveClient();
     
@@ -87,11 +73,9 @@ namespace mongo {
         }
     };
 
-#ifdef HAVE_READLOCK
-//#if 0
     class MongoMutex {
         MutexInfo _minfo;
-        boost::shared_mutex _m;
+        RWLock _m;
         ThreadLocalValue<int> _state;
 
         /* we use a separate TLS value for releasedEarly - that is ok as 
@@ -99,13 +83,16 @@ namespace mongo {
         */
         ThreadLocalValue<bool> _releasedEarly;
     public:
+        MongoMutex(const char * name) : _m(name) { }
+
         /**
          * @return
          *    > 0  write lock
          *    = 0  no lock
          *    < 0  read lock
          */
-        int getState(){ return _state.get(); }
+        int getState() { return _state.get(); }
+        bool isWriteLocked() { return getState() > 0; }
         void assertWriteLocked() { 
             assert( getState() > 0 ); 
             DEV assert( !_releasedEarly.get() );
@@ -129,7 +116,6 @@ namespace mongo {
         }
 
         void lock() { 
-            
             if ( _checkWriteLockAlready() )
                 return;
             
@@ -140,27 +126,28 @@ namespace mongo {
             curopGotLock();
 
             _minfo.entered();
+
+            MongoFile::lockAll();
         }
 
-        bool lock_try() {
+        bool lock_try( int millis ) { 
             if ( _checkWriteLockAlready() )
-                return true;            
-            
-            curopWaitingForLock( 1 );
+                return true;
 
-            boost::system_time until = get_system_time();
-            until += boost::posix_time::milliseconds(0);
-            bool got = _m.timed_lock( until );
+            curopWaitingForLock( 1 );
+            bool got = _m.lock_try( millis ); 
             curopGotLock();
             
             if ( got ){
                 _minfo.entered();
                 _state.set(1);
+                MongoFile::lockAll();
             }                
             
             return got;
         }
-        
+
+
         void unlock() { 
             //DEV cout << "UNLOCK" << endl;
             int s = _state.get();
@@ -175,6 +162,9 @@ namespace mongo {
                 }
                 massert( 12599, "internal error: attempt to unlock when wasn't in a write lock", false);
             }
+
+            MongoFile::unlockAll();
+
             _state.set(0);
             _minfo.leaving();
             _m.unlock(); 
@@ -218,10 +208,8 @@ namespace mongo {
                 lock_shared();
                 return true;
             }
-            
-            boost::system_time until = get_system_time();
-            until += boost::posix_time::milliseconds(2);
-            bool got = _m.timed_lock_shared( until );
+
+            bool got = _m.lock_shared_try( millis );
             if ( got )
                 _state.set(-1);
             return got;
@@ -246,82 +234,11 @@ namespace mongo {
         
         MutexInfo& info() { return _minfo; }
     };
-#else
-    /* this will be for old versions of boost */
-    class MongoMutex { 
-        MutexInfo _minfo;
-        boost::recursive_mutex m;
-        ThreadLocalValue<bool> _releasedEarly;
-    public:
-        MongoMutex() { }
-        void lock() { 
-#ifdef HAVE_READLOCK
-            m.lock();
-#error this should be impossible?
-#else
-            boost::detail::thread::lock_ops<boost::recursive_mutex>::lock(m);
-#endif
-            _minfo.entered();
-        }
-
-        bool lock_try(){
-            lock();
-            return true;
-        }
-
-        void releaseEarly() {
-            assertWriteLocked(); // aso must not be recursive, although we don't verify that in the old boost version
-            assert( !_releasedEarly.get() );
-            _releasedEarly.set(true);
-            _unlock();
-        }
-
-        void _unlock() {
-            _minfo.leaving();
-#ifdef HAVE_READLOCK
-            m.unlock();
-#else
-            boost::detail::thread::lock_ops<boost::recursive_mutex>::unlock(m);
-#endif
-        }
-        void unlock() { 
-            if( _releasedEarly.get() ) { 
-                _releasedEarly.set(false);
-                return;
-            }
-            _unlock();
-        }
-
-        void lock_shared() { lock(); }
-        bool lock_shared_try( int millis ) {
-            while ( millis-- ){
-                if ( getState() ){
-                    sleepmillis(1);
-                    continue;
-                }
-                lock_shared();
-                return true;
-            }
-            return false;
-        }
-                    
-        void unlock_shared() { unlock(); }
-        MutexInfo& info() { return _minfo; }
-        void assertWriteLocked() { 
-            assert( info().isLocked() );
-        }
-        void assertAtLeastReadLocked() { 
-            assert( info().isLocked() );
-        }
-        bool atLeastReadLocked() { return info().isLocked(); }
-        int getState(){ return info().isLocked() ? 1 : 0; }
-    };
-#endif
 
     extern MongoMutex &dbMutex;
 
-	void dbunlocking_write();
-	void dbunlocking_read();
+    inline void dbunlocking_write() { }
+    inline void dbunlocking_read() { }
 
     struct writelock {
         writelock(const string& ns) {
@@ -357,29 +274,36 @@ namespace mongo {
                 dbMutex.unlock_shared();
             }
         }
-        bool got(){
-            return _got;
-        }
+        bool got() const { return _got; }
+    private:
         bool _got;
     };
 
     struct writelocktry {
-        writelocktry( const string&ns ){
-            _got = dbMutex.lock_try();
+        writelocktry( const string&ns , int tryms ){
+            _got = dbMutex.lock_try( tryms );
         }
         ~writelocktry() {
             if ( _got ){
-                dbunlocking_write();
+                dbunlocking_read();
                 dbMutex.unlock();
             }
         }
-        bool got(){
-            return _got;
-        }
+        bool got() const { return _got; }
+    private:
         bool _got;
     };
 
-    
+    struct readlocktryassert : public readlocktry { 
+        readlocktryassert(const string& ns, int tryms) : 
+          readlocktry(ns,tryms) { 
+              uassert(13142, "timeout getting readlock", got());
+        }
+    };
+
+    /** assure we have at least a read lock - they key with this being
+        if you have a write lock, that's ok too.
+    */
     struct atleastreadlock {
         atleastreadlock( const string& ns ){
             _prev = dbMutex.getState();
@@ -390,7 +314,7 @@ namespace mongo {
             if ( _prev == 0 )
                 dbMutex.unlock_shared();
         }
-
+    private:
         int _prev;
     };
 
@@ -419,11 +343,9 @@ namespace mongo {
         void releaseAndWriteLock();
     };
     
-	/* use writelock and readlock instead */
+    /* use writelock and readlock instead */
     struct dblock : public writelock {
         dblock() : writelock("") { }
-        ~dblock() { 
-        }
     };
 
     // eliminate
diff --git a/db/curop.h b/db/curop.h
index 21582f2..81fa0e4 100644
--- a/db/curop.h
+++ b/db/curop.h
@@ -20,11 +20,12 @@
 
 #include "namespace.h"
 #include "client.h"
-#include "../util/atomic_int.h"
+#include "../bson/util/atomic_int.h"
 #include "db.h"
 
 namespace mongo { 
 
+    /* lifespan is different than CurOp because of recursives with DBDirectClient */
     class OpDebug {
     public:
         StringBuilder str;
@@ -55,8 +56,7 @@ namespace mongo {
         int _dbprofile; // 0=off, 1=slow, 2=all
         AtomicUInt _opNum;
         char _ns[Namespace::MaxNsLen+2];
-        struct sockaddr_in _remote;
-        
+        struct SockAddr _remote;
         char _queryBuf[256];
         
         void resetQuery(int x=0) { *((int *)_queryBuf) = x; }
@@ -81,11 +81,12 @@ namespace mongo {
         }
 
     public:
-
-        bool haveQuery() const { return *((int *) _queryBuf) != 0; }
+        
+        int querySize() const { return *((int *) _queryBuf); }
+        bool haveQuery() const { return querySize() != 0; }
 
         BSONObj query() {
-            if( *((int *) _queryBuf) == 1 ) { 
+            if( querySize() == 1 ) { 
                 return _tooBig;
             }
             BSONObj o(_queryBuf);
@@ -119,7 +120,7 @@ namespace mongo {
             resetQuery();            
         }
         
-        void reset( const sockaddr_in & remote, int op ) { 
+        void reset( const SockAddr & remote, int op ) {
             reset();
             _remote = remote;
             _op = op;
@@ -206,6 +207,10 @@ namespace mongo {
             memcpy(_queryBuf, query.objdata(), query.objsize());
         }
 
+        Client * getClient() const { 
+            return _client;
+        }
+
         CurOp( Client * client , CurOp * wrapped = 0 ) { 
             _client = client;
             _wrapped = wrapped;
@@ -237,28 +242,32 @@ namespace mongo {
             return infoNoauth();
         }
         
-        BSONObj infoNoauth();
+        BSONObj infoNoauth( int attempt = 0 );
 
-        string getRemoteString(){
-            stringstream ss;
-            ss << inet_ntoa( _remote.sin_addr ) << ":" << ntohs( _remote.sin_port );
-            return ss.str();
+        string getRemoteString( bool includePort = true ){
+            return _remote.toString(includePort);
         }
 
         ProgressMeter& setMessage( const char * msg , long long progressMeterTotal = 0 , int secondsBetween = 3 ){
-            _message = msg;
+
             if ( progressMeterTotal ){
-                assert( ! _progressMeter.isActive() );
+                if ( _progressMeter.isActive() ){
+                    cout << "about to assert, old _message: " << _message << " new message:" << msg << endl;
+                    assert( ! _progressMeter.isActive() );
+                }
                 _progressMeter.reset( progressMeterTotal , secondsBetween );
             }
             else {
                 _progressMeter.finished();
             }
+            
+            _message = msg;
+            
             return _progressMeter;
         }
-
-        string getMessage() const { return _message; }
-        ProgressMeter getProgressMeter() { return _progressMeter; }
+        
+        string getMessage() const { return _message.toString(); }
+        ProgressMeter& getProgressMeter() { return _progressMeter; }
 
         friend class Client;
     };
diff --git a/db/cursor.cpp b/db/cursor.cpp
index 29f9c97..e98cb7a 100644
--- a/db/cursor.cpp
+++ b/db/cursor.cpp
@@ -14,7 +14,7 @@
  *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "pdfile.h"
 #include "curop.h"
 
diff --git a/db/cursor.h b/db/cursor.h
index 69e5d67..db5d9a3 100644
--- a/db/cursor.h
+++ b/db/cursor.h
@@ -16,14 +16,16 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 
 #include "jsobj.h"
 #include "diskloc.h"
+#include "matcher.h"
 
 namespace mongo {
     
     class Record;
+    class CoveredIndexMatcher;
 
     /* Query cursors, base class.  This is for our internal cursors.  "ClientCursor" is a separate
        concept and is for the user's cursor.
@@ -31,13 +33,11 @@ namespace mongo {
        WARNING concurrency: the vfunctions below are called back from within a 
        ClientCursor::ccmutex.  Don't cause a deadlock, you've been warned.
     */
-    class Cursor {
+    class Cursor : boost::noncopyable {
     public:
         virtual ~Cursor() {}
         virtual bool ok() = 0;
-        bool eof() {
-            return !ok();
-        }
+        bool eof() { return !ok(); }
         virtual Record* _current() = 0;
         virtual BSONObj current() = 0;
         virtual DiskLoc currLoc() = 0;
@@ -78,10 +78,9 @@ namespace mongo {
         virtual void checkLocation() { }
         
         virtual bool supportGetMore() = 0;
-
-        virtual string toString() {
-            return "abstract?";
-        }
+        virtual bool supportYields() = 0;
+        
+        virtual string toString() { return "abstract?"; }
 
         /* used for multikey index traversal to avoid sending back dups. see Matcher::matches().
            if a multikey index traversal:
@@ -93,10 +92,21 @@ namespace mongo {
         */
         virtual bool getsetdup(DiskLoc loc) = 0;
 
-        virtual BSONObj prettyIndexBounds() const { return BSONObj(); }
+        virtual BSONObj prettyIndexBounds() const { return BSONArray(); }
 
         virtual bool capped() const { return false; }
 
+        // The implementation may return different matchers depending on the
+        // position of the cursor.  If matcher() is nonzero at the start,
+        // matcher() should be checked each time advance() is called.
+        virtual CoveredIndexMatcher *matcher() const { return 0; }
+        
+        // A convenience function for setting the value of matcher() manually
+        // so it may accessed later.  Implementations which must generate
+        // their own matcher() should assert here.
+        virtual void setMatcher( shared_ptr< CoveredIndexMatcher > matcher ) {
+            massert( 13285, "manual matcher config not allowed", false );
+        }
     };
 
     // strategy object implementing direction of traversal.
@@ -117,6 +127,7 @@ namespace mongo {
 
     private:
         bool tailable_;
+        shared_ptr< CoveredIndexMatcher > _matcher;
         void init() {
             tailable_ = false;
         }
@@ -161,6 +172,14 @@ namespace mongo {
         virtual bool getsetdup(DiskLoc loc) { return false; }
 
         virtual bool supportGetMore() { return true; }
+        virtual bool supportYields() { return true; }
+
+        virtual CoveredIndexMatcher *matcher() const { return _matcher.get(); }
+        
+        virtual void setMatcher( shared_ptr< CoveredIndexMatcher > matcher ) {
+            _matcher = matcher;
+        }
+        
     };
 
     /* used for order { $natural: -1 } */
diff --git a/db/database.cpp b/db/database.cpp
index 6361e86..dde117f 100644
--- a/db/database.cpp
+++ b/db/database.cpp
@@ -16,14 +16,64 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "pdfile.h"
 #include "database.h"
+#include "instance.h"
 
 namespace mongo {
 
     bool Database::_openAllFiles = false;
 
+    Database::Database(const char *nm, bool& newDb, const string& _path )
+        : name(nm), path(_path), namespaceIndex( path, name ) {
+        
+        { // check db name is valid
+            size_t L = strlen(nm);
+            uassert( 10028 ,  "db name is empty", L > 0 );
+            uassert( 10029 ,  "bad db name [1]", *nm != '.' );
+            uassert( 10030 ,  "bad db name [2]", nm[L-1] != '.' );
+            uassert( 10031 ,  "bad char(s) in db name", strchr(nm, ' ') == 0 );
+            uassert( 10032 ,  "db name too long", L < 64 );
+        }
+        
+        newDb = namespaceIndex.exists();
+        profile = 0;
+        profileName = name + ".system.profile";
+
+        {
+            vector<string> others;
+            getDatabaseNames( others , path );
+            
+            for ( unsigned i=0; i<others.size(); i++ ){
+
+                if ( strcasecmp( others[i].c_str() , nm ) )
+                    continue;
+
+                if ( strcmp( others[i].c_str() , nm ) == 0 )
+                    continue;
+                
+                stringstream ss;
+                ss << "db already exists with different case other: [" << others[i] << "] me [" << nm << "]";
+                uasserted( DatabaseDifferCaseCode , ss.str() );
+            }
+        }
+
+        
+        // If already exists, open.  Otherwise behave as if empty until
+        // there's a write, then open.
+        if ( ! newDb || cmdLine.defaultProfile ) {
+            namespaceIndex.init();
+            if( _openAllFiles )
+                openAllFiles();
+            
+        }
+       
+
+        magic = 781231;
+    }
+
+
     bool Database::setProfilingLevel( int newLevel , string& errmsg ){
         if ( profile == newLevel )
             return true;
@@ -61,4 +111,18 @@ namespace mongo {
         massert( 12506 , errmsg , setProfilingLevel( cmdLine.defaultProfile , errmsg ) );
     }
 
+    bool Database::validDBName( const string& ns ){
+        if ( ns.size() == 0 || ns.size() > 64 )
+            return false;
+        size_t good = strcspn( ns.c_str() , "/\\. \"" );
+        return good == ns.size();
+    }
+
+    void Database::flushFiles( bool sync ){
+        dbMutex.assertAtLeastReadLocked();
+        for ( unsigned i=0; i<files.size(); i++ ){
+            files[i]->flush( sync );
+        }
+    }
+
 } // namespace mongo
diff --git a/db/database.h b/db/database.h
index 23bca7c..ff0e814 100644
--- a/db/database.h
+++ b/db/database.h
@@ -22,7 +22,6 @@
 
 namespace mongo {
 
-
     /**
      * Database represents a database database
      * Each database database has its own set of files -- dbname.ns, dbname.0, dbname.1, ...
@@ -32,33 +31,7 @@ namespace mongo {
     public:
         static bool _openAllFiles;
         
-        Database(const char *nm, bool& newDb, const string& _path = dbpath)
-            : name(nm), path(_path), namespaceIndex( path, name ) {
-            
-            { // check db name is valid
-                size_t L = strlen(nm);
-                uassert( 10028 ,  "db name is empty", L > 0 );
-                uassert( 10029 ,  "bad db name [1]", *nm != '.' );
-                uassert( 10030 ,  "bad db name [2]", nm[L-1] != '.' );
-                uassert( 10031 ,  "bad char(s) in db name", strchr(nm, ' ') == 0 );
-                uassert( 10032 ,  "db name too long", L < 64 );
-            }
-
-            newDb = namespaceIndex.exists();
-            profile = 0;
-            profileName = name + ".system.profile";
-
-            // If already exists, open.  Otherwise behave as if empty until
-            // there's a write, then open.
-            if ( ! newDb || cmdLine.defaultProfile ) {
-                namespaceIndex.init();
-                if( _openAllFiles )
-                    openAllFiles();
-
-            }
-            
-            magic = 781231;
-        }
+        Database(const char *nm, bool& newDb, const string& _path = dbpath);
         
         ~Database() {
             magic = 0;
@@ -114,7 +87,7 @@ namespace mongo {
             namespaceIndex.init();
             if ( n < 0 || n >= DiskLoc::MaxFiles ) {
                 out() << "getFile(): n=" << n << endl;
-#if !defined(_RECSTORE)
+#if 0
                 if( n >= RecCache::Base && n <= RecCache::Base+1000 )
                     massert( 10294 , "getFile(): bad file number - using recstore db w/nonrecstore db build?", false);
 #endif
@@ -137,8 +110,8 @@ namespace mongo {
                 int minSize = 0;
                 if ( n != 0 && files[ n - 1 ] )
                     minSize = files[ n - 1 ]->getHeader()->fileLength;
-                if ( sizeNeeded + MDFHeader::headerSize() > minSize )
-                    minSize = sizeNeeded + MDFHeader::headerSize();
+                if ( sizeNeeded + DataFileHeader::HeaderSize > minSize )
+                    minSize = sizeNeeded + DataFileHeader::HeaderSize;
                 try {
                     p->open( fullNameString.c_str(), minSize, preallocateOnly );
                 }
@@ -205,6 +178,17 @@ namespace mongo {
         bool setProfilingLevel( int newLevel , string& errmsg );
 
         void finishInit();
+
+        static bool validDBName( const string& ns );
+
+        long long fileSize(){
+            long long size=0;
+            for (int n=0; exists(n); n++)
+                size += boost::filesystem::file_size( fileName(n) );
+            return size;
+        }
+
+        void flushFiles( bool sync );
         
         vector<MongoDataFile*> files;
         string name; // "alleyinsider"
diff --git a/db/db.cpp b/db/db.cpp
index 9be4031..2b91956 100644
--- a/db/db.cpp
+++ b/db/db.cpp
@@ -1,8 +1,7 @@
-// db.cpp : Defines the entry point for the console application.
-//
+// @file db.cpp : Defines the entry point for the mongod application.
 
 /**
-*    Copyright (C) 2008 10gen Inc.info
+*    Copyright (C) 2008 10gen Inc.
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
@@ -17,7 +16,7 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "db.h"
 #include "query.h"
 #include "introspect.h"
@@ -30,48 +29,57 @@
 #include "clientcursor.h"
 #include "pdfile.h"
 #include "stats/counters.h"
-#if !defined(_WIN32)
-#include <sys/file.h>
-#endif
-
-#if defined(_WIN32)
-#include "../util/ntservice.h"
-#endif
-
+#include "repl/rs.h"
 #include "../scripting/engine.h"
 #include "module.h"
 #include "cmdline.h"
 #include "stats/snapshots.h"
+#include "../util/concurrency/task.h"
+#include "../util/version.h"
+#include "client.h"
+#include "dbwebserver.h"
 
-namespace mongo {
-
-    CmdLine cmdLine;
+#if defined(_WIN32)
+# include "../util/ntservice.h"
+#else
+# include <sys/file.h>
+#endif
 
-    bool useJNI = true;
+namespace mongo {
 
     /* only off if --nocursors which is for debugging. */
     extern bool useCursors;
+
     /* only off if --nohints */
     extern bool useHints;
 
-    bool noHttpInterface = false;
-
-    extern string bind_ip;
     extern char *appsrvPath;
     extern int diagLogging;
     extern int lenForNewNsFiles;
     extern int lockFile;
-    
+    extern bool checkNsFilesOnLoad;    
     extern string repairpath;
 
+#if defined(_WIN32)
+    std::wstring windowsServiceName = L"MongoDB";
+    std::wstring windowsServiceUser = L"";
+    std::wstring windowsServicePassword = L"";
+#endif
+
     void setupSignals();
     void closeAllSockets();
+    void startReplSets(ReplSetCmdline*);
     void startReplication();
     void pairWith(const char *remoteEnd, const char *arb);
-    void setRecCacheSize(unsigned MB);
-
     void exitCleanly( ExitCode code );
 
+    CmdLine cmdLine;
+    bool useJNI = true;
+    bool noHttpInterface = false;
+    bool shouldRepairDatabases = 0;
+    bool forceRepair = 0;
+    Timer startupSrandTimer;
+
     const char *ourgetns() { 
         Client *c = currentClient.get();
         if ( ! c )
@@ -88,103 +96,115 @@ namespace mongo {
 
     QueryResult* emptyMoreResult(long long);
 
-    void testTheDb() {
-        OpDebug debug;
-        Client::Context ctx("sys.unittest.pdfile");
-
-        /* this is not validly formatted, if you query this namespace bad things will happen */
-        theDataFileMgr.insert("sys.unittest.pdfile", (void *) "hello worldx", 13);
-        theDataFileMgr.insert("sys.unittest.pdfile", (void *) "hello worldx", 13);
-
-        BSONObj j1((const char *) &js1);
-        deleteObjects("sys.unittest.delete", j1, false);
-        theDataFileMgr.insert("sys.unittest.delete", &js1, sizeof(js1));
-        deleteObjects("sys.unittest.delete", j1, false);
-        updateObjects("sys.unittest.delete", j1, j1, true,false,true,debug);
-        updateObjects("sys.unittest.delete", j1, j1, false,false,true,debug);
-
-        auto_ptr<Cursor> c = theDataFileMgr.findAll("sys.unittest.pdfile");
-        while ( c->ok() ) {
-            c->_current();
-            c->advance();
-        }
-        out() << endl;
-    }
-
-    MessagingPort *connGrab = 0;
-    void connThread();
+    void connThread( MessagingPort * p );
 
     class OurListener : public Listener {
     public:
         OurListener(const string &ip, int p) : Listener(ip, p) { }
         virtual void accepted(MessagingPort *mp) {
-            assert( connGrab == 0 );
+
             if ( ! connTicketHolder.tryAcquire() ){
-                log() << "connection refused because too many open connections" << endl;
+                log() << "connection refused because too many open connections: " << connTicketHolder.used() << endl;
                 // TODO: would be nice if we notified them...
                 mp->shutdown();
+                delete mp;
                 return;
             }
-            connGrab = mp;
+
             try {
-                boost::thread thr(connThread);
-                while ( connGrab )
-                    sleepmillis(1);
+                boost::thread thr(boost::bind(&connThread,mp));
             }
             catch ( boost::thread_resource_error& ){
                 log() << "can't create new thread, closing connection" << endl;
                 mp->shutdown();
-                connGrab = 0;
+                delete mp;
             }
             catch ( ... ){
                 log() << "unkonwn exception starting connThread" << endl;
                 mp->shutdown();
-                connGrab = 0;
+                delete mp;
             }
         }
     };
 
-    void webServerThread();
+/* todo: make this a real test.  the stuff in dbtests/ seem to do all dbdirectclient which exhaust doesn't support yet. */
+// QueryOption_Exhaust
+#define TESTEXHAUST 0
+#if( TESTEXHAUST )
+    void testExhaust() { 
+        sleepsecs(1);
+        unsigned n = 0;
+        auto f = [&n](const BSONObj& o) { 
+            assert( o.valid() );
+            //cout << o << endl;
+            n++;
+            bool testClosingSocketOnError = false;
+            if( testClosingSocketOnError )
+                assert(false);
+        };
+        DBClientConnection db(false);
+        db.connect("localhost");
+        const char *ns = "local.foo";
+        if( db.count(ns) < 10000 )
+            for( int i = 0; i < 20000; i++ ) 
+                db.insert(ns, BSON("aaa" << 3 << "b" << "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
+
+        try {
+            db.query(f, ns, Query() );
+        }
+        catch(...) { 
+            cout << "hmmm" << endl;
+        }
+
+        try {
+            db.query(f, ns, Query() );
+        }
+        catch(...) { 
+            cout << "caught" << endl;
+        }
+
+        cout << n << endl;
+    };
+#endif
 
     void listen(int port) {
-        log() << mongodVersion() << endl;
-        printGitVersion();
-        printSysInfo();
         //testTheDb();
         log() << "waiting for connections on port " << port << endl;
-        OurListener l(bind_ip, port);
+        OurListener l(cmdLine.bind_ip, port);
+        l.setAsTimeTracker();
         startReplication();
         if ( !noHttpInterface )
             boost::thread thr(webServerThread);
-        if ( l.init() ) {
-            ListeningSockets::get()->add( l.socket() );
-            l.listen();
-        }
-    }
 
-} // namespace mongo
-
-#include "client.h"
-
-namespace mongo {
+#if(TESTEXHAUST)
+        boost::thread thr(testExhaust);
+#endif
+        l.initAndListen();
+    }
 
-  void sysRuntimeInfo() {
-    out() << "sysinfo:\n";
+    void sysRuntimeInfo() {
+        out() << "sysinfo:\n";
 #if defined(_SC_PAGE_SIZE)
-    out() << "  page size: " << (int) sysconf(_SC_PAGE_SIZE) << endl;
+        out() << "  page size: " << (int) sysconf(_SC_PAGE_SIZE) << endl;
 #endif
 #if defined(_SC_PHYS_PAGES)
-    out() << "  _SC_PHYS_PAGES: " << sysconf(_SC_PHYS_PAGES) << endl;
+        out() << "  _SC_PHYS_PAGES: " << sysconf(_SC_PHYS_PAGES) << endl;
 #endif
 #if defined(_SC_AVPHYS_PAGES)
-    out() << "  _SC_AVPHYS_PAGES: " << sysconf(_SC_AVPHYS_PAGES) << endl;
+        out() << "  _SC_AVPHYS_PAGES: " << sysconf(_SC_AVPHYS_PAGES) << endl;
 #endif
-  }
+    }
+
+    /* if server is really busy, wait a bit */
+    void beNice() {
+        sleepmicros( Client::recommendedYieldMicros() );
+    }
 
     /* we create one thread for each connection from an app server database.
        app server will open a pool of threads.
+       todo: one day, asio...
     */
-    void connThread()
+    void connThread( MessagingPort * inPort )
     {
         TicketHolderReleaser connTicketReleaser( &connTicketHolder );
         Client::initThread("conn");
@@ -193,8 +213,9 @@ namespace mongo {
         LastError *le = new LastError();
         lastError.reset(le);
 
-        auto_ptr<MessagingPort> dbMsgPort( connGrab );
-        connGrab = 0;
+        auto_ptr<MessagingPort> dbMsgPort( inPort );
+
+        dbMsgPort->_logLevel = 1;
         Client& c = cc();
 
         try {
@@ -211,7 +232,7 @@ namespace mongo {
                     dbMsgPort->shutdown();
                     break;
                 }
-
+sendmore:
                 if ( inShutdown() ) {
                     log() << "got request after shutdown()" << endl;
                     break;
@@ -220,8 +241,8 @@ namespace mongo {
                 lastError.startRequest( m , le );
 
                 DbResponse dbresponse;
-                if ( !assembleResponse( m, dbresponse, dbMsgPort->farEnd.sa ) ) {
-                    out() << curTimeMillis() % 10000 << "   end msg " << dbMsgPort->farEnd.toString() << endl;
+                if ( !assembleResponse( m, dbresponse, dbMsgPort->farEnd ) ) {
+                    log() << curTimeMillis() % 10000 << "   end msg " << dbMsgPort->farEnd.toString() << endl;
                     /* todo: we may not wish to allow this, even on localhost: very low priv accounts could stop us. */
                     if ( dbMsgPort->farEnd.isLocalHost() ) {
                         dbMsgPort->shutdown();
@@ -230,17 +251,43 @@ namespace mongo {
                         dbexit(EXIT_CLEAN);
                     }
                     else {
-                        out() << "  (not from localhost, ignoring end msg)" << endl;
+                        log() << "  (not from localhost, ignoring end msg)" << endl;
                     }
                 }
 
-                if ( dbresponse.response )
+                if ( dbresponse.response ) {
                     dbMsgPort->reply(m, *dbresponse.response, dbresponse.responseTo);
+                    if( dbresponse.exhaust ) { 
+                        MsgData *header = dbresponse.response->header();
+                        QueryResult *qr = (QueryResult *) header;
+                        long long cursorid = qr->cursorId;
+                        if( cursorid ) {
+                            assert( dbresponse.exhaust && *dbresponse.exhaust != 0 );
+                            string ns = dbresponse.exhaust; // before reset() free's it...
+                            m.reset();
+                            BufBuilder b(512);
+                            b.appendNum((int) 0 /*size set later in appendData()*/);
+                            b.appendNum(header->id);
+                            b.appendNum(header->responseTo);
+                            b.appendNum((int) dbGetMore);
+                            b.appendNum((int) 0);
+                            b.appendStr(ns);
+                            b.appendNum((int) 0); // ntoreturn
+                            b.appendNum(cursorid);
+                            m.appendData(b.buf(), b.len());
+                            b.decouple();
+                            DEV log() << "exhaust=true sending more" << endl;
+                            beNice();
+                            goto sendmore;
+                        }
+                    }
+                }
             }
 
         }
-        catch ( AssertionException& ) {
-            problem() << "AssertionException in connThread, closing client connection" << endl;
+        catch ( AssertionException& e ) {
+            log() << "AssertionException in connThread, closing client connection" << endl;
+            log() << ' ' << e.what() << endl;
             dbMsgPort->shutdown();
         }
         catch ( SocketException& ) {
@@ -266,15 +313,13 @@ namespace mongo {
         globalScriptEngine->threadDone();
     }
 
-
     void msg(const char *m, const char *address, int port, int extras = 0) {
-
         SockAddr db(address, port);
 
-//  SockAddr db("127.0.0.1", DBPort);
-//  SockAddr db("192.168.37.1", MessagingPort::DBPort);
-//  SockAddr db("10.0.21.60", MessagingPort::DBPort);
-//  SockAddr db("172.16.0.179", MessagingPort::DBPort);
+        //  SockAddr db("127.0.0.1", DBPort);
+        //  SockAddr db("192.168.37.1", MessagingPort::DBPort);
+        //  SockAddr db("10.0.21.60", MessagingPort::DBPort);
+        //  SockAddr db("172.16.0.179", MessagingPort::DBPort);
 
         MessagingPort p;
         if ( !p.connect(db) ){
@@ -288,7 +333,7 @@ namespace mongo {
             Message response;
 
             send.setData( dbMsg , m);
-            int len = send.data->dataLen();
+            int len = send.header()->dataLen();
 
             for ( int i = 0; i < extras; i++ )
                 p.say(/*db, */send);
@@ -297,7 +342,7 @@ namespace mongo {
             bool ok = p.call(send, response);
             double tm = ((double) t.micros()) + 1;
             out() << " ****ok. response.data:" << ok << " time:" << tm / 1000.0 << "ms "
-                  << "len: " << len << " data: " << response.data->_data << endl;
+                  << "len: " << len << " data: " << response.singleData()->_data << endl;
 
             if (  q+1 < Loops ) {
                 out() << "\t\tSLEEP 8 then sending again as a test" << endl;
@@ -313,10 +358,7 @@ namespace mongo {
         msg(m, "127.0.0.1", CmdLine::DefaultDBPort, extras);
     }
 
-    bool shouldRepairDatabases = 0;
-    bool forceRepair = 0;
-    
-    bool doDBUpgrade( const string& dbName , string errmsg , MDFHeader * h ){
+    bool doDBUpgrade( const string& dbName , string errmsg , DataFileHeader * h ){
         static DBDirectClient db;
         
         if ( h->version == 4 && h->versionMinor == 4 ){
@@ -344,13 +386,12 @@ namespace mongo {
         return repairDatabase( dbName.c_str(), errmsg );
     }
     
-    extern bool checkNsFilesOnLoad;
-
     void repairDatabases() {
+		//        LastError * le = lastError.get( true );
         Client::GodScope gs;
-        log(1) << "enter repairDatabases" << endl;
+        log(1) << "enter repairDatabases (to check pdfile version #)" << endl;
         
-        assert(checkNsFilesOnLoad);
+        //assert(checkNsFilesOnLoad);
         checkNsFilesOnLoad = false; // we are mainly just checking the header - don't scan the whole .ns file for every db here.
 
         dblock lk;
@@ -361,7 +402,7 @@ namespace mongo {
             log(1) << "\t" << dbName << endl;
             Client::Context ctx( dbName );
             MongoDataFile *p = cc().database()->getFile( 0 );
-            MDFHeader *h = p->getHeader();
+            DataFileHeader *h = p->getHeader();
             if ( !h->currentVersion() || forceRepair ) {
                 log() << "****" << endl;
                 log() << "****" << endl;
@@ -428,10 +469,14 @@ namespace mongo {
      */
     class DataFileSync : public BackgroundJob {
     public:
+        string name() { return "DataFileSync"; }
         void run(){
-            if ( _sleepsecs > 2100 )
-                _sleepsecs = 2100;
-            log(1) << "will flush memory every: " << _sleepsecs << " seconds" << endl;
+            if( _sleepsecs == 0 )
+                log() << "warning: --syncdelay 0 is not recommended and can have strange performance" << endl;
+            else if( _sleepsecs == 1 ) 
+                log() << "--syncdelay 1" << endl;
+            else if( _sleepsecs != 60 )
+                log(1) << "--syncdelay " << _sleepsecs << endl;
             int time_flushing = 0;
             while ( ! inShutdown() ){
                 if ( _sleepsecs == 0 ){
@@ -440,61 +485,57 @@ namespace mongo {
                     continue;
                 }
 
-                sleepmillis( (int)(std::max(0.0, (_sleepsecs * 1000) - time_flushing)) );
-
+                sleepmillis( (long long) std::max(0.0, (_sleepsecs * 1000) - time_flushing) );
+                
+                if ( inShutdown() ){
+                    // occasional issue trying to flush during shutdown when sleep interrupted
+                    break;
+                }
+                
                 Date_t start = jsTime();
-                MemoryMappedFile::flushAll( true );
+                int numFiles = MemoryMappedFile::flushAll( true );
                 time_flushing = (int) (jsTime() - start);
 
                 globalFlushCounters.flushed(time_flushing);
 
-                log(1) << "flushing mmap took " << time_flushing << "ms" << endl;
+                log(1) << "flushing mmap took " << time_flushing << "ms " << " for " << numFiles << " files" << endl;
             }
         }
         
         double _sleepsecs; // default value controlled by program options
     } dataFileSync;
 
-    void show_32_warning(){
-#if BOOST_VERSION < 103500
-        cout << "\nwarning: built with boost version <= 1.34, limited concurrency" << endl;
-#endif
-
-        if ( sizeof(int*) != 4 )
-            return;
-        cout << endl;
-        cout << "** NOTE: when using MongoDB 32 bit, you are limited to about 2 gigabytes of data" << endl;
-        cout << "**       see http://blog.mongodb.org/post/137788967/32-bit-limitations for more" << endl;
-        cout << endl;
-    }
-
-    Timer startupSrandTimer;
+    void _initAndListen(int listenPort, const char *appserverLoc = NULL) {
 
-    void _initAndListen(int listenPort, const char *appserverLoc = null) {
+        bool is32bit = sizeof(int*) == 4;
 
+        {
 #if !defined(_WIN32)
-        pid_t pid = 0;
-        pid = getpid();
+            pid_t pid = getpid();
 #else
-        int pid=0;
+            DWORD pid=GetCurrentProcessId();
 #endif
-
-        bool is32bit = sizeof(int*) == 4;
-
-        log() << "Mongo DB : starting : pid = " << pid << " port = " << cmdLine.port << " dbpath = " << dbpath
-              <<  " master = " << replSettings.master << " slave = " << (int) replSettings.slave << "  " << ( is32bit ? "32" : "64" ) << "-bit " << endl;
-        DEV log() << " FULL DEBUG ENABLED " << endl;
+            Nullstream& l = log();
+            l << "MongoDB starting : pid=" << pid << " port=" << cmdLine.port << " dbpath=" << dbpath;
+            if( replSettings.master ) l << " master=" << replSettings.master;
+            if( replSettings.slave )  l << " slave=" << (int) replSettings.slave;
+            l << ( is32bit ? " 32" : " 64" ) << "-bit " << endl;
+        }
+        DEV log() << "_DEBUG build (which is slower)" << endl;
         show_32_warning();
+        log() << mongodVersion() << endl;
+        printGitVersion();
+        printSysInfo();
 
         {
             stringstream ss;
             ss << "dbpath (" << dbpath << ") does not exist";
-            massert( 10296 ,  ss.str().c_str(), boost::filesystem::exists( dbpath ) );
+            uassert( 10296 ,  ss.str().c_str(), boost::filesystem::exists( dbpath ) );
         }
         {
             stringstream ss;
             ss << "repairpath (" << repairpath << ") does not exist";
-            massert( 12590 ,  ss.str().c_str(), boost::filesystem::exists( repairpath ) );
+            uassert( 12590 ,  ss.str().c_str(), boost::filesystem::exists( repairpath ) );
         }
         
         acquirePathLock();
@@ -537,24 +578,34 @@ namespace mongo {
         srand((unsigned) (curTimeMicros() ^ startupSrandTimer.micros()));
 
         snapshotThread.go();
+        clientCursorMonitor.go();
+
+        if( !cmdLine._replSet.empty() ) {
+            replSet = true;
+            ReplSetCmdline *replSetCmdline = new ReplSetCmdline(cmdLine._replSet);
+            boost::thread t( boost::bind( &startReplSets, replSetCmdline) );
+        }
+
         listen(listenPort);
 
         // listen() will return when exit code closes its socket.
-        while( 1 )
-            sleepsecs( 100 );
+        exitCleanly(EXIT_NET_ERROR);
     }
-    void initAndListen(int listenPort, const char *appserverLoc = null) {
+
+    void testPretouch();
+
+    void initAndListen(int listenPort, const char *appserverLoc = NULL) {
         try { _initAndListen(listenPort, appserverLoc); }
         catch ( std::exception &e ) {
-            problem() << "exception in initAndListen std::exception: " << e.what() << ", terminating" << endl;
+            log() << "exception in initAndListen std::exception: " << e.what() << ", terminating" << endl;
             dbexit( EXIT_UNCAUGHT );
         }
         catch ( int& n ){
-            problem() << "exception in initAndListen int: " << n << ", terminating" << endl;
+            log() << "exception in initAndListen int: " << n << ", terminating" << endl;
             dbexit( EXIT_UNCAUGHT );
         }
         catch(...) {
-            log() << " exception in initAndListen, terminating" << endl;
+            log() << "exception in initAndListen, terminating" << endl;
             dbexit( EXIT_UNCAUGHT );
         }
     }
@@ -569,14 +620,14 @@ namespace mongo {
 
 } // namespace mongo
 
-
 using namespace mongo;
 
 #include <boost/program_options.hpp>
+#undef assert
+#define assert MONGO_assert
 
 namespace po = boost::program_options;
 
-
 void show_help_text(po::options_description options) {
     show_32_warning();
     cout << options << endl;
@@ -605,6 +656,9 @@ int main(int argc, char* argv[], char *envp[] )
     getcurns = ourgetns;
 
     po::options_description general_options("General options");
+	#if defined(_WIN32)
+	po::options_description windows_scm_options("Windows Service Control Manager options");
+	#endif
     po::options_description replication_options("Replication options");
     po::options_description sharding_options("Sharding options");
     po::options_description visible_options("Allowed options");
@@ -615,9 +669,7 @@ int main(int argc, char* argv[], char *envp[] )
     CmdLine::addGlobalOptions( general_options , hidden_options );
 
     general_options.add_options()
-        ("bind_ip", po::value<string>(&bind_ip),
-         "local ip address to bind listener - all local ips bound by default")
-        ("dbpath", po::value<string>()->default_value("/data/db/"), "directory for datafiles")
+        ("dbpath", po::value<string>() , "directory for datafiles")
         ("directoryperdb", "each database will be stored in a separate directory")
         ("repairpath", po::value<string>() , "root directory for repair files - defaults to dbpath" )
         ("cpu", "periodically show cpu and iowait utilization")
@@ -640,18 +692,28 @@ int main(int argc, char* argv[], char *envp[] )
         ("upgrade", "upgrade db if needed")
         ("repair", "run repair on all dbs")
         ("notablescan", "do not allow table scans")
-        ("syncdelay",po::value<double>(&dataFileSync._sleepsecs)->default_value(60), "seconds between disk syncs (0 for never)")
+        ("syncdelay",po::value<double>(&dataFileSync._sleepsecs)->default_value(60), "seconds between disk syncs (0=never, but not recommended)")
         ("profile",po::value<int>(), "0=off 1=slow, 2=all")
         ("slowms",po::value<int>(&cmdLine.slowMS)->default_value(100), "value of slow for profile and console log" )
         ("maxConns",po::value<int>(), "max number of simultaneous connections")
-#if defined(_WIN32)
+		#if !defined(_WIN32)
+        ("nounixsocket", "disable listening on unix sockets")
+		#endif
+        ("ipv6", "enable IPv6 support (disabled by default)")
+        ;
+	#if defined(_WIN32)
+    windows_scm_options.add_options()
         ("install", "install mongodb service")
         ("remove", "remove mongodb service")
+        ("reinstall", "reinstall mongodb service (equivilant of mongod --remove followed by mongod --install)")
         ("service", "start mongodb service")
-#endif
-        ;
+        ("serviceName", po::value<string>(), "windows service name")
+        ("serviceUser", po::value<string>(), "user name service executes as")
+        ("servicePassword", po::value<string>(), "password used to authenticate serviceUser")
+		;
+	#endif
 
-    replication_options.add_options()
+	replication_options.add_options()
         ("master", "master mode")
         ("slave", "slave mode")
         ("source", po::value<string>(), "when slave: specify master as <server:port>")
@@ -668,9 +730,12 @@ int main(int argc, char* argv[], char *envp[] )
 	sharding_options.add_options()
 		("configsvr", "declare this is a config db of a cluster")
 		("shardsvr", "declare this is a shard db of a cluster")
+        ("noMoveParanoia" , "turn off paranoid saving of data for moveChunk.  this is on by default for now, but default will switch" )
 		;
 
     hidden_options.add_options()
+        ("pretouch", po::value<int>(), "n pretouch threads for applying replicationed operations")
+        ("replSet", po::value<string>(), "specify repl set seed hostnames format <set id>/<host1>,<host2>,etc...")
         ("command", po::value< vector<string> >(), "command")
         ("cacheSize", po::value<long>(), "cache size (in MB) for rec store")
         ;
@@ -678,10 +743,14 @@ int main(int argc, char* argv[], char *envp[] )
 
     positional_options.add("command", 3);
     visible_options.add(general_options);
+	#if defined(_WIN32)
+	visible_options.add(windows_scm_options);
+	#endif
     visible_options.add(replication_options);
     visible_options.add(sharding_options);
     Module::addOptions( visible_options );
 
+    setupCoreSignals();
     setupSignals();
 
     dbExecCommand = argv[0];
@@ -698,17 +767,15 @@ int main(int argc, char* argv[], char *envp[] )
         }
     }
 
-    DEV out() << "DEV is defined (using _DEBUG), which is slower...\n";
-
     UnitTest::runTests();
 
-    if (argc == 1) {
+    if( argc == 1 )
         cout << dbExecCommand << " --help for help and startup options" << endl;
-    }
 
     {
         bool installService = false;
         bool removeService = false;
+        bool reinstallService = false;
         bool startService = false;
         po::variables_map params;
         
@@ -719,7 +786,6 @@ int main(int argc, char* argv[], char *envp[] )
             return 0;
         }
 
-
         if ( ! CmdLine::store( argc , argv , visible_options , hidden_options , positional_options , params ) )
             return 0;
 
@@ -732,7 +798,11 @@ int main(int argc, char* argv[], char *envp[] )
             printGitVersion();
             return 0;
         }
-        dbpath = params["dbpath"].as<string>();
+        if ( params.count( "dbpath" ) )
+            dbpath = params["dbpath"].as<string>();
+        else
+            dbpath = "/data/db/";
+
         if ( params.count("directoryperdb")) {
             directoryperdb = true;
         }
@@ -809,11 +879,24 @@ int main(int argc, char* argv[], char *envp[] )
             cmdLine.notablescan = true;
         }
         if (params.count("install")) {
+            if ( ! params.count( "logpath" ) ){
+                cout << "--install has to be used with --logpath" << endl;
+                ::exit(-1);
+            }
+
             installService = true;
         }
         if (params.count("remove")) {
             removeService = true;
         }
+        if (params.count("reinstall")) {
+            if ( ! params.count( "logpath" ) ){
+                cout << "--reinstall has to be used with --logpath" << endl;
+                ::exit(-1);
+            }
+
+            reinstallService = true;
+        }
         if (params.count("service")) {
             startService = true;
         }
@@ -836,10 +919,29 @@ int main(int argc, char* argv[], char *envp[] )
             /* specifies what the source in local.sources should be */
             cmdLine.source = params["source"].as<string>().c_str();
         }
+        if( params.count("pretouch") ) { 
+            cmdLine.pretouch = params["pretouch"].as<int>();
+        }
+        if (params.count("replSet")) {
+            if (params.count("slavedelay")) {
+                cout << "--slavedelay cannot be used with --replSet" << endl;
+                ::exit(-1);
+            } else if (params.count("only")) {
+                cout << "--only cannot be used with --replSet" << endl;
+                ::exit(-1);
+            }
+            /* seed list of hosts for the repl set */
+            cmdLine._replSet = params["replSet"].as<string>().c_str();
+        }
         if (params.count("only")) {
             cmdLine.only = params["only"].as<string>().c_str();
         }
         if (params.count("pairwith")) {
+            cout << "***********************************\n"
+                 << "WARNING WARNING WARNING\n"
+                 << " replica pairs are deprecated\n"
+                 << " see: http://www.mongodb.org/display/DOCS/Replica+Pairs \n" 
+                 << "***********************************" << endl;
             string paired = params["pairwith"].as<string>();
             if (params.count("arbiter")) {
                 string arbiter = params["arbiter"].as<string>();
@@ -871,7 +973,8 @@ int main(int argc, char* argv[], char *envp[] )
         if (params.count("cacheSize")) {
             long x = params["cacheSize"].as<long>();
             uassert( 10037 , "bad --cacheSize arg", x > 0);
-            setRecCacheSize(x);
+            log() << "--cacheSize option not currently supported" << endl;
+            //setRecCacheSize(x);
         }
 		if (params.count("port") == 0 ) { 
 			if( params.count("configsvr") ) {
@@ -880,8 +983,17 @@ int main(int argc, char* argv[], char *envp[] )
 			if( params.count("shardsvr") )
 				cmdLine.port = CmdLine::ShardServerPort;
 		}
-        if ( params.count("configsvr" ) && params.count( "diaglog" ) == 0 ){
-            _diaglog.level = 1;
+        else { 
+            if ( cmdLine.port <= 0 || cmdLine.port > 65535 ){
+                out() << "bad --port number" << endl;
+                dbexit( EXIT_BADOPTIONS );
+            }
+        }
+        if ( params.count("configsvr" ) ){
+            if ( params.count( "diaglog" ) == 0 )
+                _diaglog.level = 1;
+            if ( params.count( "dbpath" ) == 0 )
+                dbpath = "/data/configdb";
         }
         if ( params.count( "profile" ) ){
             cmdLine.defaultProfile = params["profile"].as<int>();
@@ -892,7 +1004,40 @@ int main(int argc, char* argv[], char *envp[] )
             uassert( 12508 , "maxConns can't be greater than 10000000" , newSize < 10000000 );
             connTicketHolder.resize( newSize );
         }
-        
+        if (params.count("nounixsocket")){
+            noUnixSocket = true;
+        }
+        if (params.count("ipv6")){
+            enableIPv6();
+        }
+        if (params.count("noMoveParanoia")){
+            cmdLine.moveParanoia = false;
+        }
+#if defined(_WIN32)
+        if (params.count("serviceName")){
+            string x = params["serviceName"].as<string>();
+            windowsServiceName = wstring(x.size(),L' ');
+            for ( size_t i=0; i<x.size(); i++) {
+                windowsServiceName[i] = x[i];
+	    }
+        }
+        if (params.count("serviceUser")){
+            string x = params["serviceUser"].as<string>();
+            windowsServiceUser = wstring(x.size(),L' ');
+            for ( size_t i=0; i<x.size(); i++) {
+                windowsServiceUser[i] = x[i];
+	    }
+        }
+        if (params.count("servicePassword")){
+            string x = params["servicePassword"].as<string>();
+            windowsServicePassword = wstring(x.size(),L' ');
+            for ( size_t i=0; i<x.size(); i++) {
+                windowsServicePassword[i] = x[i];
+	    }
+        }
+        #endif
+
+
         Module::configAll( params );
         dataFileSync.go();
 
@@ -935,24 +1080,30 @@ int main(int argc, char* argv[], char *envp[] )
         }
 
 #if defined(_WIN32)
-        if ( installService ) {
-            if ( !ServiceController::installService( L"MongoDB", L"Mongo DB", L"Mongo DB Server", argc, argv ) )
+        if ( reinstallService ) {
+            ServiceController::removeService( windowsServiceName );
+	}
+	if ( installService || reinstallService ) {
+            if ( !ServiceController::installService( windowsServiceName , L"Mongo DB", L"Mongo DB Server", windowsServiceUser, windowsServicePassword, dbpath, argc, argv ) )
                 dbexit( EXIT_NTSERVICE_ERROR );
             dbexit( EXIT_CLEAN );
         }
         else if ( removeService ) {
-            if ( !ServiceController::removeService( L"MongoDB" ) )
+            if ( !ServiceController::removeService( windowsServiceName ) )
                 dbexit( EXIT_NTSERVICE_ERROR );
             dbexit( EXIT_CLEAN );
         }
         else if ( startService ) {
-            if ( !ServiceController::startService( L"MongoDB", mongo::initService ) )
+            if ( !ServiceController::startService( windowsServiceName , mongo::initService ) )
                 dbexit( EXIT_NTSERVICE_ERROR );
             dbexit( EXIT_CLEAN );
         }
 #endif
     }
 
+    if( cmdLine.pretouch )
+        log() << "--pretouch " << cmdLine.pretouch << endl;
+
     initAndListen(cmdLine.port, appsrvPath);
     dbexit(EXIT_CLEAN);
     return 0;
@@ -1018,7 +1169,7 @@ namespace mongo {
     void interruptThread() {
         int x;
         sigwait( &asyncSignals, &x );
-        log() << "got kill or ctrl c signal " << x << " (" << strsignal( x ) << "), will terminate after current cmd ends" << endl;
+        log() << "got kill or ctrl c or hup signal " << x << " (" << strsignal( x ) << "), will terminate after current cmd ends" << endl;
         Client::initThread( "interruptThread" );
         exitCleanly( EXIT_KILL );
     }
@@ -1036,12 +1187,13 @@ namespace mongo {
         assert( signal(SIGFPE, abruptQuit) != SIG_ERR );
         assert( signal(SIGABRT, abruptQuit) != SIG_ERR );
         assert( signal(SIGBUS, abruptQuit) != SIG_ERR );
+        assert( signal(SIGQUIT, abruptQuit) != SIG_ERR );
         assert( signal(SIGPIPE, pipeSigHandler) != SIG_ERR );
-        assert( signal(SIGUSR1 , rotateLogs ) != SIG_ERR );
 
         setupSIGTRAPforGDB();
 
         sigemptyset( &asyncSignals );
+        sigaddset( &asyncSignals, SIGHUP );
         sigaddset( &asyncSignals, SIGINT );
         sigaddset( &asyncSignals, SIGTERM );
         assert( pthread_sigmask( SIG_SETMASK, &asyncSignals, 0 ) == 0 );
@@ -1083,16 +1235,22 @@ BOOL CtrlHandler( DWORD fdwCtrlType )
     }
 }
 
+    void myPurecallHandler() {
+        rawOut( "pure virtual method called, printing stack:\n" );
+        printStackTrace();
+        abort();        
+    }
+    
     void setupSignals() {
         if( SetConsoleCtrlHandler( (PHANDLER_ROUTINE) CtrlHandler, TRUE ) )
             ;
         else
             massert( 10297 , "Couldn't register Windows Ctrl-C handler", false);
+        _set_purecall_handler( myPurecallHandler );
     }
 #endif
 
 } // namespace mongo
 
-#include "recstore.h"
-#include "reccache.h"
-
+//#include "recstore.h"
+//#include "reccache.h"
diff --git a/db/db.h b/db/db.h
index 78fc98d..a261f58 100644
--- a/db/db.h
+++ b/db/db.h
@@ -16,9 +16,8 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "../util/message.h"
-#include "boost/version.hpp"
 #include "concurrency.h"
 #include "pdfile.h"
 #include "client.h"
@@ -64,7 +63,6 @@ namespace mongo {
             DBs::const_iterator it = m.find(db);
             return it != m.end();
         }
-
         
         Database * get( const string& ns , const string& path ) const {
             dbMutex.assertAtLeastReadLocked();
@@ -103,7 +101,13 @@ namespace mongo {
             }
             
             log(1) << "Accessing: " << dbname << " for the first time" << endl;
-            db = new Database( dbname.c_str() , justCreated , path );
+            try {
+                db = new Database( dbname.c_str() , justCreated , path );
+            }
+            catch ( ... ){
+                m.erase( dbname );
+                throw;
+            }
             _size++;
             return db;
         }
@@ -140,6 +144,12 @@ namespace mongo {
     private:
         
         string _todb( const string& ns ) const {
+            string d = __todb( ns );
+            uassert( 13280 , (string)"invalid db name: " + ns , Database::validDBName( d ) );            
+            return d;
+        }
+
+        string __todb( const string& ns ) const {
             size_t i = ns.find( '.' );
             if ( i == string::npos ){
                 uassert( 13074 , "db name can't be empty" , ns.size() );
@@ -190,7 +200,7 @@ namespace mongo {
             if ( _context ) _context->relocked();
         }
     };
-
+    
 
     /**
        only does a temp release if we're not nested and have a lock
@@ -213,10 +223,11 @@ namespace mongo {
             }
         }
         
+        bool unlocked(){
+            return real > 0;
+        }
     };
 
-    extern TicketHolder connTicketHolder;
-
 } // namespace mongo
 
 //#include "dbinfo.h"
diff --git a/db/db.rc b/db/db.rc
index fbfd379..b589458 100644..100755
--- a/db/db.rc
+++ b/db/db.rc
@@ -1,61 +1,12 @@
-// Microsoft Visual C++ generated resource script.

-//

-#include "resource.h"

-

-#define APSTUDIO_READONLY_SYMBOLS

-/////////////////////////////////////////////////////////////////////////////

-//

-// Generated from the TEXTINCLUDE 2 resource.

-//

-// #include "afxres.h"

-

-/////////////////////////////////////////////////////////////////////////////

-#undef APSTUDIO_READONLY_SYMBOLS

-

-/////////////////////////////////////////////////////////////////////////////

-// English (U.S.) resources

-

-#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)

-LANGUAGE 9, 1

-#pragma code_page(1252)

-

-#ifdef APSTUDIO_INVOKED

-/////////////////////////////////////////////////////////////////////////////

-//

-// TEXTINCLUDE

-//

-

-1 TEXTINCLUDE  

-BEGIN

-    "resource.h\0"

-END

-

-2 TEXTINCLUDE  

-BEGIN

-    "#include ""afxres.h""\r\n"

-    "\0"

-END

-

-3 TEXTINCLUDE  

-BEGIN

-    "\r\n"

-    "\0"

-END

-

-#endif    // APSTUDIO_INVOKED

-

-#endif    // English (U.S.) resources

-/////////////////////////////////////////////////////////////////////////////

-

-

-

-#ifndef APSTUDIO_INVOKED

-/////////////////////////////////////////////////////////////////////////////

-//

-// Generated from the TEXTINCLUDE 3 resource.

-//

-

-

-/////////////////////////////////////////////////////////////////////////////

-#endif    // not APSTUDIO_INVOKED

-

+// Microsoft Visual C++ generated resource script.
+//
+#include "resource.h"
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Icon
+//
+// Icon with lowest ID value placed first to ensure application icon
+// remains consistent on all systems.
+IDI_ICON2               ICON                    "mongo.ico"
+/////////////////////////////////////////////////////////////////////////////
\ No newline at end of file
diff --git a/db/db.sln b/db/db.sln
index 79ff2e1..b02b79d 100644
--- a/db/db.sln
+++ b/db/db.sln
@@ -4,51 +4,83 @@ Microsoft Visual Studio Solution File, Format Version 10.00
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mongod", "db.vcproj", "{215B2D68-0A70-4D10-8E75-B31010C62A91}"

 EndProject

 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "examples", "examples", "{4082881B-EB00-486F-906C-843B8EC06E18}"

-EndProject

-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "dbtests", "dbtests", "{C72EBEDD-342D-4371-8B0D-D7505902FA69}"

 	ProjectSection(SolutionItems) = preProject

-		..\dbtests\btreetests.cpp = ..\dbtests\btreetests.cpp

+		driverHelpers.cpp = driverHelpers.cpp

 	EndProjectSection

 EndProject

-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shell", "shell", "{2CABB3B8-C9A6-478D-9463-0B37799ED708}"

-EndProject

 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{2B262D59-9DC7-4BF1-A431-1BD4966899A5}"

 	ProjectSection(SolutionItems) = preProject

 		..\tools\bridge.cpp = ..\tools\bridge.cpp

+		..\tools\dump.cpp = ..\tools\dump.cpp

+		..\tools\export.cpp = ..\tools\export.cpp

+		..\tools\files.cpp = ..\tools\files.cpp

+		..\tools\import.cpp = ..\tools\import.cpp

+		..\tools\restore.cpp = ..\tools\restore.cpp

 		..\tools\sniffer.cpp = ..\tools\sniffer.cpp

+		..\tools\stat.cpp = ..\tools\stat.cpp

+		..\tools\tool.cpp = ..\tools\tool.cpp

+		..\tools\tool.h = ..\tools\tool.h

 	EndProjectSection

 EndProject

 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mongos", "..\s\dbgrid.vcproj", "{E03717ED-69B4-4D21-BC55-DF6690B585C6}"

 EndProject

 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test", "..\dbtests\test.vcproj", "{215B2D68-0A70-4D10-8E75-B33010C62A91}"

 EndProject

+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "unix files", "unix files", "{2F760952-C71B-4865-998F-AABAE96D1373}"

+	ProjectSection(SolutionItems) = preProject

+		..\util\mmap_posix.cpp = ..\util\mmap_posix.cpp

+		..\util\processinfo_darwin.cpp = ..\util\processinfo_darwin.cpp

+		..\util\processinfo_linux2.cpp = ..\util\processinfo_linux2.cpp

+		..\util\processinfo_none.cpp = ..\util\processinfo_none.cpp

+	EndProjectSection

+EndProject

+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shell", "shell", "{407B4B88-3451-433C-B74F-31B31FEB5791}"

+	ProjectSection(SolutionItems) = preProject

+		..\shell\dbshell.cpp = ..\shell\dbshell.cpp

+		..\shell\mongo_vstudio.cpp = ..\shell\mongo_vstudio.cpp

+		..\shell\utils.cpp = ..\shell\utils.cpp

+		..\shell\utils.h = ..\shell\utils.h

+	EndProjectSection

+EndProject

+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "other source files", "other source files", "{12B11474-2D74-48C3-BB3D-F03249BEA88F}"

+	ProjectSection(SolutionItems) = preProject

+		..\buildscripts\buildboost.bat = ..\buildscripts\buildboost.bat

+		..\buildscripts\buildboost64.bat = ..\buildscripts\buildboost64.bat

+		..\SConstruct = ..\SConstruct

+	EndProjectSection

+EndProject

+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bsondemo", "..\bson\bsondemo\bsondemo.vcproj", "{C9DB5EB7-81AA-4185-BAA1-DA035654402F}"

+EndProject

 Global

 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

-		Debug Recstore|Win32 = Debug Recstore|Win32

 		Debug|Win32 = Debug|Win32

 		Release|Win32 = Release|Win32

 	EndGlobalSection

 	GlobalSection(ProjectConfigurationPlatforms) = postSolution

-		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug Recstore|Win32.ActiveCfg = Debug Recstore|Win32

-		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug Recstore|Win32.Build.0 = Debug Recstore|Win32

 		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|Win32.ActiveCfg = Debug|Win32

 		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|Win32.Build.0 = Debug|Win32

 		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|Win32.ActiveCfg = Release|Win32

 		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|Win32.Build.0 = Release|Win32

-		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Debug Recstore|Win32.ActiveCfg = Debug Recstore|Win32

-		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Debug Recstore|Win32.Build.0 = Debug Recstore|Win32

 		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Debug|Win32.ActiveCfg = Debug|Win32

 		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Debug|Win32.Build.0 = Debug|Win32

 		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Release|Win32.ActiveCfg = Release|Win32

 		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Release|Win32.Build.0 = Release|Win32

-		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Debug Recstore|Win32.ActiveCfg = Debug Recstore|Win32

-		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Debug Recstore|Win32.Build.0 = Debug Recstore|Win32

 		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Debug|Win32.ActiveCfg = Debug|Win32

 		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Debug|Win32.Build.0 = Debug|Win32

 		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Release|Win32.ActiveCfg = Release|Win32

 		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Release|Win32.Build.0 = Release|Win32

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Debug|Win32.ActiveCfg = Debug|Win32

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Debug|Win32.Build.0 = Debug|Win32

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Release|Win32.ActiveCfg = Release|Win32

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Release|Win32.Build.0 = Release|Win32

 	EndGlobalSection

 	GlobalSection(SolutionProperties) = preSolution

 		HideSolutionNode = FALSE

 	EndGlobalSection

+	GlobalSection(NestedProjects) = preSolution

+		{2B262D59-9DC7-4BF1-A431-1BD4966899A5} = {12B11474-2D74-48C3-BB3D-F03249BEA88F}

+		{2F760952-C71B-4865-998F-AABAE96D1373} = {12B11474-2D74-48C3-BB3D-F03249BEA88F}

+		{407B4B88-3451-433C-B74F-31B31FEB5791} = {12B11474-2D74-48C3-BB3D-F03249BEA88F}

+		{4082881B-EB00-486F-906C-843B8EC06E18} = {12B11474-2D74-48C3-BB3D-F03249BEA88F}

+	EndGlobalSection

 EndGlobal

diff --git a/db/db.vcproj b/db/db.vcproj
index 3ea7506..2eac6eb 100644
--- a/db/db.vcproj
+++ b/db/db.vcproj
@@ -1,1891 +1,1885 @@
-<?xml version="1.0" encoding="Windows-1252"?>

-<VisualStudioProject

-	ProjectType="Visual C++"

-	Version="9.00"

-	Name="mongod"

-	ProjectGUID="{215B2D68-0A70-4D10-8E75-B31010C62A91}"

-	RootNamespace="db"

-	Keyword="Win32Proj"

-	TargetFrameworkVersion="196613"

-	>

-	<Platforms>

-		<Platform

-			Name="Win32"

-		/>

-	</Platforms>

-	<ToolFiles>

-	</ToolFiles>

-	<Configurations>

-		<Configuration

-			Name="Debug|Win32"

-			OutputDirectory="$(SolutionDir)$(ConfigurationName)"

-			IntermediateDirectory="$(ConfigurationName)"

-			ConfigurationType="1"

-			UseOfMFC="0"

-			UseOfATL="0"

-			CharacterSet="1"

-			>

-			<Tool

-				Name="VCPreBuildEventTool"

-			/>

-			<Tool

-				Name="VCCustomBuildTool"

-			/>

-			<Tool

-				Name="VCXMLDataGeneratorTool"

-			/>

-			<Tool

-				Name="VCWebServiceProxyGeneratorTool"

-			/>

-			<Tool

-				Name="VCMIDLTool"

-			/>

-			<Tool

-				Name="VCCLCompilerTool"

-				Optimization="0"

-				AdditionalIncludeDirectories="..\..\js\src;&quot;..\pcre-7.4&quot;;&quot;c:\Program Files\boost\boost_1_35_0&quot;"

-				PreprocessorDefinitions="OLDJS;STATIC_JS_API;XP_WIN;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;PCRE_STATIC"

-				MinimalRebuild="true"

-				BasicRuntimeChecks="3"

-				RuntimeLibrary="3"

-				UsePrecompiledHeader="2"

-				WarningLevel="3"

-				Detect64BitPortabilityProblems="false"

-				DebugInformationFormat="4"

-				DisableSpecificWarnings="4355;4800"

-			/>

-			<Tool

-				Name="VCManagedResourceCompilerTool"

-			/>

-			<Tool

-				Name="VCResourceCompilerTool"

-			/>

-			<Tool

-				Name="VCPreLinkEventTool"

-			/>

-			<Tool

-				Name="VCLinkerTool"

-				AdditionalDependencies="ws2_32.lib Psapi.lib"

-				LinkIncremental="2"

-				AdditionalLibraryDirectories="&quot;c:\Program Files\boost\boost_1_35_0\lib&quot;"

-				IgnoreAllDefaultLibraries="false"

-				IgnoreDefaultLibraryNames=""

-				GenerateDebugInformation="true"

-				SubSystem="1"

-				TargetMachine="1"

-			/>

-			<Tool

-				Name="VCALinkTool"

-			/>

-			<Tool

-				Name="VCManifestTool"

-			/>

-			<Tool

-				Name="VCXDCMakeTool"

-			/>

-			<Tool

-				Name="VCBscMakeTool"

-			/>

-			<Tool

-				Name="VCFxCopTool"

-			/>

-			<Tool

-				Name="VCAppVerifierTool"

-			/>

-			<Tool

-				Name="VCPostBuildEventTool"

-			/>

-		</Configuration>

-		<Configuration

-			Name="Release|Win32"

-			OutputDirectory="$(SolutionDir)$(ConfigurationName)"

-			IntermediateDirectory="$(ConfigurationName)"

-			ConfigurationType="1"

-			CharacterSet="1"

-			WholeProgramOptimization="1"

-			>

-			<Tool

-				Name="VCPreBuildEventTool"

-			/>

-			<Tool

-				Name="VCCustomBuildTool"

-			/>

-			<Tool

-				Name="VCXMLDataGeneratorTool"

-			/>

-			<Tool

-				Name="VCWebServiceProxyGeneratorTool"

-			/>

-			<Tool

-				Name="VCMIDLTool"

-			/>

-			<Tool

-				Name="VCCLCompilerTool"

-				Optimization="2"

-				EnableIntrinsicFunctions="true"

-				AdditionalIncludeDirectories="..\..\js\src;&quot;..\pcre-7.4&quot;;&quot;c:\Program Files\boost\boost_1_35_0&quot;"

-				PreprocessorDefinitions="OLDJS;STATIC_JS_API;XP_WIN;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;PCRE_STATIC"

-				RuntimeLibrary="0"

-				EnableFunctionLevelLinking="true"

-				UsePrecompiledHeader="2"

-				PrecompiledHeaderThrough="stdafx.h"

-				WarningLevel="3"

-				DebugInformationFormat="3"

-				DisableSpecificWarnings="4355;4800"

-			/>

-			<Tool

-				Name="VCManagedResourceCompilerTool"

-			/>

-			<Tool

-				Name="VCResourceCompilerTool"

-			/>

-			<Tool

-				Name="VCPreLinkEventTool"

-			/>

-			<Tool

-				Name="VCLinkerTool"

-				AdditionalDependencies="ws2_32.lib psapi.lib"

-				LinkIncremental="1"

-				AdditionalLibraryDirectories="&quot;c:\program files\boost\boost_1_35_0\lib&quot;"

-				GenerateDebugInformation="true"

-				SubSystem="1"

-				OptimizeReferences="2"

-				EnableCOMDATFolding="2"

-				TargetMachine="1"

-			/>

-			<Tool

-				Name="VCALinkTool"

-			/>

-			<Tool

-				Name="VCManifestTool"

-			/>

-			<Tool

-				Name="VCXDCMakeTool"

-			/>

-			<Tool

-				Name="VCBscMakeTool"

-			/>

-			<Tool

-				Name="VCFxCopTool"

-			/>

-			<Tool

-				Name="VCAppVerifierTool"

-			/>

-			<Tool

-				Name="VCPostBuildEventTool"

-			/>

-		</Configuration>

-		<Configuration

-			Name="release_nojni|Win32"

-			OutputDirectory="$(SolutionDir)$(ConfigurationName)"

-			IntermediateDirectory="$(ConfigurationName)"

-			ConfigurationType="1"

-			CharacterSet="1"

-			WholeProgramOptimization="1"

-			>

-			<Tool

-				Name="VCPreBuildEventTool"

-			/>

-			<Tool

-				Name="VCCustomBuildTool"

-			/>

-			<Tool

-				Name="VCXMLDataGeneratorTool"

-			/>

-			<Tool

-				Name="VCWebServiceProxyGeneratorTool"

-			/>

-			<Tool

-				Name="VCMIDLTool"

-			/>

-			<Tool

-				Name="VCCLCompilerTool"

-				Optimization="2"

-				EnableIntrinsicFunctions="true"

-				AdditionalIncludeDirectories="&quot;..\pcre-7.4&quot;;&quot;c:\Program Files\boost\boost_1_35_0&quot;;&quot;c:\program files\java\jdk\include&quot;;&quot;c:\program files\java\jdk\include\win32&quot;"

-				PreprocessorDefinitions="NOJNI;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;PCRE_STATIC"

-				RuntimeLibrary="2"

-				EnableFunctionLevelLinking="true"

-				UsePrecompiledHeader="2"

-				PrecompiledHeaderThrough="stdafx.h"

-				WarningLevel="3"

-				DebugInformationFormat="3"

-				DisableSpecificWarnings="4355;4800"

-			/>

-			<Tool

-				Name="VCManagedResourceCompilerTool"

-			/>

-			<Tool

-				Name="VCResourceCompilerTool"

-			/>

-			<Tool

-				Name="VCPreLinkEventTool"

-			/>

-			<Tool

-				Name="VCLinkerTool"

-				AdditionalDependencies="ws2_32.lib"

-				LinkIncremental="1"

-				AdditionalLibraryDirectories="&quot;c:\program files\boost\boost_1_35_0\lib&quot;"

-				GenerateDebugInformation="true"

-				SubSystem="1"

-				OptimizeReferences="2"

-				EnableCOMDATFolding="2"

-				TargetMachine="1"

-			/>

-			<Tool

-				Name="VCALinkTool"

-			/>

-			<Tool

-				Name="VCManifestTool"

-			/>

-			<Tool

-				Name="VCXDCMakeTool"

-			/>

-			<Tool

-				Name="VCBscMakeTool"

-			/>

-			<Tool

-				Name="VCFxCopTool"

-			/>

-			<Tool

-				Name="VCAppVerifierTool"

-			/>

-			<Tool

-				Name="VCPostBuildEventTool"

-			/>

-		</Configuration>

-		<Configuration

-			Name="Debug Recstore|Win32"

-			OutputDirectory="$(SolutionDir)$(ConfigurationName)"

-			IntermediateDirectory="$(ConfigurationName)"

-			ConfigurationType="1"

-			UseOfMFC="0"

-			UseOfATL="0"

-			CharacterSet="1"

-			>

-			<Tool

-				Name="VCPreBuildEventTool"

-			/>

-			<Tool

-				Name="VCCustomBuildTool"

-			/>

-			<Tool

-				Name="VCXMLDataGeneratorTool"

-			/>

-			<Tool

-				Name="VCWebServiceProxyGeneratorTool"

-			/>

-			<Tool

-				Name="VCMIDLTool"

-			/>

-			<Tool

-				Name="VCCLCompilerTool"

-				Optimization="0"

-				AdditionalIncludeDirectories="..\..\js\src;&quot;..\pcre-7.4&quot;;&quot;c:\Program Files\boost\boost_1_35_0&quot;"

-				PreprocessorDefinitions="_RECSTORE;OLDJS;STATIC_JS_API;XP_WIN;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;PCRE_STATIC"

-				MinimalRebuild="true"

-				BasicRuntimeChecks="3"

-				RuntimeLibrary="3"

-				UsePrecompiledHeader="2"

-				WarningLevel="3"

-				Detect64BitPortabilityProblems="false"

-				DebugInformationFormat="4"

-				DisableSpecificWarnings="4355;4800"

-			/>

-			<Tool

-				Name="VCManagedResourceCompilerTool"

-			/>

-			<Tool

-				Name="VCResourceCompilerTool"

-			/>

-			<Tool

-				Name="VCPreLinkEventTool"

-			/>

-			<Tool

-				Name="VCLinkerTool"

-				AdditionalDependencies="ws2_32.lib"

-				LinkIncremental="2"

-				AdditionalLibraryDirectories="&quot;c:\Program Files\boost\boost_1_35_0\lib&quot;"

-				IgnoreAllDefaultLibraries="false"

-				IgnoreDefaultLibraryNames=""

-				GenerateDebugInformation="true"

-				SubSystem="1"

-				TargetMachine="1"

-			/>

-			<Tool

-				Name="VCALinkTool"

-			/>

-			<Tool

-				Name="VCManifestTool"

-			/>

-			<Tool

-				Name="VCXDCMakeTool"

-			/>

-			<Tool

-				Name="VCBscMakeTool"

-			/>

-			<Tool

-				Name="VCFxCopTool"

-			/>

-			<Tool

-				Name="VCAppVerifierTool"

-			/>

-			<Tool

-				Name="VCPostBuildEventTool"

-			/>

-		</Configuration>

-	</Configurations>

-	<References>

-	</References>

-	<Files>

-		<Filter

-			Name="misc and third party"

-			>

-			<File

-				RelativePath="..\..\boostw\boost_1_34_1\boost\config\auto_link.hpp"

-				>

-			</File>

-			<File

-				RelativePath=".\db.rc"

-				>

-			</File>

-			<File

-				RelativePath="..\..\js\src\js.lib"

-				>

-			</File>

-			<File

-				RelativePath="..\pcre-7.4\pcrecpp.cc"

-				>

-				<FileConfiguration

-					Name="Debug|Win32"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-						UsePrecompiledHeader="0"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="Release|Win32"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-						UsePrecompiledHeader="0"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="release_nojni|Win32"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-						UsePrecompiledHeader="0"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="Debug Recstore|Win32"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-						UsePrecompiledHeader="0"

-					/>

-				</FileConfiguration>

-			</File>

-			<File

-				RelativePath="..\pcre-7.4\pcrecpp.h"

-				>

-			</File>

-			<File

-				RelativePath="..\SConstruct"

-				>

-			</File>

-			<File

-				RelativePath="..\targetver.h"

-				>

-			</File>

-			<File

-				RelativePath="..\..\boostw\boost_1_34_1\boost\version.hpp"

-				>

-			</File>

-			<Filter

-				Name="pcre"

-				>

-				<File

-					RelativePath="..\pcre-7.4\config.h"

-					>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre.h"

-					>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_chartables.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_compile.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_config.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_dfa_exec.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_exec.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_fullinfo.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_get.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_globals.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_info.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_maketables.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_newline.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_ord2utf8.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_refcount.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_scanner.cc"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_stringpiece.cc"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_study.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_tables.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_try_flipped.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_ucp_searchfuncs.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_valid_utf8.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_version.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcre_xclass.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\pcre-7.4\pcreposix.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-			</Filter>

-		</Filter>

-		<Filter

-			Name="storage related"

-			>

-			<File

-				RelativePath=".\rec.h"

-				>

-			</File>

-			<File

-				RelativePath=".\reccache.cpp"

-				>

-			</File>

-			<File

-				RelativePath=".\reccache.h"

-				>

-			</File>

-			<File

-				RelativePath=".\reci.h"

-				>

-			</File>

-			<File

-				RelativePath=".\recstore.h"

-				>

-			</File>

-			<File

-				RelativePath=".\storage.cpp"

-				>

-			</File>

-			<File

-				RelativePath=".\storage.h"

-				>

-			</File>

-		</Filter>

-		<Filter

-			Name="client"

-			>

-			<File

-				RelativePath="..\client\connpool.cpp"

-				>

-			</File>

-			<File

-				RelativePath="..\client\connpool.h"

-				>

-			</File>

-			<File

-				RelativePath="..\client\dbclient.cpp"

-				>

-			</File>

-			<File

-				RelativePath="..\client\dbclient.h"

-				>

-			</File>

-			<File

-				RelativePath="..\client\model.h"

-				>

-			</File>

-			<File

-				RelativePath="..\client\syncclusterconnection.cpp"

-				>

-			</File>

-		</Filter>

-		<Filter

-			Name="db"

-			>

-			<File

-				RelativePath=".\background.h"

-				>

-			</File>

-			<File

-				RelativePath=".\client.h"

-				>

-			</File>

-			<File

-				RelativePath=".\clientcursor.h"

-				>

-			</File>

-			<File

-				RelativePath=".\cmdline.cpp"

-				>

-			</File>

-			<File

-				RelativePath=".\cmdline.h"

-				>

-			</File>

-			<File

-				RelativePath=".\commands.h"

-				>

-			</File>

-			<File

-				RelativePath=".\concurrency.h"

-				>

-			</File>

-			<File

-				RelativePath=".\curop.h"

-				>

-			</File>

-			<File

-				RelativePath=".\cursor.h"

-				>

-			</File>

-			<File

-				RelativePath=".\database.h"

-				>

-			</File>

-			<File

-				RelativePath=".\db.h"

-				>

-			</File>

-			<File

-				RelativePath=".\dbhelpers.h"

-				>

-			</File>

-			<File

-				RelativePath=".\dbinfo.h"

-				>

-			</File>

-			<File

-				RelativePath=".\dbmessage.h"

-				>

-			</File>

-			<File

-				RelativePath=".\diskloc.h"

-				>

-			</File>

-			<File

-				RelativePath=".\index.h"

-				>

-			</File>

-			<File

-				RelativePath=".\introspect.h"

-				>

-			</File>

-			<File

-				RelativePath=".\jsobj.h"

-				>

-			</File>

-			<File

-				RelativePath=".\json.h"

-				>

-			</File>

-			<File

-				RelativePath=".\matcher.h"

-				>

-			</File>

-			<File

-				RelativePath="..\grid\message.h"

-				>

-			</File>

-			<File

-				RelativePath=".\minilex.h"

-				>

-			</File>

-			<File

-				RelativePath=".\namespace.h"

-				>

-			</File>

-			<File

-				RelativePath=".\pdfile.h"

-				>

-			</File>

-			<File

-				RelativePath="..\grid\protocol.h"

-				>

-			</File>

-			<File

-				RelativePath=".\query.h"

-				>

-			</File>

-			<File

-				RelativePath=".\queryoptimizer.h"

-				>

-			</File>

-			<File

-				RelativePath=".\queryutil.cpp"

-				>

-			</File>

-			<File

-				RelativePath=".\repl.h"

-				>

-			</File>

-			<File

-				RelativePath=".\replset.h"

-				>

-			</File>

-			<File

-				RelativePath=".\resource.h"

-				>

-			</File>

-			<File

-				RelativePath=".\scanandorder.h"

-				>

-			</File>

-			<File

-				RelativePath=".\security.h"

-				>

-			</File>

-			<File

-				RelativePath="..\stdafx.h"

-				>

-			</File>

-			<File

-				RelativePath=".\update.h"

-				>

-			</File>

-			<Filter

-				Name="cpp"

-				Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"

-				UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"

-				>

-				<File

-					RelativePath=".\client.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\clientcursor.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\cloner.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\commands.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\common.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\cursor.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\s\d_util.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\database.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\db.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\dbcommands.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\dbcommands_admin.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\dbeval.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\dbhelpers.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\dbwebserver.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\extsort.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\index.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\index_geo2d.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\instance.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\introspect.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\jsobj.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\json.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\lasterror.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\matcher.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\util\mmap_win.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\modules\mms.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\module.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\mr.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\namespace.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\nonce.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\client\parallel.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\pdfile.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\query.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\queryoptimizer.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\repl.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\security.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\security_commands.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\stdafx.cpp"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="1"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="1"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="1"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="1"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath=".\tests.cpp"

-					>

-				</File>

-				<File

-					RelativePath=".\update.cpp"

-					>

-				</File>

-			</Filter>

-		</Filter>

-		<Filter

-			Name="util"

-			>

-			<File

-				RelativePath="..\util\assert_util.h"

-				>

-			</File>

-			<File

-				RelativePath="..\util\builder.h"

-				>

-			</File>

-			<File

-				RelativePath="..\util\file.h"

-				>

-			</File>

-			<File

-				RelativePath="..\util\goodies.h"

-				>

-			</File>

-			<File

-				RelativePath="..\util\hashtab.h"

-				>

-			</File>

-			<File

-				RelativePath=".\lasterror.h"

-				>

-			</File>

-			<File

-				RelativePath="..\util\log.h"

-				>

-			</File>

-			<File

-				RelativePath="..\util\lruishmap.h"

-				>

-			</File>

-			<File

-				RelativePath="..\util\md5.h"

-				>

-			</File>

-			<File

-				RelativePath="..\util\md5.hpp"

-				>

-			</File>

-			<File

-				RelativePath="..\util\miniwebserver.h"

-				>

-			</File>

-			<File

-				RelativePath="..\util\mmap.h"

-				>

-			</File>

-			<File

-				RelativePath="..\util\sock.h"

-				>

-			</File>

-			<File

-				RelativePath="..\util\unittest.h"

-				>

-			</File>

-			<Filter

-				Name="cpp"

-				>

-				<File

-					RelativePath="..\util\assert_util.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\util\background.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\util\base64.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\util\httpclient.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\util\md5.c"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Release|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-							PrecompiledHeaderThrough=""

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="release_nojni|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-					<FileConfiguration

-						Name="Debug Recstore|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="0"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\util\md5main.cpp"

-					>

-					<FileConfiguration

-						Name="Debug|Win32"

-						>

-						<Tool

-							Name="VCCLCompilerTool"

-							UsePrecompiledHeader="2"

-						/>

-					</FileConfiguration>

-				</File>

-				<File

-					RelativePath="..\util\message.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\util\miniwebserver.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\util\mmap.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\util\ntservice.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\util\processinfo_win32.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\util\sock.cpp"

-					>

-				</File>

-				<File

-					RelativePath="..\util\util.cpp"

-					>

-				</File>

-			</Filter>

-		</Filter>

-		<Filter

-			Name="shard"

-			>

-			<File

-				RelativePath="..\s\d_logic.cpp"

-				>

-			</File>

-		</Filter>

-		<Filter

-			Name="scripting"

-			>

-			<File

-				RelativePath="..\scripting\engine.cpp"

-				>

-			</File>

-			<File

-				RelativePath="..\scripting\engine_spidermonkey.cpp"

-				>

-			</File>

-			<File

-				RelativePath="..\shell\mongo_vstudio.cpp"

-				>

-				<FileConfiguration

-					Name="Debug|Win32"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-						UsePrecompiledHeader="0"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="Release|Win32"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-						UsePrecompiledHeader="0"

-					/>

-				</FileConfiguration>

-				<FileConfiguration

-					Name="Debug Recstore|Win32"

-					>

-					<Tool

-						Name="VCCLCompilerTool"

-						UsePrecompiledHeader="0"

-					/>

-				</FileConfiguration>

-			</File>

-			<File

-				RelativePath="..\scripting\utils.cpp"

-				>

-			</File>

-		</Filter>

-		<Filter

-			Name="stats"

-			>

-			<File

-				RelativePath=".\stats\counters.cpp"

-				>

-			</File>

-			<File

-				RelativePath=".\stats\snapshots.cpp"

-				>

-			</File>

-			<File

-				RelativePath=".\stats\top.cpp"

-				>

-			</File>

-		</Filter>

-		<Filter

-			Name="btree related"

-			>

-			<File

-				RelativePath=".\btree.cpp"

-				>

-			</File>

-			<File

-				RelativePath=".\btree.h"

-				>

-			</File>

-			<File

-				RelativePath=".\btreecursor.cpp"

-				>

-			</File>

-		</Filter>

-	</Files>

-	<Globals>

-	</Globals>

-</VisualStudioProject>

+<?xml version="1.0" encoding="Windows-1252"?>
+<VisualStudioProject
+	ProjectType="Visual C++"
+	Version="9.00"
+	Name="mongod"
+	ProjectGUID="{215B2D68-0A70-4D10-8E75-B31010C62A91}"
+	RootNamespace="db"
+	Keyword="Win32Proj"
+	TargetFrameworkVersion="196613"
+	>
+	<Platforms>
+		<Platform
+			Name="Win32"
+		/>
+	</Platforms>
+	<ToolFiles>
+	</ToolFiles>
+	<Configurations>
+		<Configuration
+			Name="Debug|Win32"
+			OutputDirectory="$(SolutionDir)$(ConfigurationName)"
+			IntermediateDirectory="$(ConfigurationName)"
+			ConfigurationType="1"
+			UseOfMFC="0"
+			UseOfATL="0"
+			CharacterSet="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="&quot;c:\program files\boost\latest&quot;;..\..\js\src;&quot;..\pcre-7.4&quot;;c:\boost;\boost"
+				PreprocessorDefinitions="MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;PCRE_STATIC"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="3"
+				UsePrecompiledHeader="2"
+				PrecompiledHeaderThrough="pch.h"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="false"
+				DebugInformationFormat="4"
+				DisableSpecificWarnings="4355;4800"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				AdditionalDependencies="ws2_32.lib Psapi.lib"
+				LinkIncremental="2"
+				AdditionalLibraryDirectories="&quot;c:\program files\boost\latest\lib&quot;;c:\boost\lib;\boost\lib"
+				IgnoreAllDefaultLibraries="false"
+				IgnoreDefaultLibraryNames=""
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|Win32"
+			OutputDirectory="$(SolutionDir)$(ConfigurationName)"
+			IntermediateDirectory="$(ConfigurationName)"
+			ConfigurationType="1"
+			CharacterSet="1"
+			WholeProgramOptimization="1"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="2"
+				EnableIntrinsicFunctions="true"
+				AdditionalIncludeDirectories="&quot;c:\program files\boost\latest&quot;;..\..\js\src;&quot;..\pcre-7.4&quot;;c:\boost;\boost"
+				PreprocessorDefinitions="MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;PCRE_STATIC"
+				RuntimeLibrary="0"
+				EnableFunctionLevelLinking="true"
+				UsePrecompiledHeader="2"
+				PrecompiledHeaderThrough="pch.h"
+				WarningLevel="3"
+				DebugInformationFormat="3"
+				DisableSpecificWarnings="4355;4800"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				AdditionalDependencies="ws2_32.lib psapi.lib"
+				LinkIncremental="1"
+				AdditionalLibraryDirectories="&quot;c:\program files\boost\latest\lib&quot;;c:\boost\lib;\boost\lib"
+				GenerateDebugInformation="true"
+				SubSystem="1"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+	</Configurations>
+	<References>
+	</References>
+	<Files>
+		<Filter
+			Name="libs"
+			>
+			<File
+				RelativePath=".\db.rc"
+				>
+			</File>
+			<File
+				RelativePath="..\..\js\src\js.lib"
+				>
+			</File>
+			<File
+				RelativePath="..\pcre-7.4\pcrecpp.cc"
+				>
+				<FileConfiguration
+					Name="Debug|Win32"
+					>
+					<Tool
+						Name="VCCLCompilerTool"
+						UsePrecompiledHeader="0"
+					/>
+				</FileConfiguration>
+				<FileConfiguration
+					Name="Release|Win32"
+					>
+					<Tool
+						Name="VCCLCompilerTool"
+						UsePrecompiledHeader="0"
+					/>
+				</FileConfiguration>
+				<FileConfiguration
+					Name="release_nojni|Win32"
+					>
+					<Tool
+						Name="VCCLCompilerTool"
+						UsePrecompiledHeader="0"
+					/>
+				</FileConfiguration>
+			</File>
+			<File
+				RelativePath="..\SConstruct"
+				>
+			</File>
+			<File
+				RelativePath="..\targetver.h"
+				>
+			</File>
+			<Filter
+				Name="pcre"
+				>
+				<File
+					RelativePath="..\pcre-7.4\config.h"
+					>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre.h"
+					>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_chartables.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_compile.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_config.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_dfa_exec.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_exec.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_fullinfo.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_get.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_globals.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_info.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_maketables.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_newline.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_ord2utf8.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_refcount.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_scanner.cc"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_stringpiece.cc"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_study.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Debug Recstore|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_tables.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Debug Recstore|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_try_flipped.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Debug Recstore|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_ucp_searchfuncs.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Debug Recstore|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_valid_utf8.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Debug Recstore|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_version.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Debug Recstore|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcre_xclass.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Debug Recstore|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\pcre-7.4\pcreposix.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Debug Recstore|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+			</Filter>
+			<Filter
+				Name="old_repl"
+				>
+				<File
+					RelativePath=".\oplog.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\repl.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\repl.h"
+					>
+				</File>
+				<File
+					RelativePath=".\repl_block.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\replpair.h"
+					>
+				</File>
+			</Filter>
+		</Filter>
+		<Filter
+			Name="storage related"
+			>
+			<File
+				RelativePath=".\rec.h"
+				>
+			</File>
+			<File
+				RelativePath=".\reccache.h"
+				>
+			</File>
+			<File
+				RelativePath=".\reci.h"
+				>
+			</File>
+			<File
+				RelativePath=".\recstore.h"
+				>
+			</File>
+			<File
+				RelativePath=".\storage.cpp"
+				>
+			</File>
+			<File
+				RelativePath=".\storage.h"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="client"
+			>
+			<File
+				RelativePath="..\client\connpool.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\client\connpool.h"
+				>
+			</File>
+			<File
+				RelativePath="..\client\dbclient.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\client\dbclient.h"
+				>
+			</File>
+			<File
+				RelativePath="..\client\dbclientcursor.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\client\model.h"
+				>
+			</File>
+			<File
+				RelativePath="..\client\redef_macros.h"
+				>
+			</File>
+			<File
+				RelativePath="..\client\syncclusterconnection.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\client\syncclusterconnection.h"
+				>
+			</File>
+			<File
+				RelativePath="..\client\undef_macros.h"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="db"
+			>
+			<File
+				RelativePath="..\pch.cpp"
+				>
+				<FileConfiguration
+					Name="Debug|Win32"
+					>
+					<Tool
+						Name="VCCLCompilerTool"
+						UsePrecompiledHeader="1"
+					/>
+				</FileConfiguration>
+				<FileConfiguration
+					Name="Release|Win32"
+					>
+					<Tool
+						Name="VCCLCompilerTool"
+						UsePrecompiledHeader="1"
+					/>
+				</FileConfiguration>
+			</File>
+			<Filter
+				Name="cpp"
+				Filter="cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx"
+				UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
+				>
+				<File
+					RelativePath=".\client.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\clientcursor.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\cloner.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\commands.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\common.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\cursor.cpp"
+					>
+				</File>
+				<File
+					RelativePath="..\s\d_util.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\database.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\db.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\dbcommands.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\dbcommands_admin.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\dbeval.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\dbhelpers.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\dbwebserver.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\extsort.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\index.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\index_geo2d.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\indexkey.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\instance.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\introspect.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\jsobj.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\json.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\lasterror.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\matcher.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\matcher_covered.cpp"
+					>
+				</File>
+				<File
+					RelativePath="..\util\mmap_win.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\modules\mms.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\module.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\mr.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\namespace.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\nonce.cpp"
+					>
+				</File>
+				<File
+					RelativePath="..\client\parallel.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\pdfile.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\query.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\queryoptimizer.cpp"
+					>
+				</File>
+				<File
+					RelativePath="..\util\ramstore.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\security.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\security_commands.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\tests.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\update.cpp"
+					>
+				</File>
+			</Filter>
+			<Filter
+				Name="h"
+				>
+				<File
+					RelativePath=".\background.h"
+					>
+				</File>
+				<File
+					RelativePath=".\client.h"
+					>
+				</File>
+				<File
+					RelativePath=".\clientcursor.h"
+					>
+				</File>
+				<File
+					RelativePath=".\cmdline.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\cmdline.h"
+					>
+				</File>
+				<File
+					RelativePath=".\commands.h"
+					>
+				</File>
+				<File
+					RelativePath=".\concurrency.h"
+					>
+				</File>
+				<File
+					RelativePath=".\curop.h"
+					>
+				</File>
+				<File
+					RelativePath=".\cursor.h"
+					>
+				</File>
+				<File
+					RelativePath=".\database.h"
+					>
+				</File>
+				<File
+					RelativePath=".\db.h"
+					>
+				</File>
+				<File
+					RelativePath=".\dbhelpers.h"
+					>
+				</File>
+				<File
+					RelativePath=".\dbinfo.h"
+					>
+				</File>
+				<File
+					RelativePath=".\dbmessage.h"
+					>
+				</File>
+				<File
+					RelativePath=".\diskloc.h"
+					>
+				</File>
+				<File
+					RelativePath=".\index.h"
+					>
+				</File>
+				<File
+					RelativePath=".\indexkey.h"
+					>
+				</File>
+				<File
+					RelativePath=".\introspect.h"
+					>
+				</File>
+				<File
+					RelativePath=".\json.h"
+					>
+				</File>
+				<File
+					RelativePath=".\matcher.h"
+					>
+				</File>
+				<File
+					RelativePath="..\grid\message.h"
+					>
+				</File>
+				<File
+					RelativePath=".\minilex.h"
+					>
+				</File>
+				<File
+					RelativePath=".\namespace.h"
+					>
+				</File>
+				<File
+					RelativePath="..\pch.h"
+					>
+				</File>
+				<File
+					RelativePath=".\pdfile.h"
+					>
+				</File>
+				<File
+					RelativePath="..\grid\protocol.h"
+					>
+				</File>
+				<File
+					RelativePath=".\query.h"
+					>
+				</File>
+				<File
+					RelativePath=".\queryoptimizer.h"
+					>
+				</File>
+				<File
+					RelativePath=".\queryutil.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\resource.h"
+					>
+				</File>
+				<File
+					RelativePath=".\scanandorder.h"
+					>
+				</File>
+				<File
+					RelativePath=".\security.h"
+					>
+				</File>
+				<File
+					RelativePath=".\update.h"
+					>
+				</File>
+			</Filter>
+		</Filter>
+		<Filter
+			Name="util"
+			>
+			<File
+				RelativePath="..\util\allocator.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\array.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\assert_util.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\util\assert_util.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\background.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\util\background.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\base64.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\util\base64.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\builder.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\debug_util.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\embedded_builder.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\file.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\file_allocator.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\goodies.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\hashtab.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\hex.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\interlocked.h"
+				>
+			</File>
+			<File
+				RelativePath=".\lasterror.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\rwlock.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\log.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\lruishmap.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\mmap.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\util\mmap.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\mvar.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\ntservice.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\util\ntservice.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\optime.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\processinfo.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\processinfo_win32.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\util\queue.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\ramstore.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\thread_pool.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\text.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\util\text.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\unittest.h"
+				>
+			</File>
+			<File
+				RelativePath="..\util\util.cpp"
+				>
+			</File>
+			<Filter
+				Name="concurrency"
+				>
+				<File
+					RelativePath="..\util\concurrency\list.h"
+					>
+				</File>
+				<File
+					RelativePath="..\util\concurrency\msg.h"
+					>
+				</File>
+				<File
+					RelativePath="..\util\concurrency\task.cpp"
+					>
+				</File>
+				<File
+					RelativePath="..\util\concurrency\task.h"
+					>
+				</File>
+				<File
+					RelativePath="..\util\concurrency\value.h"
+					>
+				</File>
+				<File
+					RelativePath="..\util\concurrency\vars.cpp"
+					>
+				</File>
+			</Filter>
+			<Filter
+				Name="web"
+				>
+				<File
+					RelativePath="..\util\web\html.h"
+					>
+				</File>
+				<File
+					RelativePath="..\util\httpclient.cpp"
+					>
+				</File>
+				<File
+					RelativePath="..\util\httpclient.h"
+					>
+				</File>
+				<File
+					RelativePath="..\util\miniwebserver.cpp"
+					>
+				</File>
+				<File
+					RelativePath="..\util\miniwebserver.h"
+					>
+				</File>
+			</Filter>
+			<Filter
+				Name="md5"
+				>
+				<File
+					RelativePath="..\util\md5.c"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Release|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+							PrecompiledHeaderThrough=""
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="release_nojni|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+					<FileConfiguration
+						Name="Debug Recstore|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="0"
+						/>
+					</FileConfiguration>
+				</File>
+				<File
+					RelativePath="..\util\md5.h"
+					>
+				</File>
+				<File
+					RelativePath="..\util\md5.hpp"
+					>
+				</File>
+				<File
+					RelativePath="..\util\md5main.cpp"
+					>
+					<FileConfiguration
+						Name="Debug|Win32"
+						>
+						<Tool
+							Name="VCCLCompilerTool"
+							UsePrecompiledHeader="2"
+						/>
+					</FileConfiguration>
+				</File>
+			</Filter>
+			<Filter
+				Name="net"
+				>
+				<File
+					RelativePath="..\util\message.cpp"
+					>
+				</File>
+				<File
+					RelativePath="..\util\message.h"
+					>
+				</File>
+				<File
+					RelativePath="..\util\message_server.h"
+					>
+				</File>
+				<File
+					RelativePath="..\util\message_server_port.cpp"
+					>
+				</File>
+				<File
+					RelativePath="..\util\sock.cpp"
+					>
+				</File>
+				<File
+					RelativePath="..\util\sock.h"
+					>
+				</File>
+			</Filter>
+		</Filter>
+		<Filter
+			Name="shard"
+			>
+			<File
+				RelativePath="..\s\d_logic.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\s\shardconnection.cpp"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="scripting"
+			>
+			<File
+				RelativePath="..\scripting\engine.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\scripting\engine.h"
+				>
+			</File>
+			<File
+				RelativePath="..\scripting\engine_java.h"
+				>
+			</File>
+			<File
+				RelativePath="..\scripting\engine_spidermonkey.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\scripting\engine_spidermonkey.h"
+				>
+			</File>
+			<File
+				RelativePath="..\scripting\engine_v8.h"
+				>
+			</File>
+			<File
+				RelativePath="..\shell\mongo_vstudio.cpp"
+				>
+				<FileConfiguration
+					Name="Debug|Win32"
+					>
+					<Tool
+						Name="VCCLCompilerTool"
+						UsePrecompiledHeader="0"
+					/>
+				</FileConfiguration>
+				<FileConfiguration
+					Name="Release|Win32"
+					>
+					<Tool
+						Name="VCCLCompilerTool"
+						UsePrecompiledHeader="0"
+					/>
+				</FileConfiguration>
+				<FileConfiguration
+					Name="Debug Recstore|Win32"
+					>
+					<Tool
+						Name="VCCLCompilerTool"
+						UsePrecompiledHeader="0"
+					/>
+				</FileConfiguration>
+			</File>
+			<File
+				RelativePath="..\scripting\utils.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\scripting\v8_db.h"
+				>
+			</File>
+			<File
+				RelativePath="..\scripting\v8_utils.h"
+				>
+			</File>
+			<File
+				RelativePath="..\scripting\v8_wrapper.h"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="stats"
+			>
+			<File
+				RelativePath=".\stats\counters.cpp"
+				>
+			</File>
+			<File
+				RelativePath=".\stats\snapshots.cpp"
+				>
+			</File>
+			<File
+				RelativePath=".\stats\top.cpp"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="btree"
+			>
+			<File
+				RelativePath=".\btree.cpp"
+				>
+			</File>
+			<File
+				RelativePath=".\btree.h"
+				>
+			</File>
+			<File
+				RelativePath=".\btreecursor.cpp"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="replsets"
+			>
+			<File
+				RelativePath=".\repl\connections.h"
+				>
+			</File>
+			<File
+				RelativePath=".\repl\consensus.cpp"
+				>
+			</File>
+			<File
+				RelativePath=".\repl\health.cpp"
+				>
+			</File>
+			<File
+				RelativePath=".\repl\health.h"
+				>
+			</File>
+			<File
+				RelativePath=".\repl\heartbeat.cpp"
+				>
+			</File>
+			<File
+				RelativePath="..\util\hostandport.h"
+				>
+			</File>
+			<File
+				RelativePath=".\repl\manager.cpp"
+				>
+			</File>
+			<File
+				RelativePath=".\repl\replset.cpp"
+				>
+			</File>
+			<File
+				RelativePath=".\repl\replset.h"
+				>
+			</File>
+			<File
+				RelativePath=".\repl\replset_commands.cpp"
+				>
+			</File>
+			<File
+				RelativePath=".\repl\rs_config.cpp"
+				>
+			</File>
+			<File
+				RelativePath=".\repl\rs_sync.cpp"
+				>
+			</File>
+			<File
+				RelativePath=".\repl\rs_config.h"
+				>
+			</File>
+			<File
+				RelativePath=".\repl\rs_initiate.cpp"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="bson"
+			>
+			<File
+				RelativePath="..\bson\bsonelement.h"
+				>
+			</File>
+			<File
+				RelativePath="..\bson\bsoninlines.h"
+				>
+			</File>
+			<File
+				RelativePath="..\bson\bsonmisc.h"
+				>
+			</File>
+			<File
+				RelativePath="..\bson\bsonobj.h"
+				>
+			</File>
+			<File
+				RelativePath="..\bson\bsonobjbuilder.h"
+				>
+			</File>
+			<File
+				RelativePath="..\bson\bsonobjiterator.h"
+				>
+			</File>
+			<File
+				RelativePath="..\bson\bsontypes.h"
+				>
+			</File>
+			<File
+				RelativePath=".\jsobj.h"
+				>
+			</File>
+			<File
+				RelativePath="..\bson\oid.h"
+				>
+			</File>
+			<File
+				RelativePath="..\bson\ordering.h"
+				>
+			</File>
+		</Filter>
+	</Files>
+	<Globals>
+	</Globals>
+</VisualStudioProject>
diff --git a/db/db.vcxproj b/db/db.vcxproj
index 878b52c..61f81fe 100644
--- a/db/db.vcxproj
+++ b/db/db.vcxproj
@@ -1,18 +1,22 @@
 <?xml version="1.0" encoding="utf-8"?>

 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">

   <ItemGroup Label="ProjectConfigurations">

-    <ProjectConfiguration Include="Debug Recstore|Win32">

-      <Configuration>Debug Recstore</Configuration>

-      <Platform>Win32</Platform>

-    </ProjectConfiguration>

     <ProjectConfiguration Include="Debug|Win32">

       <Configuration>Debug</Configuration>

       <Platform>Win32</Platform>

     </ProjectConfiguration>

+    <ProjectConfiguration Include="Debug|x64">

+      <Configuration>Debug</Configuration>

+      <Platform>x64</Platform>

+    </ProjectConfiguration>

     <ProjectConfiguration Include="Release|Win32">

       <Configuration>Release</Configuration>

       <Platform>Win32</Platform>

     </ProjectConfiguration>

+    <ProjectConfiguration Include="Release|x64">

+      <Configuration>Release</Configuration>

+      <Platform>x64</Platform>

+    </ProjectConfiguration>

   </ItemGroup>

   <PropertyGroup Label="Globals">

     <ProjectName>mongod</ProjectName>

@@ -21,13 +25,12 @@
     <Keyword>Win32Proj</Keyword>

   </PropertyGroup>

   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />

-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'" Label="Configuration">

+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">

     <ConfigurationType>Application</ConfigurationType>

-    <UseOfMfc>false</UseOfMfc>

-    <UseOfAtl>false</UseOfAtl>

     <CharacterSet>Unicode</CharacterSet>

+    <WholeProgramOptimization>true</WholeProgramOptimization>

   </PropertyGroup>

-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">

+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">

     <ConfigurationType>Application</ConfigurationType>

     <CharacterSet>Unicode</CharacterSet>

     <WholeProgramOptimization>true</WholeProgramOptimization>

@@ -38,47 +41,73 @@
     <UseOfAtl>false</UseOfAtl>

     <CharacterSet>Unicode</CharacterSet>

   </PropertyGroup>

+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">

+    <ConfigurationType>Application</ConfigurationType>

+    <UseOfMfc>false</UseOfMfc>

+    <UseOfAtl>false</UseOfAtl>

+    <CharacterSet>Unicode</CharacterSet>

+  </PropertyGroup>

   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />

   <ImportGroup Label="ExtensionSettings">

   </ImportGroup>

-  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'" Label="PropertySheets">

-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" />

-  </ImportGroup>

   <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">

-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" />

+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />

+  </ImportGroup>

+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">

+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />

   </ImportGroup>

   <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">

-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" />

+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />

+  </ImportGroup>

+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">

+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />

   </ImportGroup>

   <PropertyGroup Label="UserMacros" />

   <PropertyGroup>

-    <_ProjectFileVersion>10.0.21006.1</_ProjectFileVersion>

+    <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>

     <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(SolutionDir)$(Configuration)\</OutDir>

+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(SolutionDir)$(Configuration)\</OutDir>

     <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(Configuration)\</IntDir>

+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">$(Configuration)\</IntDir>

     <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</LinkIncremental>

+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</LinkIncremental>

     <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(SolutionDir)$(Configuration)\</OutDir>

+    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(SolutionDir)$(Configuration)\</OutDir>

     <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(Configuration)\</IntDir>

+    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|x64'">$(Configuration)\</IntDir>

     <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>

-    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">$(SolutionDir)$(Configuration)\</OutDir>

-    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">$(Configuration)\</IntDir>

-    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">true</LinkIncremental>

+    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</LinkIncremental>

+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>

+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AllRules.ruleset</CodeAnalysisRuleSet>

+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />

+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />

+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />

+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />

+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>

+    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>

+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />

+    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />

+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />

+    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />

   </PropertyGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

     <ClCompile>

       <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\js\src;..\pcre-7.4;c:\Program Files\boost\boost_1_41_0;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>OLDJS;STATIC_JS_API;XP_WIN;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;PCRE_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>

-      <MinimalRebuild>true</MinimalRebuild>

+      <AdditionalIncludeDirectories>..\..\js\src;..\pcre-7.4;c:\boost;\boost</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>_UNICODE;UNICODE;SUPPORT_UCP;SUPPORT_UTF8;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;PCRE_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <MinimalRebuild>No</MinimalRebuild>

       <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>

       <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>

       <PrecompiledHeader>Use</PrecompiledHeader>

+      <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>

       <WarningLevel>Level3</WarningLevel>

       <DebugInformationFormat>EditAndContinue</DebugInformationFormat>

       <DisableSpecificWarnings>4355;4800;%(DisableSpecificWarnings)</DisableSpecificWarnings>

+      <MultiProcessorCompilation>true</MultiProcessorCompilation>

     </ClCompile>

     <Link>

-      <AdditionalDependencies>ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>

-      <AdditionalLibraryDirectories>c:\Program Files\boost\boost_1_41_0\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>

+      <AdditionalDependencies>ws2_32.lib;Psapi.lib;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalLibraryDirectories>c:\boost\lib\vs2010_32;\boost\lib\vs2010_32;\boost\lib</AdditionalLibraryDirectories>

       <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>

       <IgnoreSpecificDefaultLibraries>%(IgnoreSpecificDefaultLibraries)</IgnoreSpecificDefaultLibraries>

       <GenerateDebugInformation>true</GenerateDebugInformation>

@@ -86,343 +115,393 @@
       <TargetMachine>MachineX86</TargetMachine>

     </Link>

   </ItemDefinitionGroup>

+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+    <ClCompile>

+      <Optimization>Disabled</Optimization>

+      <AdditionalIncludeDirectories>..\..\js\src;..\pcre-7.4;c:\boost;\boost</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>SUPPORT_UCP;SUPPORT_UTF8;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;PCRE_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>

+      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>

+      <PrecompiledHeader>Use</PrecompiledHeader>

+      <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>

+      <WarningLevel>Level3</WarningLevel>

+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>

+      <DisableSpecificWarnings>4355;4800;4267;4244;%(DisableSpecificWarnings)</DisableSpecificWarnings>

+      <MultiProcessorCompilation>true</MultiProcessorCompilation>

+      <MinimalRebuild>No</MinimalRebuild>

+    </ClCompile>

+    <Link>

+      <AdditionalDependencies>ws2_32.lib;Psapi.lib;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalLibraryDirectories>c:\boost\lib\vs2010_64;\boost\lib\vs2010_64;\boost\lib</AdditionalLibraryDirectories>

+      <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>

+      <IgnoreSpecificDefaultLibraries>%(IgnoreSpecificDefaultLibraries)</IgnoreSpecificDefaultLibraries>

+      <GenerateDebugInformation>true</GenerateDebugInformation>

+      <SubSystem>Console</SubSystem>

+    </Link>

+  </ItemDefinitionGroup>

   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

     <ClCompile>

       <Optimization>MaxSpeed</Optimization>

       <IntrinsicFunctions>true</IntrinsicFunctions>

-      <AdditionalIncludeDirectories>..\..\js\src;..\pcre-7.4;c:\Program Files\boost\boost_1_41_0;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>OLDJS;STATIC_JS_API;XP_WIN;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;PCRE_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <AdditionalIncludeDirectories>..\..\js\src;..\pcre-7.4;c:\boost;\boost</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>_UNICODE;UNICODE;SUPPORT_UCP;SUPPORT_UTF8;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;PCRE_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>

       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>

       <FunctionLevelLinking>true</FunctionLevelLinking>

       <PrecompiledHeader>Use</PrecompiledHeader>

-      <PrecompiledHeaderFile>stdafx.h</PrecompiledHeaderFile>

+      <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>

       <WarningLevel>Level3</WarningLevel>

       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>

       <DisableSpecificWarnings>4355;4800;%(DisableSpecificWarnings)</DisableSpecificWarnings>

+      <MultiProcessorCompilation>true</MultiProcessorCompilation>

+      <MinimalRebuild>No</MinimalRebuild>

     </ClCompile>

     <Link>

-      <AdditionalDependencies>ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>

-      <AdditionalLibraryDirectories>c:\Program Files\boost\boost_1_41_0\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>

+      <AdditionalDependencies>ws2_32.lib;psapi.lib;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalLibraryDirectories>c:\boost\lib\vs2010_32;\boost\lib\vs2010_32;\boost\lib</AdditionalLibraryDirectories>

       <GenerateDebugInformation>true</GenerateDebugInformation>

       <SubSystem>Console</SubSystem>

       <OptimizeReferences>true</OptimizeReferences>

       <EnableCOMDATFolding>true</EnableCOMDATFolding>

       <TargetMachine>MachineX86</TargetMachine>

+      <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>

     </Link>

   </ItemDefinitionGroup>

-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

     <ClCompile>

-      <Optimization>Disabled</Optimization>

-      <AdditionalIncludeDirectories>..\..\js\src;..\pcre-7.4;c:\Program Files\boost\boost_1_41_0;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>

-      <PreprocessorDefinitions>_RECSTORE;OLDJS;STATIC_JS_API;XP_WIN;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;PCRE_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>

-      <MinimalRebuild>true</MinimalRebuild>

-      <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>

-      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>

+      <Optimization>MaxSpeed</Optimization>

+      <IntrinsicFunctions>true</IntrinsicFunctions>

+      <AdditionalIncludeDirectories>..\..\js\src;..\pcre-7.4;c:\boost;\boost</AdditionalIncludeDirectories>

+      <PreprocessorDefinitions>SUPPORT_UCP;SUPPORT_UTF8;MONGO_EXPOSE_MACROS;OLDJS;STATIC_JS_API;XP_WIN;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;HAVE_CONFIG_H;PCRE_STATIC;%(PreprocessorDefinitions)</PreprocessorDefinitions>

+      <RuntimeLibrary>MultiThreaded</RuntimeLibrary>

+      <FunctionLevelLinking>true</FunctionLevelLinking>

       <PrecompiledHeader>Use</PrecompiledHeader>

+      <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>

       <WarningLevel>Level3</WarningLevel>

-      <DebugInformationFormat>EditAndContinue</DebugInformationFormat>

-      <DisableSpecificWarnings>4355;4800;%(DisableSpecificWarnings)</DisableSpecificWarnings>

+      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>

+      <DisableSpecificWarnings>4355;4800;4267;4244;%(DisableSpecificWarnings)</DisableSpecificWarnings>

+      <MultiProcessorCompilation>true</MultiProcessorCompilation>

+      <MinimalRebuild>No</MinimalRebuild>

     </ClCompile>

     <Link>

-      <AdditionalDependencies>ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>

-      <AdditionalLibraryDirectories>c:\Program Files\boost\boost_1_41_0\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>

-      <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>

-      <IgnoreSpecificDefaultLibraries>%(IgnoreSpecificDefaultLibraries)</IgnoreSpecificDefaultLibraries>

+      <AdditionalDependencies>ws2_32.lib;psapi.lib;%(AdditionalDependencies)</AdditionalDependencies>

+      <AdditionalLibraryDirectories>c:\boost\lib\vs2010_64;\boost\lib\vs2010_64;\boost\lib</AdditionalLibraryDirectories>

       <GenerateDebugInformation>true</GenerateDebugInformation>

       <SubSystem>Console</SubSystem>

-      <TargetMachine>MachineX86</TargetMachine>

+      <OptimizeReferences>true</OptimizeReferences>

+      <EnableCOMDATFolding>true</EnableCOMDATFolding>

     </Link>

   </ItemDefinitionGroup>

   <ItemGroup>

-    <ClInclude Include="..\pcre-7.4\pcrecpp.h" />

-    <ClInclude Include="..\targetver.h" />

-    <ClInclude Include="..\pcre-7.4\config.h" />

-    <ClInclude Include="..\pcre-7.4\pcre.h" />

-    <ClInclude Include="rec.h" />

-    <ClInclude Include="reccache.h" />

-    <ClInclude Include="reci.h" />

-    <ClInclude Include="recstore.h" />

-    <ClInclude Include="storage.h" />

-    <ClInclude Include="..\client\connpool.h" />

-    <ClInclude Include="..\client\dbclient.h" />

-    <ClInclude Include="..\client\model.h" />

-    <ClInclude Include="btree.h" />

-    <ClInclude Include="client.h" />

-    <ClInclude Include="clientcursor.h" />

-    <ClInclude Include="cmdline.h" />

-    <ClInclude Include="commands.h" />

-    <ClInclude Include="concurrency.h" />

-    <ClInclude Include="curop.h" />

-    <ClInclude Include="cursor.h" />

-    <ClInclude Include="database.h" />

-    <ClInclude Include="db.h" />

-    <ClInclude Include="dbhelpers.h" />

-    <ClInclude Include="dbinfo.h" />

-    <ClInclude Include="dbmessage.h" />

-    <ClInclude Include="introspect.h" />

-    <ClInclude Include="jsobj.h" />

-    <ClInclude Include="json.h" />

-    <ClInclude Include="matcher.h" />

-    <ClInclude Include="..\grid\message.h" />

-    <ClInclude Include="minilex.h" />

-    <ClInclude Include="namespace.h" />

-    <ClInclude Include="pdfile.h" />

-    <ClInclude Include="..\grid\protocol.h" />

-    <ClInclude Include="query.h" />

-    <ClInclude Include="queryoptimizer.h" />

-    <ClInclude Include="repl.h" />

-    <ClInclude Include="replset.h" />

-    <ClInclude Include="resource.h" />

-    <ClInclude Include="scanandorder.h" />

-    <ClInclude Include="security.h" />

-    <ClInclude Include="..\stdafx.h" />

-    <ClInclude Include="..\util\builder.h" />

-    <ClInclude Include="..\util\file.h" />

-    <ClInclude Include="..\util\goodies.h" />

-    <ClInclude Include="..\util\hashtab.h" />

-    <ClInclude Include="lasterror.h" />

-    <ClInclude Include="..\util\log.h" />

-    <ClInclude Include="..\util\lruishmap.h" />

-    <ClInclude Include="..\util\md5.h" />

-    <ClInclude Include="..\util\md5.hpp" />

-    <ClInclude Include="..\util\miniwebserver.h" />

-    <ClInclude Include="..\util\mmap.h" />

-    <ClInclude Include="..\util\sock.h" />

-    <ClInclude Include="..\util\unittest.h" />

-  </ItemGroup>

-  <ItemGroup>

-    <ResourceCompile Include="db.rc" />

-  </ItemGroup>

-  <ItemGroup>

-    <CustomBuild Include="..\..\js\js\Release\js.lib">

-      <FileType>Document</FileType>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">true</ExcludedFromBuild>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>

-    </CustomBuild>

-    <CustomBuild Include="..\..\js\js\Debug\js.lib">

-      <FileType>Document</FileType>

-      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>

-    </CustomBuild>

-  </ItemGroup>

-  <ItemGroup>

+    <ClCompile Include="..\client\dbclientcursor.cpp" />

+    <ClCompile Include="..\client\distlock.cpp" />

+    <ClCompile Include="..\client\model.cpp" />

     <ClCompile Include="..\pcre-7.4\pcrecpp.cc">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_chartables.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_compile.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_config.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_dfa_exec.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_exec.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_fullinfo.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_get.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_globals.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_info.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_maketables.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_newline.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_ord2utf8.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_refcount.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_scanner.cc">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_stringpiece.cc">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_study.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_tables.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_try_flipped.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_ucp_searchfuncs.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_valid_utf8.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_version.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcre_xclass.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\pcre-7.4\pcreposix.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

-    <ClCompile Include="reccache.cpp" />

+    <ClCompile Include="..\s\chunk.cpp" />

+    <ClCompile Include="..\s\config.cpp" />

+    <ClCompile Include="..\s\d_migrate.cpp" />

+    <ClCompile Include="..\s\d_split.cpp" />

+    <ClCompile Include="..\s\d_state.cpp" />

+    <ClCompile Include="..\s\d_writeback.cpp" />

+    <ClCompile Include="..\s\grid.cpp" />

+    <ClCompile Include="..\s\shard.cpp" />

+    <ClCompile Include="..\s\shardconnection.cpp" />

+    <ClCompile Include="..\s\shardkey.cpp" />

+    <ClCompile Include="..\util\concurrency\task.cpp" />

+    <ClCompile Include="..\util\concurrency\thread_pool.cpp" />

+    <ClCompile Include="..\util\concurrency\vars.cpp" />

+    <ClCompile Include="..\util\log.cpp" />

+    <ClCompile Include="..\util\processinfo.cpp" />

+    <ClCompile Include="..\util\stringutils.cpp" />

+    <ClCompile Include="..\util\text.cpp" />

+    <ClCompile Include="..\util\version.cpp" />

+    <ClCompile Include="cap.cpp" />

+    <ClCompile Include="dbcommands_generic.cpp" />

+    <ClCompile Include="geo\2d.cpp" />

+    <ClCompile Include="geo\haystack.cpp" />

+    <ClCompile Include="oplog.cpp" />

+    <ClCompile Include="repl.cpp" />

+    <ClCompile Include="repl\consensus.cpp" />

+    <ClCompile Include="repl\heartbeat.cpp" />

+    <ClCompile Include="repl\manager.cpp" />

+    <ClCompile Include="repl\rs_initialsync.cpp" />

+    <ClCompile Include="repl\rs_initiate.cpp" />

+    <ClCompile Include="repl\rs_rollback.cpp" />

+    <ClCompile Include="repl\rs_sync.cpp" />

+    <ClCompile Include="repl_block.cpp" />

+    <ClCompile Include="restapi.cpp" />

     <ClCompile Include="storage.cpp" />

     <ClCompile Include="..\client\connpool.cpp" />

     <ClCompile Include="..\client\dbclient.cpp" />

-    <ClCompile Include="btree.cpp" />

-    <ClCompile Include="btreecursor.cpp" />

-    <ClCompile Include="queryutil.cpp" />

+    <ClCompile Include="..\client\syncclusterconnection.cpp" />

+    <ClCompile Include="..\pch.cpp">

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Create</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Create</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Create</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Create</PrecompiledHeader>

+    </ClCompile>

     <ClCompile Include="client.cpp" />

     <ClCompile Include="clientcursor.cpp" />

     <ClCompile Include="cloner.cpp" />

     <ClCompile Include="commands.cpp" />

+    <ClCompile Include="common.cpp" />

     <ClCompile Include="cursor.cpp" />

     <ClCompile Include="..\s\d_util.cpp" />

+    <ClCompile Include="database.cpp" />

     <ClCompile Include="db.cpp" />

     <ClCompile Include="dbcommands.cpp" />

+    <ClCompile Include="dbcommands_admin.cpp" />

     <ClCompile Include="dbeval.cpp" />

     <ClCompile Include="dbhelpers.cpp" />

-    <ClCompile Include="dbinfo.cpp" />

     <ClCompile Include="dbwebserver.cpp" />

     <ClCompile Include="extsort.cpp" />

+    <ClCompile Include="index.cpp" />

+    <ClCompile Include="indexkey.cpp" />

     <ClCompile Include="instance.cpp" />

     <ClCompile Include="introspect.cpp" />

     <ClCompile Include="jsobj.cpp" />

     <ClCompile Include="json.cpp" />

     <ClCompile Include="lasterror.cpp" />

     <ClCompile Include="matcher.cpp" />

+    <ClCompile Include="matcher_covered.cpp" />

     <ClCompile Include="..\util\mmap_win.cpp" />

     <ClCompile Include="modules\mms.cpp" />

     <ClCompile Include="module.cpp" />

@@ -433,55 +512,232 @@
     <ClCompile Include="pdfile.cpp" />

     <ClCompile Include="query.cpp" />

     <ClCompile Include="queryoptimizer.cpp" />

-    <ClCompile Include="repl.cpp" />

+    <ClCompile Include="..\util\ramstore.cpp" />

     <ClCompile Include="security.cpp" />

     <ClCompile Include="security_commands.cpp" />

-    <ClCompile Include="..\stdafx.cpp">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">Create</PrecompiledHeader>

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Create</PrecompiledHeader>

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">Create</PrecompiledHeader>

-    </ClCompile>

     <ClCompile Include="tests.cpp" />

     <ClCompile Include="update.cpp" />

+    <ClCompile Include="cmdline.cpp" />

+    <ClCompile Include="queryutil.cpp" />

     <ClCompile Include="..\util\assert_util.cpp" />

     <ClCompile Include="..\util\background.cpp" />

     <ClCompile Include="..\util\base64.cpp" />

+    <ClCompile Include="..\util\mmap.cpp" />

+    <ClCompile Include="..\util\ntservice.cpp" />

+    <ClCompile Include="..\util\processinfo_win32.cpp" />

+    <ClCompile Include="..\util\util.cpp" />

     <ClCompile Include="..\util\httpclient.cpp" />

+    <ClCompile Include="..\util\miniwebserver.cpp" />

     <ClCompile Include="..\util\md5.c">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeaderFile Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeaderFile>

+      <PrecompiledHeaderFile Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeaderFile>

     </ClCompile>

     <ClCompile Include="..\util\md5main.cpp">

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Use</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Use</PrecompiledHeader>

     </ClCompile>

     <ClCompile Include="..\util\message.cpp" />

-    <ClCompile Include="..\util\miniwebserver.cpp" />

-    <ClCompile Include="..\util\mmap.cpp" />

-    <ClCompile Include="..\util\ntservice.cpp" />

-    <ClCompile Include="..\util\processinfo_none.cpp" />

+    <ClCompile Include="..\util\message_server_port.cpp" />

     <ClCompile Include="..\util\sock.cpp" />

-    <ClCompile Include="..\util\util.cpp" />

     <ClCompile Include="..\s\d_logic.cpp" />

     <ClCompile Include="..\scripting\engine.cpp" />

     <ClCompile Include="..\scripting\engine_spidermonkey.cpp" />

     <ClCompile Include="..\shell\mongo_vstudio.cpp">

-      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug Recstore|Win32'">

-      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">

+      </PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">

       </PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

+      </PrecompiledHeader>

     </ClCompile>

+    <ClCompile Include="..\scripting\utils.cpp" />

+    <ClCompile Include="stats\counters.cpp" />

+    <ClCompile Include="stats\snapshots.cpp" />

+    <ClCompile Include="stats\top.cpp" />

+    <ClCompile Include="btree.cpp" />

+    <ClCompile Include="btreecursor.cpp" />

+    <ClCompile Include="repl\health.cpp" />

+    <ClCompile Include="repl\rs.cpp" />

+    <ClCompile Include="repl\replset_commands.cpp" />

+    <ClCompile Include="repl\rs_config.cpp" />

   </ItemGroup>

   <ItemGroup>

+    <None Include="..\jstests\rs\rs_basic.js" />

+    <None Include="..\jstests\rs\test_framework.js" />

     <None Include="..\SConstruct" />

-    <None Include="ClassDiagram1.cd" />

+    <None Include="..\util\mongoutils\README" />

+    <None Include="mongo.ico" />

+    <None Include="repl\notes.txt" />

+    <None Include="repl\test.html" />

+    <None Include="repl\testing.js" />

+  </ItemGroup>

+  <ItemGroup>

+    <ClInclude Include="..\client\dbclientcursor.h" />

+    <ClInclude Include="..\client\distlock.h" />

+    <ClInclude Include="..\client\gridfs.h" />

+    <ClInclude Include="..\client\parallel.h" />

+    <ClInclude Include="..\s\d_logic.h" />

+    <ClInclude Include="..\targetver.h" />

+    <ClInclude Include="..\pcre-7.4\config.h" />

+    <ClInclude Include="..\pcre-7.4\pcre.h" />

+    <ClInclude Include="..\util\concurrency\rwlock.h" />

+    <ClInclude Include="..\util\concurrency\msg.h" />

+    <ClInclude Include="..\util\concurrency\mutex.h" />

+    <ClInclude Include="..\util\concurrency\mvar.h" />

+    <ClInclude Include="..\util\concurrency\task.h" />

+    <ClInclude Include="..\util\concurrency\thread_pool.h" />

+    <ClInclude Include="..\util\mongoutils\html.h" />

+    <ClInclude Include="..\util\mongoutils\str.h" />

+    <ClInclude Include="..\util\ramlog.h" />

+    <ClInclude Include="..\util\text.h" />

+    <ClInclude Include="geo\core.h" />

+    <ClInclude Include="helpers\dblogger.h" />

+    <ClInclude Include="instance.h" />

+    <ClInclude Include="oplogreader.h" />

+    <ClInclude Include="repl.h" />

+    <ClInclude Include="replpair.h" />

+    <ClInclude Include="rec.h" />

+    <ClInclude Include="reccache.h" />

+    <ClInclude Include="reci.h" />

+    <ClInclude Include="recstore.h" />

+    <ClInclude Include="repl\connections.h" />

+    <ClInclude Include="repl\multicmd.h" />

+    <ClInclude Include="repl\rsmember.h" />

+    <ClInclude Include="repl\rs_optime.h" />

+    <ClInclude Include="stats\counters.h" />

+    <ClInclude Include="stats\snapshots.h" />

+    <ClInclude Include="stats\top.h" />

+    <ClInclude Include="..\client\connpool.h" />

+    <ClInclude Include="..\client\dbclient.h" />

+    <ClInclude Include="..\client\model.h" />

+    <ClInclude Include="..\client\redef_macros.h" />

+    <ClInclude Include="..\client\syncclusterconnection.h" />

+    <ClInclude Include="..\client\undef_macros.h" />

+    <ClInclude Include="background.h" />

+    <ClInclude Include="client.h" />

+    <ClInclude Include="clientcursor.h" />

+    <ClInclude Include="cmdline.h" />

+    <ClInclude Include="commands.h" />

+    <ClInclude Include="concurrency.h" />

+    <ClInclude Include="curop.h" />

+    <ClInclude Include="cursor.h" />

+    <ClInclude Include="database.h" />

+    <ClInclude Include="db.h" />

+    <ClInclude Include="dbhelpers.h" />

+    <ClInclude Include="dbinfo.h" />

+    <ClInclude Include="dbmessage.h" />

+    <ClInclude Include="diskloc.h" />

+    <ClInclude Include="index.h" />

+    <ClInclude Include="indexkey.h" />

+    <ClInclude Include="introspect.h" />

+    <ClInclude Include="json.h" />

+    <ClInclude Include="matcher.h" />

+    <ClInclude Include="minilex.h" />

+    <ClInclude Include="namespace.h" />

+    <ClInclude Include="..\pch.h" />

+    <ClInclude Include="pdfile.h" />

+    <ClInclude Include="..\grid\protocol.h" />

+    <ClInclude Include="query.h" />

+    <ClInclude Include="queryoptimizer.h" />

+    <ClInclude Include="resource.h" />

+    <ClInclude Include="scanandorder.h" />

+    <ClInclude Include="security.h" />

+    <ClInclude Include="update.h" />

+    <ClInclude Include="..\util\allocator.h" />

+    <ClInclude Include="..\util\array.h" />

+    <ClInclude Include="..\util\assert_util.h" />

+    <ClInclude Include="..\util\background.h" />

+    <ClInclude Include="..\util\base64.h" />

+    <ClInclude Include="..\util\builder.h" />

+    <ClInclude Include="..\util\debug_util.h" />

+    <ClInclude Include="..\util\embedded_builder.h" />

+    <ClInclude Include="..\util\file.h" />

+    <ClInclude Include="..\util\file_allocator.h" />

+    <ClInclude Include="..\util\goodies.h" />

+    <ClInclude Include="..\util\hashtab.h" />

+    <ClInclude Include="..\util\hex.h" />

+    <ClInclude Include="lasterror.h" />

+    <ClInclude Include="..\util\log.h" />

+    <ClInclude Include="..\util\lruishmap.h" />

+    <ClInclude Include="..\util\mmap.h" />

+    <ClInclude Include="..\util\ntservice.h" />

+    <ClInclude Include="..\util\optime.h" />

+    <ClInclude Include="..\util\processinfo.h" />

+    <ClInclude Include="..\util\queue.h" />

+    <ClInclude Include="..\util\ramstore.h" />

+    <ClInclude Include="..\util\unittest.h" />

+    <ClInclude Include="..\util\concurrency\list.h" />

+    <ClInclude Include="..\util\concurrency\value.h" />

+    <ClInclude Include="..\util\web\html.h" />

+    <ClInclude Include="..\util\httpclient.h" />

+    <ClInclude Include="..\util\miniwebserver.h" />

+    <ClInclude Include="..\util\md5.h" />

+    <ClInclude Include="..\util\md5.hpp" />

+    <ClInclude Include="..\util\message.h" />

+    <ClInclude Include="..\util\message_server.h" />

+    <ClInclude Include="..\util\sock.h" />

+    <ClInclude Include="..\scripting\engine.h" />

+    <ClInclude Include="..\scripting\engine_spidermonkey.h" />

+    <ClInclude Include="..\scripting\engine_v8.h" />

+    <ClInclude Include="..\scripting\v8_db.h" />

+    <ClInclude Include="..\scripting\v8_utils.h" />

+    <ClInclude Include="..\scripting\v8_wrapper.h" />

+    <ClInclude Include="btree.h" />

+    <ClInclude Include="repl\health.h" />

+    <ClInclude Include="..\util\hostandport.h" />

+    <ClInclude Include="repl\rs.h" />

+    <ClInclude Include="repl\rs_config.h" />

+    <ClInclude Include="..\bson\bsonelement.h" />

+    <ClInclude Include="..\bson\bsoninlines.h" />

+    <ClInclude Include="..\bson\bsonmisc.h" />

+    <ClInclude Include="..\bson\bsonobj.h" />

+    <ClInclude Include="..\bson\bsonobjbuilder.h" />

+    <ClInclude Include="..\bson\bsonobjiterator.h" />

+    <ClInclude Include="..\bson\bsontypes.h" />

+    <ClInclude Include="jsobj.h" />

+    <ClInclude Include="..\bson\oid.h" />

+    <ClInclude Include="..\bson\ordering.h" />

+  </ItemGroup>

+  <ItemGroup>

+    <Library Include="..\..\js\js32d.lib">

+      <FileType>Document</FileType>

+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>

+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>

+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>

+    </Library>

+    <Library Include="..\..\js\js32r.lib">

+      <FileType>Document</FileType>

+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>

+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>

+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>

+    </Library>

+    <Library Include="..\..\js\js64d.lib">

+      <FileType>Document</FileType>

+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>

+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>

+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>

+    </Library>

+    <Library Include="..\..\js\js64r.lib">

+      <FileType>Document</FileType>

+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>

+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>

+      <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>

+    </Library>

+  </ItemGroup>

+  <ItemGroup>

+    <ResourceCompile Include="db.rc" />

   </ItemGroup>

   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />

   <ImportGroup Label="ExtensionTargets">

diff --git a/db/db.vcxproj.filters b/db/db.vcxproj.filters
new file mode 100755
index 0000000..f8d72a6
--- /dev/null
+++ b/db/db.vcxproj.filters
@@ -0,0 +1,892 @@
+<?xml version="1.0" encoding="utf-8"?>

+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">

+  <ItemGroup>

+    <ClCompile Include="repl\replset_commands.cpp">

+      <Filter>replSets</Filter>

+    </ClCompile>

+    <ClCompile Include="repl\rs_config.cpp">

+      <Filter>replSets</Filter>

+    </ClCompile>

+    <ClCompile Include="repl\health.cpp">

+      <Filter>replSets</Filter>

+    </ClCompile>

+    <ClCompile Include="btree.cpp">

+      <Filter>db\btree</Filter>

+    </ClCompile>

+    <ClCompile Include="btreecursor.cpp">

+      <Filter>db\btree</Filter>

+    </ClCompile>

+    <ClCompile Include="repl\consensus.cpp">

+      <Filter>replSets</Filter>

+    </ClCompile>

+    <ClCompile Include="..\client\connpool.cpp">

+      <Filter>client</Filter>

+    </ClCompile>

+    <ClCompile Include="..\client\dbclient.cpp">

+      <Filter>client</Filter>

+    </ClCompile>

+    <ClCompile Include="..\client\dbclientcursor.cpp">

+      <Filter>client</Filter>

+    </ClCompile>

+    <ClCompile Include="repl\manager.cpp">

+      <Filter>replSets</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcreposix.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_chartables.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_compile.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_config.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_dfa_exec.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_exec.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_fullinfo.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_get.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_globals.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_info.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_maketables.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_newline.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_ord2utf8.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_refcount.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_scanner.cc">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_stringpiece.cc">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_study.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_tables.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_try_flipped.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_ucp_searchfuncs.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_valid_utf8.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_version.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcre_xclass.c">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pcre-7.4\pcrecpp.cc">

+      <Filter>util\pcre</Filter>

+    </ClCompile>

+    <ClCompile Include="repl\rs_initiate.cpp">

+      <Filter>replSets</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\concurrency\vars.cpp">

+      <Filter>util\concurrency</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\concurrency\task.cpp">

+      <Filter>util\concurrency</Filter>

+    </ClCompile>

+    <ClCompile Include="repl\heartbeat.cpp">

+      <Filter>replSets</Filter>

+    </ClCompile>

+    <ClCompile Include="..\scripting\engine_spidermonkey.cpp">

+      <Filter>scripting</Filter>

+    </ClCompile>

+    <ClCompile Include="..\scripting\engine.cpp">

+      <Filter>scripting</Filter>

+    </ClCompile>

+    <ClCompile Include="..\scripting\utils.cpp">

+      <Filter>scripting</Filter>

+    </ClCompile>

+    <ClCompile Include="..\client\syncclusterconnection.cpp">

+      <Filter>client</Filter>

+    </ClCompile>

+    <ClCompile Include="repl\rs.cpp">

+      <Filter>replSets</Filter>

+    </ClCompile>

+    <ClCompile Include="oplog.cpp">

+      <Filter>repl_old</Filter>

+    </ClCompile>

+    <ClCompile Include="client.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="clientcursor.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="cloner.cpp">

+      <Filter>repl_old</Filter>

+    </ClCompile>

+    <ClCompile Include="cmdline.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="commands.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="common.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="stats\counters.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="cursor.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\s\d_util.cpp">

+      <Filter>sharding</Filter>

+    </ClCompile>

+    <ClCompile Include="..\s\d_logic.cpp">

+      <Filter>sharding</Filter>

+    </ClCompile>

+    <ClCompile Include="database.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="db.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="dbeval.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="dbcommands.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="dbcommands_admin.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="dbhelpers.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="extsort.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="dbwebserver.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\httpclient.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="index.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="indexkey.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="instance.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="introspect.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="jsobj.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="json.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\md5.c">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="matcher_covered.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="lasterror.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="matcher.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\message_server_port.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\md5main.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\message.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="modules\mms.cpp">

+      <Filter>db\modules</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\mmap.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\mmap_win.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\shell\mongo_vstudio.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="module.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="mr.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="namespace.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="nonce.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\ntservice.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="queryutil.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\client\parallel.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="pdfile.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="query.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="queryoptimizer.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\ramstore.cpp">

+      <Filter>db\storage engine</Filter>

+    </ClCompile>

+    <ClCompile Include="repl_block.cpp">

+      <Filter>repl_old</Filter>

+    </ClCompile>

+    <ClCompile Include="repl.cpp">

+      <Filter>repl_old</Filter>

+    </ClCompile>

+    <ClCompile Include="..\s\shardconnection.cpp">

+      <Filter>sharding</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\sock.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\util.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="security.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="security_commands.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="stats\snapshots.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="storage.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="tests.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="stats\top.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="update.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\background.cpp">

+      <Filter>util\concurrency</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\assert_util.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\base64.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\miniwebserver.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\processinfo_win32.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\version.cpp">

+      <Filter>db</Filter>

+    </ClCompile>

+    <ClCompile Include="..\pch.cpp">

+      <Filter>db</Filter>

+    </ClCompile>

+    <ClCompile Include="..\s\d_writeback.cpp">

+      <Filter>sharding</Filter>

+    </ClCompile>

+    <ClCompile Include="..\s\d_state.cpp">

+      <Filter>sharding</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\text.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="geo\2d.cpp">

+      <Filter>db\geo</Filter>

+    </ClCompile>

+    <ClCompile Include="dbcommands_generic.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\s\config.cpp">

+      <Filter>sharding</Filter>

+    </ClCompile>

+    <ClCompile Include="..\s\chunk.cpp">

+      <Filter>sharding</Filter>

+    </ClCompile>

+    <ClCompile Include="..\s\shard.cpp">

+      <Filter>sharding</Filter>

+    </ClCompile>

+    <ClCompile Include="..\s\shardkey.cpp">

+      <Filter>sharding</Filter>

+    </ClCompile>

+    <ClCompile Include="..\client\model.cpp">

+      <Filter>client</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\stringutils.cpp">

+      <Filter>util\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\concurrency\thread_pool.cpp">

+      <Filter>util\concurrency</Filter>

+    </ClCompile>

+    <ClCompile Include="..\client\distlock.cpp">

+      <Filter>client</Filter>

+    </ClCompile>

+    <ClCompile Include="..\s\d_migrate.cpp">

+      <Filter>sharding</Filter>

+    </ClCompile>

+    <ClCompile Include="..\s\d_split.cpp">

+      <Filter>sharding</Filter>

+    </ClCompile>

+    <ClCompile Include="repl\rs_sync.cpp">

+      <Filter>replSets</Filter>

+    </ClCompile>

+    <ClCompile Include="repl\rs_initialsync.cpp">

+      <Filter>replSets</Filter>

+    </ClCompile>

+    <ClCompile Include="repl\rs_rollback.cpp">

+      <Filter>replSets</Filter>

+    </ClCompile>

+    <ClCompile Include="geo\haystack.cpp">

+      <Filter>db\geo</Filter>

+    </ClCompile>

+    <ClCompile Include="cap.cpp">

+      <Filter>db\core</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\log.cpp">

+      <Filter>util</Filter>

+    </ClCompile>

+    <ClCompile Include="..\util\processinfo.cpp">

+      <Filter>util</Filter>

+    </ClCompile>

+    <ClCompile Include="restapi.cpp">

+      <Filter>db</Filter>

+    </ClCompile>

+    <ClCompile Include="..\s\grid.cpp" />

+  </ItemGroup>

+  <ItemGroup>

+    <ClInclude Include="repl\rs_config.h">

+      <Filter>replSets</Filter>

+    </ClInclude>

+    <ClInclude Include="repl\health.h">

+      <Filter>replSets</Filter>

+    </ClInclude>

+    <ClInclude Include="..\bson\bsonelement.h">

+      <Filter>bson</Filter>

+    </ClInclude>

+    <ClInclude Include="..\bson\bsontypes.h">

+      <Filter>bson</Filter>

+    </ClInclude>

+    <ClInclude Include="..\bson\bsoninlines.h">

+      <Filter>bson</Filter>

+    </ClInclude>

+    <ClInclude Include="..\bson\bsonmisc.h">

+      <Filter>bson</Filter>

+    </ClInclude>

+    <ClInclude Include="..\bson\bsonobj.h">

+      <Filter>bson</Filter>

+    </ClInclude>

+    <ClInclude Include="..\bson\bsonobjbuilder.h">

+      <Filter>bson</Filter>

+    </ClInclude>

+    <ClInclude Include="..\bson\bsonobjiterator.h">

+      <Filter>bson</Filter>

+    </ClInclude>

+    <ClInclude Include="btree.h">

+      <Filter>db\btree</Filter>

+    </ClInclude>

+    <ClInclude Include="repl\connections.h">

+      <Filter>replSets</Filter>

+    </ClInclude>

+    <ClInclude Include="..\client\connpool.h">

+      <Filter>client</Filter>

+    </ClInclude>

+    <ClInclude Include="..\client\dbclient.h">

+      <Filter>client</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\mongoutils\html.h">

+      <Filter>util\mongoutils</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\mongoutils\str.h">

+      <Filter>util\mongoutils</Filter>

+    </ClInclude>

+    <ClInclude Include="..\pcre-7.4\pcre.h">

+      <Filter>util\pcre</Filter>

+    </ClInclude>

+    <ClInclude Include="repl\rsmember.h">

+      <Filter>replSets</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\concurrency\list.h">

+      <Filter>util\concurrency</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\concurrency\value.h">

+      <Filter>util\concurrency</Filter>

+    </ClInclude>

+    <ClInclude Include="..\client\dbclientcursor.h">

+      <Filter>client</Filter>

+    </ClInclude>

+    <ClInclude Include="..\client\gridfs.h">

+      <Filter>client</Filter>

+    </ClInclude>

+    <ClInclude Include="..\client\parallel.h">

+      <Filter>client</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\concurrency\task.h">

+      <Filter>util\concurrency</Filter>

+    </ClInclude>

+    <ClInclude Include="repl\multicmd.h">

+      <Filter>replSets</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\concurrency\msg.h">

+      <Filter>util\concurrency</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\concurrency\mutex.h">

+      <Filter>util\concurrency</Filter>

+    </ClInclude>

+    <ClInclude Include="stats\counters.h">

+      <Filter>stats</Filter>

+    </ClInclude>

+    <ClInclude Include="stats\snapshots.h">

+      <Filter>stats</Filter>

+    </ClInclude>

+    <ClInclude Include="stats\top.h">

+      <Filter>stats</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\concurrency\rwlock.h">

+      <Filter>util\concurrency</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\concurrency\mvar.h">

+      <Filter>util\concurrency</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\concurrency\thread_pool.h">

+      <Filter>util\concurrency</Filter>

+    </ClInclude>

+    <ClInclude Include="..\scripting\v8_db.h">

+      <Filter>scripting</Filter>

+    </ClInclude>

+    <ClInclude Include="..\scripting\v8_wrapper.h">

+      <Filter>scripting</Filter>

+    </ClInclude>

+    <ClInclude Include="..\scripting\v8_utils.h">

+      <Filter>scripting</Filter>

+    </ClInclude>

+    <ClInclude Include="..\scripting\engine.h">

+      <Filter>scripting</Filter>

+    </ClInclude>

+    <ClInclude Include="..\scripting\engine_spidermonkey.h">

+      <Filter>scripting</Filter>

+    </ClInclude>

+    <ClInclude Include="..\scripting\engine_v8.h">

+      <Filter>scripting</Filter>

+    </ClInclude>

+    <ClInclude Include="..\client\syncclusterconnection.h">

+      <Filter>client</Filter>

+    </ClInclude>

+    <ClInclude Include="repl\rs_optime.h">

+      <Filter>replSets</Filter>

+    </ClInclude>

+    <ClInclude Include="repl\rs.h">

+      <Filter>replSets</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\optime.h">

+      <Filter>repl_old</Filter>

+    </ClInclude>

+    <ClInclude Include="client.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="clientcursor.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="cursor.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="cmdline.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="commands.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="concurrency.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\pcre-7.4\config.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="curop.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="database.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="db.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\embedded_builder.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\debug_util.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="diskloc.h">

+      <Filter>db\storage engine</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\hashtab.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\file.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\file_allocator.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\goodies.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="dbhelpers.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="dbinfo.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="helpers\dblogger.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="dbmessage.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\hostandport.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\hex.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\web\html.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\httpclient.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="introspect.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="index.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="indexkey.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="json.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="jsobj.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\log.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\md5.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="lasterror.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="matcher.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\md5.hpp">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\message.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\message_server.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="minilex.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\mmap.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\client\model.h">

+      <Filter>client</Filter>

+    </ClInclude>

+    <ClInclude Include="namespace.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\ntservice.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\bson\oid.h">

+      <Filter>bson</Filter>

+    </ClInclude>

+    <ClInclude Include="..\bson\ordering.h">

+      <Filter>bson</Filter>

+    </ClInclude>

+    <ClInclude Include="pdfile.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\grid\protocol.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="query.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="queryoptimizer.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\queue.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\ramstore.h">

+      <Filter>db\storage engine</Filter>

+    </ClInclude>

+    <ClInclude Include="recstore.h">

+      <Filter>db\storage engine</Filter>

+    </ClInclude>

+    <ClInclude Include="rec.h">

+      <Filter>db\storage engine</Filter>

+    </ClInclude>

+    <ClInclude Include="reccache.h">

+      <Filter>db\storage engine</Filter>

+    </ClInclude>

+    <ClInclude Include="reci.h">

+      <Filter>db\storage engine</Filter>

+    </ClInclude>

+    <ClInclude Include="repl.h">

+      <Filter>repl_old</Filter>

+    </ClInclude>

+    <ClInclude Include="replpair.h">

+      <Filter>repl_old</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\sock.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\client\redef_macros.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="update.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="resource.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="scanandorder.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="security.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\targetver.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\client\undef_macros.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\unittest.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="background.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\background.h">

+      <Filter>util\concurrency</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\ramlog.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\allocator.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\array.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\assert_util.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\base64.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\builder.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\lruishmap.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\miniwebserver.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\processinfo.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\pch.h">

+      <Filter>db</Filter>

+    </ClInclude>

+    <ClInclude Include="..\util\text.h">

+      <Filter>util\core</Filter>

+    </ClInclude>

+    <ClInclude Include="geo\core.h">

+      <Filter>db\geo</Filter>

+    </ClInclude>

+    <ClInclude Include="instance.h">

+      <Filter>db\core</Filter>

+    </ClInclude>

+    <ClInclude Include="..\client\distlock.h">

+      <Filter>client</Filter>

+    </ClInclude>

+    <ClInclude Include="..\s\d_logic.h">

+      <Filter>sharding</Filter>

+    </ClInclude>

+    <ClInclude Include="oplogreader.h">

+      <Filter>repl_old</Filter>

+    </ClInclude>

+  </ItemGroup>

+  <ItemGroup>

+    <Filter Include="libs">

+      <UniqueIdentifier>{4b29c82d-d30a-4bf1-9c78-19f59c5777ba}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="util">

+      <UniqueIdentifier>{d2c3db88-7fb7-4365-a63b-b7ad45d316ae}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="util\concurrency">

+      <UniqueIdentifier>{8e6fe846-2833-45bb-b13b-c0f0d4d38593}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="util\mongoutils">

+      <UniqueIdentifier>{cc5d96e6-1805-422b-804d-adcb367dc721}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="util\pcre">

+      <UniqueIdentifier>{fa527226-9b03-4f17-8e4c-80d31fb1e449}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="client">

+      <UniqueIdentifier>{932baf83-ba80-49e5-8280-f1b9c8dbbde6}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="stats">

+      <UniqueIdentifier>{88f4374a-9d55-44a2-a234-c758cc4affa9}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="db">

+      <UniqueIdentifier>{6204f40e-3a9c-44e2-a88b-0e1b6fd9a510}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="db\btree">

+      <UniqueIdentifier>{37b238b2-21ec-4788-bdf9-a59b43490454}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="scripting">

+      <UniqueIdentifier>{6b78f34f-e6b0-49e4-b04e-6478c3a3c077}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="db\storage engine">

+      <UniqueIdentifier>{d565a775-7a99-4860-b25f-441e1655b7c6}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="db\modules">

+      <UniqueIdentifier>{466f15bb-4d5b-4634-ba6b-05a282e0a174}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="db\core">

+      <UniqueIdentifier>{d7f08f93-36bf-49cd-9e1c-ba1fec3234ce}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="repl_old">

+      <UniqueIdentifier>{e899caa1-9a90-4604-ac2e-68d5ca12425c}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="util\core">

+      <UniqueIdentifier>{9775f24c-3a29-4e0d-b5de-991c592cf376}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="Resource Files">

+      <UniqueIdentifier>{9aea1b83-cdcb-48a8-97e6-47805cacdc29}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="bson">

+      <UniqueIdentifier>{aff20a87-2efe-4861-930f-8780c08cbea5}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="db\geo">

+      <UniqueIdentifier>{2a0924a5-9bd9-4c86-a149-0df09dcb5548}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="sharding">

+      <UniqueIdentifier>{03b0d798-b13d-48f4-930d-ca827e2a3f00}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="replSets">

+      <UniqueIdentifier>{3b73f786-d352-446f-a5f5-df49384baf7a}</UniqueIdentifier>

+    </Filter>

+    <Filter Include="replSets\test stuff">

+      <UniqueIdentifier>{4a1ea357-1077-4ad1-85b4-db48a6e1eb46}</UniqueIdentifier>

+    </Filter>

+  </ItemGroup>

+  <ItemGroup>

+    <None Include="repl\notes.txt">

+      <Filter>replSets</Filter>

+    </None>

+    <None Include="..\util\mongoutils\README">

+      <Filter>util\mongoutils</Filter>

+    </None>

+    <None Include="repl\testing.js">

+      <Filter>replSets\test stuff</Filter>

+    </None>

+    <None Include="repl\test.html">

+      <Filter>replSets\test stuff</Filter>

+    </None>

+    <None Include="..\SConstruct">

+      <Filter>db</Filter>

+    </None>

+    <None Include="..\jstests\rs\rs_basic.js">

+      <Filter>replSets\test stuff</Filter>

+    </None>

+    <None Include="..\jstests\rs\test_framework.js">

+      <Filter>replSets\test stuff</Filter>

+    </None>

+    <None Include="mongo.ico">

+      <Filter>Resource Files</Filter>

+    </None>

+  </ItemGroup>

+  <ItemGroup>

+    <Library Include="..\..\js\js64r.lib">

+      <Filter>libs</Filter>

+    </Library>

+    <Library Include="..\..\js\js32d.lib">

+      <Filter>libs</Filter>

+    </Library>

+    <Library Include="..\..\js\js32r.lib">

+      <Filter>libs</Filter>

+    </Library>

+    <Library Include="..\..\js\js64d.lib">

+      <Filter>libs</Filter>

+    </Library>

+  </ItemGroup>

+  <ItemGroup>

+    <ResourceCompile Include="db.rc">

+      <Filter>Resource Files</Filter>

+    </ResourceCompile>

+  </ItemGroup>

+</Project>
\ No newline at end of file
diff --git a/db/db_10.sln b/db/db_10.sln
index 76e8fe9..0aa382f 100644
--- a/db/db_10.sln
+++ b/db/db_10.sln
@@ -1,45 +1,148 @@
 

 Microsoft Visual Studio Solution File, Format Version 11.00

 # Visual Studio 2010

+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "examples", "examples", "{4082881B-EB00-486F-906C-843B8EC06E18}"

+	ProjectSection(SolutionItems) = preProject

+		driverHelpers.cpp = driverHelpers.cpp

+	EndProjectSection

+EndProject

+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{2B262D59-9DC7-4BF1-A431-1BD4966899A5}"

+	ProjectSection(SolutionItems) = preProject

+		..\tools\bridge.cpp = ..\tools\bridge.cpp

+		..\tools\bsondump.cpp = ..\tools\bsondump.cpp

+		..\tools\dump.cpp = ..\tools\dump.cpp

+		..\tools\export.cpp = ..\tools\export.cpp

+		..\tools\files.cpp = ..\tools\files.cpp

+		..\tools\import.cpp = ..\tools\import.cpp

+		..\tools\restore.cpp = ..\tools\restore.cpp

+		..\tools\sniffer.cpp = ..\tools\sniffer.cpp

+		..\tools\stat.cpp = ..\tools\stat.cpp

+		..\tools\tool.cpp = ..\tools\tool.cpp

+		..\tools\tool.h = ..\tools\tool.h

+	EndProjectSection

+EndProject

+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "unix files", "unix files", "{2F760952-C71B-4865-998F-AABAE96D1373}"

+	ProjectSection(SolutionItems) = preProject

+		..\util\processinfo_darwin.cpp = ..\util\processinfo_darwin.cpp

+		..\util\processinfo_linux2.cpp = ..\util\processinfo_linux2.cpp

+		..\util\processinfo_none.cpp = ..\util\processinfo_none.cpp

+	EndProjectSection

+EndProject

+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shell", "shell", "{407B4B88-3451-433C-B74F-31B31FEB5791}"

+	ProjectSection(SolutionItems) = preProject

+		..\shell\utils.h = ..\shell\utils.h

+	EndProjectSection

+EndProject

+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "other", "other", "{12B11474-2D74-48C3-BB3D-F03249BEA88F}"

+EndProject

 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mongod", "db.vcxproj", "{215B2D68-0A70-4D10-8E75-B31010C62A91}"

 EndProject

 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mongos", "..\s\dbgrid.vcxproj", "{E03717ED-69B4-4D21-BC55-DF6690B585C6}"

 EndProject

 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test", "..\dbtests\test.vcxproj", "{215B2D68-0A70-4D10-8E75-B33010C62A91}"

 EndProject

+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bsondemo", "..\bson\bsondemo\bsondemo.vcxproj", "{C9DB5EB7-81AA-4185-BAA1-DA035654402F}"

+EndProject

+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mongoutils test program", "..\util\mongoutils\mongoutils.vcxproj", "{7B84584E-92BC-4DB9-971B-A1A8F93E5053}"

+EndProject

+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "jstests", "jstests", "{F5ABFB2C-A34F-48C1-9B5F-01D456AF6C57}"

+	ProjectSection(SolutionItems) = preProject

+		..\jstests\index_many.js = ..\jstests\index_many.js

+		..\jstests\indexapi.js = ..\jstests\indexapi.js

+		..\jstests\objid5.js = ..\jstests\objid5.js

+	EndProjectSection

+EndProject

 Global

 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

-		Debug Recstore|Win32 = Debug Recstore|Win32

+		Debug|Any CPU = Debug|Any CPU

+		Debug|Mixed Platforms = Debug|Mixed Platforms

 		Debug|Win32 = Debug|Win32

-		release_nojni|Win32 = release_nojni|Win32

+		Debug|x64 = Debug|x64

+		Release|Any CPU = Release|Any CPU

+		Release|Mixed Platforms = Release|Mixed Platforms

 		Release|Win32 = Release|Win32

+		Release|x64 = Release|x64

 	EndGlobalSection

 	GlobalSection(ProjectConfigurationPlatforms) = postSolution

-		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug Recstore|Win32.ActiveCfg = Debug Recstore|Win32

-		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug Recstore|Win32.Build.0 = Debug Recstore|Win32

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|Any CPU.ActiveCfg = Debug|x64

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|Mixed Platforms.ActiveCfg = Debug|x64

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|Mixed Platforms.Build.0 = Debug|x64

 		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|Win32.ActiveCfg = Debug|Win32

 		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|Win32.Build.0 = Debug|Win32

-		{215B2D68-0A70-4D10-8E75-B31010C62A91}.release_nojni|Win32.ActiveCfg = Release|Win32

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|x64.ActiveCfg = Debug|x64

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Debug|x64.Build.0 = Debug|x64

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|Any CPU.ActiveCfg = Release|x64

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|Mixed Platforms.ActiveCfg = Release|x64

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|Mixed Platforms.Build.0 = Release|x64

 		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|Win32.ActiveCfg = Release|Win32

 		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|Win32.Build.0 = Release|Win32

-		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Debug Recstore|Win32.ActiveCfg = Debug Recstore|Win32

-		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Debug Recstore|Win32.Build.0 = Debug Recstore|Win32

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|x64.ActiveCfg = Release|x64

+		{215B2D68-0A70-4D10-8E75-B31010C62A91}.Release|x64.Build.0 = Release|x64

+		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Debug|Any CPU.ActiveCfg = Debug|x64

+		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Debug|Mixed Platforms.ActiveCfg = Debug|x64

+		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Debug|Mixed Platforms.Build.0 = Debug|x64

 		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Debug|Win32.ActiveCfg = Debug|Win32

 		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Debug|Win32.Build.0 = Debug|Win32

-		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.release_nojni|Win32.ActiveCfg = Release|Win32

-		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.release_nojni|Win32.Build.0 = Release|Win32

+		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Debug|x64.ActiveCfg = Debug|x64

+		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Debug|x64.Build.0 = Debug|x64

+		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Release|Any CPU.ActiveCfg = Release|x64

+		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Release|Mixed Platforms.ActiveCfg = Release|x64

+		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Release|Mixed Platforms.Build.0 = Release|x64

 		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Release|Win32.ActiveCfg = Release|Win32

 		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Release|Win32.Build.0 = Release|Win32

-		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Debug Recstore|Win32.ActiveCfg = Debug Recstore|Win32

-		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Debug Recstore|Win32.Build.0 = Debug Recstore|Win32

+		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Release|x64.ActiveCfg = Release|x64

+		{E03717ED-69B4-4D21-BC55-DF6690B585C6}.Release|x64.Build.0 = Release|x64

+		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Debug|Any CPU.ActiveCfg = Debug|x64

+		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Debug|Mixed Platforms.ActiveCfg = Debug|x64

+		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Debug|Mixed Platforms.Build.0 = Debug|x64

 		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Debug|Win32.ActiveCfg = Debug|Win32

 		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Debug|Win32.Build.0 = Debug|Win32

-		{215B2D68-0A70-4D10-8E75-B33010C62A91}.release_nojni|Win32.ActiveCfg = release_nojni|Win32

-		{215B2D68-0A70-4D10-8E75-B33010C62A91}.release_nojni|Win32.Build.0 = release_nojni|Win32

+		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Debug|x64.ActiveCfg = Debug|x64

+		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Debug|x64.Build.0 = Debug|x64

+		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Release|Any CPU.ActiveCfg = Release|x64

+		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Release|Mixed Platforms.ActiveCfg = Release|x64

+		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Release|Mixed Platforms.Build.0 = Release|x64

 		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Release|Win32.ActiveCfg = Release|Win32

 		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Release|Win32.Build.0 = Release|Win32

+		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Release|x64.ActiveCfg = Release|x64

+		{215B2D68-0A70-4D10-8E75-B33010C62A91}.Release|x64.Build.0 = Release|x64

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Debug|Any CPU.ActiveCfg = Debug|x64

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Debug|Mixed Platforms.ActiveCfg = Debug|x64

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Debug|Mixed Platforms.Build.0 = Debug|x64

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Debug|Win32.ActiveCfg = Debug|Win32

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Debug|Win32.Build.0 = Debug|Win32

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Debug|x64.ActiveCfg = Debug|x64

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Debug|x64.Build.0 = Debug|x64

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Release|Any CPU.ActiveCfg = Release|x64

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Release|Mixed Platforms.ActiveCfg = Release|x64

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Release|Mixed Platforms.Build.0 = Release|x64

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Release|Win32.ActiveCfg = Release|Win32

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Release|Win32.Build.0 = Release|Win32

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Release|x64.ActiveCfg = Release|x64

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F}.Release|x64.Build.0 = Release|x64

+		{7B84584E-92BC-4DB9-971B-A1A8F93E5053}.Debug|Any CPU.ActiveCfg = Debug|Win32

+		{7B84584E-92BC-4DB9-971B-A1A8F93E5053}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32

+		{7B84584E-92BC-4DB9-971B-A1A8F93E5053}.Debug|Mixed Platforms.Build.0 = Debug|Win32

+		{7B84584E-92BC-4DB9-971B-A1A8F93E5053}.Debug|Win32.ActiveCfg = Debug|Win32

+		{7B84584E-92BC-4DB9-971B-A1A8F93E5053}.Debug|Win32.Build.0 = Debug|Win32

+		{7B84584E-92BC-4DB9-971B-A1A8F93E5053}.Debug|x64.ActiveCfg = Debug|Win32

+		{7B84584E-92BC-4DB9-971B-A1A8F93E5053}.Release|Any CPU.ActiveCfg = Release|Win32

+		{7B84584E-92BC-4DB9-971B-A1A8F93E5053}.Release|Mixed Platforms.ActiveCfg = Release|Win32

+		{7B84584E-92BC-4DB9-971B-A1A8F93E5053}.Release|Mixed Platforms.Build.0 = Release|Win32

+		{7B84584E-92BC-4DB9-971B-A1A8F93E5053}.Release|Win32.ActiveCfg = Release|Win32

+		{7B84584E-92BC-4DB9-971B-A1A8F93E5053}.Release|Win32.Build.0 = Release|Win32

+		{7B84584E-92BC-4DB9-971B-A1A8F93E5053}.Release|x64.ActiveCfg = Release|Win32

 	EndGlobalSection

 	GlobalSection(SolutionProperties) = preSolution

 		HideSolutionNode = FALSE

 	EndGlobalSection

+	GlobalSection(NestedProjects) = preSolution

+		{2B262D59-9DC7-4BF1-A431-1BD4966899A5} = {12B11474-2D74-48C3-BB3D-F03249BEA88F}

+		{2F760952-C71B-4865-998F-AABAE96D1373} = {12B11474-2D74-48C3-BB3D-F03249BEA88F}

+		{407B4B88-3451-433C-B74F-31B31FEB5791} = {12B11474-2D74-48C3-BB3D-F03249BEA88F}

+		{4082881B-EB00-486F-906C-843B8EC06E18} = {12B11474-2D74-48C3-BB3D-F03249BEA88F}

+		{C9DB5EB7-81AA-4185-BAA1-DA035654402F} = {12B11474-2D74-48C3-BB3D-F03249BEA88F}

+		{7B84584E-92BC-4DB9-971B-A1A8F93E5053} = {12B11474-2D74-48C3-BB3D-F03249BEA88F}

+		{F5ABFB2C-A34F-48C1-9B5F-01D456AF6C57} = {12B11474-2D74-48C3-BB3D-F03249BEA88F}

+	EndGlobalSection

 EndGlobal

diff --git a/db/dbcommands.cpp b/db/dbcommands.cpp
index 85c695d..94edf0a 100644
--- a/db/dbcommands.cpp
+++ b/db/dbcommands.cpp
@@ -15,11 +15,11 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "query.h"
 #include "pdfile.h"
 #include "jsobj.h"
-#include "../util/builder.h"
+#include "../bson/util/builder.h"
 #include <time.h>
 #include "introspect.h"
 #include "btree.h"
@@ -28,7 +28,8 @@
 #include "../util/processinfo.h"
 #include "json.h"
 #include "repl.h"
-#include "replset.h"
+#include "repl_block.h"
+#include "replpair.h"
 #include "commands.h"
 #include "db.h"
 #include "instance.h"
@@ -38,38 +39,13 @@
 #include "../scripting/engine.h"
 #include "stats/counters.h"
 #include "background.h"
+#include "../util/version.h"
 
 namespace mongo {
 
-    TicketHolder connTicketHolder( 20000 );
-
     extern int otherTraceLevel;
     void flushOpLog( stringstream &ss );
 
-    class CmdShutdown : public Command {
-    public:
-        virtual bool requiresAuth() { return true; }
-        virtual bool adminOnly() { return true; }
-        virtual bool localHostOnlyIfNoAuth(const BSONObj& cmdObj) { return true; }
-        virtual bool logTheOp() {
-            return false;
-        }
-        virtual bool slaveOk() {
-            return true;
-        }
-        virtual LockType locktype(){ return WRITE; } 
-        virtual void help( stringstream& help ) const {
-            help << "shutdown the database.  must be ran against admin db and either (1) ran from localhost or (2) authenticated.\n";
-        }
-        CmdShutdown() : Command("shutdown") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            cc().shutdown();
-            log() << "terminating, shutdown command received" << endl;
-            dbexit( EXIT_CLEAN ); // this never returns
-            return true;
-        }
-    } cmdShutdown;
-
     /* reset any errors so that getlasterror comes back clean.
 
        useful before performing a long series of operations where we want to
@@ -78,19 +54,19 @@ namespace mongo {
     */
     class CmdResetError : public Command {
     public:
-        virtual LockType locktype(){ return NONE; } 
+        virtual LockType locktype() const { return NONE; } 
         virtual bool requiresAuth() { return false; }
         virtual bool logTheOp() {
             return false;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
         virtual void help( stringstream& help ) const {
             help << "reset error state (used with getpreverror)";
         }
-        CmdResetError() : Command("reseterror") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        CmdResetError() : Command("resetError", false, "reseterror") {}
+        bool run(const string& db, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             LastError *le = lastError.get();
             assert( le );
             le->reset();
@@ -98,77 +74,70 @@ namespace mongo {
         }
     } cmdResetError;
 
-    /* for diagnostic / testing purposes. */
-    class CmdSleep : public Command { 
-    public:
-        virtual LockType locktype(){ return READ; } 
-        virtual bool adminOnly() { return true; }
-        virtual bool logTheOp() {
-            return false;
-        }
-        virtual bool slaveOk() {
-            return true;
-        }
-        virtual void help( stringstream& help ) const {
-            help << "internal / make db block for 100 seconds";
-        }
-        CmdSleep() : Command("sleep") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            sleepsecs(100);
-            return true;
-        }
-    } cmdSleep;
-
     class CmdGetLastError : public Command {
     public:
-        virtual LockType locktype(){ return NONE; } 
+        virtual LockType locktype() const { return NONE; } 
         virtual bool requiresAuth() { return false; }
         virtual bool logTheOp() {
             return false;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
         virtual void help( stringstream& help ) const {
-            help << "return error status of the last operation";
+            help << "return error status of the last operation on this connection";
         }
-        CmdGetLastError() : Command("getlasterror") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        CmdGetLastError() : Command("getLastError", false, "getlasterror") {}
+        bool run(const string& dbnamne, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             LastError *le = lastError.disableForCommand();
             if ( le->nPrev != 1 )
                 LastError::noError.appendSelf( result );
             else
                 le->appendSelf( result );
             
+            Client& c = cc();
+            c.appendLastOp( result );
+
             if ( cmdObj["fsync"].trueValue() ){
                 log() << "fsync from getlasterror" << endl;
                 result.append( "fsyncFiles" , MemoryMappedFile::flushAll( true ) );
             }
             
-            return true;
-        }
-    } cmdGetLastError;
+            BSONElement e = cmdObj["w"];
+            if ( e.isNumber() ){
+                int timeout = cmdObj["wtimeout"].numberInt();
+                Timer t;
+
+                int w = e.numberInt();
+
+                long long passes = 0;
+                char buf[32];
+                while ( 1 ){
+                    if ( opReplicatedEnough( c.getLastOp() , w ) )
+                        break;
+                    
+                    if ( timeout > 0 && t.millis() >= timeout ){
+                        result.append( "wtimeout" , true );
+                        errmsg = "timed out waiting for slaves";
+                        result.append( "waited" , t.millis() );
+                        return false;
+                    }
 
-    /* for testing purposes only */
-    class CmdForceError : public Command {
-    public:
-        virtual bool logTheOp() {
-            return false;
-        }
-        virtual bool slaveOk() {
-            return true;
-        }
-        virtual LockType locktype(){ return NONE; } 
-        CmdForceError() : Command("forceerror") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            uassert( 10038 , "forced error", false);
+                    assert( sprintf( buf , "w block pass: %lld" , ++passes ) < 30 );
+                    c.curop()->setMessage( buf );
+                    sleepmillis(1);
+                    killCurrentOp.checkForInterrupt();
+                }
+                result.appendNumber( "wtime" , t.millis() );
+            }
+            
             return true;
         }
-    } cmdForceError;
+    } cmdGetLastError;
 
     class CmdGetPrevError : public Command {
     public:
-        virtual LockType locktype(){ return NONE; } 
+        virtual LockType locktype() const { return NONE; } 
         virtual bool requiresAuth() { return false; }
         virtual bool logTheOp() {
             return false;
@@ -176,11 +145,11 @@ namespace mongo {
         virtual void help( stringstream& help ) const {
             help << "check for errors since last reseterror commandcal";
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        CmdGetPrevError() : Command("getpreverror") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        CmdGetPrevError() : Command("getPrevError", false, "getpreverror") {}
+        bool run(const string& dbnamne, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             LastError *le = lastError.disableForCommand();
             le->appendSelf( result );
             if ( le->valid )
@@ -200,12 +169,12 @@ namespace mongo {
         virtual void help( stringstream& help ) const {
             help << "convert to id based errors rather than connection based";
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        virtual LockType locktype(){ return NONE; } 
-        CmdSwitchToClientErrors() : Command("switchtoclienterrors") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        virtual LockType locktype() const { return NONE; } 
+        CmdSwitchToClientErrors() : Command("switchToClientErrors", false, "switchtoclienterrors") {}
+        bool run(const string& dbnamne , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             if ( lastError.getID() ){
                 errmsg = "already in client id mode";
                 return false;
@@ -225,19 +194,19 @@ namespace mongo {
         virtual void help( stringstream& help ) const {
             help << "drop (delete) this database";
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; } 
+        virtual LockType locktype() const { return WRITE; } 
         CmdDropDatabase() : Command("dropDatabase") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            BSONElement e = cmdObj.getField(name);
-            log() << "dropDatabase " << ns << endl;
+        bool run(const string& dbnamne, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            BSONElement e = cmdObj.firstElement();
+            log() << "dropDatabase " << dbnamne << endl;
             int p = (int) e.number();
             if ( p != 1 )
                 return false;
-            dropDatabase(ns);
-            result.append( "dropped" , ns );
+            dropDatabase(dbnamne);
+            result.append( "dropped" , dbnamne );
             return true;
         }
     } cmdDropDatabase;
@@ -247,17 +216,17 @@ namespace mongo {
         virtual bool logTheOp() {
             return false;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
         virtual void help( stringstream& help ) const {
             help << "repair database.  also compacts. note: slow.";
         }
-        virtual LockType locktype(){ return WRITE; } 
+        virtual LockType locktype() const { return WRITE; } 
         CmdRepairDatabase() : Command("repairDatabase") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            BSONElement e = cmdObj.getField(name);
-            log() << "repairDatabase " << ns << endl;
+        bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            BSONElement e = cmdObj.firstElement();
+            log() << "repairDatabase " << dbname << endl;
             int p = (int) e.number();
             if ( p != 1 )
                 return false;
@@ -265,7 +234,7 @@ namespace mongo {
             bool preserveClonedFilesOnFailure = e.isBoolean() && e.boolean();
             e = cmdObj.getField( "backupOriginalFiles" );
             bool backupOriginalFiles = e.isBoolean() && e.boolean();
-            return repairDatabase( ns, errmsg, preserveClonedFilesOnFailure, backupOriginalFiles );
+            return repairDatabase( dbname, errmsg, preserveClonedFilesOnFailure, backupOriginalFiles );
         }
     } cmdRepairDatabase;
     
@@ -275,19 +244,25 @@ namespace mongo {
     */
     class CmdProfile : public Command {
     public:
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
         virtual void help( stringstream& help ) const {
-            help << "enable or disable performance profiling";
+            help << "enable or disable performance profiling\n";
+            help << "{ profile : <n> }\n";
+            help << "0=off 1=log slow ops 2=log all\n";
+            help << "http://www.mongodb.org/display/DOCS/Database+Profiler";
         }
-        virtual LockType locktype(){ return WRITE; } 
+        virtual LockType locktype() const { return WRITE; } 
         CmdProfile() : Command("profile") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            BSONElement e = cmdObj.getField(name);
-            result.append("was", (double) cc().database()->profile);
+        bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            BSONElement e = cmdObj.firstElement();
+            result.append("was", cc().database()->profile);
+            result.append("slowms", cmdLine.slowMS );
+
             int p = (int) e.number();
             bool ok = false;
+            
             if ( p == -1 )
                 ok = true;
             else if ( p >= 0 && p <= 2 ) {
@@ -304,21 +279,30 @@ namespace mongo {
 
     class CmdServerStatus : public Command {
     public:
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        CmdServerStatus() : Command("serverStatus") {
+        CmdServerStatus() : Command("serverStatus", true) {
             started = time(0);
         }
         
-        virtual LockType locktype(){ return NONE; } 
+        virtual LockType locktype() const { return NONE; } 
+
+        virtual void help( stringstream& help ) const {
+            help << "returns lots of administrative server statistics";
+        }
 
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             
+            long long start = Listener::getElapsedTimeMillis();
+            BSONObjBuilder timeBuilder(128);
+
+
 			bool authed = cc().getAuthenticationInfo()->isAuthorizedReads("admin");
 
             result.append("version", versionString);
             result.append("uptime",(double) (time(0)-started));
+            result.append("uptimeEstimate",(double) (start/1000));
             result.appendDate( "localTime" , jsTime() );
 
             {
@@ -333,9 +317,18 @@ namespace mongo {
                 t.append("lockTime", tl);
                 t.append("ratio", (tt ? tl/tt : 0));
                 
+                BSONObjBuilder ttt( t.subobjStart( "currentQueue" ) );
+                int w=0, r=0;
+                Client::recommendedYieldMicros( &w , &r );
+                ttt.append( "total" , w + r );
+                ttt.append( "readers" , r );
+                ttt.append( "writers" , w );
+                ttt.done();
+
                 result.append( "globalLock" , t.obj() );
             }
-            
+            timeBuilder.appendNumber( "after basic" , Listener::getElapsedTimeMillis() - start );
+
             if ( authed ){
                 
                 BSONObjBuilder t( result.subobjStart( "mem" ) );
@@ -358,6 +351,7 @@ namespace mongo {
                 t.done();
                     
             }
+            timeBuilder.appendNumber( "after is authed" , Listener::getElapsedTimeMillis() - start );
             
             {
                 BSONObjBuilder bb( result.subobjStart( "connections" ) );
@@ -365,6 +359,7 @@ namespace mongo {
                 bb.append( "available" , connTicketHolder.available() );
                 bb.done();
             }
+            timeBuilder.appendNumber( "after connections" , Listener::getElapsedTimeMillis() - start );
             
             if ( authed ){
                 BSONObjBuilder bb( result.subobjStart( "extra_info" ) );
@@ -372,26 +367,31 @@ namespace mongo {
                 ProcessInfo p;
                 p.getExtraInfo(bb);
                 bb.done();
+                timeBuilder.appendNumber( "after extra info" , Listener::getElapsedTimeMillis() - start );
+            
             }
 
-
             {
                 BSONObjBuilder bb( result.subobjStart( "indexCounters" ) );
                 globalIndexCounters.append( bb );
                 bb.done();
             }
-
+            
             {
                 BSONObjBuilder bb( result.subobjStart( "backgroundFlushing" ) );
                 globalFlushCounters.append( bb );
                 bb.done();
             }
-            
+
+            timeBuilder.appendNumber( "after counters" , Listener::getElapsedTimeMillis() - start );            
+
             if ( anyReplEnabled() ){
                 BSONObjBuilder bb( result.subobjStart( "repl" ) );
                 appendReplicationInfo( bb , authed , cmdObj["repl"].numberInt() );
                 bb.done();
             }
+
+            timeBuilder.appendNumber( "after repl" , Listener::getElapsedTimeMillis() - start );            
             
             result.append( "opcounters" , globalOpCounters.getObj() );
             
@@ -405,44 +405,28 @@ namespace mongo {
                 asserts.done();
             }
 
+            timeBuilder.appendNumber( "after asserts" , Listener::getElapsedTimeMillis() - start );            
+
             if ( ! authed )
                 result.append( "note" , "run against admin for more info" );
+            
+            if ( Listener::getElapsedTimeMillis() - start > 1000 )
+                result.append( "timing" , timeBuilder.obj() );
 
             return true;
         }
         time_t started;
     } cmdServerStatus;
 
-    /* just to check if the db has asserted */
-    class CmdAssertInfo : public Command {
-    public:
-        virtual bool slaveOk() {
-            return true;
-        }
-        virtual void help( stringstream& help ) const {
-            help << "check if any asserts have occurred on the server";
-        }
-        virtual LockType locktype(){ return WRITE; } 
-        CmdAssertInfo() : Command("assertinfo") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            result.appendBool("dbasserted", lastAssert[0].isSet() || lastAssert[1].isSet() || lastAssert[2].isSet());
-            result.appendBool("asserted", lastAssert[0].isSet() || lastAssert[1].isSet() || lastAssert[2].isSet() || lastAssert[3].isSet());
-            result.append("assert", lastAssert[AssertRegular].toString());
-            result.append("assertw", lastAssert[AssertW].toString());
-            result.append("assertmsg", lastAssert[AssertMsg].toString());
-            result.append("assertuser", lastAssert[AssertUser].toString());
-            return true;
-        }
-    } cmdAsserts;
-
     class CmdGetOpTime : public Command {
     public:
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        virtual LockType locktype(){ return NONE; } 
+        virtual void help( stringstream& help ) const { help << "internal"; }
+        virtual LockType locktype() const { return NONE; } 
         CmdGetOpTime() : Command("getoptime") { }
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             writelock l( "" );
             result.appendDate("optime", OpTime::now().asDate());
             return true;
@@ -453,7 +437,7 @@ namespace mongo {
     class Cmd : public Command {
     public:
         Cmd() : Command("") { }
-        bool adminOnly() { return true; }
+        bool adminOnly() const { return true; }
         bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result) {
             return true;
         }
@@ -462,21 +446,22 @@ namespace mongo {
 
     class CmdDiagLogging : public Command {
     public:
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
         CmdDiagLogging() : Command("diagLogging") { }
-        bool adminOnly() {
+        bool adminOnly() const {
             return true;
         }
-        virtual LockType locktype(){ return WRITE; } 
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
+        void help(stringstream& h) const { h << "http://www.mongodb.org/display/DOCS/Monitoring+and+Diagnostics#MonitoringandDiagnostics-DatabaseRecord%2FReplay"; }
+        virtual LockType locktype() const { return WRITE; } 
+        bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
             int was = _diaglog.setLevel( cmdObj.firstElement().numberInt() );
             stringstream ss;
             flushOpLog( ss );
             out() << ss.str() << endl;
             if ( !cmdLine.quiet )
-                log() << "CMD: diagLogging set to " << _diaglog.level << " from: " << was << endl;
+                tlog() << "CMD: diagLogging set to " << _diaglog.level << " from: " << was << endl;
             result.append( "was" , was );
             return true;
         }
@@ -584,18 +569,19 @@ namespace mongo {
         virtual bool logTheOp() {
             return true;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return false;
         }
-        virtual bool adminOnly() {
+        virtual bool adminOnly() const {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; } 
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
-            string nsToDrop = cc().database()->name + '.' + cmdObj.getField(name).valuestr();
+        virtual void help( stringstream& help ) const { help << "drop a collection\n{drop : <collectionName>}"; }
+        virtual LockType locktype() const { return WRITE; } 
+        virtual bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
+            string nsToDrop = dbname + '.' + cmdObj.firstElement().valuestr();
             NamespaceDetails *d = nsdetails(nsToDrop.c_str());
             if ( !cmdLine.quiet )
-                log() << "CMD: drop " << nsToDrop << endl;
+                tlog() << "CMD: drop " << nsToDrop << endl;
             if ( d == 0 ) {
                 errmsg = "ns not found";
                 return false;
@@ -609,23 +595,24 @@ namespace mongo {
     /* select count(*) */
     class CmdCount : public Command {
     public:
-        virtual LockType locktype(){ return READ; } 
+        virtual LockType locktype() const { return READ; } 
         CmdCount() : Command("count") { }
         virtual bool logTheOp() {
             return false;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             // ok on --slave setups, not ok for nonmaster of a repl pair (unless override)
             return replSettings.slave == SimpleSlave;
         }
         virtual bool slaveOverrideOk() {
             return true;
         }
-        virtual bool adminOnly() {
+        virtual bool adminOnly() const {
             return false;
         }
-        virtual bool run(const char *_ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
-            string ns = cc().database()->name + '.' + cmdObj.getField(name).valuestr();
+        virtual void help( stringstream& help ) const { help << "count objects in collection"; }
+        virtual bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
+            string ns = dbname + '.' + cmdObj.firstElement().valuestr();
             string err;
             long long n = runCount(ns.c_str(), cmdObj, err);
             long long nn = n;
@@ -652,18 +639,18 @@ namespace mongo {
         virtual bool logTheOp() {
             return false;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return false;
         }
-        virtual bool adminOnly() {
+        virtual bool adminOnly() const {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; } 
+        virtual LockType locktype() const { return WRITE; } 
         virtual void help( stringstream& help ) const {
             help << "create a collection";
         }
-        virtual bool run(const char *_ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
-            string ns = cc().database()->name + '.' + cmdObj.getField(name).valuestr();
+        virtual bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
+            string ns = dbname + '.' + cmdObj.firstElement().valuestr();
             string err;
             bool ok = userCreateNS(ns.c_str(), cmdObj, err, true);
             if ( !ok && !err.empty() )
@@ -678,20 +665,20 @@ namespace mongo {
         virtual bool logTheOp() {
             return true;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; } 
+        virtual LockType locktype() const { return WRITE; } 
         virtual void help( stringstream& help ) const {
             help << "drop indexes for a collection";
         }
-        CmdDropIndexes(const char *cmdname = "dropIndexes") : Command(cmdname) { }
-        bool run(const char *ns, BSONObj& jsobj, string& errmsg, BSONObjBuilder& anObjBuilder, bool /*fromRepl*/) {
-            BSONElement e = jsobj.getField(name.c_str());
-            string toDeleteNs = cc().database()->name + '.' + e.valuestr();
+        CmdDropIndexes() : Command("dropIndexes", false, "deleteIndexes") { }
+        bool run(const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& anObjBuilder, bool /*fromRepl*/) {
+            BSONElement e = jsobj.firstElement();
+            string toDeleteNs = dbname + '.' + e.valuestr();
             NamespaceDetails *d = nsdetails(toDeleteNs.c_str());
             if ( !cmdLine.quiet )
-                log() << "CMD: dropIndexes " << toDeleteNs << endl;
+                tlog() << "CMD: dropIndexes " << toDeleteNs << endl;
             if ( d ) {
                 BSONElement f = jsobj.getField("index");
                 if ( f.type() == String ) {
@@ -701,7 +688,7 @@ namespace mongo {
                     int idxId = d->findIndexByKeyPattern( f.embeddedObject() );
                     if ( idxId < 0 ){
                         errmsg = "can't find index with key:";
-                        errmsg += f.embeddedObject();
+                        errmsg += f.embeddedObject().toString();
                         return false;
                     }
                     else {
@@ -721,33 +708,28 @@ namespace mongo {
             }
         }
     } cmdDropIndexes;
-    class CmdDeleteIndexes : public CmdDropIndexes { 
-    public:
-        CmdDeleteIndexes() : CmdDropIndexes("deleteIndexes") { }
-    } cmdDeleteIndexes;
 
     class CmdReIndex : public Command {
     public:
         virtual bool logTheOp() {
             return true;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; } 
+        virtual LockType locktype() const { return WRITE; } 
         virtual void help( stringstream& help ) const {
             help << "re-index a collection";
         }
         CmdReIndex() : Command("reIndex") { }
-        bool run(const char *ns, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) {
-            BackgroundOperation::assertNoBgOpInProgForNs(ns);
-
+        bool run(const string& dbname , BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) {
             static DBDirectClient db;
 
-            BSONElement e = jsobj.getField(name.c_str());
-            string toDeleteNs = cc().database()->name + '.' + e.valuestr();
+            BSONElement e = jsobj.firstElement();
+            string toDeleteNs = dbname + '.' + e.valuestr();
             NamespaceDetails *d = nsdetails(toDeleteNs.c_str());
-            log() << "CMD: reIndex " << toDeleteNs << endl;
+            tlog() << "CMD: reIndex " << toDeleteNs << endl;
+            BackgroundOperation::assertNoBgOpInProgForNs(toDeleteNs.c_str());
 
             if ( ! d ){
                 errmsg = "ns not found";
@@ -772,7 +754,7 @@ namespace mongo {
 
             for ( list<BSONObj>::iterator i=all.begin(); i!=all.end(); i++ ){
                 BSONObj o = *i;
-                db.insert( Namespace( toDeleteNs.c_str() ).getSisterNS( "system.indexes" ).c_str() , o );
+                theDataFileMgr.insertWithObjMod( Namespace( toDeleteNs.c_str() ).getSisterNS( "system.indexes" ).c_str() , o , true );
             }
 
             result.append( "ok" , 1 );
@@ -787,18 +769,19 @@ namespace mongo {
         virtual bool logTheOp() {
             return false;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
         virtual bool slaveOverrideOk() {
             return true;
         }
-        virtual bool adminOnly() {
+        virtual bool adminOnly() const {
             return true;
         }
-        virtual LockType locktype(){ return WRITE; } 
+        virtual LockType locktype() const { return READ; } 
+        virtual void help( stringstream& help ) const { help << "list databases on this server"; }
         CmdListDatabases() : Command("listDatabases") {}
-        bool run(const char *ns, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) {
+        bool run(const string& dbname , BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) {
             vector< string > dbNames;
             getDatabaseNames( dbNames );
             vector< BSONObj > dbInfos;
@@ -846,11 +829,13 @@ namespace mongo {
        */
     class CmdCloseAllDatabases : public Command {
     public:
-        virtual bool adminOnly() { return true; }
-        virtual bool slaveOk() { return false; }
-        virtual LockType locktype(){ return WRITE; } 
+        virtual void help( stringstream& help ) const { help << "Close all database files.\nA new request will cause an immediate reopening; thus, this is mostly for testing purposes."; }
+        virtual bool adminOnly() const { return true; }
+        virtual bool slaveOk() const { return false; }
+        virtual LockType locktype() const { return WRITE; } 
+
         CmdCloseAllDatabases() : Command( "closeAllDatabases" ) {}
-        bool run(const char *ns, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) {
+        bool run(const string& dbname , BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) {
             return dbHolder.closeAll( dbpath , result, false );
         }
     } cmdCloseAllDatabases;
@@ -858,17 +843,15 @@ namespace mongo {
     class CmdFileMD5 : public Command {
     public:
         CmdFileMD5() : Command( "filemd5" ){}
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
         virtual void help( stringstream& help ) const {
-            help << " example: { filemd5 : ObjectId(aaaaaaa) , key : { ts : 1 } }";
+            help << " example: { filemd5 : ObjectId(aaaaaaa) , root : \"fs\" }";
         }
-        virtual LockType locktype(){ return READ; } 
-        bool run(const char *dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
-            static DBDirectClient db;
-
-            string ns = nsToDatabase( dbname );
+        virtual LockType locktype() const { return READ; } 
+        bool run(const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
+            string ns = dbname;
             ns += ".";
             {
                 string root = jsobj.getStringField( "root" );
@@ -878,41 +861,71 @@ namespace mongo {
             }
             ns += ".chunks"; // make this an option in jsobj
 
-            BSONObjBuilder query;
-            query.appendAs( jsobj["filemd5"] , "files_id" );
-            Query q( query.obj() );
-            q.sort( BSON( "files_id" << 1 << "n" << 1 ) );
-
             md5digest d;
             md5_state_t st;
             md5_init(&st);
 
-            dbtemprelease temp;
+            BSONObj query = BSON( "files_id" << jsobj["filemd5"] );
+            BSONObj sort = BSON( "files_id" << 1 << "n" << 1 );
+
+            shared_ptr<Cursor> cursor = bestGuessCursor(ns.c_str(), query, sort);
+            scoped_ptr<ClientCursor> cc (new ClientCursor(QueryOption_NoCursorTimeout, cursor, ns.c_str()));
 
-            auto_ptr<DBClientCursor> cursor = db.query( ns.c_str() , q );
             int n = 0;
-            while ( cursor->more() ){
-                BSONObj c = cursor->next();
-                int myn = c.getIntField( "n" );
-                if ( n != myn ){
-                    log() << "should have chunk: " << n << " have:" << myn << endl;
-                    uassert( 10040 ,  "chunks out of order" , n == myn );
+            while ( cursor->ok() ){
+                if ( ! cursor->matcher()->matchesCurrent( cursor.get() ) ){
+                    log() << "**** NOT MATCHING ****" << endl;
+                    PRINT(cursor->current());
+                    cursor->advance();
+                    continue;
                 }
 
-                int len;
-                const char * data = c["data"].binData( len );
-                md5_append( &st , (const md5_byte_t*)(data + 4) , len - 4 );
+                BSONObj obj = cursor->current();
+                cursor->advance();
+
+                ClientCursor::YieldLock yield (cc);
+                try {
+
+                    BSONElement ne = obj["n"];
+                    assert(ne.isNumber());
+                    int myn = ne.numberInt();
+                    if ( n != myn ){
+                        log() << "should have chunk: " << n << " have:" << myn << endl;
+
+                        DBDirectClient client;
+                        Query q(query);
+                        q.sort(sort);
+                        auto_ptr<DBClientCursor> c = client.query(ns, q);
+                        while(c->more())
+                            PRINT(c->nextSafe());
+
+                        uassert( 10040 ,  "chunks out of order" , n == myn );
+                    }
+
+                    int len;
+                    const char * data = obj["data"].binDataClean( len );
+                    md5_append( &st , (const md5_byte_t*)(data) , len );
+
+                    n++;
+                } catch (...) {
+                    yield.relock(); // needed before yield goes out of scope
+                    throw;
+                }
 
-                n++;
+                if ( ! yield.stillOk() ){
+                    uasserted(13281, "File deleted during filemd5 command");
+                }
             }
+
             md5_finish(&st, d);
 
+            result.append( "numChunks" , n );
             result.append( "md5" , digestToString( d ) );
             return true;
         }
     } cmdFileMD5;
 
-    IndexDetails *cmdIndexDetailsForRange( const char *ns, string &errmsg, BSONObj &min, BSONObj &max, BSONObj &keyPattern ) {
+    static IndexDetails *cmdIndexDetailsForRange( const char *ns, string &errmsg, BSONObj &min, BSONObj &max, BSONObj &keyPattern ) {
         if ( ns[ 0 ] == '\0' || min.isEmpty() || max.isEmpty() ) {
             errmsg = "invalid command syntax (note: min and max are required)";
             return 0;
@@ -920,103 +933,71 @@ namespace mongo {
         return indexDetailsForRange( ns, errmsg, min, max, keyPattern );
     }
 
-    class CmdMedianKey : public Command {
-    public:
-        CmdMedianKey() : Command( "medianKey" ) {}
-        virtual bool slaveOk() { return true; }
-        virtual LockType locktype(){ return READ; } 
-        virtual void help( stringstream &help ) const {
-            help << " example: { medianKey:\"blog.posts\", keyPattern:{x:1}, min:{x:10}, max:{x:55} }\n"
-                "NOTE: This command may take awhile to run";
-        }
-        bool run(const char *dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
-            const char *ns = jsobj.getStringField( "medianKey" );
-            BSONObj min = jsobj.getObjectField( "min" );
-            BSONObj max = jsobj.getObjectField( "max" );
-            BSONObj keyPattern = jsobj.getObjectField( "keyPattern" );
-            
-            Client::Context ctx( ns );
-
-            IndexDetails *id = cmdIndexDetailsForRange( ns, errmsg, min, max, keyPattern );
-            if ( id == 0 )
-                return false;
-
-            Timer t;
-            int num = 0;
-            NamespaceDetails *d = nsdetails(ns);
-            int idxNo = d->idxNo(*id);
-            for( BtreeCursor c( d, idxNo, *id, min, max, false, 1 ); c.ok(); c.advance(), ++num );
-            num /= 2;
-            BtreeCursor c( d, idxNo, *id, min, max, false, 1 );
-            for( ; num; c.advance(), --num );
-            int ms = t.millis();
-            if ( ms > cmdLine.slowMS ) {
-                out() << "Finding median for index: " << keyPattern << " between " << min << " and " << max << " took " << ms << "ms." << endl;
-            }
-
-            if ( !c.ok() ) {
-                errmsg = "no index entries in the specified range";
-                return false;
-            }
-
-            result.append( "median", c.prettyKey( c.currKey() ) );
-            return true;
-        }
-    } cmdMedianKey;
-
     class CmdDatasize : public Command {
     public:
-        CmdDatasize() : Command( "datasize" ) {}
-        virtual bool slaveOk() { return true; }
-        virtual LockType locktype(){ return READ; } 
+        CmdDatasize() : Command( "dataSize", false, "datasize" ) {}
+        virtual bool slaveOk() const { return true; }
+        virtual LockType locktype() const { return READ; } 
         virtual void help( stringstream &help ) const {
             help <<
-                "\ndetermine data size for a set of data in a certain range"
+                "determine data size for a set of data in a certain range"
                 "\nexample: { datasize:\"blog.posts\", keyPattern:{x:1}, min:{x:10}, max:{x:55} }"
                 "\nkeyPattern, min, and max parameters are optional."
-                "\nnot: This command may take a while to run";
+                "\nnote: This command may take a while to run";
         }
-        bool run(const char *dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
-            const char *ns = jsobj.getStringField( "datasize" );
+        bool run(const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
+            string ns = jsobj.firstElement().String();
             BSONObj min = jsobj.getObjectField( "min" );
             BSONObj max = jsobj.getObjectField( "max" );
             BSONObj keyPattern = jsobj.getObjectField( "keyPattern" );
 
             Client::Context ctx( ns );
             
-            auto_ptr< Cursor > c;
+            shared_ptr<Cursor> c;
             if ( min.isEmpty() && max.isEmpty() ) {
-                c = theDataFileMgr.findAll( ns );
-            } else if ( min.isEmpty() || max.isEmpty() ) {
+                c = theDataFileMgr.findAll( ns.c_str() );
+            } 
+            else if ( min.isEmpty() || max.isEmpty() ) {
                 errmsg = "only one of min or max specified";
                 return false;
-            } else {
-                IndexDetails *idx = cmdIndexDetailsForRange( ns, errmsg, min, max, keyPattern );
+            } 
+            else {
+                IndexDetails *idx = cmdIndexDetailsForRange( ns.c_str(), errmsg, min, max, keyPattern );
                 if ( idx == 0 )
                     return false;
-                NamespaceDetails *d = nsdetails(ns);
+                NamespaceDetails *d = nsdetails(ns.c_str());
                 c.reset( new BtreeCursor( d, d->idxNo(*idx), *idx, min, max, false, 1 ) );
             }
+            
+            long long maxSize = jsobj["maxSize"].numberLong();
+            long long maxObjects = jsobj["maxObjects"].numberLong();
 
-            Timer t;
+            Timer timer;
             long long size = 0;
             long long numObjects = 0;
             while( c->ok() ) {
-                size += c->current().objsize();
-                c->advance();
+                size += c->currLoc().rec()->netLength();
                 numObjects++;
-            }
-            int ms = t.millis();
-            if ( ms > cmdLine.slowMS ) {
-                if ( min.isEmpty() ) {
-                    out() << "Finding size for ns: " << ns << " took " << ms << "ms." << endl;
-                } else {
-                    out() << "Finding size for ns: " << ns << " between " << min << " and " << max << " took " << ms << "ms." << endl;
+                
+                if ( ( maxSize && size > maxSize ) || 
+                     ( maxObjects && numObjects > maxObjects ) ){
+                    result.appendBool( "maxReached" , true );
+                    break;
                 }
+
+                c->advance();
             }
 
+            ostringstream os;
+            os <<  "Finding size for ns: " << ns;
+            if ( ! min.isEmpty() ){
+                os << " between " << min << " and " << max;
+            }
+            logIfSlow( timer , os.str() );
+
             result.append( "size", (double)size );
             result.append( "numObjects" , (double)numObjects );
+            result.append( "millis" , timer.millis() );
             return true;
         }
     } cmdDatasize;
@@ -1050,19 +1031,16 @@ namespace mongo {
 
     class CollectionStats : public Command {
     public:
-        CollectionStats() : Command( "collstats" ) {}
-        virtual bool slaveOk() { return true; }
-        virtual LockType locktype(){ return READ; } 
+        CollectionStats() : Command( "collStats", false, "collstats" ) {}
+        virtual bool slaveOk() const { return true; }
+        virtual LockType locktype() const { return READ; } 
         virtual void help( stringstream &help ) const {
-            help << " example: { collstats:\"blog.posts\" } ";
+            help << "{ collStats:\"blog.posts\" , scale : 1 } scale divides sizes e.g. for KB use 1024";
         }
-        bool run(const char *dbname_c, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
-            string dbname = dbname_c;
-            if ( dbname.find( "." ) != string::npos )
-                dbname = dbname.substr( 0 , dbname.find( "." ) );
-            
+        bool run(const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
             string ns = dbname + "." + jsobj.firstElement().valuestr();
-
+            Client::Context cx( ns );
+            
             NamespaceDetails * nsd = nsdetails( ns.c_str() );
             if ( ! nsd ){
                 errmsg = "ns not found";
@@ -1072,11 +1050,23 @@ namespace mongo {
             result.append( "ns" , ns.c_str() );
             
             int scale = 1;
-            if ( jsobj["scale"].isNumber() )
+            if ( jsobj["scale"].isNumber() ){
                 scale = jsobj["scale"].numberInt();
+                if ( scale <= 0 ){
+                    errmsg = "scale has to be > 0";
+                    return false;
+                }
+                    
+            }
+            else if ( jsobj["scale"].trueValue() ){
+                errmsg = "scale has to be a number > 0";
+                return false;
+            }
 
+            long long size = nsd->datasize / scale;
             result.appendNumber( "count" , nsd->nrecords );
-            result.appendNumber( "size" , nsd->datasize / scale );
+            result.appendNumber( "size" , size );
+            result.append      ( "avgObjSize" , double(size) / double(nsd->nrecords) );
             int numExtents;
             result.appendNumber( "storageSize" , nsd->storageSize( &numExtents ) / scale );
             result.append( "numExtents" , numExtents );
@@ -1098,22 +1088,19 @@ namespace mongo {
         }
     } cmdCollectionStatis;
 
-
     class DBStats : public Command {
     public:
-        DBStats() : Command( "dbstats" ) {}
-        virtual bool slaveOk() { return true; }
-        virtual LockType locktype(){ return READ; } 
+        DBStats() : Command( "dbStats", false, "dbstats" ) {}
+        virtual bool slaveOk() const { return true; }
+        virtual LockType locktype() const { return READ; } 
         virtual void help( stringstream &help ) const {
             help << " example: { dbstats:1 } ";
         }
-        bool run(const char *dbname_c, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
-            string dbname = dbname_c;
-            if ( dbname.find( "." ) != string::npos )
-                dbname = dbname.substr( 0 , dbname.find( "." ) );
-
-            DBDirectClient client;
-            const list<string> collections = client.getCollectionNames(dbname);
+        bool run(const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
+            list<string> collections;
+            Database* d = cc().database();
+            if ( d )
+                d->namespaceIndex.getNamespaces( collections );
 
             long long ncollections = 0;
             long long objects = 0;
@@ -1128,8 +1115,9 @@ namespace mongo {
 
                 NamespaceDetails * nsd = nsdetails( ns.c_str() );
                 if ( ! nsd ){
-                    // should this assert here?
-                    continue;
+                    errmsg = "missing ns: ";
+                    errmsg += ns;
+                    return false;
                 }
 
                 ncollections += 1;
@@ -1146,42 +1134,28 @@ namespace mongo {
 
             result.appendNumber( "collections" , ncollections );
             result.appendNumber( "objects" , objects );
+            result.append      ( "avgObjSize" , double(size) / double(objects) );
             result.appendNumber( "dataSize" , size );
             result.appendNumber( "storageSize" , storageSize);
             result.appendNumber( "numExtents" , numExtents );
             result.appendNumber( "indexes" , indexes );
             result.appendNumber( "indexSize" , indexSize );
+            result.appendNumber( "fileSize" , d->fileSize() );
 
                 return true;
         }
     } cmdDBStats;
 
-    class CmdBuildInfo : public Command {
-    public:
-        CmdBuildInfo() : Command( "buildinfo" ) {}
-        virtual bool slaveOk() { return true; }
-        virtual bool adminOnly() { return true; }
-        virtual LockType locktype(){ return NONE; } 
-        virtual void help( stringstream &help ) const {
-            help << "example: { buildinfo:1 }";
-        }
-        bool run(const char *dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
-            result << "version" << versionString << "gitVersion" << gitVersion() << "sysInfo" << sysInfo();
-            result << "bits" << ( sizeof( int* ) == 4 ? 32 : 64 );
-            return true;
-        }
-    } cmdBuildInfo;
-
     /* convertToCapped seems to use this */
     class CmdCloneCollectionAsCapped : public Command {
     public:
         CmdCloneCollectionAsCapped() : Command( "cloneCollectionAsCapped" ) {}
-        virtual bool slaveOk() { return false; }
-        virtual LockType locktype(){ return WRITE; } 
+        virtual bool slaveOk() const { return false; }
+        virtual LockType locktype() const { return WRITE; } 
         virtual void help( stringstream &help ) const {
-            help << "example: { cloneCollectionAsCapped:<fromName>, toCollection:<toName>, size:<sizeInBytes> }";
+            help << "{ cloneCollectionAsCapped:<fromName>, toCollection:<toName>, size:<sizeInBytes> }";
         }
-        bool run(const char *dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
+        bool run(const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
             string from = jsobj.getStringField( "cloneCollectionAsCapped" );
             string to = jsobj.getStringField( "toCollection" );
             long long size = (long long)jsobj.getField( "size" ).number();
@@ -1191,11 +1165,8 @@ namespace mongo {
                 return false;
             }
 
-            char realDbName[256];
-            nsToDatabase( dbname, realDbName );
-
-            string fromNs = string( realDbName ) + "." + from;
-            string toNs = string( realDbName ) + "." + to;
+            string fromNs = dbname + "." + from;
+            string toNs = dbname + "." + to;
             NamespaceDetails *nsd = nsdetails( fromNs.c_str() );
             massert( 10301 ,  "source collection " + fromNs + " does not exist", nsd );
             long long excessSize = nsd->datasize - size * 2; // datasize and extentSize can't be compared exactly, so add some padding to 'size'
@@ -1209,9 +1180,8 @@ namespace mongo {
 
             CursorId id;
             {
-                auto_ptr< Cursor > c = theDataFileMgr.findAll( fromNs.c_str(), startLoc );
-                ClientCursor *cc = new ClientCursor(c, fromNs.c_str(), true);
-                cc->matcher.reset( new CoveredIndexMatcher( BSONObj(), fromjson( "{$natural:1}" ) ) );
+                shared_ptr<Cursor> c = theDataFileMgr.findAll( fromNs.c_str(), startLoc );
+                ClientCursor *cc = new ClientCursor(0, c, fromNs.c_str());
                 id = cc->cursorid;
             }
 
@@ -1241,13 +1211,13 @@ namespace mongo {
     class CmdConvertToCapped : public Command {
     public:
         CmdConvertToCapped() : Command( "convertToCapped" ) {}
-        virtual bool slaveOk() { return false; }
-        virtual LockType locktype(){ return WRITE; } 
+        virtual bool slaveOk() const { return false; }
+        virtual LockType locktype() const { return WRITE; } 
         virtual void help( stringstream &help ) const {
-            help << "example: { convertToCapped:<fromCollectionName>, size:<sizeInBytes> }";
+            help << "{ convertToCapped:<fromCollectionName>, size:<sizeInBytes> }";
         }
-        bool run(const char *dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
-            BackgroundOperation::assertNoBgOpInProgForDb(dbname);
+        bool run(const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
+            BackgroundOperation::assertNoBgOpInProgForDb(dbname.c_str());
 
             string from = jsobj.getStringField( "convertToCapped" );
             long long size = (long long)jsobj.getField( "size" ).number();
@@ -1257,29 +1227,27 @@ namespace mongo {
                 return false;
             }
 
-            char realDbName[256];
-            nsToDatabase( dbname, realDbName );
-
             DBDirectClient client;
-            client.dropCollection( string( realDbName ) + "." + from + ".$temp_convertToCapped" );
+            client.dropCollection( dbname + "." + from + ".$temp_convertToCapped" );
 
             BSONObj info;
-            if ( !client.runCommand( realDbName,
+            if ( !client.runCommand( dbname ,
                                     BSON( "cloneCollectionAsCapped" << from << "toCollection" << ( from + ".$temp_convertToCapped" ) << "size" << double( size ) ),
                                     info ) ) {
-                errmsg = "cloneCollectionAsCapped failed: " + string(info);
+                errmsg = "cloneCollectionAsCapped failed: " + info.toString();
                 return false;
             }
 
-            if ( !client.dropCollection( string( realDbName ) + "." + from ) ) {
+            if ( !client.dropCollection( dbname + "." + from ) ) {
                 errmsg = "failed to drop original collection";
                 return false;
             }
 
             if ( !client.runCommand( "admin",
-                                    BSON( "renameCollection" << ( string( realDbName ) + "." + from + ".$temp_convertToCapped" ) << "to" << ( string( realDbName ) + "." + from ) ),
+                                    BSON( "renameCollection" << ( dbname + "." + from + ".$temp_convertToCapped" ) 
+                                          << "to" << ( dbname + "." + from ) ),
                                     info ) ) {
-                errmsg = "renameCollection failed: " + string(info);
+                errmsg = "renameCollection failed: " + info.toString();
                 return false;
             }
 
@@ -1290,10 +1258,11 @@ namespace mongo {
     class GroupCommand : public Command {
     public:
         GroupCommand() : Command("group"){}
-        virtual LockType locktype(){ return READ; } 
-        virtual bool slaveOk() { return true; }
+        virtual LockType locktype() const { return READ; } 
+        virtual bool slaveOk() const { return true; }
+        virtual bool slaveOverrideOk() { return true; }
         virtual void help( stringstream &help ) const {
-            help << "see http://www.mongodb.org/display/DOCS/Aggregation";
+            help << "http://www.mongodb.org/display/DOCS/Aggregation";
         }
 
         BSONObj getKey( const BSONObj& obj , const BSONObj& keyPattern , ScriptingFunction func , double avgSize , Scope * s ){
@@ -1309,7 +1278,7 @@ namespace mongo {
             return obj.extractFields( keyPattern , true );
         }
 
-        bool group( string realdbname , auto_ptr<DBClientCursor> cursor ,
+        bool group( string realdbname , const string& ns , const BSONObj& query , 
                     BSONObj keyPattern , string keyFunctionCode , string reduceCode , const char * reduceScope ,
                     BSONObj initial , string finalize ,
                     string& errmsg , BSONObjBuilder& result ){
@@ -1349,8 +1318,17 @@ namespace mongo {
             map<BSONObj,int,BSONObjCmp> map;
             list<BSONObj> blah;
 
-            while ( cursor->more() ){
-                BSONObj obj = cursor->next();
+            shared_ptr<Cursor> cursor = bestGuessCursor(ns.c_str() , query , BSONObj() );
+
+            while ( cursor->ok() ){
+                if ( cursor->matcher() && ! cursor->matcher()->matchesCurrent( cursor.get() ) ){
+                    cursor->advance();
+                    continue;
+                }
+
+                BSONObj obj = cursor->current();
+                cursor->advance();
+
                 BSONObj key = getKey( obj , keyPattern , keyFunction , keysize / keynum , s.get() );
                 keysize += key.objsize();
                 keynum++;
@@ -1392,8 +1370,7 @@ namespace mongo {
             return true;
         }
 
-        bool run(const char *dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
-            static DBDirectClient db;
+        bool run(const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
 
             /* db.$cmd.findOne( { group : <p> } ) */
             const BSONObj& p = jsobj.firstElement().embeddedObjectUserCheck();
@@ -1406,18 +1383,12 @@ namespace mongo {
             else 
                 q = getQuery( p );
 
-            string ns = dbname;
-            ns = ns.substr( 0 , ns.size() - 4 );
-            string realdbname = ns.substr( 0 , ns.size() - 1 );
-
             if ( p["ns"].type() != String ){
                 errmsg = "ns has to be set";
                 return false;
             }
-
-            ns += p["ns"].valuestr();
-
-            auto_ptr<DBClientCursor> cursor = db.query( ns , q );
+            
+            string ns = dbname + "." + p["ns"].String();
 
             BSONObj key;
             string keyf;
@@ -1429,7 +1400,7 @@ namespace mongo {
                 }
             }
             else if ( p["$keyf"].type() ){
-                keyf = p["$keyf"].ascode();
+                keyf = p["$keyf"]._asCode();
             }
             else {
                 // no key specified, will use entire object as key
@@ -1450,10 +1421,10 @@ namespace mongo {
 
             string finalize;
             if (p["finalize"].type())
-                finalize = p["finalize"].ascode();
+                finalize = p["finalize"]._asCode();
 
-            return group( realdbname , cursor ,
-                          key , keyf , reduce.ascode() , reduce.type() != CodeWScope ? 0 : reduce.codeWScopeScopeData() ,
+            return group( dbname , ns , q ,
+                          key , keyf , reduce._asCode() , reduce.type() != CodeWScope ? 0 : reduce.codeWScopeScopeData() ,
                           initial.embeddedObject() , finalize ,
                           errmsg , result );
         }
@@ -1464,44 +1435,43 @@ namespace mongo {
     class DistinctCommand : public Command {
     public:
         DistinctCommand() : Command("distinct"){}
-        virtual bool slaveOk() { return true; }
-        virtual LockType locktype(){ return READ; } 
+        virtual bool slaveOk() const { return true; }
+        virtual LockType locktype() const { return READ; } 
         virtual void help( stringstream &help ) const {
-            help << "{ distinct : 'collection name' , key : 'a.b' }";
+            help << "{ distinct : 'collection name' , key : 'a.b' , query : {} }";
         }
 
-        bool run(const char *dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
-            static DBDirectClient db;
+        bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
+            string ns = dbname + '.' + cmdObj.firstElement().valuestr();
 
-            string ns = cc().database()->name + '.' + cmdObj.getField(name).valuestr();
             string key = cmdObj["key"].valuestrsafe();
-
             BSONObj keyPattern = BSON( key << 1 );
 
-            set<BSONObj,BSONObjCmp> map;
-
-            long long size = 0;
+            BSONObj query = getQuery( cmdObj );
+            
+            BSONElementSet values;
+            shared_ptr<Cursor> cursor = bestGuessCursor(ns.c_str() , query , BSONObj() );
 
-            auto_ptr<DBClientCursor> cursor = db.query( ns , getQuery( cmdObj ) , 0 , 0 , &keyPattern );
-            while ( cursor->more() ){
-                BSONObj o = cursor->next();
-                BSONObj value = o.extractFields( keyPattern );
-                if ( value.isEmpty() )
+            while ( cursor->ok() ){
+                if ( cursor->matcher() && ! cursor->matcher()->matchesCurrent( cursor.get() ) ){
+                    cursor->advance();
                     continue;
-                if ( map.insert( value ).second ){
-                    size += o.objsize() + 20;
-                    uassert( 10044 ,  "distinct too big, 4mb cap" , size < 4 * 1024 * 1024 );
                 }
+
+                BSONObj o = cursor->current();
+                cursor->advance();
+                
+                o.getFieldsDotted( key.c_str(), values );
             }
 
-            assert( size <= 0x7fffffff );
-            BSONObjBuilder b( (int) size );
-            int n=0;
-            for ( set<BSONObj,BSONObjCmp>::iterator i = map.begin() ; i != map.end(); i++ ){
-                b.appendAs( i->firstElement() , b.numStr( n++ ).c_str() );
+            BSONArrayBuilder b( result.subarrayStart( "values" ) );
+            for ( BSONElementSet::iterator i = values.begin() ; i != values.end(); i++ ){
+                b.append( *i );
             }
+            BSONObj arr = b.done();
 
-            result.appendArray( "values" , b.obj() );
+            uassert(10044,  "distinct too big, 4mb cap",
+                    (arr.objsize() + 1024) < (4 * 1024 * 1024));
 
             return true;
         }
@@ -1511,48 +1481,88 @@ namespace mongo {
     /* Find and Modify an object returning either the old (default) or new value*/
     class CmdFindAndModify : public Command {
     public:
-        /* {findandmodify: "collection", query: {processed:false}, update: {$set: {processed:true}}, new: true}
-         * {findandmodify: "collection", query: {processed:false}, remove: true, sort: {priority:-1}}
-         * 
-         * either update or remove is required, all other fields have default values
-         * output is in the "value" field
-         */
-        CmdFindAndModify() : Command("findandmodify") { }
+        virtual void help( stringstream &help ) const {
+            help << 
+                "{ findandmodify: \"collection\", query: {processed:false}, update: {$set: {processed:true}}, new: true}\n"
+                "{ findandmodify: \"collection\", query: {processed:false}, remove: true, sort: {priority:-1}}\n"
+                "Either update or remove is required, all other fields have default values.\n"
+                "Output is in the \"value\" field\n";
+        }
+
+        CmdFindAndModify() : Command("findAndModify", false, "findandmodify") { }
         virtual bool logTheOp() {
             return false; // the modification will be logged directly
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; } 
-        virtual bool run(const char *dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
+        virtual LockType locktype() const { return WRITE; } 
+        virtual bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
             static DBDirectClient db;
 
-            string ns = nsToDatabase(dbname) + '.' + cmdObj.firstElement().valuestr();
+            string ns = dbname + '.' + cmdObj.firstElement().valuestr();
 
-            Query q (cmdObj.getObjectField("query")); // defaults to {}
+            BSONObj origQuery = cmdObj.getObjectField("query"); // defaults to {}
+            Query q (origQuery);
             BSONElement sort = cmdObj["sort"];
             if (!sort.eoo())
                 q.sort(sort.embeddedObjectUserCheck());
 
-            BSONObj out = db.findOne(ns, q);
-            if (out.firstElement().eoo()){
-                errmsg = "No matching object found";
-                return false;
-            }
+            bool upsert = cmdObj["upsert"].trueValue();
 
-            q = QUERY( "_id" << out["_id"]);
+            BSONObj fieldsHolder (cmdObj.getObjectField("fields"));
+            const BSONObj* fields = (fieldsHolder.isEmpty() ? NULL : &fieldsHolder);
+
+            BSONObj out = db.findOne(ns, q, fields);
+            if (out.isEmpty()){
+                if (!upsert){
+                    errmsg = "No matching object found";
+                    return false;
+                }
 
-            if (cmdObj["remove"].trueValue()){
-                uassert(12515, "can't remove and update", cmdObj["update"].eoo());
-                db.remove(ns, q, 1);
-            } else {
                 BSONElement update = cmdObj["update"];
-                uassert(12516, "must specify remove or update", !update.eoo());
-                db.update(ns, q, update.embeddedObjectUserCheck());
+                uassert(13329, "upsert mode requires update field", !update.eoo());
+                uassert(13330, "upsert mode requires query field", !origQuery.isEmpty());
+                db.update(ns, origQuery, update.embeddedObjectUserCheck(), true);
+
+                if (cmdObj["new"].trueValue()){
+                    BSONObj gle = db.getLastErrorDetailed();
+
+                    BSONElement _id = gle["upserted"];
+                    if (_id.eoo())
+                        _id = origQuery["_id"];
+
+                    out = db.findOne(ns, QUERY("_id" << _id), fields);
+                }
+
+            } else {
 
-                if (cmdObj["new"].trueValue())
-                    out = db.findOne(ns, q);
+                Query idQuery = QUERY( "_id" << out["_id"]);
+
+                if (cmdObj["remove"].trueValue()){
+                    uassert(12515, "can't remove and update", cmdObj["update"].eoo());
+                    db.remove(ns, idQuery, 1);
+
+                } else { // update
+
+                    // need to include original query for $ positional operator
+                    BSONObjBuilder b;
+                    b.append(out["_id"]);
+                    BSONObjIterator it(origQuery);
+                    while (it.more()){
+                        BSONElement e = it.next();
+                        if (strcmp(e.fieldName(), "_id"))
+                            b.append(e);
+                    }
+                    q = Query(b.obj());
+
+                    BSONElement update = cmdObj["update"];
+                    uassert(12516, "must specify remove or update", !update.eoo());
+                    db.update(ns, q, update.embeddedObjectUserCheck());
+
+                    if (cmdObj["new"].trueValue())
+                        out = db.findOne(ns, idQuery, fields);
+                }
             }
 
             result.append("value", out);
@@ -1568,17 +1578,17 @@ namespace mongo {
         virtual bool logTheOp() {
             return false; // the modification will be logged directly
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        virtual LockType locktype(){ return NONE; } 
+        virtual LockType locktype() const { return NONE; } 
         virtual bool requiresAuth() {
             return false;
         }
         virtual void help( stringstream &help ) const {
             help << "{whatsmyuri:1}";
         }        
-        virtual bool run(const char *dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
+        virtual bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
             BSONObj info = cc().curop()->infoNoauth();
             result << "you" << info[ "client" ];
             return true;
@@ -1592,35 +1602,36 @@ namespace mongo {
         virtual bool logTheOp() {
             return true;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return false;
         }
-        virtual LockType locktype() { return WRITE; }
+        virtual LockType locktype() const { return WRITE; }
         virtual bool requiresAuth() {
             return true;
         }
         virtual void help( stringstream &help ) const {
-            help << "[for testing only]";
+            help << "internal. for testing only.";
         }        
-        virtual bool run(const char *dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
+        virtual bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
             string coll = cmdObj[ "godinsert" ].valuestrsafe();
             uassert( 13049, "godinsert must specify a collection", !coll.empty() );
-            string ns = nsToDatabase( dbname ) + "." + coll;
+            string ns = dbname + "." + coll;
             BSONObj obj = cmdObj[ "obj" ].embeddedObjectUserCheck();
-            DiskLoc loc = theDataFileMgr.insert( ns.c_str(), obj, true );
+            DiskLoc loc = theDataFileMgr.insertWithObjMod( ns.c_str(), obj, true );
             return true;
         }
     } cmdGodInsert;
 
     class DBHashCmd : public Command {
     public:
-        DBHashCmd() : Command( "dbhash" ){}
-        virtual bool slaveOk() { return true; }
-        virtual LockType locktype() { return READ; }
-        virtual bool run(const char * badns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool){
-            string dbname = nsToDatabase( badns );
-            
-            list<string> colls = _db.getCollectionNames( dbname );
+        DBHashCmd() : Command( "dbHash", false, "dbhash" ){}
+        virtual bool slaveOk() const { return true; }
+        virtual LockType locktype() const { return READ; }
+        virtual bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool){
+            list<string> colls;
+            Database* db = cc().database();
+            if ( db )
+                db->namespaceIndex.getNamespaces( colls );
             colls.sort();
             
             result.appendNumber( "numCollections" , (long long)colls.size() );
@@ -1634,9 +1645,22 @@ namespace mongo {
                 if ( c.find( ".system.profil" ) != string::npos )
                     continue;
                 
-                auto_ptr<Cursor> cursor;
+                shared_ptr<Cursor> cursor;
 
                 NamespaceDetails * nsd = nsdetails( c.c_str() );
+                
+                // debug SERVER-761
+                NamespaceDetails::IndexIterator ii = nsd->ii();
+                while( ii.more() ) {
+                    const IndexDetails &idx = ii.next();
+                    if ( !idx.head.isValid() || !idx.info.isValid() ) {
+                        log() << "invalid index for ns: " << c << " " << idx.head << " " << idx.info;
+                        if ( idx.info.isValid() )
+                            log() << " " << idx.info.obj();
+                        log() << endl;
+                    }
+                }
+                
                 int idNum = nsd->findIdIndex();
                 if ( idNum >= 0 ){
                     cursor.reset( new BtreeCursor( nsd , idNum , nsd->idx( idNum ) , BSONObj() , BSONObj() , false , 1 ) );
@@ -1682,8 +1706,91 @@ namespace mongo {
             return 1;
         }
 
-        DBDirectClient _db;
     } dbhashCmd;
+
+    /* for diagnostic / testing purposes. */
+    class CmdSleep : public Command { 
+    public:
+        virtual LockType locktype() const { return NONE; } 
+        virtual bool adminOnly() const { return true; }
+        virtual bool logTheOp() {
+            return false;
+        }
+        virtual bool slaveOk() const {
+            return true;
+        }
+        virtual void help( stringstream& help ) const {
+            help << "internal testing command.  Makes db block (in a read lock) for 100 seconds\n";
+            help << "w:true write lock";
+        }
+        CmdSleep() : Command("sleep") { }
+        bool run(const string& ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            if( cmdObj.getBoolField("w") ) { 
+                writelock lk("");
+                sleepsecs(100);
+            }
+            else {
+                readlock lk("");
+                sleepsecs(100);
+            }
+            return true;
+        }
+    } cmdSleep;
+
+    class AvailableQueryOptions : public Command {
+    public:
+        AvailableQueryOptions() : Command( "availablequeryoptions" ){}
+        virtual bool slaveOk() const { return true; }
+        virtual LockType locktype() const { return NONE; }
+        virtual bool requiresAuth() { return false; }
+        virtual bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool){
+            result << "options" << QueryOption_AllSupported;
+            return true;
+        }
+    } availableQueryOptionsCmd;    
+    
+    // just for testing
+    class CapTrunc : public Command {
+    public:
+        CapTrunc() : Command( "captrunc" ){}
+        virtual bool slaveOk() const { return false; }
+        virtual LockType locktype() const { return WRITE; }
+        virtual bool requiresAuth() { return true; }
+        virtual bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool){
+            string coll = cmdObj[ "captrunc" ].valuestrsafe();
+            uassert( 13416, "captrunc must specify a collection", !coll.empty() );
+            string ns = dbname + "." + coll;
+            int n = cmdObj.getIntField( "n" );
+            bool inc = cmdObj.getBoolField( "inc" );
+            NamespaceDetails *nsd = nsdetails( ns.c_str() );
+            ReverseCappedCursor c( nsd );
+            massert( 13417, "captrunc invalid collection", c.ok() );
+            for( int i = 0; i < n; ++i ) {
+                massert( 13418, "captrunc invalid n", c.advance() );
+            }
+            DiskLoc end = c.currLoc();
+            nsd->cappedTruncateAfter( ns.c_str(), end, inc );
+            return true;
+        }
+    } capTruncCmd;    
+    
+    // just for testing
+    class EmptyCapped : public Command {
+    public:
+        EmptyCapped() : Command( "emptycapped" ){}
+        virtual bool slaveOk() const { return false; }
+        virtual LockType locktype() const { return WRITE; }
+        virtual bool requiresAuth() { return true; }
+        virtual bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool){
+            string coll = cmdObj[ "emptycapped" ].valuestrsafe();
+            uassert( 13428, "emptycapped must specify a collection", !coll.empty() );
+            string ns = dbname + "." + coll;
+            NamespaceDetails *nsd = nsdetails( ns.c_str() );
+            massert( 13429, "emptycapped no such collection", nsd );
+            nsd->emptyCappedCollection( ns.c_str() );
+            return true;
+        }
+    } emptyCappedCmd;    
     
     /** 
      * this handles
@@ -1694,11 +1801,11 @@ namespace mongo {
     */
     bool execCommand( Command * c ,
                       Client& client , int queryOptions , 
-                      const char *ns, BSONObj& cmdObj , 
+                      const char *cmdns, BSONObj& cmdObj , 
                       BSONObjBuilder& result, 
                       bool fromRepl ){
         
-        string dbname = nsToDatabase( ns );
+        string dbname = nsToDatabase( cmdns );
         
         AuthenticationInfo *ai = client.getAuthenticationInfo();    
 
@@ -1711,7 +1818,7 @@ namespace mongo {
         
 
         if ( c->adminOnly() && ! fromRepl && dbname != "admin" ) {
-            result.append( "errmsg" ,  "access denied" );
+            result.append( "errmsg" ,  "access denied- use admin db" );
             log() << "command denied: " << cmdObj.toString() << endl;
             return false;
         }        
@@ -1735,11 +1842,14 @@ namespace mongo {
             result.append( "errmsg" , "not master" );
             return false;
         }
+
+        if ( c->adminOnly() )
+            log( 2 ) << "command: " << cmdObj << endl;
         
         if ( c->locktype() == Command::NONE ){
             // we also trust that this won't crash
             string errmsg;
-            int ok = c->run( ns , cmdObj , errmsg , result , fromRepl );
+            int ok = c->run( dbname , cmdObj , errmsg , result , fromRepl );
             if ( ! ok )
                 result.append( "errmsg" , errmsg );
             return ok;
@@ -1747,40 +1857,30 @@ namespace mongo {
      
         bool needWriteLock = c->locktype() == Command::WRITE;
         
-        if ( ! c->requiresAuth() && 
-             ( ai->isAuthorizedReads( dbname ) && 
-               ! ai->isAuthorized( dbname ) ) ){
-            // this means that they can read, but not write
-            // so only get a read lock
-            needWriteLock = false;
-        }
-        
         if ( ! needWriteLock ){
             assert( ! c->logTheOp() );
         }
 
         mongolock lk( needWriteLock );
-        Client::Context ctx( ns , dbpath , &lk , c->requiresAuth() );
-        
-        if ( c->adminOnly() )
-            log( 2 ) << "command: " << cmdObj << endl;
+        Client::Context ctx( dbname , dbpath , &lk , c->requiresAuth() );
         
         try {
             string errmsg;
-            if ( ! c->run(ns, cmdObj, errmsg, result, fromRepl ) ){
+            if ( ! c->run(dbname, cmdObj, errmsg, result, fromRepl ) ){
                 result.append( "errmsg" , errmsg );
                 return false;
             }
         }
-        catch ( AssertionException& e ){
+        catch ( DBException& e ){
             stringstream ss;
-            ss << "assertion: " << e.what();
+            ss << "exception: " << e.what();
             result.append( "errmsg" , ss.str() );
+            result.append( "code" , e.getCode() );
             return false;
         }
         
         if ( c->logTheOp() && ! fromRepl ){
-            logOp("c", ns, cmdObj);
+            logOp("c", cmdns, cmdObj);
         }
         
         return true;
@@ -1795,6 +1895,7 @@ namespace mongo {
        returns true if ran a cmd
     */
     bool _runCommands(const char *ns, BSONObj& _cmdobj, BufBuilder &b, BSONObjBuilder& anObjBuilder, bool fromRepl, int queryOptions) {
+        cc().curop()->ensureStarted();
         string dbname = nsToDatabase( ns );
 
         if( logLevel >= 1 ) 
@@ -1821,6 +1922,7 @@ namespace mongo {
         BSONElement e = jsobj.firstElement();
         
         Command * c = e.type() ? Command::findCommand( e.fieldName() ) : 0;
+
         if ( c ){
             ok = execCommand( c , client , queryOptions , ns , jsobj , anObjBuilder , fromRepl );
         }
@@ -1828,10 +1930,14 @@ namespace mongo {
             anObjBuilder.append("errmsg", "no such cmd");
             anObjBuilder.append("bad cmd" , _cmdobj );
         }
+
+        // switch to bool, but wait a bit longer before switching?
+        // anObjBuilder.append("ok", ok);
         anObjBuilder.append("ok", ok?1.0:0.0);
         BSONObj x = anObjBuilder.done();
-        b.append((void*) x.objdata(), x.objsize());
+        b.appendBuf((void*) x.objdata(), x.objsize());
+
         return true;
     }
-
+    
 } // namespace mongo
diff --git a/db/dbcommands_admin.cpp b/db/dbcommands_admin.cpp
index 7265002..2d08ac8 100644
--- a/db/dbcommands_admin.cpp
+++ b/db/dbcommands_admin.cpp
@@ -22,7 +22,7 @@
 */
 
 
-#include "stdafx.h"
+#include "pch.h"
 #include "jsobj.h"
 #include "pdfile.h"
 #include "namespace.h"
@@ -35,37 +35,20 @@
 
 namespace mongo {
 
-    class FeaturesCmd : public Command {
-    public:
-        FeaturesCmd() : Command( "features" ){}
-
-        virtual bool slaveOk(){ return true; }
-        virtual bool readOnly(){ return true; }
-        virtual LockType locktype(){ return READ; } 
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl){
-            result.append( "readlock" , readLockSupported() );
-            if ( globalScriptEngine ){
-                BSONObjBuilder bb( result.subobjStart( "js" ) );
-                result.append( "utf8" , globalScriptEngine->utf8Ok() );
-                bb.done();
-            }
-            return true;
-        }
-        
-    } featuresCmd;
-
     class CleanCmd : public Command {
     public:
         CleanCmd() : Command( "clean" ){}
 
-        virtual bool slaveOk(){ return true; }
-        virtual LockType locktype(){ return WRITE; } 
+        virtual bool slaveOk() const { return true; }
+        virtual LockType locktype() const { return WRITE; } 
         
-        bool run(const char *nsRaw, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
-            string dropns = cc().database()->name + "." + cmdObj.firstElement().valuestrsafe();
+        virtual void help(stringstream& h) const { h << "internal"; }
+
+        bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
+            string dropns = dbname + "." + cmdObj.firstElement().valuestrsafe();
             
             if ( !cmdLine.quiet )
-                log() << "CMD: clean " << dropns << endl;
+                tlog() << "CMD: clean " << dropns << endl;
             
             NamespaceDetails *d = nsdetails(dropns.c_str());
             
@@ -87,18 +70,20 @@ namespace mongo {
     public:
         ValidateCmd() : Command( "validate" ){}
 
-        virtual bool slaveOk(){
+        virtual bool slaveOk() const {
             return true;
         }
         
-        virtual LockType locktype(){ return WRITE; } 
+        virtual void help(stringstream& h) const { h << "Validate contents of a namespace by scanning its data structures for correctness.  Slow."; }
+
+        virtual LockType locktype() const { return READ; } 
         //{ validate: "collectionnamewithoutthedbpart" [, scandata: <bool>] } */
         
-        bool run(const char *nsRaw, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
-            string ns = cc().database()->name + "." + cmdObj.firstElement().valuestrsafe();
+        bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
+            string ns = dbname + "." + cmdObj.firstElement().valuestrsafe();
             NamespaceDetails * d = nsdetails( ns.c_str() );
             if ( !cmdLine.quiet )
-                log() << "CMD: validate " << ns << endl;
+                tlog() << "CMD: validate " << ns << endl;
 
             if ( ! d ){
                 errmsg = "ns not found";
@@ -118,7 +103,7 @@ namespace mongo {
             bool valid = true;
             stringstream ss;
             ss << "\nvalidate\n";
-            ss << "  details: " << hex << d << " ofs:" << nsindex(ns)->detailsOffset(d) << dec << endl;
+            //ss << "  details: " << hex << d << " ofs:" << nsindex(ns)->detailsOffset(d) << dec << endl;
             if ( d->capped )
                 ss << "  capped:" << d->capped << " max:" << d->max << '\n';
             
@@ -158,7 +143,7 @@ namespace mongo {
 
                 set<DiskLoc> recs;
                 if( scanData ) {
-                    auto_ptr<Cursor> c = theDataFileMgr.findAll(ns);
+                    shared_ptr<Cursor> c = theDataFileMgr.findAll(ns);
                     int n = 0;
                     long long len = 0;
                     long long nlen = 0;
@@ -190,7 +175,7 @@ namespace mongo {
                         else ss << " (OK)";
                         ss << '\n';
                     }
-                    ss << "  " << n << " objects found, nobj:" << d->nrecords << "\n";
+                    ss << "  " << n << " objects found, nobj:" << d->nrecords << '\n';
                     ss << "  " << len << " bytes data w/headers\n";
                     ss << "  " << nlen << " bytes data wout/headers\n";
                 }
@@ -281,8 +266,8 @@ namespace mongo {
     public:
         UnlockCommand() : Command( "unlock" ) { }
         virtual bool readOnly() { return true; }
-        virtual bool slaveOk(){ return true; }
-        virtual bool adminOnly(){ return true; }
+        virtual bool slaveOk() const { return true; }
+        virtual bool adminOnly() const { return true; }
         virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             if( lockedForWriting ) { 
 				log() << "command: unlock requested" << endl;
@@ -304,6 +289,7 @@ namespace mongo {
     class FSyncCommand : public Command {
         class LockDBJob : public BackgroundJob { 
         protected:
+            string name() { return "lockdbjob"; }
             void run() { 
                 Client::initThread("fsyncjob");
                 Client& c = cc();
@@ -337,14 +323,15 @@ namespace mongo {
         };
     public:
         FSyncCommand() : Command( "fsync" ){}
-        virtual LockType locktype(){ return WRITE; } 
-        virtual bool slaveOk(){ return true; }
-        virtual bool adminOnly(){ return true; }
+        virtual LockType locktype() const { return WRITE; } 
+        virtual bool slaveOk() const { return true; }
+        virtual bool adminOnly() const { return true; }
         /*virtual bool localHostOnlyIfNoAuth(const BSONObj& cmdObj) { 
             string x = cmdObj["exec"].valuestrsafe();
             return !x.empty();
         }*/
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        virtual void help(stringstream& h) const { h << "http://www.mongodb.org/display/DOCS/fsync+Command"; }
+        virtual bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             /* async means do an fsync, but return immediately */
             bool sync = ! cmdObj["async"].trueValue();
             bool lock = cmdObj["lock"].trueValue();
@@ -376,18 +363,8 @@ namespace mongo {
         }
         
     } fsyncCmd;
+    
+
 
-    class LogRotateCmd : public Command {
-    public:
-        LogRotateCmd() : Command( "logRotate" ){}
-        virtual LockType locktype(){ return NONE; } 
-        virtual bool slaveOk(){ return true; }
-        virtual bool adminOnly(){ return true; }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            rotateLogs();
-            return 1;
-        }        
-        
-    } logRotateCmd;
 }
 
diff --git a/db/dbcommands_generic.cpp b/db/dbcommands_generic.cpp
new file mode 100644
index 0000000..6274394
--- /dev/null
+++ b/db/dbcommands_generic.cpp
@@ -0,0 +1,222 @@
+// dbcommands_generic.cpp
+
+/**
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/**
+ * commands suited for any mongo server
+ */
+
+#include "pch.h"
+#include "query.h"
+#include "pdfile.h"
+#include "jsobj.h"
+#include "../bson/util/builder.h"
+#include <time.h>
+#include "introspect.h"
+#include "btree.h"
+#include "../util/lruishmap.h"
+#include "../util/md5.hpp"
+#include "../util/processinfo.h"
+#include "json.h"
+#include "repl.h"
+#include "repl_block.h"
+#include "replpair.h"
+#include "commands.h"
+#include "db.h"
+#include "instance.h"
+#include "lasterror.h"
+#include "security.h"
+#include "queryoptimizer.h"
+#include "../scripting/engine.h"
+#include "stats/counters.h"
+#include "background.h"
+#include "../util/version.h"
+
+namespace mongo {
+
+    class CmdBuildInfo : public Command {
+    public:
+        CmdBuildInfo() : Command( "buildInfo", true, "buildinfo" ) {}
+        virtual bool slaveOk() const { return true; }
+        virtual bool adminOnly() const { return true; }
+        virtual LockType locktype() const { return NONE; } 
+        virtual void help( stringstream &help ) const {
+            help << "get version #, etc.\n";
+            help << "{ buildinfo:1 }";
+        }
+        bool run(const string& dbname, BSONObj& jsobj, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
+            result << "version" << versionString << "gitVersion" << gitVersion() << "sysInfo" << sysInfo();
+            result << "bits" << ( sizeof( int* ) == 4 ? 32 : 64 );
+            return true;
+        }
+    } cmdBuildInfo;
+
+
+    /* just to check if the db has asserted */
+    class CmdAssertInfo : public Command {
+    public:
+        virtual bool slaveOk() const {
+            return true;
+        }
+        virtual void help( stringstream& help ) const {
+            help << "check if any asserts have occurred on the server";
+        }
+        virtual LockType locktype() const { return WRITE; } 
+        CmdAssertInfo() : Command("assertInfo",true,"assertinfo") {}
+        bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            result.appendBool("dbasserted", lastAssert[0].isSet() || lastAssert[1].isSet() || lastAssert[2].isSet());
+            result.appendBool("asserted", lastAssert[0].isSet() || lastAssert[1].isSet() || lastAssert[2].isSet() || lastAssert[3].isSet());
+            result.append("assert", lastAssert[AssertRegular].toString());
+            result.append("assertw", lastAssert[AssertW].toString());
+            result.append("assertmsg", lastAssert[AssertMsg].toString());
+            result.append("assertuser", lastAssert[AssertUser].toString());
+            return true;
+        }
+    } cmdAsserts;
+
+    class PingCommand : public Command {
+    public:
+        PingCommand() : Command( "ping" ){}
+        virtual bool slaveOk() const { return true; }
+        virtual void help( stringstream &help ) const { help << "a way to check that the server is alive. responds immediately even if server is in a db lock."; }
+        virtual LockType locktype() const { return NONE; }
+        virtual bool requiresAuth() { return false; }
+        virtual bool run(const string& badns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool){
+            // IMPORTANT: Don't put anything in here that might lock db - including authentication
+            return true;
+        }
+    } pingCmd;
+    
+    class FeaturesCmd : public Command {
+    public:
+        FeaturesCmd() : Command( "features", true ){}
+        void help(stringstream& h) const { h << "return on build level feature settings"; }
+        virtual bool slaveOk() const { return true; }
+        virtual bool readOnly(){ return true; }
+        virtual LockType locktype() const { return READ; } 
+        virtual bool run(const string& ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl){
+            if ( globalScriptEngine ){
+                BSONObjBuilder bb( result.subobjStart( "js" ) );
+                result.append( "utf8" , globalScriptEngine->utf8Ok() );
+                bb.done();
+            }
+            if ( cmdObj["oidReset"].trueValue() ){
+                result.append( "oidMachineOld" , OID::staticMachine() );
+                OID::newState();
+            }
+            result.append( "oidMachine" , OID::staticMachine() );
+            return true;
+        }
+        
+    } featuresCmd;
+
+    class LogRotateCmd : public Command {
+    public:
+        LogRotateCmd() : Command( "logRotate" ){}
+        virtual LockType locktype() const { return NONE; } 
+        virtual bool slaveOk() const { return true; }
+        virtual bool adminOnly() const { return true; }
+        virtual bool run(const string& ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            rotateLogs();
+            return 1;
+        }        
+        
+    } logRotateCmd;
+    
+    class ListCommandsCmd : public Command {
+    public:
+        virtual void help( stringstream &help ) const { help << "get a list of all db commands"; }
+        ListCommandsCmd() : Command( "listCommands", false ){}
+        virtual LockType locktype() const { return NONE; } 
+        virtual bool slaveOk() const { return true; }
+        virtual bool adminOnly() const { return false; }
+        virtual bool run(const string& ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            BSONObjBuilder b( result.subobjStart( "commands" ) );
+            for ( map<string,Command*>::iterator i=_commands->begin(); i!=_commands->end(); ++i ){
+                Command * c = i->second;
+
+                // don't show oldnames
+                if (i->first != c->name)
+                    continue;
+
+                BSONObjBuilder temp( b.subobjStart( c->name.c_str() ) );
+
+                {
+                    stringstream help;
+                    c->help( help );
+                    temp.append( "help" , help.str() );
+                }
+                temp.append( "lockType" , c->locktype() );
+                temp.append( "slaveOk" , c->slaveOk() );
+                temp.append( "adminOnly" , c->adminOnly() );
+                temp.done();
+            }
+            b.done();
+
+            return 1;
+        }        
+
+    } listCommandsCmd;
+    
+    class CmdShutdown : public Command {
+    public:
+        virtual bool requiresAuth() { return true; }
+        virtual bool adminOnly() const { return true; }
+        virtual bool localHostOnlyIfNoAuth(const BSONObj& cmdObj) { return true; }
+        virtual bool logTheOp() {
+            return false;
+        }
+        virtual bool slaveOk() const {
+            return true;
+        }
+        virtual LockType locktype() const { return WRITE; } 
+        virtual void help( stringstream& help ) const {
+            help << "shutdown the database.  must be ran against admin db and either (1) ran from localhost or (2) authenticated.\n";
+        }
+        CmdShutdown() : Command("shutdown") {}
+        bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            Client * c = currentClient.get();
+            if ( c )
+                c->shutdown();
+            log() << "terminating, shutdown command received" << endl;
+            dbexit( EXIT_CLEAN ); // this never returns
+            return true;
+        }
+    } cmdShutdown;
+
+    /* for testing purposes only */
+    class CmdForceError : public Command {
+    public:
+        virtual void help( stringstream& help ) const {
+            help << "for testing purposes only.  forces a user assertion exception";
+        }
+        virtual bool logTheOp() {
+            return false;
+        }
+        virtual bool slaveOk() const {
+            return true;
+        }
+        virtual LockType locktype() const { return NONE; } 
+        CmdForceError() : Command("forceerror") {}
+        bool run(const string& dbnamne, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            uassert( 10038 , "forced error", false);
+            return true;
+        }
+    } cmdForceError;
+
+    
+
+}
diff --git a/db/dbeval.cpp b/db/dbeval.cpp
index a3be894..e8a42b2 100644
--- a/db/dbeval.cpp
+++ b/db/dbeval.cpp
@@ -17,11 +17,11 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "query.h"
 #include "pdfile.h"
 #include "jsobj.h"
-#include "../util/builder.h"
+#include "../bson/util/builder.h"
 #include <time.h>
 #include "introspect.h"
 #include "btree.h"
@@ -108,20 +108,30 @@ namespace mongo {
 
     class CmdEval : public Command {
     public:
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return false;
         }
+        virtual void help( stringstream &help ) const {
+            help << "Evaluate javascript at the server.\n" "http://www.mongodb.org/display/DOCS/Server-side+Code+Execution";
+        }
         // We need at least read only access to run db.eval - auth for eval'd writes will be checked
         // as they are requested.
         virtual bool requiresAuth() {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; }
-        CmdEval() : Command("$eval") { }
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        virtual LockType locktype() const { return NONE; }
+        CmdEval() : Command("eval", false, "$eval") { }
+        bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            
             AuthenticationInfo *ai = cc().getAuthenticationInfo();
-            uassert( 12598 , "$eval reads unauthorized", ai->isAuthorizedReads(cc().database()->name.c_str()));
-            return dbEval(ns, cmdObj, result, errmsg);
+            uassert( 12598 , "$eval reads unauthorized", ai->isAuthorizedReads(dbname.c_str()) );
+            
+            // write security will be enforced in DBDirectClient
+            mongolock lk( ai->isAuthorized( dbname.c_str() ) );
+            Client::Context ctx( dbname );
+            
+
+            return dbEval(dbname.c_str(), cmdObj, result, errmsg);
         }
     } cmdeval;
 
diff --git a/db/dbhelpers.cpp b/db/dbhelpers.cpp
index ee221ab..124deb8 100644
--- a/db/dbhelpers.cpp
+++ b/db/dbhelpers.cpp
@@ -16,7 +16,7 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "db.h"
 #include "dbhelpers.h"
 #include "query.h"
@@ -24,10 +24,11 @@
 #include "queryoptimizer.h"
 #include "btree.h"
 #include "pdfile.h"
+#include "oplog.h"
 
 namespace mongo {
 
-    CursorIterator::CursorIterator( auto_ptr<Cursor> c , BSONObj filter )
+    CursorIterator::CursorIterator( shared_ptr<Cursor> c , BSONObj filter )
         : _cursor( c ){
             if ( ! filter.isEmpty() )
                 _matcher.reset( new CoveredIndexMatcher( filter , BSONObj() ) );
@@ -93,51 +94,65 @@ namespace mongo {
     class FindOne : public QueryOp {
     public:
         FindOne( bool requireIndex ) : requireIndex_( requireIndex ) {}
-        virtual void init() {
+        virtual void _init() {
             if ( requireIndex_ && strcmp( qp().indexKey().firstElement().fieldName(), "$natural" ) == 0 )
                 throw MsgAssertionException( 9011 , "Not an index cursor" );
             c_ = qp().newCursor();
-            if ( !c_->ok() )
+            if ( !c_->ok() ) {
                 setComplete();
-            else
-                matcher_.reset( new CoveredIndexMatcher( qp().query(), qp().indexKey() ) );
+            }
         }
         virtual void next() {
             if ( !c_->ok() ) {
                 setComplete();
                 return;
             }
-            if ( matcher_->matches( c_->currKey(), c_->currLoc() ) ) {
+            if ( matcher()->matches( c_->currKey(), c_->currLoc() ) ) {
                 one_ = c_->current();
-                setComplete();
+                loc_ = c_->currLoc();
+                setStop();
             } else {
                 c_->advance();
             }
         }
         virtual bool mayRecordPlan() const { return false; }
-        virtual QueryOp *clone() const { return new FindOne( requireIndex_ ); }
+        virtual QueryOp *_createChild() const { return new FindOne( requireIndex_ ); }
         BSONObj one() const { return one_; }
+        DiskLoc loc() const { return loc_; }
     private:
         bool requireIndex_;
-        auto_ptr< Cursor > c_;
-        auto_ptr< CoveredIndexMatcher > matcher_;
+        shared_ptr<Cursor> c_;
         BSONObj one_;
+        DiskLoc loc_;
     };
     
     /* fetch a single object from collection ns that matches query 
        set your db SavedContext first
     */
-    bool Helpers::findOne(const char *ns, BSONObj query, BSONObj& result, bool requireIndex) { 
-        QueryPlanSet s( ns, query, BSONObj(), 0, !requireIndex );
+    bool Helpers::findOne(const char *ns, const BSONObj &query, BSONObj& result, bool requireIndex) { 
+        MultiPlanScanner s( ns, query, BSONObj(), 0, !requireIndex );
         FindOne original( requireIndex );
         shared_ptr< FindOne > res = s.runOp( original );
-        massert( 10302 ,  res->exceptionMessage(), res->complete() );
+        if ( ! res->complete() )
+            throw MsgAssertionException( res->exception() );
         if ( res->one().isEmpty() )
             return false;
         result = res->one();
         return true;
     }
 
+    /* fetch a single object from collection ns that matches query 
+       set your db SavedContext first
+    */
+    DiskLoc Helpers::findOne(const char *ns, const BSONObj &query, bool requireIndex) { 
+        MultiPlanScanner s( ns, query, BSONObj(), 0, !requireIndex );
+        FindOne original( requireIndex );
+        shared_ptr< FindOne > res = s.runOp( original );
+        if ( ! res->complete() )
+            throw MsgAssertionException( res->exception() );
+        return res->loc();
+    }
+
     auto_ptr<CursorIterator> Helpers::find( const char *ns , BSONObj query , bool requireIndex ){
         uassert( 10047 ,  "requireIndex not supported in Helpers::find yet" , ! requireIndex );
         auto_ptr<CursorIterator> i;
@@ -145,9 +160,9 @@ namespace mongo {
         return i;
     }
     
-    
     bool Helpers::findById(Client& c, const char *ns, BSONObj query, BSONObj& result ,
                            bool * nsFound , bool * indexFound ){
+        dbMutex.assertAtLeastReadLocked();
         Database *database = c.database();
         assert( database );
         NamespaceDetails *d = database->namespaceIndex.details(ns);
@@ -173,6 +188,20 @@ namespace mongo {
         return true;
     }
 
+     DiskLoc Helpers::findById(NamespaceDetails *d, BSONObj idquery) {
+         int idxNo = d->findIdIndex();
+         uassert(13430, "no _id index", idxNo>=0);
+         IndexDetails& i = d->idx( idxNo );        
+         BSONObj key = i.getKeyFromQuery( idquery );
+         return i.head.btree()->findSingle( i , i.head , key );
+    }
+
+    bool Helpers::isEmpty(const char *ns) {
+        Client::Context context(ns);
+        shared_ptr<Cursor> c = DataFileMgr::findAll(ns);
+        return !c->ok();
+    }
+
     /* Get the first object from a collection.  Generally only useful if the collection
        only ever has a single object -- which is a "singleton collection.
 
@@ -181,7 +210,7 @@ namespace mongo {
     bool Helpers::getSingleton(const char *ns, BSONObj& result) {
         Client::Context context(ns);
 
-        auto_ptr<Cursor> c = DataFileMgr::findAll(ns);
+        shared_ptr<Cursor> c = DataFileMgr::findAll(ns);
         if ( !c->ok() )
             return false;
 
@@ -189,10 +218,92 @@ namespace mongo {
         return true;
     }
 
+    bool Helpers::getLast(const char *ns, BSONObj& result) {
+        Client::Context ctx(ns);
+        shared_ptr<Cursor> c = findTableScan(ns, reverseNaturalObj);
+        if( !c->ok() ) 
+            return false;
+        result = c->current();
+        return true;
+    }
+
+    void Helpers::upsert( const string& ns , const BSONObj& o ){
+        BSONElement e = o["_id"];
+        assert( e.type() );
+        BSONObj id = e.wrap();
+        
+        OpDebug debug;
+        Client::Context context(ns);
+        updateObjects(ns.c_str(), o, /*pattern=*/id, /*upsert=*/true, /*multi=*/false , /*logtheop=*/true , debug );
+    }
+
     void Helpers::putSingleton(const char *ns, BSONObj obj) {
         OpDebug debug;
         Client::Context context(ns);
-        updateObjects(ns, obj, /*pattern=*/BSONObj(), /*upsert=*/true, /*multi=*/false , true , debug );
+        updateObjects(ns, obj, /*pattern=*/BSONObj(), /*upsert=*/true, /*multi=*/false , /*logtheop=*/true , debug );
+    }
+
+    void Helpers::putSingletonGod(const char *ns, BSONObj obj, bool logTheOp) {
+        OpDebug debug;
+        Client::Context context(ns);
+        _updateObjects(/*god=*/true, ns, obj, /*pattern=*/BSONObj(), /*upsert=*/true, /*multi=*/false , logTheOp , debug );
+    }
+
+    BSONObj Helpers::toKeyFormat( const BSONObj& o , BSONObj& key ){
+        BSONObjBuilder me;
+        BSONObjBuilder k;
+
+        BSONObjIterator i( o );
+        while ( i.more() ){
+            BSONElement e = i.next();
+            k.append( e.fieldName() , 1 );
+            me.appendAs( e , "" );
+        }
+        key = k.obj();
+        return me.obj();
+    }
+    
+    long long Helpers::removeRange( const string& ns , const BSONObj& min , const BSONObj& max , bool yield , bool maxInclusive , RemoveCallback * callback ){
+        BSONObj keya , keyb;
+        BSONObj minClean = toKeyFormat( min , keya );
+        BSONObj maxClean = toKeyFormat( max , keyb );
+        assert( keya == keyb );
+
+        Client::Context ctx(ns);
+        NamespaceDetails* nsd = nsdetails( ns.c_str() );
+        if ( ! nsd )
+            return 0;
+
+        int ii = nsd->findIndexByKeyPattern( keya );
+        assert( ii >= 0 );
+        
+        long long num = 0;
+        
+        IndexDetails& i = nsd->idx( ii );
+
+        shared_ptr<Cursor> c( new BtreeCursor( nsd , ii , i , minClean , maxClean , maxInclusive, 1 ) );
+        auto_ptr<ClientCursor> cc( new ClientCursor( QueryOption_NoCursorTimeout , c , ns ) );
+        cc->setDoingDeletes( true );
+        
+        while ( c->ok() ){
+            DiskLoc rloc = c->currLoc();
+            BSONObj key = c->currKey();
+
+            if ( callback )
+                callback->goingToDelete( c->current() );
+            
+            c->advance();
+            c->noteLocation();
+            
+            logOp( "d" , ns.c_str() , rloc.obj()["_id"].wrap() );
+            theDataFileMgr.deleteRecord(ns.c_str() , rloc.rec(), rloc);
+            num++;
+
+            c->checkLocation();
+
+        }
+
+        return num;
     }
 
     void Helpers::emptyCollection(const char *ns) {
@@ -241,7 +352,7 @@ namespace mongo {
         if ( val ) {
             try {
                 BSONObj k = obj;
-                theDataFileMgr.insert( name_.c_str(), k, false );
+                theDataFileMgr.insertWithObjMod( name_.c_str(), k, false );
             } catch ( DBException& ) {
                 // dup key - already in set
             }
@@ -249,5 +360,48 @@ namespace mongo {
             deleteObjects( name_.c_str(), obj, true, false, false );
         }                        
     }
+
+    RemoveSaver::RemoveSaver( const string& a , const string& b , const string& why) : _out(0){
+        static int NUM = 0;
+        
+        _root = dbpath;
+        if ( a.size() )
+            _root /= a;
+        if ( b.size() )
+            _root /= b;
+        assert( a.size() || b.size() );
+        
+        _file = _root;
+        
+        stringstream ss;
+        ss << why << "." << terseCurrentTime(false) << "." << NUM++ << ".bson";
+        _file /= ss.str();
+
+    }
+    
+    RemoveSaver::~RemoveSaver(){
+        if ( _out ){
+            _out->close();
+            delete _out;
+            _out = 0;
+        }
+    }
+    
+    void RemoveSaver::goingToDelete( const BSONObj& o ){
+        if ( ! _out ){
+            create_directories( _root );
+            _out = new ofstream();
+            _out->open( _file.string().c_str() , ios_base::out | ios_base::binary );
+            if ( ! _out->good() ){
+                log( LL_WARNING ) << "couldn't create file: " << _file.string() << " for remove saving" << endl;
+                delete _out;
+                _out = 0;
+                return;
+            }
+            
+        }
+        _out->write( o.objdata() , o.objsize() );
+    }
+    
         
 } // namespace mongo
diff --git a/db/dbhelpers.h b/db/dbhelpers.h
index 3c223d8..ee9a59c 100644
--- a/db/dbhelpers.h
+++ b/db/dbhelpers.h
@@ -1,4 +1,8 @@
-// dbhelpers.h
+/* @file dbhelpers.h
+
+   db helpers are helper functions and classes that let us easily manipulate the local
+   database instance in-proc.
+*/
 
 /**
 *    Copyright (C) 2008 10gen Inc.
@@ -16,31 +20,29 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-/* db helpers are helper functions and classes that let us easily manipulate the local
-   database instance.
-*/
-
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "client.h"
 #include "db.h"
 
 namespace mongo {
 
+    const BSONObj reverseNaturalObj = BSON( "$natural" << -1 );
+
     class Cursor;
     class CoveredIndexMatcher;
 
     class CursorIterator {
     public:
-        CursorIterator( auto_ptr<Cursor> c , BSONObj filter = BSONObj() );
+        CursorIterator( shared_ptr<Cursor> c , BSONObj filter = BSONObj() );
         BSONObj next();
         bool hasNext();
 
     private:
         void _advance();
 
-        auto_ptr<Cursor> _cursor;
+        shared_ptr<Cursor> _cursor;
         auto_ptr<CoveredIndexMatcher> _matcher;
         BSONObj _o;
     };
@@ -66,13 +68,16 @@ namespace mongo {
         /* fetch a single object from collection ns that matches query.
            set your db SavedContext first.
 
+           @param query - the query to perform.  note this is the low level portion of query so "orderby : ..." 
+                          won't work.
+
            @param requireIndex if true, complain if no index for the query.  a way to guard against
            writing a slow query.
 
            @return true if object found
         */
-        static bool findOne(const char *ns, BSONObj query, BSONObj& result, bool requireIndex = false);
-        
+        static bool findOne(const char *ns, const BSONObj &query, BSONObj& result, bool requireIndex = false);
+        static DiskLoc findOne(const char *ns, const BSONObj &query, bool requireIndex);        
 
         /**
          * @param foundIndex if passed in will be set to 1 if ns and index found
@@ -81,18 +86,47 @@ namespace mongo {
         static bool findById(Client&, const char *ns, BSONObj query, BSONObj& result , 
                              bool * nsFound = 0 , bool * indexFound = 0 );
 
+        /* uasserts if no _id index. 
+           @return null loc if not found */
+        static DiskLoc findById(NamespaceDetails *d, BSONObj query);
+
         static auto_ptr<CursorIterator> find( const char *ns , BSONObj query = BSONObj() , bool requireIndex = false );
 
-        /* Get/put the first object from a collection.  Generally only useful if the collection
-           only ever has a single object -- which is a "singleton collection".
+        /** Get/put the first (or last) object from a collection.  Generally only useful if the collection
+            only ever has a single object -- which is a "singleton collection".
+
+            You do not need to set the database (Context) before calling.
 
-		   You do not need to set the database before calling.
-		   
-		   Returns: true if object exists.
+            @return true if object exists.
         */
         static bool getSingleton(const char *ns, BSONObj& result);
         static void putSingleton(const char *ns, BSONObj obj);
+        static void putSingletonGod(const char *ns, BSONObj obj, bool logTheOp);
+        static bool getFirst(const char *ns, BSONObj& result) { return getSingleton(ns, result); }
+        static bool getLast(const char *ns, BSONObj& result); // get last object int he collection; e.g. {$natural : -1}        
+
+        /**
+         * you have to lock
+         * you do not have to have Context set
+         * o has to have an _id field or will assert
+         */
+        static void upsert( const string& ns , const BSONObj& o );
+
+        /** You do not need to set the database before calling.
+            @return true if collection is empty.
+        */
+        static bool isEmpty(const char *ns);
+
+        // TODO: this should be somewhere else probably
+        static BSONObj toKeyFormat( const BSONObj& o , BSONObj& key );
 
+        class RemoveCallback {
+        public:
+            virtual ~RemoveCallback(){}
+            virtual void goingToDelete( const BSONObj& o ) = 0;
+        };
+        /* removeRange: operation is oplog'd */
+        static long long removeRange( const string& ns , const BSONObj& min , const BSONObj& max , bool yield = false , bool maxInclusive = false , RemoveCallback * callback = 0 );
 
         /* Remove all objects from a collection.
         You do not need to set the database before calling.
@@ -118,5 +152,24 @@ namespace mongo {
         string name_;
         BSONObj key_;
     };
+
+
+    /**
+     * user for saving deletd bson objects to a flat file
+     */
+    class RemoveSaver : public Helpers::RemoveCallback , boost::noncopyable {
+    public:
+        RemoveSaver( const string& type , const string& ns , const string& why);
+        ~RemoveSaver();
+
+        void goingToDelete( const BSONObj& o );
+        
+    private:
+        path _root;
+        path _file;
+        ofstream* _out;
+        
+    };
+
     
 } // namespace mongo
diff --git a/db/dbmessage.h b/db/dbmessage.h
index ba5cf94..2849de8 100644
--- a/db/dbmessage.h
+++ b/db/dbmessage.h
@@ -20,6 +20,7 @@
 #include "jsobj.h"
 #include "namespace.h"
 #include "../util/message.h"
+#include "../client/constants.h"
 
 namespace mongo {
 
@@ -37,24 +38,6 @@ namespace mongo {
     
 #pragma pack(1)
     struct QueryResult : public MsgData {
-        enum ResultFlagType {
-            /* returned, with zero results, when getMore is called but the cursor id 
-               is not valid at the server. */
-            ResultFlag_CursorNotFound = 1,   
-
-            /* { $err : ... } is being returned */
-            ResultFlag_ErrSet = 2,           
-
-            /* Have to update config from the server, usually $err is also set */
-            ResultFlag_ShardConfigStale = 4,  
-
-            /* for backward compatability: this let's us know the server supports 
-               the QueryOption_AwaitData option. if it doesn't, a repl slave client should sleep 
-               a little between getMore's.
-            */
-            ResultFlag_AwaitCapable = 8
-        };
-
         long long cursorId;
         int startingFrom;
         int nReturned;
@@ -68,41 +51,57 @@ namespace mongo {
             return dataAsInt();
         }
         void setResultFlagsToOk() { 
-            _resultFlags() = 0; // ResultFlag_AwaitCapable
+            _resultFlags() = ResultFlag_AwaitCapable;
         }
     };
 #pragma pack()
 
     /* For the database/server protocol, these objects and functions encapsulate
        the various messages transmitted over the connection.
-    */
 
+       See http://www.mongodb.org/display/DOCS/Mongo+Wire+Protocol
+    */
     class DbMessage {
     public:
-        DbMessage(const Message& _m) : m(_m) {
-            theEnd = _m.data->_data + _m.data->dataLen();
-            int *r = (int *) _m.data->_data;
-            reserved = *r;
-            r++;
-            data = (const char *) r;
+        DbMessage(const Message& _m) : m(_m)
+        {
+            // for received messages, Message has only one buffer
+            theEnd = _m.singleData()->_data + _m.header()->dataLen();
+            char *r = _m.singleData()->_data;
+            reserved = (int *) r;
+            data = r + 4;
             nextjsobj = data;
         }
 
-        const char * getns() {
+        /** the 32 bit field before the ns */
+        int& reservedField() { return *reserved; }
+
+        const char * getns() const {
             return data;
         }
-        void getns(Namespace& ns) {
+        void getns(Namespace& ns) const {
             ns = data;
         }
+
+        const char * afterNS() const {
+            return data + strlen( data ) + 1;
+        }
         
-        
-        void resetPull(){
-            nextjsobj = data;
+        int getInt( int num ) const {
+            const int * foo = (const int*)afterNS();
+            return foo[num];
+        }
+
+        int getQueryNToReturn() const {
+            return getInt( 1 );
         }
-        int pullInt() {
+
+        void resetPull(){ nextjsobj = data; }
+        int pullInt() const { return pullInt(); }
+        int& pullInt() {
             if ( nextjsobj == data )
                 nextjsobj += strlen(data) + 1; // skip namespace
-            int i = *((int *)nextjsobj);
+            int& i = *((int *)nextjsobj);
             nextjsobj += 4;
             return i;
         }
@@ -117,7 +116,7 @@ namespace mongo {
             return i;
         }
 
-        OID* getOID() {
+        OID* getOID() const {
             return (OID *) (data + strlen(data) + 1); // skip namespace
         }
 
@@ -129,7 +128,7 @@ namespace mongo {
         }
 
         /* for insert and update msgs */
-        bool moreJSObjs() {
+        bool moreJSObjs() const {
             return nextjsobj != 0;
         }
         BSONObj nextJsObj() {
@@ -137,13 +136,13 @@ namespace mongo {
                 nextjsobj += strlen(data) + 1; // skip namespace
                 massert( 13066 ,  "Message contains no documents", theEnd > nextjsobj );
             }
-            massert( 10304 ,  "Remaining data too small for BSON object", theEnd - nextjsobj > 3 );
+            massert( 10304 ,  "Client Error: Remaining data too small for BSON object", theEnd - nextjsobj > 3 );
             BSONObj js(nextjsobj);
-            massert( 10305 ,  "Invalid object size", js.objsize() > 3 );
-            massert( 10306 ,  "Next object larger than available space",
+            massert( 10305 ,  "Client Error: Invalid object size", js.objsize() > 3 );
+            massert( 10306 ,  "Client Error: Next object larger than space left in message",
                     js.objsize() < ( theEnd - data ) );
             if ( objcheck && !js.valid() ) {
-                massert( 10307 , "bad object in message", false);
+                massert( 10307 , "Client Error: bad object in message", false);
             }            
             nextjsobj += js.objsize();
             if ( nextjsobj >= theEnd )
@@ -151,9 +150,7 @@ namespace mongo {
             return js;
         }
 
-        const Message& msg() {
-            return m;
-        }
+        const Message& msg() const { return m; }
 
         void markSet(){
             mark = nextjsobj;
@@ -165,7 +162,7 @@ namespace mongo {
 
     private:
         const Message& m;
-        int reserved;
+        int* reserved;
         const char *data;
         const char *nextjsobj;
         const char *theEnd;
@@ -193,7 +190,7 @@ namespace mongo {
             if ( d.moreJSObjs() ) {
                 fields = d.nextJsObj();
             }
-            queryOptions = d.msg().data->dataAsInt();
+            queryOptions = d.msg().header()->dataAsInt();
         }
     };
 
@@ -211,7 +208,7 @@ namespace mongo {
                             ) {
         BufBuilder b(32768);
         b.skip(sizeof(QueryResult));
-        b.append(data, size);
+        b.appendBuf(data, size);
         QueryResult *qr = (QueryResult *) b.buf();
         qr->_resultFlags() = queryResultFlags;
         qr->len = b.len();
@@ -221,12 +218,13 @@ namespace mongo {
         qr->nReturned = nReturned;
         b.decouple();
         Message resp(qr, true);
-        p->reply(requestMsg, resp, requestMsg.data->id);
+        p->reply(requestMsg, resp, requestMsg.header()->id);
     }
 
 } // namespace mongo
 
 //#include "bsonobj.h"
+
 #include "instance.h"
 
 namespace mongo {
@@ -245,7 +243,7 @@ namespace mongo {
     inline void replyToQuery(int queryResultFlags, Message &m, DbResponse &dbresponse, BSONObj obj) {
         BufBuilder b;
         b.skip(sizeof(QueryResult));
-        b.append((void*) obj.objdata(), obj.objsize());
+        b.appendBuf((void*) obj.objdata(), obj.objsize());
         QueryResult* msgdata = (QueryResult *) b.buf();
         b.decouple();
         QueryResult *qr = msgdata;
@@ -258,7 +256,9 @@ namespace mongo {
         Message *resp = new Message();
         resp->setData(msgdata, true); // transport will free
         dbresponse.response = resp;
-        dbresponse.responseTo = m.data->id;
+        dbresponse.responseTo = m.header()->id;
     }
 
+    string debugString( Message& m );
+
 } // namespace mongo
diff --git a/db/dbwebserver.cpp b/db/dbwebserver.cpp
index c55c8a6..7e45d8f 100644
--- a/db/dbwebserver.cpp
+++ b/db/dbwebserver.cpp
@@ -19,45 +19,31 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "../util/miniwebserver.h"
+#include "../util/mongoutils/html.h"
 #include "../util/md5.hpp"
 #include "db.h"
-#include "repl.h"
-#include "replset.h"
 #include "instance.h"
 #include "security.h"
 #include "stats/snapshots.h"
 #include "background.h"
 #include "commands.h"
-
+#include "../util/version.h"
+#include "../util/ramlog.h"
 #include <pcrecpp.h>
+#include "dbwebserver.h"
 #include <boost/date_time/posix_time/posix_time.hpp>
 #undef assert
-#define assert xassert
+#define assert MONGO_assert
 
 namespace mongo {
 
-    extern string bind_ip;
-    extern const char *replInfo;
-
-    bool getInitialSyncCompleted();
+    using namespace mongoutils::html;
+    using namespace bson;
 
     time_t started = time(0);
 
-    /*
-        string toString() {
-            stringstream ss;
-            unsigned long long dt = last - start;
-            ss << dt/1000;
-            ss << '\t';
-            ss << timeLocked/1000 << '\t';
-            if( dt )
-                ss << (timeLocked*100)/dt << '%';
-            return ss.str();
-        }
-    */
-
     struct Timing {
         Timing() {
             start = timeLocked = 0;
@@ -65,187 +51,39 @@ namespace mongo {
         unsigned long long start, timeLocked;
     };
 
-    bool _bold;
-    string bold(bool x) {
-        _bold = x;
-        return x ? "<b>" : "";
-    }
-    string bold() {
-        return _bold ? "</b>" : "";
-    }
+    bool execCommand( Command * c ,
+                      Client& client , int queryOptions , 
+                      const char *ns, BSONObj& cmdObj , 
+                      BSONObjBuilder& result, 
+                      bool fromRepl );
 
     class DbWebServer : public MiniWebServer {
     public:
-        // caller locks
-        void doLockedStuff(stringstream& ss) {
-            ss << "# databases: " << dbHolder.size() << '\n';
-
-            ss << bold(ClientCursor::byLocSize()>10000) << "Cursors byLoc.size(): " << ClientCursor::byLocSize() << bold() << '\n';
-            ss << "\n<b>replication</b>\n";
-            ss << "master: " << replSettings.master << '\n';
-            ss << "slave:  " << replSettings.slave << '\n';
-            if ( replPair ) {
-                ss << "replpair:\n";
-                ss << replPair->getInfo();
-            }
-            bool seemCaughtUp = getInitialSyncCompleted();
-            if ( !seemCaughtUp ) ss << "<b>";
-            ss <<   "initialSyncCompleted: " << seemCaughtUp;
-            if ( !seemCaughtUp ) ss << "</b>";
-            ss << '\n';
-            
-            auto_ptr<SnapshotDelta> delta = statsSnapshots.computeDelta();
-            if ( delta.get() ){
-                ss << "\n<b>DBTOP  (occurences|percent of elapsed)</b>\n";
-                ss << "<table border=1>";
-                ss << "<tr align='left'>";
-                ss << "<th>NS</th>"
-                      "<th colspan=2>total</th>"
-                      "<th colspan=2>Reads</th>"
-                      "<th colspan=2>Writes</th>"
-                      "<th colspan=2>Queries</th>"
-                      "<th colspan=2>GetMores</th>"
-                      "<th colspan=2>Inserts</th>"
-                      "<th colspan=2>Updates</th>"
-                      "<th colspan=2>Removes</th>";
-                ss << "</tr>";
-                
-                display( ss , (double) delta->elapsed() , "GLOBAL" , delta->globalUsageDiff() );
-                
-                Top::UsageMap usage = delta->collectionUsageDiff();
-                for ( Top::UsageMap::iterator i=usage.begin(); i != usage.end(); i++ ){
-                    display( ss , (double) delta->elapsed() , i->first , i->second );
-                }
-                
-                ss << "</table>";
-            }
-
-            statsSnapshots.outputLockInfoHTML( ss );
-
-            BackgroundOperation::dump(ss);
-        }
-
-        void display( stringstream& ss , double elapsed , const Top::UsageData& usage ){
-            ss << "<td>";
-            ss << usage.count;
-            ss << "</td><td>";
-            double per = 100 * ((double)usage.time)/elapsed;
-            ss << setprecision(2) << fixed << per << "%";
-            ss << "</td>";
-        }
-
-        void display( stringstream& ss , double elapsed , const string& ns , const Top::CollectionData& data ){
-            if ( ns != "GLOBAL" && data.total.count == 0 )
-                return;
-            ss << "<tr><th>" << ns << "</th>";
-            
-            display( ss , elapsed , data.total );
-
-            display( ss , elapsed , data.readLock );
-            display( ss , elapsed , data.writeLock );
-
-            display( ss , elapsed , data.queries );
-            display( ss , elapsed , data.getmore );
-            display( ss , elapsed , data.insert );
-            display( ss , elapsed , data.update );
-            display( ss , elapsed , data.remove );
-            
-            ss << "</tr>";
+        DbWebServer(const string& ip, int port) : MiniWebServer(ip, port) {
+            WebStatusPlugin::initAll();
         }
 
-        void tablecell( stringstream& ss , bool b ){
-            ss << "<td>" << (b ? "<b>X</b>" : "") << "</td>";
-        }
-        
+    private:
 
-        template< typename T> 
-        void tablecell( stringstream& ss , const T& t ){
-            ss << "<td>" << t << "</td>";
-        }
-        
         void doUnlockedStuff(stringstream& ss) {
             /* this is in the header already ss << "port:      " << port << '\n'; */
-            ss << mongodVersion() << "\n";
-            ss << "git hash: " << gitVersion() << "\n";
-            ss << "sys info: " << sysInfo() << "\n";
-            ss << "\n";
-            ss << "dbwritelocked:  " << dbMutex.info().isLocked() << " (initial)\n";
-            ss << "uptime:    " << time(0)-started << " seconds\n";
-            if ( replAllDead )
-                ss << "<b>replication replAllDead=" << replAllDead << "</b>\n";
-            ss << "\nassertions:\n";
-            for ( int i = 0; i < 4; i++ ) {
-                if ( lastAssert[i].isSet() ) {
-                    ss << "<b>";
-                    if ( i == 3 ) ss << "usererr";
-                    else ss << i;
-                    ss << "</b>" << ' ' << lastAssert[i].toString();
-                }
-            }
-
-            ss << "\nreplInfo:  " << replInfo << "\n\n";
-
-            ss << "Clients:\n";
-            ss << "<table border=1>";
-            ss << "<tr align='left'>"
-               << "<th>Thread</th>" 
-             
-               << "<th>OpId</th>" 
-               << "<th>Active</th>" 
-               << "<th>LockType</th>"
-               << "<th>Waiting</th>"
-               << "<th>SecsRunning</th>"
-               << "<th>Op</th>"
-               << "<th>NameSpace</th>"
-               << "<th>Query</th>"
-               << "<th>client</th>"
-               << "<th>msg</th>"
-               << "<th>progress</th>"
-
-               << "</tr>\n";
-            {
-                scoped_lock bl(Client::clientsMutex);
-                for( set<Client*>::iterator i = Client::clients.begin(); i != Client::clients.end(); i++ ) { 
-                    Client *c = *i;
-                    CurOp& co = *(c->curop());
-                    ss << "<tr><td>" << c->desc() << "</td>";
-                    
-                    tablecell( ss , co.opNum() );
-                    tablecell( ss , co.active() );
-                    tablecell( ss , co.getLockType() );
-                    tablecell( ss , co.isWaitingForLock() );
-                    if ( co.active() )
-                        tablecell( ss , co.elapsedSeconds() );
-                    else
-                        tablecell( ss , "" );
-                    tablecell( ss , co.getOp() );
-                    tablecell( ss , co.getNS() );
-                    if ( co.haveQuery() )
-                        tablecell( ss , co.query() );
-                    else
-                        tablecell( ss , "" );
-                    tablecell( ss , co.getRemoteString() );
-
-                    tablecell( ss , co.getMessage() );
-                    tablecell( ss , co.getProgressMeter().toString() );
-
-
-                    ss << "</tr>";
-                }
-            }
-            ss << "</table>\n";
+            ss << "<pre>";
+            ss << mongodVersion() << '\n';
+            ss << "git hash: " << gitVersion() << '\n';
+            ss << "sys info: " << sysInfo() << '\n';
+            ss << "uptime: " << time(0)-started << " seconds\n";
+            ss << "</pre>";
         }
+
+    private:
         
-        bool allowed( const char * rq , vector<string>& headers, const SockAddr &from ){
-            
-            if ( from.localhost() )
+        bool allowed( const char * rq , vector<string>& headers, const SockAddr &from ) {
+            if ( from.isLocalHost() )
                 return true;
-            
-            Client::GodScope gs;
 
-            if ( db.findOne( "admin.system.users" , BSONObj() , 0 , QueryOption_SlaveOk ).isEmpty() )
+            if ( ! webHaveAdminUsers() )
                 return true;
-            
+
             string auth = getHeader( rq , "Authorization" );
 
             if ( auth.size() > 0 && auth.find( "Digest " ) == 0 ){
@@ -260,25 +98,26 @@ namespace mongo {
                     parms[name] = val;
                 }
 
-                BSONObj user = db.findOne( "admin.system.users" , BSON( "user" << parms["username"] ) );
+                BSONObj user = webGetAdminUser( parms["username"] );
                 if ( ! user.isEmpty() ){
                     string ha1 = user["pwd"].str();
                     string ha2 = md5simpledigest( (string)"GET" + ":" + parms["uri"] );
                     
-                    string r = ha1 + ":" + parms["nonce"];
+                    stringstream r;
+                    r << ha1 << ':' << parms["nonce"];
                     if ( parms["nc"].size() && parms["cnonce"].size() && parms["qop"].size() ){
-                        r += ":";
-                        r += parms["nc"];
-                        r += ":";
-                        r += parms["cnonce"];
-                        r += ":";
-                        r += parms["qop"];
+                        r << ':';
+                        r << parms["nc"];
+                        r << ':';
+                        r << parms["cnonce"];
+                        r << ':';
+                        r << parms["qop"];
                     }
-                    r += ":";
-                    r += ha2;
-                    r = md5simpledigest( r );
+                    r << ':';
+                    r << ha2;
+                    string r1 = md5simpledigest( r.str() );
                     
-                    if ( r == parms["response"] )
+                    if ( r1 == parms["response"] )
                         return true;
                 }
 
@@ -307,85 +146,239 @@ namespace mongo {
             const SockAddr &from
         )
         {
-            //out() << "url [" << url << "]" << endl;
-            
             if ( url.size() > 1 ) {
                 
-                if ( url.find( "/_status" ) == 0 ){
-                    if ( ! allowed( rq , headers, from ) ){
-                        responseCode = 401;
-                        responseMsg = "not allowed\n";
-                        return;
-                    }              
-                    headers.push_back( "Content-Type: application/json" );
-                    generateServerStatus( url , responseMsg );
-                    responseCode = 200;
+                if ( ! allowed( rq , headers, from ) ) {
+                    responseCode = 401;
+                    headers.push_back( "Content-Type: text/plain" );
+                    responseMsg = "not allowed\n";
                     return;
+                }              
+
+                {
+                    DbWebHandler * handler = DbWebHandler::findHandler( url );
+                    if ( handler ){
+                        if ( handler->requiresREST( url ) && ! cmdLine.rest )
+                            _rejectREST( responseMsg , responseCode , headers );
+                        else
+                            handler->handle( rq , url , responseMsg , responseCode , headers , from );
+                        return;
+                    }
                 }
 
-                if ( ! cmdLine.rest ){
-                    responseCode = 403;
-                    responseMsg = "rest is not enabled.  use --rest to turn on";
+
+                if ( ! cmdLine.rest ) {
+                    _rejectREST( responseMsg , responseCode , headers );
                     return;
                 }
-                if ( ! allowed( rq , headers, from ) ){
-                    responseCode = 401;
-                    responseMsg = "not allowed\n";
-                    return;
-                }                
-                handleRESTRequest( rq , url , responseMsg , responseCode , headers );
+                
+                responseCode = 404;
+                headers.push_back( "Content-Type: text/html" );
+                responseMsg = "<html><body>unknown url</body></html>\n";
                 return;
             }
+            
+            // generate home page
 
+            if ( ! allowed( rq , headers, from ) ){
+                responseCode = 401;
+                responseMsg = "not allowed\n";
+                return;
+            }            
 
             responseCode = 200;
             stringstream ss;
-            ss << "<html><head><title>";
-
             string dbname;
             {
                 stringstream z;
-                z << "mongodb " << getHostName() << ':' << mongo::cmdLine.port << ' ';
+                z << "mongod " << prettyHostName();
                 dbname = z.str();
             }
-            ss << dbname << "</title></head><body><h2>" << dbname << "</h2><p>\n<pre>";
-
-            doUnlockedStuff(ss);
+            ss << start(dbname) << h2(dbname);
+            ss << "<p><a href=\"/_commands\">List all commands</a> | \n";
+            ss << "<a href=\"/_replSet\">Replica set status</a></p>\n";
 
+            //ss << "<a href=\"/_status\">_status</a>";
             {
-                Timer t;
-                readlocktry lk( "" , 2000 );
-                if ( lk.got() ){
-                    ss << "time to get dblock: " << t.millis() << "ms\n";
-                    doLockedStuff(ss);
-                }
-                else {
-                    ss << "\n<b>timed out getting dblock</b>\n";
+                const map<string, Command*> *m = Command::webCommands();
+                if( m ) {
+                    ss << a("", "These read-only context-less commands can be executed from the web interface.  Results are json format, unless ?text is appended in which case the result is output as text for easier human viewing", "Commands") << ": ";
+                    for( map<string, Command*>::const_iterator i = m->begin(); i != m->end(); i++ ) { 
+                        stringstream h;
+                        i->second->help(h);
+                        string help = h.str();
+                        ss << "<a href=\"/" << i->first << "?text\"";
+                        if( help != "no help defined" )
+                            ss << " title=\"" << help << '"';
+                        ss << ">" << i->first << "</a> ";
+                    }
+                    ss << '\n';
                 }
             }
-            
+            ss << '\n';
+	    /*
+            ss << "HTTP <a "
+                "title=\"click for documentation on this http interface\""
+                "href=\"http://www.mongodb.org/display/DOCS/Http+Interface\">admin port</a>:" << _port << "<p>\n";
+	    */
 
-            ss << "</pre></body></html>";
+            doUnlockedStuff(ss);
+
+            WebStatusPlugin::runAll( ss );
+            
+            ss << "</body></html>\n";
             responseMsg = ss.str();
 
-            // we want to return SavedContext from before the authentication was performed
-            if ( ! allowed( rq , headers, from ) ){
-                responseCode = 401;
-                responseMsg = "not allowed\n";
-                return;
-            }            
+
         }
 
-        void generateServerStatus( string url , string& responseMsg ){
+        void _rejectREST( string& responseMsg , int& responseCode, vector<string>& headers ){
+                                responseCode = 403;
+                    stringstream ss;
+                    ss << "REST is not enabled.  use --rest to turn on.\n";
+                    ss << "check that port " << _port << " is secured for the network too.\n";
+                    responseMsg = ss.str();
+                    headers.push_back( "Content-Type: text/plain" );
+        }
+
+    };
+    // ---
+    
+    bool prisort( const Prioritizable * a , const Prioritizable * b ){
+        return a->priority() < b->priority();
+    }
+
+    // -- status framework ---
+    WebStatusPlugin::WebStatusPlugin( const string& secionName , double priority , const string& subheader ) 
+        : Prioritizable(priority), _name( secionName ) , _subHeading( subheader ) {
+        if ( ! _plugins )
+            _plugins = new vector<WebStatusPlugin*>();
+        _plugins->push_back( this );
+    }
+
+    void WebStatusPlugin::initAll(){
+        if ( ! _plugins )
+            return;
+        
+        sort( _plugins->begin(), _plugins->end() , prisort );
+        
+        for ( unsigned i=0; i<_plugins->size(); i++ )
+            (*_plugins)[i]->init();
+    }
+
+    void WebStatusPlugin::runAll( stringstream& ss ){
+        if ( ! _plugins )
+            return;
+        
+        for ( unsigned i=0; i<_plugins->size(); i++ ){
+            WebStatusPlugin * p = (*_plugins)[i];
+            ss << "<hr>\n" 
+               << "<b>" << p->_name << "</b>";
+            
+            ss << " " << p->_subHeading;
+
+            ss << "<br>\n";
+            
+            p->run(ss);
+        }
+
+    }
+
+    vector<WebStatusPlugin*> * WebStatusPlugin::_plugins = 0;
+
+    // -- basic statuc plugins --
+
+    class LogPlugin : public WebStatusPlugin {
+    public:
+        LogPlugin() : WebStatusPlugin( "Log" , 100 ), _log(0){
+        }
+        
+        virtual void init(){
+            assert( ! _log );
+            _log = new RamLog();
+            Logstream::get().addGlobalTee( _log );
+        }
+
+        virtual void run( stringstream& ss ){
+            _log->toHTML( ss );
+        }
+        RamLog * _log;
+    };
+      
+    LogPlugin * logPlugin = new LogPlugin();
+
+    // -- handler framework ---
+
+    DbWebHandler::DbWebHandler( const string& name , double priority , bool requiresREST )
+        : Prioritizable(priority), _name(name) , _requiresREST(requiresREST){
+
+        { // setup strings
+            _defaultUrl = "/";
+            _defaultUrl += name;
+
+            stringstream ss;
+            ss << name << " priority: " << priority << " rest: " << requiresREST;
+            _toString = ss.str();
+        }
+        
+        { // add to handler list
+            if ( ! _handlers )
+                _handlers = new vector<DbWebHandler*>();
+            _handlers->push_back( this );
+            sort( _handlers->begin() , _handlers->end() , prisort );
+        }
+    }
+
+    DbWebHandler * DbWebHandler::findHandler( const string& url ){
+        if ( ! _handlers )
+            return 0;
+        
+        for ( unsigned i=0; i<_handlers->size(); i++ ){
+            DbWebHandler * h = (*_handlers)[i];
+            if ( h->handles( url ) )
+                return h;
+        }
+
+        return 0;
+    }
+    
+    vector<DbWebHandler*> * DbWebHandler::_handlers = 0;
+
+    // --- basic handlers ---
+
+    class FavIconHandler : public DbWebHandler {
+    public:
+        FavIconHandler() : DbWebHandler( "favicon.ico" , 0 , false ){}
+
+        virtual void handle( const char *rq, string url, 
+                             string& responseMsg, int& responseCode,
+                             vector<string>& headers,  const SockAddr &from ){
+            responseCode = 404;
+            headers.push_back( "Content-Type: text/plain" );
+            responseMsg = "no favicon\n";
+        }
+
+    } faviconHandler;
+    
+    class StatusHandler : public DbWebHandler {
+    public:
+        StatusHandler() : DbWebHandler( "_status" , 1 , false ){}
+        
+        virtual void handle( const char *rq, string url, 
+                             string& responseMsg, int& responseCode,
+                             vector<string>& headers,  const SockAddr &from ){
+            headers.push_back( "Content-Type: application/json" );
+            responseCode = 200;
+            
             static vector<string> commands;
             if ( commands.size() == 0 ){
                 commands.push_back( "serverStatus" );
                 commands.push_back( "buildinfo" );
             }
-
+            
             BSONObj params;
             if ( url.find( "?" ) != string::npos ) {
-                parseParams( params , url.substr( url.find( "?" ) + 1 ) );
+                MiniWebServer::parseParams( params , url.substr( url.find( "?" ) + 1 ) );
             }
             
             BSONObjBuilder buf(1024);
@@ -400,7 +393,7 @@ namespace mongo {
                 BSONObj co;
                 {
                     BSONObjBuilder b;
-                    b.append( cmd.c_str() , 1 );
+                    b.append( cmd , 1 );
                     
                     if ( cmd == "serverStatus" && params["repl"].type() ){
                         b.append( "repl" , atoi( params["repl"].valuestr() ) );
@@ -413,188 +406,143 @@ namespace mongo {
                 
                 BSONObjBuilder sub;
                 if ( ! c->run( "admin.$cmd" , co , errmsg , sub , false ) )
-                    buf.append( cmd.c_str() , errmsg );
+                    buf.append( cmd , errmsg );
                 else
-                    buf.append( cmd.c_str() , sub.obj() );
+                    buf.append( cmd , sub.obj() );
             }
             
             responseMsg = buf.obj().jsonString();
-        }
 
-        void handleRESTRequest( const char *rq, // the full request
-                                string url,
-                                string& responseMsg,
-                                int& responseCode,
-                                vector<string>& headers // if completely empty, content-type: text/html will be added
-                              ) {
-
-            string::size_type first = url.find( "/" , 1 );
-            if ( first == string::npos ) {
-                responseCode = 400;
-                return;
-            }
-
-            string method = parseMethod( rq );
-            string dbname = url.substr( 1 , first - 1 );
-            string coll = url.substr( first + 1 );
-            string action = "";
-
-            BSONObj params;
-            if ( coll.find( "?" ) != string::npos ) {
-                parseParams( params , coll.substr( coll.find( "?" ) + 1 ) );
-                coll = coll.substr( 0 , coll.find( "?" ) );
-            }
-
-            string::size_type last = coll.find_last_of( "/" );
-            if ( last == string::npos ) {
-                action = coll;
-                coll = "_defaultCollection";
-            }
-            else {
-                action = coll.substr( last + 1 );
-                coll = coll.substr( 0 , last );
-            }
-
-            for ( string::size_type i=0; i<coll.size(); i++ )
-                if ( coll[i] == '/' )
-                    coll[i] = '.';
-
-            string fullns = urlDecode(dbname + "." + coll);
+        }
 
-            headers.push_back( (string)"x-action: " + action );
-            headers.push_back( (string)"x-ns: " + fullns );
-            headers.push_back( "Content-Type: text/plain;charset=utf-8" );
+    } statusHandler;
 
+    class CommandListHandler : public DbWebHandler {
+    public:
+        CommandListHandler() : DbWebHandler( "_commands" , 1 , true ){}
+        
+        virtual void handle( const char *rq, string url, 
+                             string& responseMsg, int& responseCode,
+                             vector<string>& headers,  const SockAddr &from ){
+            headers.push_back( "Content-Type: text/html" );
+            responseCode = 200;
+            
             stringstream ss;
-
-            if ( method == "GET" ) {
-                responseCode = 200;
-                handleRESTQuery( fullns , action , params , responseCode , ss  );
-            }
-            else if ( method == "POST" ) {
-                responseCode = 201;
-                handlePost( fullns , body( rq ) , params , responseCode , ss  );
-            }
-            else {
-                responseCode = 400;
-                headers.push_back( "X_err: bad request" );
-                ss << "don't know how to handle a [" << method << "]";
-                out() << "don't know how to handle a [" << method << "]" << endl;
-            }
-
+            ss << start("Commands List");
+            ss << p( a("/", "back", "Home") );
+            ss << p( "<b>MongoDB List of <a href=\"http://www.mongodb.org/display/DOCS/Commands\">Commands</a></b>\n" );
+            const map<string, Command*> *m = Command::commandsByBestName();
+            ss << "S:slave-only  N:no-lock  R:read-lock  W:write-lock  A:admin-only<br>\n";
+            ss << table();
+            ss << "<tr><th>Command</th><th>Attributes</th><th>Help</th></tr>\n";
+            for( map<string, Command*>::const_iterator i = m->begin(); i != m->end(); i++ ) 
+                i->second->htmlHelp(ss);
+            ss << _table() << _end();
+            
             responseMsg = ss.str();
         }
+    } commandListHandler;
 
-        void handleRESTQuery( string ns , string action , BSONObj & params , int & responseCode , stringstream & out ) {
-            Timer t;
-
-            int skip = _getOption( params["skip"] , 0 );
-            int num = _getOption( params["limit"] , _getOption( params["count" ] , 1000 ) ); // count is old, limit is new
-
-            int one = 0;
-            if ( params["one"].type() == String && tolower( params["one"].valuestr()[0] ) == 't' ) {
-                num = 1;
-                one = 1;
-            }
-
-            BSONObjBuilder queryBuilder;
-
-            BSONObjIterator i(params);
-            while ( i.more() ){
-                BSONElement e = i.next();
-                string name = e.fieldName();
-                if ( ! name.find( "filter_" ) == 0 )
-                    continue;
-
-                const char * field = name.substr( 7 ).c_str();
-                const char * val = e.valuestr();
-
-                char * temp;
-
-                // TODO: this is how i guess if something is a number.  pretty lame right now
-                double number = strtod( val , &temp );
-                if ( temp != val )
-                    queryBuilder.append( field , number );
-                else
-                    queryBuilder.append( field , val );
+    class CommandsHandler : public DbWebHandler {
+    public:
+        CommandsHandler() : DbWebHandler( "DUMMY COMMANDS" , 2 , true ){}
+        
+        bool _cmd( const string& url , string& cmd , bool& text ) const {
+            const char * x = url.c_str();
+            
+            if ( x[0] != '/' ){
+                // this should never happen
+                return false;
             }
+            
+            if ( strchr( x + 1 , '/' ) )
+                return false;
+            
+            x++;
 
-            BSONObj query = queryBuilder.obj();
-
-            auto_ptr<DBClientCursor> cursor = db.query( ns.c_str() , query, num , skip );
-            uassert( 13085 , "query failed for dbwebserver" , cursor.get() );
-            if ( one ) {
-                if ( cursor->more() ) {
-                    BSONObj obj = cursor->next();
-                    out << obj.jsonString() << "\n";
-                }
-                else {
-                    responseCode = 404;
-                }
-                return;
+            const char * end = strstr( x , "?text" );
+            if ( end ){
+                text = true;
+                cmd = string( x , end - x );
             }
-
-            out << "{\n";
-            out << "  \"offset\" : " << skip << ",\n";
-            out << "  \"rows\": [\n";
-
-            int howMany = 0;
-            while ( cursor->more() ) {
-                if ( howMany++ )
-                    out << " ,\n";
-                BSONObj obj = cursor->next();
-                out << "    " << obj.jsonString();
-
+            else {
+                text = false;
+                cmd = string(x);
             }
-            out << "\n  ],\n\n";
+             
+            return true;
+        }
 
-            out << "  \"total_rows\" : " << howMany << " ,\n";
-            out << "  \"query\" : " << query.jsonString() << " ,\n";
-            out << "  \"millis\" : " << t.millis() << "\n";
-            out << "}\n";
+        Command * _cmd( const string& cmd ) const {
+            const map<string,Command*> *m = Command::webCommands();
+            if( ! m )
+                return 0;
+            
+            map<string,Command*>::const_iterator i = m->find(cmd);
+            if ( i == m->end() )
+                return 0;
+            
+            return i->second;
         }
 
-        // TODO Generate id and revision per couch POST spec
-        void handlePost( string ns, const char *body, BSONObj& params, int & responseCode, stringstream & out ) {
-            try {
-                BSONObj obj = fromjson( body );
-                db.insert( ns.c_str(), obj );
-            } catch ( ... ) {
-                responseCode = 400; // Bad Request.  Seems reasonable for now.
-                out << "{ \"ok\" : false }";
-                return;
-            }
+        virtual bool handles( const string& url ) const { 
+            string cmd;
+            bool text;
+            if ( ! _cmd( url , cmd , text ) )
+                return false;
 
-            responseCode = 201;
-            out << "{ \"ok\" : true }";
+            return _cmd( cmd );
         }
+        
+        virtual void handle( const char *rq, string url, 
+                             string& responseMsg, int& responseCode,
+                             vector<string>& headers,  const SockAddr &from ){
+            
+            string cmd;
+            bool text = false;
+            assert( _cmd( url , cmd , text ) );
+            Command * c = _cmd( cmd );
+            assert( c );
+
+            BSONObj cmdObj = BSON( cmd << 1 );
+            Client& client = cc();
+            
+            BSONObjBuilder result;
+            execCommand(c, client, 0, "admin.", cmdObj , result, false);
+            
+            responseCode = 200;
+            
+            string j = result.done().jsonString(JS, text );
+            responseMsg = j;
+            
+            if( text ){
+                headers.push_back( "Content-Type: text/plain" );
+                responseMsg += '\n';
+            }
+            else {
+                headers.push_back( "Content-Type: application/json" );
+            }
 
-        int _getOption( BSONElement e , int def ) {
-            if ( e.isNumber() )
-                return e.numberInt();
-            if ( e.type() == String )
-                return atoi( e.valuestr() );
-            return def;
         }
+        
+    } commandsHandler;
 
-    private:
-        static DBDirectClient db;
-    };
+    // --- external ----
 
-    DBDirectClient DbWebServer::db;
+    string prettyHostName() { 
+        stringstream s;
+        s << getHostName();
+        if( mongo::cmdLine.port != CmdLine::DefaultDBPort ) 
+            s << ':' << mongo::cmdLine.port;
+        return s.str();
+    }
 
     void webServerThread() {
         Client::initThread("websvr");
-        DbWebServer mini;
-        int p = cmdLine.port + 1000;
-        if ( mini.init(bind_ip, p) ) {
-            ListeningSockets::get()->add( mini.socket() );
-            log() << "web admin interface listening on port " << p << endl;
-            mini.run();
-        }
-        else { 
-            log() << "warning: web admin interface failed to initialize on port " << p << endl;
-        }
+        const int p = cmdLine.port + 1000;
+        DbWebServer mini(cmdLine.bind_ip, p);
+        log() << "web admin interface listening on port " << p << endl;
+        mini.initAndListen();
         cc().shutdown();
     }
 
diff --git a/db/dbwebserver.h b/db/dbwebserver.h
new file mode 100644
index 0000000..d1a2f0d
--- /dev/null
+++ b/db/dbwebserver.h
@@ -0,0 +1,90 @@
+/** @file dbwebserver.h
+ */
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+namespace mongo {
+
+    class Prioritizable {
+    public:
+        Prioritizable( double p ) : _priority(p){}
+        double priority() const { return _priority; }
+    private:
+        double _priority;
+    };
+    
+    class DbWebHandler : public Prioritizable {
+    public:
+        DbWebHandler( const string& name , double priority , bool requiresREST );
+        virtual ~DbWebHandler(){}
+
+        virtual bool handles( const string& url ) const { return url == _defaultUrl; }
+
+        virtual bool requiresREST( const string& url ) const { return _requiresREST; }
+
+        virtual void handle( const char *rq, // the full request
+                             string url,
+                             // set these and return them:
+                             string& responseMsg,
+                             int& responseCode,
+                             vector<string>& headers, // if completely empty, content-type: text/html will be added
+                             const SockAddr &from
+                             ) = 0;
+        
+        string toString() const { return _toString; }
+        static DbWebHandler * findHandler( const string& url );
+
+    private:
+        string _name;
+        bool _requiresREST;
+        
+        string _defaultUrl;
+        string _toString;
+
+        static vector<DbWebHandler*> * _handlers;
+    };
+
+    class WebStatusPlugin : public Prioritizable {
+    public:
+        WebStatusPlugin( const string& secionName , double priority , const string& subheader = "" );
+        virtual ~WebStatusPlugin(){}
+        
+        virtual void run( stringstream& ss ) = 0;
+        /** called when web server stats up */
+        virtual void init() = 0;
+
+        static void initAll();
+        static void runAll( stringstream& ss );
+    private:
+        string _name;
+        string _subHeading;
+        static vector<WebStatusPlugin*> * _plugins;
+        
+    };
+
+    void webServerThread();
+    string prettyHostName();
+    
+    /** @return if there are any admin users.  this should not block for long and throw if can't get a lock if needed */
+    bool webHaveAdminUsers();
+    
+    /** @return admin user with this name.  this should not block for long and throw if can't get a lock if needed */
+    BSONObj webGetAdminUser( const string& username );
+
+};
+
+
diff --git a/db/diskloc.h b/db/diskloc.h
index cc29e60..2747abd 100644
--- a/db/diskloc.h
+++ b/db/diskloc.h
@@ -22,17 +22,18 @@
 
 #pragma once
 
+#include "jsobj.h"
+
 namespace mongo {
 
-#pragma pack(1)
 
     class Record;
     class DeletedRecord;
     class Extent;
     class BtreeBucket;
-    class BSONObj;
     class MongoDataFile;
 
+#pragma pack(1)
     class DiskLoc {
         int fileNo; /* this will be volume, file #, etc. */
         int ofs;
@@ -85,7 +86,10 @@ namespace mongo {
             ss << hex << fileNo << ':' << ofs;
             return ss.str();
         }
-        operator string() const { return toString(); }
+
+        BSONObj toBSONObj() const {
+            return BSON( "file" << fileNo << "offset" << ofs );
+        }
 
         int& GETOFS() {
             return ofs;
@@ -146,7 +150,6 @@ namespace mongo {
 
         MongoDataFile& pdf() const;
     };
-
 #pragma pack()
 
     const DiskLoc minDiskLoc(0, 1);
diff --git a/db/driverHelpers.cpp b/db/driverHelpers.cpp
index c2d1b9d..d8971ad 100644
--- a/db/driverHelpers.cpp
+++ b/db/driverHelpers.cpp
@@ -21,7 +21,7 @@
 */
 
 
-#include "stdafx.h"
+#include "pch.h"
 #include "jsobj.h"
 #include "pdfile.h"
 #include "namespace.h"
@@ -38,16 +38,15 @@ namespace mongo {
     public:
         BasicDriverHelper( const char * name ) : Command( name ){}
         
-        virtual LockType locktype(){ return NONE; }
-        virtual bool slaveOk(){ return true; }
-        virtual bool slaveOverrideOk(){ return true; }
-        
+        virtual LockType locktype() const { return NONE; }
+        virtual bool slaveOk() const { return true; }
+        virtual bool slaveOverrideOk(){ return true; }        
     };
 
     class ObjectIdTest : public BasicDriverHelper {
     public:
         ObjectIdTest() : BasicDriverHelper( "driverOIDTest" ){}
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl){
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl){
             if ( cmdObj.firstElement().type() != jstOID ){
                 errmsg = "not oid";
                 return false;
diff --git a/db/extsort.cpp b/db/extsort.cpp
index a0b9f7a..68e6b52 100644
--- a/db/extsort.cpp
+++ b/db/extsort.cpp
@@ -16,7 +16,7 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 
 #include "extsort.h"
 #include "namespace.h"
@@ -126,7 +126,7 @@ namespace mongo {
         
         ofstream out;
         out.open( file.c_str() , ios_base::out | ios_base::binary );
-        ASSERT_STREAM_GOOD( 10051 ,  (string)"couldn't open file: " + file , out );
+        assertStreamGood( 10051 ,  (string)"couldn't open file: " + file , out );
         
         int num = 0;
         for ( InMemory::iterator i=_cur->begin(); i != _cur->end(); ++i ){
@@ -221,7 +221,7 @@ namespace mongo {
         long length;
         _buf = (char*)_file.map( file.c_str() , length , MemoryMappedFile::SEQUENTIAL );
         massert( 10308 ,  "mmap failed" , _buf );
-        assert( (unsigned long)length == file_size( file ) );
+        assert( (unsigned long long)length == (unsigned long long)file_size( file ) );
         _end = _buf + length;
     }
     BSONObjExternalSorter::FileIterator::~FileIterator(){
diff --git a/db/extsort.h b/db/extsort.h
index 60ee423..fa0eca4 100644
--- a/db/extsort.h
+++ b/db/extsort.h
@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "jsobj.h"
 #include "namespace.h"
 #include "curop.h"
diff --git a/db/flushtest.cpp b/db/flushtest.cpp
index 00cebcf..2009d92 100644
--- a/db/flushtest.cpp
+++ b/db/flushtest.cpp
@@ -14,7 +14,7 @@
  *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "stdafx.h"
+#include "pch.h"
 #include <stdio.h>
 #include "../util/goodies.h"
 #include <fcntl.h>
diff --git a/db/index_geo2d.cpp b/db/geo/2d.cpp
index 5ebf65a..19efafd 100644
--- a/db/index_geo2d.cpp
+++ b/db/geo/2d.cpp
@@ -16,301 +16,54 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
-#include "namespace.h"
-#include "jsobj.h"
-#include "index.h"
-#include "../util/unittest.h"
-#include "commands.h"
-#include "pdfile.h"
-#include "btree.h"
-#include "curop.h"
-#include "matcher.h"
-
-//#define GEODEBUG(x) cout << x << endl;
-#define GEODEBUG(x) 
+#include "pch.h"
+#include "../namespace.h"
+#include "../jsobj.h"
+#include "../index.h"
+#include "../../util/unittest.h"
+#include "../commands.h"
+#include "../pdfile.h"
+#include "../btree.h"
+#include "../curop.h"
+#include "../matcher.h"
+
+#include "core.h"
 
 namespace mongo {
 
-    const string GEO2DNAME = "2d";
-
-    class GeoBitSets {
-    public:
-        GeoBitSets(){
-            for ( int i=0; i<32; i++ ){
-                masks32[i] = ( 1 << ( 31 - i ) );
-            }
-            for ( int i=0; i<64; i++ ){
-                masks64[i] = ( 1LL << ( 63 - i ) );
-            }
+#if 0
+# define GEODEBUG(x) cout << x << endl;
+    inline void PREFIXDEBUG(GeoHash prefix, const GeoConvert* g){
+        if (!prefix.constrains()) {
+            cout << "\t empty prefix" << endl;
+            return ;
         }
-        int masks32[32];
-        long long masks64[64];
-    } geoBitSets;
 
-    
-    class GeoHash {
-    public:
-        GeoHash()
-            : _hash(0),_bits(0){
-        }
+        Point ll (g, prefix); // lower left
+        prefix.move(1,1);
+        Point tr (g, prefix); // top right
 
-        GeoHash( const char * hash ){
-            init( hash );
-        }
+        Point center ( (ll._x+tr._x)/2, (ll._y+tr._y)/2 );
+        double radius = fabs(ll._x - tr._x) / 2;
 
-        GeoHash( const string& hash ){
-            init( hash );
-        }
+        cout << "\t ll: " << ll.toString() << " tr: " << tr.toString() 
+             << " center: " << center.toString() << " radius: " << radius << endl;
 
-        GeoHash( const BSONElement& e , unsigned bits=32 ){
-            _bits = bits;
-            if ( e.type() == BinData ){
-                int len = 0;
-                _copy( (char*)&_hash , e.binData( len ) );
-                assert( len == 8 );
-                _bits = bits;
-            }
-            else {
-                cout << "GeoHash cons e : " << e << endl;
-                uassert(13047,"wrong type for geo index. if you're using a pre-release version, need to rebuild index",0);
-            }
-            _fix();
-        }
-        
-        GeoHash( unsigned x , unsigned y , unsigned bits=32){
-            init( x , y , bits );
-        }
-
-        GeoHash( const GeoHash& old ){
-            _hash = old._hash;
-            _bits = old._bits;
-        }
-
-        GeoHash( long long hash , unsigned bits )
-            : _hash( hash ) , _bits( bits ){
-            _fix();
-        }
-
-        void init( unsigned x , unsigned y , unsigned bits ){
-            assert( bits <= 32 );
-            _hash = 0;
-            _bits = bits;
-            for ( unsigned i=0; i<bits; i++ ){
-                if ( isBitSet( x , i ) ) _hash |= geoBitSets.masks64[i*2];
-                if ( isBitSet( y , i ) ) _hash |= geoBitSets.masks64[(i*2)+1];
-            }
-        }
-
-        void unhash( unsigned& x , unsigned& y ) const {
-            x = 0;
-            y = 0;
-            for ( unsigned i=0; i<_bits; i++ ){
-                if ( getBitX(i) )
-                    x |= geoBitSets.masks32[i];
-                if ( getBitY(i) )
-                    y |= geoBitSets.masks32[i];
-            }
-        }
-
-        /**
-         * @param 0 = high
-         */
-        static bool isBitSet( unsigned val , unsigned  bit ){
-            return geoBitSets.masks32[bit] & val;
-        }
-        
-        GeoHash up() const {
-            return GeoHash( _hash , _bits - 1 );
-        }
-        
-        bool hasPrefix( const GeoHash& other ) const {
-            assert( other._bits <= _bits );
-            if ( other._bits == 0 )
-                return true;
-            long long x = other._hash ^ _hash;
-            x = x >> (64-(other._bits*2));
-            return x == 0;
-        }
-        
-
-        string toString() const { 
-            StringBuilder buf( _bits * 2 );
-            for ( unsigned x=0; x<_bits*2; x++ )
-                buf.append( _hash & geoBitSets.masks64[x] ? "1" : "0" );
-            return buf.str();
-        }
-
-        string toStringHex1() const {
-            stringstream ss;
-            ss << hex << _hash;
-            return ss.str();
-        }
-
-        void init( const string& s ){
-            _hash = 0;
-            _bits = s.size() / 2;
-            for ( unsigned pos=0; pos<s.size(); pos++ )
-                if ( s[pos] == '1' )
-                    setBit( pos , 1 );
-        }
-
-        void setBit( unsigned pos , bool one ){
-            assert( pos < _bits * 2 );
-            if ( one )
-                _hash |= geoBitSets.masks64[pos];
-            else if ( _hash & geoBitSets.masks64[pos] )
-                _hash &= ~geoBitSets.masks64[pos];
-        }
-        
-        bool getBit( unsigned pos ) const {
-            return _hash & geoBitSets.masks64[pos];
-        }
-
-        bool getBitX( unsigned pos ) const {
-            assert( pos < 32 );
-            return getBit( pos * 2 );
-        }
-
-        bool getBitY( unsigned pos ) const {
-            assert( pos < 32 );
-            return getBit( ( pos * 2 ) + 1 );
-        }
-        
-        BSONObj wrap() const {
-            BSONObjBuilder b(20);
-            append( b , "" );
-            BSONObj o = b.obj();
-            assert( o.objsize() == 20 );
-            return o;
-        }
-
-        bool constrains() const {
-            return _bits > 0;
-        }
-        
-        void move( int x , int y ){
-            assert( _bits );
-            _move( 0 , x );
-            _move( 1 , y );
-        }
-
-        void _move( unsigned offset , int d ){
-            if ( d == 0 )
-                return;
-            assert( d <= 1 && d>= -1 ); // TEMP
-            
-            bool from, to;
-            if ( d > 0 ){
-                from = 0;
-                to = 1;
-            }
-            else {
-                from = 1;
-                to = 0;
-            }
-
-            unsigned pos = ( _bits * 2 ) - 1;
-            if ( offset == 0 )
-                pos--;
-            while ( true ){
-                if ( getBit(pos) == from ){
-                    setBit( pos , to );
-                    return;
-                }
-
-                if ( pos < 2 ){
-                    // overflow
-                    for ( ; pos < ( _bits * 2 ) ; pos += 2 ){
-                        setBit( pos , from );
-                    }
-                    return;
-                }
-                
-                setBit( pos , from );
-                pos -= 2;
-            }
-            
-            assert(0);
-        }
-
-        GeoHash& operator=(const GeoHash& h) { 
-            _hash = h._hash;
-            _bits = h._bits;
-            return *this;
-        }
-        
-        bool operator==(const GeoHash& h ){
-            return _hash == h._hash && _bits == h._bits;
-        }
-
-        GeoHash& operator+=( const char * s ) {
-            unsigned pos = _bits * 2;
-            _bits += strlen(s) / 2;
-            assert( _bits <= 32 );
-            while ( s[0] ){
-                if ( s[0] == '1' )
-                    setBit( pos , 1 );
-                pos++;
-                s++;
-            }
-
-            return *this;
-        }
-
-        GeoHash operator+( const char * s ) const {
-            GeoHash n = *this;
-            n+=s;
-            return n;
-        }
-      
-        void _fix(){
-            if ( ( _hash << ( _bits * 2 ) ) == 0 )
-                return;
-            long long mask = 0;
-            for ( unsigned i=0; i<_bits*2; i++ )
-                mask |= geoBitSets.masks64[i];
-            _hash &= mask;
-        }
-        
-        void append( BSONObjBuilder& b , const char * name ) const {
-            char buf[8];
-            _copy( buf , (char*)&_hash );
-            b.appendBinData( name , 8 , bdtCustom , buf );
-        }
-        
-        long long getHash() const {
-            return _hash;
-        }
+    }
+#else
+# define GEODEBUG(x) 
+# define PREFIXDEBUG(x, y) 
+#endif
 
-        GeoHash commonPrefix( const GeoHash& other ) const {
-            unsigned i=0;
-            for ( ; i<_bits && i<other._bits; i++ ){
-                if ( getBitX( i ) == other.getBitX( i ) &&
-                     getBitY( i ) == other.getBitY( i ) )
-                    continue;
-                break;
-            }
-            return GeoHash(_hash,i);
-        }
-    private:
+    double EARTH_RADIUS_KM = 6371;
+    double EARTH_RADIUS_MILES = EARTH_RADIUS_KM * 0.621371192;
 
-        void _copy( char * dst , const char * src ) const {
-            for ( unsigned a=0; a<8; a++ ){
-                dst[a] = src[7-a];
-            }
-        }
 
-        long long _hash;
-        unsigned _bits; // bits per field, so 1 to 32
-    };
+    GeoBitSets geoBitSets;
 
-    ostream& operator<<( ostream &s, const GeoHash &h ){
-        s << h.toString();
-        return s;
-    } // end GeoHash
+    const string GEO2DNAME = "2d";
 
-    class Geo2dType : public IndexType {
+    class Geo2dType : public IndexType , public GeoConvert {
     public:
         Geo2dType( const IndexPlugin * plugin , const IndexSpec* spec )
             : IndexType( plugin , spec ){
@@ -343,6 +96,11 @@ namespace mongo {
             _scaling = (1024*1024*1024*4.0)/(_max-_min);
 
             _order = orderBuilder.obj();
+
+            GeoHash a(0, 0, _bits);
+            GeoHash b = a;
+            b.move(1, 1);
+            _error = distance(a, b);
         }
 
         int _configval( const IndexSpec* spec , const string& name , int def ){
@@ -417,12 +175,12 @@ namespace mongo {
             uassert( 13068 , "geo field only has 1 element" , i.more() );
             BSONElement y = i.next();
             
-            uassert( 13026 , "geo values have to be numbers" , x.isNumber() && y.isNumber() );
+            uassert( 13026 , "geo values have to be numbers: " + o.toString() , x.isNumber() && y.isNumber() );
 
-            return _hash( x.number() , y.number() );
+            return hash( x.number() , y.number() );
         }
 
-        GeoHash _hash( double x , double y ) const {
+        GeoHash hash( double x , double y ) const {
             return GeoHash( _convert(x), _convert(y) , _bits );
         }
 
@@ -436,7 +194,7 @@ namespace mongo {
         }
         
         unsigned _convert( double in ) const {
-            uassert( 13027 , "point not in range" , in <= _max && in >= _min );
+            uassert( 13027 , "point not in range" , in <= (_max + _error) && in >= (_min - _error) );
             in -= _min;
             assert( in > 0 );
             return (unsigned)(in * _scaling);
@@ -448,8 +206,8 @@ namespace mongo {
             x += _min;
             return x;
         }
-
-        void _unconvert( const GeoHash& h , double& x , double& y ) const {
+        
+        void unhash( const GeoHash& h , double& x , double& y ) const {
             unsigned a,b;
             h.unhash(a,b);
             x = _unconvert( a );
@@ -458,8 +216,8 @@ namespace mongo {
         
         double distance( const GeoHash& a , const GeoHash& b ) const {
             double ax,ay,bx,by;
-            _unconvert( a , ax , ay );
-            _unconvert( b , bx , by );
+            unhash( a , ax , ay );
+            unhash( b , bx , by );
             
             double dx = bx - ax;
             double dy = by - ay;
@@ -467,17 +225,26 @@ namespace mongo {
             return sqrt( ( dx * dx ) + ( dy * dy ) );
         }
 
-        double size( const GeoHash& a ) const {
+        double sizeDiag( const GeoHash& a ) const {
             GeoHash b = a;
             b.move( 1 , 1 );
             return distance( a , b );
         }
 
+        double sizeEdge( const GeoHash& a ) const {
+            double ax,ay,bx,by;
+            GeoHash b = a;
+            b.move( 1 , 1 );
+            unhash( a, ax, ay );
+            unhash( b, bx, by );
+            return (fabs(ax-bx));
+        }
+
         const IndexDetails* getDetails() const {
             return _spec->getDetails();
         }
 
-        virtual auto_ptr<Cursor> newCursor( const BSONObj& query , const BSONObj& order , int numWanted ) const;
+        virtual shared_ptr<Cursor> newCursor( const BSONObj& query , const BSONObj& order , int numWanted ) const;
 
         virtual IndexSuitability suitability( const BSONObj& query , const BSONObj& order ) const {
             BSONElement e = query.getFieldDotted(_geo.c_str());
@@ -507,35 +274,7 @@ namespace mongo {
         double _scaling;
 
         BSONObj _order;
-    };
-
-    class Point {
-    public:
-        
-        Point( const Geo2dType * g , const GeoHash& hash ){
-            g->_unconvert( hash , _x , _y );
-        }
-
-        Point( double x , double y )
-            : _x( x ) , _y( y ){
-        }
-        
-        Point() : _x(0),_y(0){
-        }
-
-        GeoHash hash( const Geo2dType * g ){
-            return g->_hash( _x , _y );
-        }
-        
-        string toString() const {
-            StringBuilder buf(32);
-            buf << "(" << _x << "," << _y << ")";
-            return buf.str();
-  
-        }
-
-        double _x;
-        double _y;
+        double _error;
     };
 
     class Box {
@@ -543,7 +282,7 @@ namespace mongo {
         
         Box( const Geo2dType * g , const GeoHash& hash )
             : _min( g , hash ) , 
-              _max( _min._x + g->size( hash ) , _min._y + g->size( hash ) ){
+              _max( _min._x + g->sizeEdge( hash ) , _min._y + g->sizeEdge( hash ) ){
         }
         
         Box( double x , double y , double size )
@@ -563,17 +302,13 @@ namespace mongo {
             return buf.str();
         }
         
-        operator string() const {
-            return toString();
-        }
-
         bool between( double min , double max , double val , double fudge=0) const {
             return val + fudge >= min && val <= max + fudge;
         }
         
         bool mid( double amin , double amax , double bmin , double bmax , bool min , double& res ) const {
-            assert( amin < amax );
-            assert( bmin < bmax );
+            assert( amin <= amax );
+            assert( bmin <= bmax );
 
             if ( amin < bmin ){
                 if ( amax < bmin )
@@ -644,7 +379,7 @@ namespace mongo {
             return (int)(.5+(d*1000));
         }
         
-#define GEOHEQ(a,b) if ( a.toString() != b ){ cout << "[" << a.toString() << "] != [" << b << "]" << endl; assert( a == b ); }
+#define GEOHEQ(a,b) if ( a.toString() != b ){ cout << "[" << a.toString() << "] != [" << b << "]" << endl; assert( a == GeoHash(b) ); }
 
         void run(){
             assert( ! GeoHash::isBitSet( 0 , 0 ) );
@@ -702,11 +437,11 @@ namespace mongo {
             }
 
             {
-                GeoHash a = g._hash( 1 , 1 );
-                GeoHash b = g._hash( 4 , 5 );
+                GeoHash a = g.hash( 1 , 1 );
+                GeoHash b = g.hash( 4 , 5 );
                 assert( 5 == (int)(g.distance( a , b ) ) );
-                a = g._hash( 50 , 50 );
-                b = g._hash( 42 , 44 );
+                a = g.hash( 50 , 50 );
+                b = g.hash( 42 , 44 );
                 assert( round(10) == round(g.distance( a , b )) );
             }
             
@@ -729,9 +464,9 @@ namespace mongo {
 
             { 
                 
-                GeoHash a = g._hash( 5 , 5 );
-                GeoHash b = g._hash( 5 , 7 );
-                GeoHash c = g._hash( 100 , 100 );
+                GeoHash a = g.hash( 5 , 5 );
+                GeoHash b = g.hash( 5 , 7 );
+                GeoHash c = g.hash( 100 , 100 );
                 /*
                 cout << "a: " << a << endl;
                 cout << "b: " << b << endl;
@@ -769,15 +504,15 @@ namespace mongo {
                 GeoHash entry(  "1100110000011100000111000001110000011100000111000001000000000000" );
                 assert( ! entry.hasPrefix( prefix ) );
 
-                entry = "1100110000001100000111000001110000011100000111000001000000000000";
+                entry = GeoHash("1100110000001100000111000001110000011100000111000001000000000000");
                 assert( entry.toString().find( prefix.toString() ) == 0 );
                 assert( entry.hasPrefix( GeoHash( "1100" ) ) );
                 assert( entry.hasPrefix( prefix ) );
             }
             
             {
-                GeoHash a = g._hash( 50 , 50 );
-                GeoHash b = g._hash( 48 , 54 );
+                GeoHash a = g.hash( 50 , 50 );
+                GeoHash b = g.hash( 48 , 54 );
                 assert( round( 4.47214 ) == round( g.distance( a , b ) ) );
             }
             
@@ -791,10 +526,76 @@ namespace mongo {
 
             {
                 GeoHash a( "11001111" );
-                assert( GeoHash( "11" ) == a.commonPrefix( "11" ) );
-                assert( GeoHash( "11" ) == a.commonPrefix( "11110000" ) );
+                assert( GeoHash( "11" ) == a.commonPrefix( GeoHash("11") ) );
+                assert( GeoHash( "11" ) == a.commonPrefix( GeoHash("11110000") ) );
+            }
+
+            {
+                int N = 10000;
+                {
+                    Timer t;
+                    for ( int i=0; i<N; i++ ){
+                        unsigned x = (unsigned)rand();
+                        unsigned y = (unsigned)rand();
+                        GeoHash h( x , y );
+                        unsigned a,b;
+                        h.unhash_slow( a,b );
+                        assert( a == x );
+                        assert( b == y );
+                    }
+                    //cout << "slow: " << t.millis() << endl;
+                }
+
+                {
+                    Timer t;
+                    for ( int i=0; i<N; i++ ){
+                        unsigned x = (unsigned)rand();
+                        unsigned y = (unsigned)rand();
+                        GeoHash h( x , y );
+                        unsigned a,b;
+                        h.unhash_fast( a,b );
+                        assert( a == x );
+                        assert( b == y );
+                    }
+                    //cout << "fast: " << t.millis() << endl;
+                }
+
+            }
+
+            {
+                // see http://en.wikipedia.org/wiki/Great-circle_distance#Worked_example
+
+                {
+                    Point BNA (-86.67, 36.12);
+                    Point LAX (-118.40, 33.94);
+
+                    double dist1 = spheredist_deg(BNA, LAX);
+                    double dist2 = spheredist_deg(LAX, BNA);
+
+                    // target is 0.45306
+                    assert( 0.45305 <= dist1 && dist1 <= 0.45307 );
+                    assert( 0.45305 <= dist2 && dist2 <= 0.45307 );
+                }
+                {
+                    Point BNA (-1.5127, 0.6304);
+                    Point LAX (-2.0665, 0.5924);
+                    
+                    double dist1 = spheredist_rad(BNA, LAX);
+                    double dist2 = spheredist_rad(LAX, BNA);
+
+                    // target is 0.45306
+                    assert( 0.45305 <= dist1 && dist1 <= 0.45307 );
+                    assert( 0.45305 <= dist2 && dist2 <= 0.45307 );
+                }
+                {
+                    Point JFK (-73.77694444, 40.63861111 );
+                    Point LAX (-118.40, 33.94);
+                    
+                    double dist = spheredist_deg(JFK, LAX) * EARTH_RADIUS_MILES;
+                    assert( dist > 2469 && dist < 2470 );
+                }
+
             }
-            
         }
     } geoUnitTest;
     
@@ -839,11 +640,11 @@ namespace mongo {
 
         virtual void add( const KeyNode& node ){
             // when looking at other boxes, don't want to look at some object twice
-            if ( _seen.count( node.recordLoc ) ){
+            pair<set<DiskLoc>::iterator,bool> seenBefore = _seen.insert( node.recordLoc );
+            if ( ! seenBefore.second ){
                 GEODEBUG( "\t\t\t\t already seen : " << node.recordLoc.obj()["_id"] );
                 return;
             }
-            _seen.insert( node.recordLoc );
             _lookedAt++;
             
             // distance check
@@ -851,7 +652,8 @@ namespace mongo {
             if ( ! checkDistance( GeoHash( node.key.firstElement() ) , d ) ){
                 GEODEBUG( "\t\t\t\t bad distance : " << node.recordLoc.obj()  << "\t" << d );
                 return;
-            }
+            } 
+            GEODEBUG( "\t\t\t\t good distance : " << node.recordLoc.obj()  << "\t" << d );
             
             // matcher
             MatchDetails details;
@@ -895,7 +697,7 @@ namespace mongo {
 
         GeoHopper( const Geo2dType * g , unsigned max , const GeoHash& n , const BSONObj& filter = BSONObj() , double maxDistance = numeric_limits<double>::max() )
             : GeoAccumulator( g , filter ) , _max( max ) , _near( n ), _maxDistance( maxDistance ) {
-
+            _farthest = -1;
         }
 
         virtual bool checkDistance( const GeoHash& h , double& d ){
@@ -912,24 +714,23 @@ namespace mongo {
             if ( _points.size() > _max ){
                 _points.erase( --_points.end() );
             }
-        }
 
-        double farthest(){
-            if ( _points.size() == 0 )
-                return -1;
-                
             Holder::iterator i = _points.end();
             i--;
-            return i->_distance;
+            _farthest = i->_distance;
+        }
+
+        double farthest() const {
+            return _farthest;
         }
 
         unsigned _max;
         GeoHash _near;
         Holder _points;
         double _maxDistance;
-
+        double _farthest;
     };
-
+    
     struct BtreeLocation {
         int pos;
         bool found;
@@ -954,7 +755,10 @@ namespace mongo {
                 return false;
             bucket = bucket.btree()->advance( bucket , pos , direction , "btreelocation" );
             
-            return checkCur( totalFound , all );
+            if ( all )
+                return checkCur( totalFound , all );
+            
+            return ! bucket.isNull();
         }
 
         bool checkCur( int& totalFound , GeoAccumulator* all ){
@@ -981,16 +785,19 @@ namespace mongo {
         static bool initial( const IndexDetails& id , const Geo2dType * spec , 
                              BtreeLocation& min , BtreeLocation&  max , 
                              GeoHash start ,
-                             int & found , GeoAccumulator * hopper ){
+                             int & found , GeoAccumulator * hopper )
+        {
             
+            Ordering ordering = Ordering::make(spec->_order);
+
             min.bucket = id.head.btree()->locate( id , id.head , start.wrap() , 
-                                                  spec->_order , min.pos , min.found , minDiskLoc );
+                                                  ordering , min.pos , min.found , minDiskLoc );
             min.checkCur( found , hopper );
             max = min;
             
-            if ( min.bucket.isNull() ){
+            if ( min.bucket.isNull() || ( !(hopper->found()) ) ){
                 min.bucket = id.head.btree()->locate( id , id.head , start.wrap() , 
-                                                      spec->_order , min.pos , min.found , minDiskLoc , -1 );
+                                                      ordering , min.pos , min.found , minDiskLoc , -1 );
                 min.checkCur( found , hopper );
             }
             
@@ -1053,14 +860,20 @@ namespace mongo {
             if ( _found && _prefix.constrains() ){
                 // 2
                 Point center( _spec , _n );
-                double boxSize = _spec->size( _prefix );
                 double farthest = hopper->farthest();
-                if ( farthest > boxSize )
-                    boxSize = farthest;
-                Box want( center._x - ( boxSize / 2 ) , center._y - ( boxSize / 2 ) , boxSize );
-                while ( _spec->size( _prefix ) < boxSize )
+                // Phase 1 might not have found any points.
+                if (farthest == -1)
+                    farthest = _spec->sizeDiag( _prefix );
+                Box want( center._x - farthest , center._y - farthest , farthest * 2 );
+                _prefix = _n;
+                while ( _spec->sizeEdge( _prefix ) < ( farthest / 2 ) ){
                     _prefix = _prefix.up();
-                log(1) << "want: " << want << " found:" << _found << " hash size:" << _spec->size( _prefix ) << endl;
+                }
+                
+                if ( logLevel > 0 ){
+                    log(1) << "want: " << want << " found:" << _found << " nscanned: " << _nscanned << " hash size:" << _spec->sizeEdge( _prefix ) 
+                           << " farthest: " << farthest << " using box: " << Box( _spec , _prefix ).toString() << endl;
+                }
                 
                 for ( int x=-1; x<=1; x++ ){
                     for ( int y=-1; y<=1; y++ ){
@@ -1078,28 +891,37 @@ namespace mongo {
 
         void doBox( const IndexDetails& id , const Box& want , const GeoHash& toscan , int depth = 0 ){
             Box testBox( _spec , toscan );
-            if ( logLevel > 0 ) log(1) << "\t doBox: " << testBox << "\t" << toscan.toString() << endl;
+            if ( logLevel > 2 ){
+                cout << "\t";
+                for ( int i=0; i<depth; i++ )
+                    cout << "\t";
+                cout << " doBox: " << testBox.toString() << "\t" << toscan.toString() << " scanned so far: " << _nscanned << endl;
+            }
 
             double intPer = testBox.intersects( want );
-
+            
             if ( intPer <= 0 )
                 return;
             
-            if ( intPer < .5 && depth < 3 ){
-                doBox( id , want , toscan + "00" , depth + 1);
-                doBox( id , want , toscan + "01" , depth + 1);
-                doBox( id , want , toscan + "10" , depth + 1);
-                doBox( id , want , toscan + "11" , depth + 1);
-                return;
-            }
+            bool goDeeper = intPer < .5 && depth < 2;
 
+            long long myscanned = 0;
+            
             BtreeLocation loc;
-            loc.bucket = id.head.btree()->locate( id , id.head , toscan.wrap() , _spec->_order , 
+            loc.bucket = id.head.btree()->locate( id , id.head , toscan.wrap() , Ordering::make(_spec->_order) , 
                                                         loc.pos , loc.found , minDiskLoc );
             loc.checkCur( _found , _hopper.get() );
-            while ( loc.hasPrefix( toscan ) && loc.advance( 1 , _found , _hopper.get() ) )
+            while ( loc.hasPrefix( toscan ) && loc.advance( 1 , _found , _hopper.get() ) ){
                 _nscanned++;
-
+                if ( ++myscanned > 100 && goDeeper ){
+                    doBox( id , want , toscan + "00" , depth + 1);
+                    doBox( id , want , toscan + "01" , depth + 1);
+                    doBox( id , want , toscan + "10" , depth + 1);
+                    doBox( id , want , toscan + "11" , depth + 1);
+                    return;        
+                }
+            }
+            
         }
 
 
@@ -1140,6 +962,7 @@ namespace mongo {
         }
 
         virtual bool supportGetMore() { return false; }
+        virtual bool supportYields() { return false; }
 
         virtual bool getsetdup(DiskLoc loc){
             return false;
@@ -1273,10 +1096,12 @@ namespace mongo {
             
             uassert( 13060 , "$center needs 2 fields (middle,max distance)" , circle.nFields() == 2 );
             BSONObjIterator i(circle);
-            _start = g->_tohash( i.next() );
+            _startPt = Point(i.next());
+            _start = _startPt.hash(g);
             _prefix = _start;
             _maxDistance = i.next().numberDouble();
             uassert( 13061 , "need a max distance > 0 " , _maxDistance > 0 );
+            _maxDistance += g->_error;
 
             _state = START;
             _found = 0;
@@ -1297,45 +1122,112 @@ namespace mongo {
                 }
                 _state = DOING_EXPAND;
             }
+
+
+            if ( _state == DOING_AROUND ){
+                // TODO could rework and return rather than looping
+                for (int i=-1; i<=1; i++){
+                    for (int j=-1; j<=1; j++){
+                        if (i == 0 && j == 0)
+                            continue; // main box
+
+                        GeoHash newBox = _prefix;
+                        newBox.move(i, j);
+
+                        PREFIXDEBUG(newBox, _g);
+                        if (needToCheckBox(newBox)){
+                            // TODO consider splitting into quadrants
+                            getPointsForPrefix(newBox);
+                        } else  {
+                            GEODEBUG("skipping box");
+                        }
+                    }
+                }
+
+                _state = DONE;
+                return;
+            }
             
-            if ( _state == DOING_EXPAND ){
+            if (_state == DOING_EXPAND){
                 GEODEBUG( "circle prefix [" << _prefix << "]" );
+                PREFIXDEBUG(_prefix, _g);
+
                 while ( _min.hasPrefix( _prefix ) && _min.advance( -1 , _found , this ) );
                 while ( _max.hasPrefix( _prefix ) && _max.advance( 1 , _found , this ) );
-                
+
                 if ( ! _prefix.constrains() ){
                     GEODEBUG( "\t exhausted the btree" );
                     _state = DONE;
                     return;
                 }
                 
-                if ( _g->distance( _prefix , _start ) > _maxDistance ){
-                    GEODEBUG( "\tpast circle bounds" );
-                    GeoHash tr = _prefix;
-                    tr.move( 1 , 1 );
-                    if ( _g->distance( tr , _start ) > _maxDistance )
+
+                Point ll (_g, _prefix);
+                GeoHash trHash = _prefix;
+                trHash.move( 1 , 1 );
+                Point tr (_g, trHash);
+                double sideLen = fabs(tr._x - ll._x);
+
+                if (sideLen > _maxDistance){ // circle must be contained by surrounding squares
+                    if ( (ll._x + _maxDistance < _startPt._x && ll._y + _maxDistance < _startPt._y) && 
+                         (tr._x - _maxDistance > _startPt._x && tr._y - _maxDistance > _startPt._y) )
+                    {
+                        GEODEBUG("square fully contains circle");
+                        _state = DONE;
+                    } else if (_prefix.getBits() > 1){
+                        GEODEBUG("checking surrounding squares");
                         _state = DOING_AROUND;
-                    else
+                    } else {
+                        GEODEBUG("using simple search");
                         _prefix = _prefix.up();
-                }
-                else
+                    }
+                } else {
                     _prefix = _prefix.up();
+                }
+
                 return;
             }
             
-            if ( _state == DOING_AROUND ){
-                _state = DONE;
+            /* Clients are expected to use moreToDo before calling
+             * fillStack, so DONE is checked for there. If any more
+             * State values are defined, you should handle them
+             * here. */ 
+            assert(0);
+        }
+
+        bool needToCheckBox(const GeoHash& prefix){
+            Point ll (_g, prefix);
+            if (fabs(ll._x - _startPt._x) <= _maxDistance) return true;
+            if (fabs(ll._y - _startPt._y) <= _maxDistance) return true;
+
+            GeoHash trHash = _prefix;
+            trHash.move( 1 , 1 );
+            Point tr (_g, trHash);
+
+            if (fabs(tr._x - _startPt._x) <= _maxDistance) return true;
+            if (fabs(tr._y - _startPt._y) <= _maxDistance) return true;
+
+            return false;
+        }
+
+        void getPointsForPrefix(const GeoHash& prefix){
+            if ( ! BtreeLocation::initial( *_id , _spec , _min , _max , prefix , _found , this ) ){
                 return;
             }
+
+            while ( _min.hasPrefix( prefix ) && _min.advance( -1 , _found , this ) );
+            while ( _max.hasPrefix( prefix ) && _max.advance( 1 , _found , this ) );
         }
+
         
         virtual bool checkDistance( const GeoHash& h , double& d ){
             d = _g->distance( _start , h );
             GEODEBUG( "\t " << h << "\t" << d );
-            return d <= ( _maxDistance + .01 );
+            return d <= _maxDistance;
         }
 
         GeoHash _start;
+        Point _startPt;
         double _maxDistance;
         
         int _found;
@@ -1372,7 +1264,7 @@ namespace mongo {
             _found = 0;
 
             Point center = _want.center();
-            _prefix = _g->_hash( center._x , center._y );
+            _prefix = _g->hash( center._x , center._y );
             
             GEODEBUG( "center : " << center.toString() << "\t" << _prefix );
 
@@ -1465,7 +1357,7 @@ namespace mongo {
     };    
 
 
-    auto_ptr<Cursor> Geo2dType::newCursor( const BSONObj& query , const BSONObj& order , int numWanted ) const {
+    shared_ptr<Cursor> Geo2dType::newCursor( const BSONObj& query , const BSONObj& order , int numWanted ) const {
         if ( numWanted < 0 )
             numWanted = numWanted * -1;
         else if ( numWanted == 0 )
@@ -1483,7 +1375,8 @@ namespace mongo {
             
             switch ( e.embeddedObject().firstElement().getGtLtOp() ){
             case BSONObj::opNEAR: {
-                e = e.embeddedObject().firstElement();
+                BSONObj n = e.embeddedObject();
+                e = n.firstElement();
                 double maxDistance = numeric_limits<double>::max();
                 if ( e.isABSONObj() && e.embeddedObject().nFields() > 2 ){
                     BSONObjIterator i(e.embeddedObject());
@@ -1493,9 +1386,14 @@ namespace mongo {
                     if ( e.isNumber() )
                         maxDistance = e.numberDouble();
                 }
+                {
+                    BSONElement e = n["$maxDistance"];
+                    if ( e.isNumber() )
+                        maxDistance = e.numberDouble();
+                }
                 shared_ptr<GeoSearch> s( new GeoSearch( this , _tohash(e) , numWanted , query , maxDistance ) );
                 s->exec();
-                auto_ptr<Cursor> c;
+                shared_ptr<Cursor> c;
                 c.reset( new GeoSearchCursor( s ) );
                 return c;   
             }
@@ -1506,13 +1404,13 @@ namespace mongo {
                 string type = e.fieldName();
                 if ( type == "$center" ){
                     uassert( 13059 , "$center has to take an object or array" , e.isABSONObj() );
-                    auto_ptr<Cursor> c;
+                    shared_ptr<Cursor> c;
                     c.reset( new GeoCircleBrowse( this , e.embeddedObjectUserCheck() , query ) );
                     return c;   
                 }
                 else if ( type == "$box" ){
                     uassert( 13065 , "$box has to take an object or array" , e.isABSONObj() );
-                    auto_ptr<Cursor> c;
+                    shared_ptr<Cursor> c;
                     c.reset( new GeoBoxBrowse( this , e.embeddedObjectUserCheck() , query ) );
                     return c;   
                 }
@@ -1533,11 +1431,12 @@ namespace mongo {
     class Geo2dFindNearCmd : public Command {
     public:
         Geo2dFindNearCmd() : Command( "geoNear" ){}
-        virtual LockType locktype(){ return READ; } 
-        bool slaveOk() { return true; }
+        virtual LockType locktype() const { return READ; } 
+        bool slaveOk() const { return true; }
+        void help(stringstream& h) const { h << "http://www.mongodb.org/display/DOCS/Geospatial+Indexing#GeospatialIndexing-geoNearCommand"; }
         bool slaveOverrideOk() { return true; }
-        bool run(const char * stupidns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl){
-            string ns = nsToDatabase( stupidns ) + "." + cmdObj.firstElement().valuestr();
+        bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl){
+            string ns = dbname + "." + cmdObj.firstElement().valuestr();
 
             NamespaceDetails * d = nsdetails( ns.c_str() );
             if ( ! d ){
@@ -1545,25 +1444,20 @@ namespace mongo {
                 return false;
             }
 
-            int geoIdx = -1;
-            {
-                NamespaceDetails::IndexIterator ii = d->ii();
-                while ( ii.more() ){
-                    IndexDetails& id = ii.next();
-                    if ( id.getSpec().getTypeName() == GEO2DNAME ){
-                        if ( geoIdx >= 0 ){
-                            errmsg = "2 geo indexes :(";
-                            return false;
-                        }
-                        geoIdx = ii.pos() - 1;
-                    }
-                }
+            vector<int> idxs;
+            d->findIndexByType( GEO2DNAME , idxs );
+            
+            if ( idxs.size() > 1 ){
+                errmsg = "more than 1 geo indexes :(";
+                return false;
             }
             
-            if ( geoIdx < 0 ){
+            if ( idxs.size() == 0 ){
                 errmsg = "no geo index :(";
                 return false;
             }
+
+            int geoIdx = idxs[0];
             
             result.append( "ns" , ns );
 
@@ -1590,7 +1484,7 @@ namespace mongo {
             GeoSearch gs( g , n , numWanted , filter , maxDistance );
 
             if ( cmdObj["start"].type() == String){
-                GeoHash start = (string) cmdObj["start"].valuestr();
+                GeoHash start ((string) cmdObj["start"].valuestr());
                 gs._start = start;
             }
             
@@ -1624,6 +1518,7 @@ namespace mongo {
             stats.appendNumber( "nscanned" , gs._hopper->_lookedAt );
             stats.appendNumber( "objectsLoaded" , gs._hopper->_objectsLoaded );
             stats.append( "avgDistance" , totalDistance / x );
+            stats.append( "maxDistance" , gs._hopper->farthest() );
             stats.done();
             
             return true;
@@ -1634,11 +1529,11 @@ namespace mongo {
     class GeoWalkCmd : public Command {
     public:
         GeoWalkCmd() : Command( "geoWalk" ){}
-        virtual LockType locktype(){ return READ; } 
-        bool slaveOk() { return true; }
+        virtual LockType locktype() const { return READ; } 
+        bool slaveOk() const { return true; }
         bool slaveOverrideOk() { return true; }
-        bool run(const char * stupidns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl){
-            string ns = nsToDatabase( stupidns ) + "." + cmdObj.firstElement().valuestr();
+        bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl){
+            string ns = dbname + "." + cmdObj.firstElement().valuestr();
 
             NamespaceDetails * d = nsdetails( ns.c_str() );
             if ( ! d ){
diff --git a/db/geo/core.h b/db/geo/core.h
new file mode 100644
index 0000000..13f3636
--- /dev/null
+++ b/db/geo/core.h
@@ -0,0 +1,427 @@
+// core.h
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include "../../pch.h"
+#include "../jsobj.h"
+
+#include <cmath>
+
+#ifndef M_PI
+#  define M_PI 3.14159265358979323846
+#endif
+
+namespace mongo {
+
+    class GeoBitSets {
+    public:
+        GeoBitSets(){
+            for ( int i=0; i<32; i++ ){
+                masks32[i] = ( 1 << ( 31 - i ) );
+            }
+            for ( int i=0; i<64; i++ ){
+                masks64[i] = ( 1LL << ( 63 - i ) );
+            }
+            
+            for ( unsigned i=0; i<16; i++ ){
+                unsigned fixed = 0;
+                for ( int j=0; j<4; j++ ){
+                    if ( i & ( 1 << j ) )
+                        fixed |= ( 1 << ( j * 2 ) );
+                }
+                hashedToNormal[fixed] = i;
+            }
+            
+        }
+        int masks32[32];
+        long long masks64[64];
+
+        unsigned hashedToNormal[256];
+    };
+
+    extern GeoBitSets geoBitSets;
+    
+    class GeoHash {
+    public:
+        GeoHash()
+            : _hash(0),_bits(0){
+        }
+
+        explicit GeoHash( const char * hash ){
+            init( hash );
+        }
+
+        explicit GeoHash( const string& hash ){
+            init( hash );
+        }
+
+        explicit GeoHash( const BSONElement& e , unsigned bits=32 ){
+            _bits = bits;
+            if ( e.type() == BinData ){
+                int len = 0;
+                _copy( (char*)&_hash , e.binData( len ) );
+                assert( len == 8 );
+                _bits = bits;
+            }
+            else {
+                cout << "GeoHash cons e : " << e << endl;
+                uassert(13047,"wrong type for geo index. if you're using a pre-release version, need to rebuild index",0);
+            }
+            _fix();
+        }
+        
+        GeoHash( unsigned x , unsigned y , unsigned bits=32){
+            init( x , y , bits );
+        }
+
+        GeoHash( const GeoHash& old ){
+            _hash = old._hash;
+            _bits = old._bits;
+        }
+
+        GeoHash( long long hash , unsigned bits )
+            : _hash( hash ) , _bits( bits ){
+            _fix();
+        }
+
+        void init( unsigned x , unsigned y , unsigned bits ){
+            assert( bits <= 32 );
+            _hash = 0;
+            _bits = bits;
+            for ( unsigned i=0; i<bits; i++ ){
+                if ( isBitSet( x , i ) ) _hash |= geoBitSets.masks64[i*2];
+                if ( isBitSet( y , i ) ) _hash |= geoBitSets.masks64[(i*2)+1];
+            }
+        }
+
+        void unhash_fast( unsigned& x , unsigned& y ) const {
+            x = 0;
+            y = 0;
+            char * c = (char*)(&_hash);
+            for ( int i=0; i<8; i++ ){
+                unsigned t = (unsigned)(c[i]) & 0x55;
+                y |= ( geoBitSets.hashedToNormal[t] << (4*(i)) );
+
+                t = ( (unsigned)(c[i]) >> 1 ) & 0x55;
+                x |= ( geoBitSets.hashedToNormal[t] << (4*(i)) );
+            }
+        }
+
+        void unhash_slow( unsigned& x , unsigned& y ) const {
+            x = 0;
+            y = 0;
+            for ( unsigned i=0; i<_bits; i++ ){
+                if ( getBitX(i) )
+                    x |= geoBitSets.masks32[i];
+                if ( getBitY(i) )
+                    y |= geoBitSets.masks32[i];
+            }
+        }
+
+        void unhash( unsigned& x , unsigned& y ) const {
+            unhash_fast( x , y );
+        }
+
+        /**
+         * @param 0 = high
+         */
+        static bool isBitSet( unsigned val , unsigned  bit ){
+            return geoBitSets.masks32[bit] & val;
+        }
+        
+        GeoHash up() const {
+            return GeoHash( _hash , _bits - 1 );
+        }
+        
+        bool hasPrefix( const GeoHash& other ) const {
+            assert( other._bits <= _bits );
+            if ( other._bits == 0 )
+                return true;
+            long long x = other._hash ^ _hash;
+            x = x >> (64-(other._bits*2));
+            return x == 0;
+        }
+        
+
+        string toString() const { 
+            StringBuilder buf( _bits * 2 );
+            for ( unsigned x=0; x<_bits*2; x++ )
+                buf.append( _hash & geoBitSets.masks64[x] ? "1" : "0" );
+            return buf.str();
+        }
+
+        string toStringHex1() const {
+            stringstream ss;
+            ss << hex << _hash;
+            return ss.str();
+        }
+
+        void init( const string& s ){
+            _hash = 0;
+            _bits = s.size() / 2;
+            for ( unsigned pos=0; pos<s.size(); pos++ )
+                if ( s[pos] == '1' )
+                    setBit( pos , 1 );
+        }
+
+        void setBit( unsigned pos , bool one ){
+            assert( pos < _bits * 2 );
+            if ( one )
+                _hash |= geoBitSets.masks64[pos];
+            else if ( _hash & geoBitSets.masks64[pos] )
+                _hash &= ~geoBitSets.masks64[pos];
+        }
+        
+        bool getBit( unsigned pos ) const {
+            return _hash & geoBitSets.masks64[pos];
+        }
+
+        bool getBitX( unsigned pos ) const {
+            assert( pos < 32 );
+            return getBit( pos * 2 );
+        }
+
+        bool getBitY( unsigned pos ) const {
+            assert( pos < 32 );
+            return getBit( ( pos * 2 ) + 1 );
+        }
+        
+        BSONObj wrap() const {
+            BSONObjBuilder b(20);
+            append( b , "" );
+            BSONObj o = b.obj();
+            assert( o.objsize() == 20 );
+            return o;
+        }
+
+        bool constrains() const {
+            return _bits > 0;
+        }
+        
+        void move( int x , int y ){
+            assert( _bits );
+            _move( 0 , x );
+            _move( 1 , y );
+        }
+
+        void _move( unsigned offset , int d ){
+            if ( d == 0 )
+                return;
+            assert( d <= 1 && d>= -1 ); // TEMP
+            
+            bool from, to;
+            if ( d > 0 ){
+                from = 0;
+                to = 1;
+            }
+            else {
+                from = 1;
+                to = 0;
+            }
+
+            unsigned pos = ( _bits * 2 ) - 1;
+            if ( offset == 0 )
+                pos--;
+            while ( true ){
+                if ( getBit(pos) == from ){
+                    setBit( pos , to );
+                    return;
+                }
+
+                if ( pos < 2 ){
+                    // overflow
+                    for ( ; pos < ( _bits * 2 ) ; pos += 2 ){
+                        setBit( pos , from );
+                    }
+                    return;
+                }
+                
+                setBit( pos , from );
+                pos -= 2;
+            }
+            
+            assert(0);
+        }
+
+        GeoHash& operator=(const GeoHash& h) { 
+            _hash = h._hash;
+            _bits = h._bits;
+            return *this;
+        }
+        
+        bool operator==(const GeoHash& h ){
+            return _hash == h._hash && _bits == h._bits;
+        }
+
+        GeoHash& operator+=( const char * s ) {
+            unsigned pos = _bits * 2;
+            _bits += strlen(s) / 2;
+            assert( _bits <= 32 );
+            while ( s[0] ){
+                if ( s[0] == '1' )
+                    setBit( pos , 1 );
+                pos++;
+                s++;
+            }
+
+            return *this;
+        }
+
+        GeoHash operator+( const char * s ) const {
+            GeoHash n = *this;
+            n+=s;
+            return n;
+        }
+      
+        void _fix(){
+            static long long FULL = 0xFFFFFFFFFFFFFFFFLL;
+            long long mask = FULL << ( 64 - ( _bits * 2 ) );
+            _hash &= mask;
+        }
+        
+        void append( BSONObjBuilder& b , const char * name ) const {
+            char buf[8];
+            _copy( buf , (char*)&_hash );
+            b.appendBinData( name , 8 , bdtCustom , buf );
+        }
+        
+        long long getHash() const {
+            return _hash;
+        }
+
+        unsigned getBits() const {
+            return _bits;
+        }
+
+        GeoHash commonPrefix( const GeoHash& other ) const {
+            unsigned i=0;
+            for ( ; i<_bits && i<other._bits; i++ ){
+                if ( getBitX( i ) == other.getBitX( i ) &&
+                     getBitY( i ) == other.getBitY( i ) )
+                    continue;
+                break;
+            }
+            return GeoHash(_hash,i);
+        }
+
+    private:
+
+        void _copy( char * dst , const char * src ) const {
+            for ( unsigned a=0; a<8; a++ ){
+                dst[a] = src[7-a];
+            }
+        }
+
+        long long _hash;
+        unsigned _bits; // bits per field, so 1 to 32
+    };
+
+    inline ostream& operator<<( ostream &s, const GeoHash &h ){
+        s << h.toString();
+        return s;
+    } 
+
+    class GeoConvert {
+    public:
+        virtual ~GeoConvert(){}
+
+        virtual void unhash( const GeoHash& h , double& x , double& y ) const = 0;
+        virtual GeoHash hash( double x , double y ) const = 0;
+    };
+
+    class Point {
+    public:
+        
+        Point( const GeoConvert * g , const GeoHash& hash ){
+            g->unhash( hash , _x , _y );
+        }
+        
+        explicit Point( const BSONElement& e ){
+            BSONObjIterator i(e.Obj());
+            _x = i.next().number();
+            _y = i.next().number();
+        }
+
+        explicit Point( const BSONObj& o ){
+            BSONObjIterator i(o);
+            _x = i.next().number();
+            _y = i.next().number();
+        }
+
+        Point( double x , double y )
+            : _x( x ) , _y( y ){
+        }
+        
+        Point() : _x(0),_y(0){
+        }
+
+        GeoHash hash( const GeoConvert * g ){
+            return g->hash( _x , _y );
+        }
+
+        double distance( const Point& p ) const {
+            double a = _x - p._x;
+            double b = _y - p._y;
+            return sqrt( ( a * a ) + ( b * b ) );
+        }
+        
+        string toString() const {
+            StringBuilder buf(32);
+            buf << "(" << _x << "," << _y << ")";
+            return buf.str();
+  
+        }
+
+        double _x;
+        double _y;
+    };
+
+
+    extern double EARTH_RADIUS_KM;
+    extern double EARTH_RADIUS_MILES;
+
+    // WARNING: _x and _y MUST be longitude and latitude in that order
+    // note: multiply by earth radius for distance
+    inline double spheredist_rad( const Point& p1, const Point& p2 ) {
+        // this uses the n-vector formula: http://en.wikipedia.org/wiki/N-vector
+        // If you try to match the code to the formula, note that I inline the cross-product.
+        // TODO: optimize with SSE
+
+        double sin_x1(sin(p1._x)), cos_x1(cos(p1._x));
+        double sin_y1(sin(p1._y)), cos_y1(cos(p1._y));
+        double sin_x2(sin(p2._x)), cos_x2(cos(p2._x));
+        double sin_y2(sin(p2._y)), cos_y2(cos(p2._y));
+        
+        double cross_prod = 
+            (cos_y1*cos_x1 * cos_y2*cos_x2) +
+            (cos_y1*sin_x1 * cos_y2*sin_x2) +
+            (sin_y1        * sin_y2);
+
+        return acos(cross_prod);
+    }
+
+    // note: return is still in radians as that can be multiplied by radius to get arc length
+    inline double spheredist_deg( const Point& p1, const Point& p2 ) {
+        return spheredist_rad(
+                    Point( p1._x * (M_PI/180), p1._y * (M_PI/180)),
+                    Point( p2._x * (M_PI/180), p2._y * (M_PI/180))
+               );
+    }
+
+}
diff --git a/db/geo/haystack.cpp b/db/geo/haystack.cpp
new file mode 100644
index 0000000..4a1d4a7
--- /dev/null
+++ b/db/geo/haystack.cpp
@@ -0,0 +1,317 @@
+// db/geo/haystack.cpp
+
+/**
+ *    Copyright (C) 2008 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "pch.h"
+#include "../namespace.h"
+#include "../jsobj.h"
+#include "../index.h"
+#include "../../util/unittest.h"
+#include "../commands.h"
+#include "../pdfile.h"
+#include "../btree.h"
+#include "../curop.h"
+#include "../matcher.h"
+#include "core.h"
+
+#define GEOQUADDEBUG(x)
+//#define GEOQUADDEBUG(x) cout << x << endl
+
+/**
+ * this is a geo based search piece, which is different than regular geo lookup
+ * this is useful when you want to look for something within a region where the ratio is low
+ * works well for search for restaurants withing 25 miles with a certain name
+ * should not be used for finding the closest restaurants that are open
+ */
+namespace mongo {
+    
+    string GEOSEARCHNAME = "geoHaystack";
+    
+    class GeoHaystackSearchHopper {
+    public:
+        GeoHaystackSearchHopper( const BSONObj& n , double maxDistance , unsigned limit , const string& geoField )
+            : _near( n ) , _maxDistance( maxDistance ) , _limit( limit ) , _geoField(geoField){
+            
+        }
+        
+        void got( const DiskLoc& loc ){
+            Point p( loc.obj().getFieldDotted( _geoField ) );
+            if ( _near.distance( p ) > _maxDistance )
+                return;
+            _locs.push_back( loc );
+        }
+
+        int append( BSONArrayBuilder& b ){
+            for ( unsigned i=0; i<_locs.size() && i<_limit; i++ )
+                b.append( _locs[i].obj() );
+            return _locs.size();
+        }
+        
+        Point _near;
+        double _maxDistance;
+        unsigned _limit;
+        string _geoField;
+
+        vector<DiskLoc> _locs;
+    };
+
+    class GeoHaystackSearchIndex : public IndexType {
+        
+    public:
+            
+        GeoHaystackSearchIndex( const IndexPlugin* plugin , const IndexSpec* spec )
+            : IndexType( plugin , spec ){
+             
+            BSONElement e = spec->info["bucketSize"];
+            uassert( 13321 , "need bucketSize" , e.isNumber() );
+            _bucketSize = e.numberDouble();
+            
+            BSONObjBuilder orderBuilder;
+            
+            BSONObjIterator i( spec->keyPattern );
+            while ( i.more() ){
+                BSONElement e = i.next();
+                if ( e.type() == String && GEOSEARCHNAME == e.valuestr() ){
+                    uassert( 13314 , "can't have 2 geo fields" , _geo.size() == 0 );
+                    uassert( 13315 , "2d has to be first in index" , _other.size() == 0 );
+                    _geo = e.fieldName();
+                }
+                else {
+                    _other.push_back( e.fieldName() );
+                }
+                orderBuilder.append( "" , 1 );
+            }
+            
+            uassert( 13316 , "no geo field specified" , _geo.size() );
+            uassert( 13317 , "no other fields specified" , _other.size() );
+            uassert( 13326 , "quadrant search can only have 1 other field for now" , _other.size() == 1 );
+            _order = orderBuilder.obj();
+        }
+        
+        int hash( const BSONElement& e ) const {
+            uassert( 13322 , "not a number" , e.isNumber() );
+            return hash( e.numberDouble() );
+        }
+
+        int hash( double d ) const {
+            d += 180;
+            d /= _bucketSize;
+            return (int)d;
+        }
+
+        string makeString( int hashedX , int hashedY ) const {
+            stringstream ss;
+            ss << hashedX << "_" << hashedY;
+            return ss.str();
+        }
+
+        void _add( const BSONObj& obj, const string& root , const BSONElement& e , BSONObjSetDefaultOrder& keys ) const {
+            BSONObjBuilder buf;
+            buf.append( "" , root );
+            if ( e.eoo() )
+                buf.appendNull( "" );
+            else
+                buf.appendAs( e , "" );
+            
+            BSONObj key = buf.obj();
+            GEOQUADDEBUG( obj << "\n\t" << root << "\n\t" << key );
+            keys.insert( key );
+        }
+
+        void getKeys( const BSONObj &obj, BSONObjSetDefaultOrder &keys ) const {
+            
+            BSONElement loc = obj.getFieldDotted( _geo );
+            if ( loc.eoo() )
+                return;
+                
+            uassert( 13323 , "latlng not an array" , loc.isABSONObj() );
+            string root;
+            {
+                BSONObjIterator i( loc.Obj() );
+                BSONElement x = i.next();
+                BSONElement y = i.next();
+                root = makeString( hash(x) , hash(y) );
+            }
+            
+            
+            assert( _other.size() == 1 );
+            
+            BSONElementSet all;
+            obj.getFieldsDotted( _other[0] , all );
+            
+            if ( all.size() == 0 ){
+                _add( obj , root , BSONElement() , keys );
+            }
+            else {
+                for ( BSONElementSet::iterator i=all.begin(); i!=all.end(); ++i ){
+                    _add( obj , root , *i , keys );
+                }
+            }
+            
+        }
+            
+        shared_ptr<Cursor> newCursor( const BSONObj& query , const BSONObj& order , int numWanted ) const {
+            shared_ptr<Cursor> c;
+            assert(0);
+            return c;
+        }
+            
+        void searchCommand( NamespaceDetails* nsd , int idxNo ,  
+                            const BSONObj& n /*near*/ , double maxDistance , const BSONObj& search , 
+                            BSONObjBuilder& result , unsigned limit ){
+         
+            Timer t;
+
+            log(1) << "SEARCH near:" << n << " maxDistance:" << maxDistance << " search: " << search << endl;
+            int x,y;
+            {
+                BSONObjIterator i( n );
+                x = hash( i.next() );
+                y = hash( i.next() );
+            }
+            int scale = (int)ceil( maxDistance / _bucketSize );
+                
+            GeoHaystackSearchHopper hopper(n,maxDistance,limit,_geo);
+                
+            long long btreeMatches = 0;
+
+            for ( int a=-scale; a<=scale; a++ ){
+                for ( int b=-scale; b<=scale; b++ ){
+
+                    BSONObjBuilder bb;
+                    bb.append( "" , makeString( x + a , y + b ) );
+                    for ( unsigned i=0; i<_other.size(); i++ ){
+                        BSONElement e = search.getFieldDotted( _other[i] );
+                        if ( e.eoo() )
+                            bb.appendNull( "" );
+                        else
+                            bb.appendAs( e , "" );
+                    }
+                    
+                    BSONObj key = bb.obj();
+                    
+                    GEOQUADDEBUG( "KEY: " << key );
+                    
+                    set<DiskLoc> thisPass;
+                    BtreeCursor cursor( nsd , idxNo , *getDetails() , key , key , true , 1 );
+                    while ( cursor.ok() ){
+                        pair<set<DiskLoc>::iterator, bool> p = thisPass.insert( cursor.currLoc() );
+                        if ( p.second ){
+                            hopper.got( cursor.currLoc() );
+                            GEOQUADDEBUG( "\t" << cursor.current() );
+                            btreeMatches++;
+                        }
+                        cursor.advance();
+                    }
+                }
+
+            }
+
+            BSONArrayBuilder arr( result.subarrayStart( "results" ) );                
+            int num = hopper.append( arr );
+            arr.done();
+            
+            {
+                BSONObjBuilder b( result.subobjStart( "stats" ) );
+                b.append( "time" , t.millis() );
+                b.appendNumber( "btreeMatches" , btreeMatches );
+                b.append( "n" , num );
+                b.done();
+            }
+        }
+
+        const IndexDetails* getDetails() const {
+            return _spec->getDetails();
+        }
+            
+        string _geo;
+        vector<string> _other;
+        
+        BSONObj _order;
+
+        double _bucketSize;
+    };
+        
+    class GeoHaystackSearchIndexPlugin : public IndexPlugin {
+    public:
+        GeoHaystackSearchIndexPlugin() : IndexPlugin( GEOSEARCHNAME ){
+        }
+        
+        virtual IndexType* generate( const IndexSpec* spec ) const {
+            return new GeoHaystackSearchIndex( this , spec );
+        }
+
+    } nameIndexPlugin;
+
+
+    class GeoHaystackSearchCommand : public Command {
+        public:
+        GeoHaystackSearchCommand() : Command( "geoSearch" ){}
+        virtual LockType locktype() const { return READ; } 
+        bool slaveOk() const { return true; }
+        bool slaveOverrideOk() const { return true; }
+        bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl){
+                
+            string ns = dbname + "." + cmdObj.firstElement().valuestr();
+                
+            NamespaceDetails * d = nsdetails( ns.c_str() );
+            if ( ! d ){
+                errmsg = "can't find ns";
+                return false;
+            }
+            
+            vector<int> idxs;
+            d->findIndexByType( GEOSEARCHNAME , idxs );
+            if ( idxs.size() == 0 ){
+                errmsg = "no geoSearch index";
+                return false;
+            }
+            if ( idxs.size() > 1 ){
+                errmsg = "more than 1 geosearch index";
+                return false;
+            }
+                
+            int idxNum = idxs[0];
+            
+            IndexDetails& id = d->idx( idxNum );
+            GeoHaystackSearchIndex * si = (GeoHaystackSearchIndex*)id.getSpec().getType();
+            assert( &id == si->getDetails() );         
+                
+            BSONElement n = cmdObj["near"];
+            BSONElement maxDistance = cmdObj["maxDistance"];
+            BSONElement search = cmdObj["search"];
+
+            uassert( 13318 , "near needs to be an array" , n.isABSONObj() );
+            uassert( 13319 , "maxDistance needs a number" , maxDistance.isNumber() );
+            uassert( 13320 , "search needs to be an object" , search.type() == Object );
+            
+            unsigned limit = 50;
+            if ( cmdObj["limit"].isNumber() )
+                limit = (unsigned)cmdObj["limit"].numberInt();
+
+            si->searchCommand( d , idxNum , n.Obj() , maxDistance.numberDouble() , search.Obj() , result , limit );
+            
+            return 1;
+        }
+        
+        } nameSearchCommand;
+
+
+        
+
+    
+}
diff --git a/db/helpers/dblogger.h b/db/helpers/dblogger.h
new file mode 100644
index 0000000..572169b
--- /dev/null
+++ b/db/helpers/dblogger.h
@@ -0,0 +1,31 @@
+// @file db.logger.h
+
+/*
+ *    Copyright (C) 2010 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+namespace mongo { 
+
+    /** helper to log (and read log) of a capped collection in the database */
+    class DBLogger {
+        bool _inited;
+    public:
+        const string _ns;
+        DBLogger(string ns) : _inited(false), _ns(ns){ }
+    };
+
+}
diff --git a/db/index.cpp b/db/index.cpp
index 6931d93..04eca73 100644
--- a/db/index.cpp
+++ b/db/index.cpp
@@ -16,7 +16,7 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "namespace.h"
 #include "index.h"
 #include "btree.h"
@@ -25,32 +25,6 @@
 
 namespace mongo {
 
-    map<string,IndexPlugin*> * IndexPlugin::_plugins;
-
-    IndexType::IndexType( const IndexPlugin * plugin , const IndexSpec * spec )
-        : _plugin( plugin ) , _spec( spec ){
-        
-    }
-
-    IndexType::~IndexType(){
-    }
-    
-    const BSONObj& IndexType::keyPattern() const { 
-        return _spec->keyPattern; 
-    }
-
-    IndexPlugin::IndexPlugin( const string& name )
-        : _name( name ){
-        if ( ! _plugins )
-            _plugins = new map<string,IndexPlugin*>();
-        (*_plugins)[name] = this;
-    }
-    
-    int IndexType::compare( const BSONObj& l , const BSONObj& r ) const {
-        return l.woCompare( r , _spec->keyPattern );
-    }
-
-
     int removeFromSysIndexes(const char *ns, const char *idxName) { 
         string system_indexes = cc().database()->name + ".system.indexes";
         BSONObjBuilder b;
@@ -111,172 +85,6 @@ namespace mongo {
         wassert( n == 1 );
     }
     
-    void IndexSpec::reset( const IndexDetails * details ){
-        _details = details;
-        reset( details->info );
-    }
-
-    void IndexSpec::reset( const DiskLoc& loc ){
-        info = loc.obj();
-        keyPattern = info["key"].embeddedObjectUserCheck();
-        if ( keyPattern.objsize() == 0 ) {
-            out() << info.toString() << endl;
-            assert(false);
-        }
-        _init();
-    }
-
-
-    void IndexSpec::_init(){
-        assert( keyPattern.objsize() );
-        
-        string pluginName = "";
-
-        BSONObjIterator i( keyPattern );
-        BSONObjBuilder nullKeyB;
-        while( i.more() ) {
-            BSONElement e = i.next();
-            _fieldNames.push_back( e.fieldName() );
-            _fixed.push_back( BSONElement() );
-            nullKeyB.appendNull( "" );
-            if ( e.type() == String ){
-                uassert( 13007 , "can only have 1 index plugin / bad index key pattern" , pluginName.size() == 0 );
-                pluginName = e.valuestr();
-            }
-                
-        }
-        
-        _nullKey = nullKeyB.obj();
-
-        BSONObjBuilder b;
-        b.appendNull( "" );
-        _nullObj = b.obj();
-        _nullElt = _nullObj.firstElement();
-        
-        if ( pluginName.size() ){
-            IndexPlugin * plugin = IndexPlugin::get( pluginName );
-            if ( ! plugin ){
-                log() << "warning: can't find plugin [" << pluginName << "]" << endl;
-            }
-            else {
-                _indexType.reset( plugin->generate( this ) );
-            }
-        }
-        _finishedInit = true;
-    }
-
-    
-    void IndexSpec::getKeys( const BSONObj &obj, BSONObjSetDefaultOrder &keys ) const {
-        if ( _indexType.get() ){
-            _indexType->getKeys( obj , keys );
-            return;
-        }
-        vector<const char*> fieldNames( _fieldNames );
-        vector<BSONElement> fixed( _fixed );
-        _getKeys( fieldNames , fixed , obj, keys );
-        if ( keys.empty() )
-            keys.insert( _nullKey );
-    }
-
-    void IndexSpec::_getKeys( vector<const char*> fieldNames , vector<BSONElement> fixed , const BSONObj &obj, BSONObjSetDefaultOrder &keys ) const {
-        BSONElement arrElt;
-        unsigned arrIdx = ~0;
-        for( unsigned i = 0; i < fieldNames.size(); ++i ) {
-            if ( *fieldNames[ i ] == '\0' )
-                continue;
-            BSONElement e = obj.getFieldDottedOrArray( fieldNames[ i ] );
-            if ( e.eoo() )
-                e = _nullElt; // no matching field
-            if ( e.type() != Array )
-                fieldNames[ i ] = ""; // no matching field or non-array match
-            if ( *fieldNames[ i ] == '\0' )
-                fixed[ i ] = e; // no need for further object expansion (though array expansion still possible)
-            if ( e.type() == Array && arrElt.eoo() ) { // we only expand arrays on a single path -- track the path here
-                arrIdx = i;
-                arrElt = e;
-            }
-            // enforce single array path here
-            uassert( 10088 ,  "cannot index parallel arrays", e.type() != Array || e.rawdata() == arrElt.rawdata() );
-        }
-
-        bool allFound = true; // have we found elements for all field names in the key spec?
-        for( vector<const char*>::const_iterator i = fieldNames.begin(); i != fieldNames.end(); ++i ){
-            if ( **i != '\0' ){
-                allFound = false;
-                break;
-            }
-        }
-        
-        bool insertArrayNull = false;
-
-        if ( allFound ) {
-            if ( arrElt.eoo() ) {
-                // no terminal array element to expand
-                BSONObjBuilder b(_sizeTracker);
-                for( vector< BSONElement >::iterator i = fixed.begin(); i != fixed.end(); ++i )
-                    b.appendAs( *i, "" );
-                keys.insert( b.obj() );
-            } 
-            else {
-                // terminal array element to expand, so generate all keys
-                BSONObjIterator i( arrElt.embeddedObject() );
-                if ( i.more() ){
-                    while( i.more() ) {
-                        BSONObjBuilder b(_sizeTracker);
-                        for( unsigned j = 0; j < fixed.size(); ++j ) {
-                            if ( j == arrIdx )
-                                b.appendAs( i.next(), "" );
-                            else
-                                b.appendAs( fixed[ j ], "" );
-                        }
-                        keys.insert( b.obj() );
-                    }
-                }
-                else if ( fixed.size() > 1 ){
-                    insertArrayNull = true;
-                }
-            }
-        } else {
-            // nonterminal array element to expand, so recurse
-            assert( !arrElt.eoo() );
-            BSONObjIterator i( arrElt.embeddedObject() );
-            if ( i.more() ){
-                while( i.more() ) {
-                    BSONElement e = i.next();
-                    if ( e.type() == Object )
-                        _getKeys( fieldNames, fixed, e.embeddedObject(), keys );
-                }
-            }
-            else {
-                insertArrayNull = true;
-            }
-        }
-        
-        if ( insertArrayNull ){
-            // x : [] - need to insert undefined
-            BSONObjBuilder b(_sizeTracker);
-            for( unsigned j = 0; j < fixed.size(); ++j ) {
-                if ( j == arrIdx ){
-                    b.appendUndefined( "" );
-                }
-                else {
-                    BSONElement e = fixed[j];
-                    if ( e.eoo() )
-                        b.appendNull( "" );
-                    else
-                        b.appendAs( e , "" );
-                }
-            }
-            keys.insert( b.obj() );
-        }
-
-    }
-    
-    /* Pull out the relevant key objects from obj, so we
-       can index them.  Note that the set is multiple elements
-       only when it's a "multikey" array.
-       Keys will be left empty if key not found in the object.
-    */
     void IndexDetails::getKeysFromObject( const BSONObj& obj, BSONObjSetDefaultOrder& keys) const {
         getSpec().getKeys( obj, keys );
     }
@@ -297,7 +105,7 @@ namespace mongo {
         }
     }
 
-    void getIndexChanges(vector<IndexChanges>& v, NamespaceDetails& d, BSONObj newObj, BSONObj oldObj) { 
+    void getIndexChanges(vector<IndexChanges>& v, NamespaceDetails& d, BSONObj newObj, BSONObj oldObj, bool &changedId) { 
         int z = d.nIndexesBeingBuilt();
         v.resize(z);
         NamespaceDetails::IndexIterator i = d.ii();
@@ -311,6 +119,9 @@ namespace mongo {
                 d.setIndexIsMultikey(i);
             setDifference(ch.oldkeys, ch.newkeys, ch.removed);
             setDifference(ch.newkeys, ch.oldkeys, ch.added);
+            if ( ch.removed.size() > 0 && ch.added.size() > 0 && idx.isIdIndex() ) {
+                changedId = true;
+            }
         }
     }
 
@@ -390,7 +201,7 @@ namespace mongo {
                 return false;
             }
             sourceCollection = nsdetails(sourceNS.c_str());
-            log() << "info: creating collection " << sourceNS << " on add index\n";
+            tlog() << "info: creating collection " << sourceNS << " on add index\n";
             assert( sourceCollection );
         }
 
@@ -422,40 +233,20 @@ namespace mongo {
         return true;
     }
 
-    bool anyElementNamesMatch( const BSONObj& a , const BSONObj& b ){
-        BSONObjIterator x(a);
-        while ( x.more() ){
-            BSONElement e = x.next();
-            BSONObjIterator y(b);
-            while ( y.more() ){
-                BSONElement f = y.next();
-                FieldCompareResult res = compareDottedFieldNames( e.fieldName() , f.fieldName() );
-                if ( res == SAME || res == LEFT_SUBFIELD || res == RIGHT_SUBFIELD )
-                    return true;
-            }
-        }
-        return false;
-    }
-    
-    IndexSuitability IndexSpec::suitability( const BSONObj& query , const BSONObj& order ) const {
-        if ( _indexType.get() )
-            return _indexType->suitability( query , order );
-        return _suitability( query , order );
-    }
-    
-    IndexSuitability IndexSpec::_suitability( const BSONObj& query , const BSONObj& order ) const {
-        // TODO: optimize
-        if ( anyElementNamesMatch( keyPattern , query ) == 0 && anyElementNamesMatch( keyPattern , order ) == 0 )
-            return USELESS;
-        return HELPFUL;
-    }
 
-    IndexSuitability IndexType::suitability( const BSONObj& query , const BSONObj& order ) const {
-        return _spec->_suitability( query , order );
+    void IndexSpec::reset( const IndexDetails * details ){
+        _details = details;
+        reset( details->info );
     }
 
-    bool IndexType::scanAndOrderRequired( const BSONObj& query , const BSONObj& order ) const {
-        return ! order.isEmpty();
+    void IndexSpec::reset( const DiskLoc& loc ){
+        info = loc.obj();
+        keyPattern = info["key"].embeddedObjectUserCheck();
+        if ( keyPattern.objsize() == 0 ) {
+            out() << info.toString() << endl;
+            assert(false);
+        }
+        _init();
     }
 
 }
diff --git a/db/index.h b/db/index.h
index 6965f11..a2d7e7e 100644
--- a/db/index.h
+++ b/db/index.h
@@ -18,157 +18,13 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "diskloc.h"
 #include "jsobj.h"
-#include <map>
+#include "indexkey.h"
 
 namespace mongo {
 
-    class IndexSpec;
-    class IndexType; // TODO: this name sucks
-    class IndexPlugin;
-    class IndexDetails;
-
-    enum IndexSuitability { USELESS = 0 , HELPFUL = 1 , OPTIMAL = 2 };
-
-    /**
-     * this represents an instance of a index plugin
-     * done this way so parsing, etc... can be cached
-     * so if there is a FTS IndexPlugin, for each index using FTS
-     * there will be 1 of these, and it can have things pre-parsed, etc...
-     */
-    class IndexType : boost::noncopyable {
-    public:
-        IndexType( const IndexPlugin * plugin , const IndexSpec * spec );
-        virtual ~IndexType();
-
-        virtual void getKeys( const BSONObj &obj, BSONObjSetDefaultOrder &keys ) const = 0;
-        virtual auto_ptr<Cursor> newCursor( const BSONObj& query , const BSONObj& order , int numWanted ) const = 0;
-        
-        /** optional op : changes query to match what's in the index */
-        virtual BSONObj fixKey( const BSONObj& in ) { return in; }
-
-        /** optional op : compare 2 objects with regards to this index */
-        virtual int compare( const BSONObj& l , const BSONObj& r ) const;        
-
-        /** @return plugin */
-        const IndexPlugin * getPlugin() const { return _plugin; }
-        
-        const BSONObj& keyPattern() const;
-
-        virtual IndexSuitability suitability( const BSONObj& query , const BSONObj& order ) const ;
-
-        virtual bool scanAndOrderRequired( const BSONObj& query , const BSONObj& order ) const ;
-
-    protected:
-        const IndexPlugin * _plugin;
-        const IndexSpec * _spec;
-    };
-    
-    /**
-     * this represents a plugin
-     * a plugin could be something like full text search, sparse index, etc...
-     * 1 of these exists per type of index per server
-     * 1 IndexType is created per index using this plugin
-     */
-    class IndexPlugin : boost::noncopyable {
-    public:
-        IndexPlugin( const string& name );
-        virtual ~IndexPlugin(){}
-        
-        virtual IndexType* generate( const IndexSpec * spec ) const = 0;
-
-        static IndexPlugin* get( const string& name ){
-            if ( ! _plugins )
-                return 0;
-            map<string,IndexPlugin*>::iterator i = _plugins->find( name );
-            if ( i == _plugins->end() )
-                return 0;
-            return i->second;
-        }
-
-        string getName() const { return _name; }
-    private:
-        string _name;
-        static map<string,IndexPlugin*> * _plugins;
-    };
-
-    /* precomputed details about an index, used for inserting keys on updates
-       stored/cached in NamespaceDetailsTransient, or can be used standalone
-       */
-    class IndexSpec {
-    public:
-        BSONObj keyPattern; // e.g., { name : 1 }
-        BSONObj info; // this is the same as IndexDetails::info.obj()
-        
-        IndexSpec()
-            : _details(0) , _finishedInit(false){
-        }
-
-        IndexSpec( const BSONObj& k , const BSONObj& m = BSONObj() )
-            : keyPattern(k) , info(m) , _details(0) , _finishedInit(false){
-            _init();
-        }
-        
-        /**
-           this is a DiscLoc of an IndexDetails info
-           should have a key field 
-         */
-        IndexSpec( const DiskLoc& loc ){
-            reset( loc );
-        }
-        
-        void reset( const DiskLoc& loc );
-        void reset( const IndexDetails * details );
-        
-        void getKeys( const BSONObj &obj, BSONObjSetDefaultOrder &keys ) const;
-
-        BSONElement missingField() const { return _nullElt; }
-        
-        string getTypeName() const {
-            if ( _indexType.get() )
-                return _indexType->getPlugin()->getName();
-            return "";
-        }
-
-        IndexType* getType() const {
-            return _indexType.get();
-        }
-
-        const IndexDetails * getDetails() const {
-            return _details;
-        }
-
-        IndexSuitability suitability( const BSONObj& query , const BSONObj& order ) const ;
-
-    protected:
-
-        IndexSuitability _suitability( const BSONObj& query , const BSONObj& order ) const ;
-
-        void _getKeys( vector<const char*> fieldNames , vector<BSONElement> fixed , const BSONObj &obj, BSONObjSetDefaultOrder &keys ) const;
-        
-        BSONSizeTracker _sizeTracker;
-
-        vector<const char*> _fieldNames;
-        vector<BSONElement> _fixed;
-        BSONObj _nullKey;
-        
-        BSONObj _nullObj;
-        BSONElement _nullElt;
-        
-        shared_ptr<IndexType> _indexType;
-
-        const IndexDetails * _details;
-        
-        void _init();
-
-    public:
-        bool _finishedInit;
-
-        friend class IndexType;
-    };
-
 	/* Details about a particular index. There is one of these effectively for each object in 
 	   system.namespaces (although this also includes the head pointer, which is not in that 
 	   collection).
@@ -275,7 +131,7 @@ namespace mongo {
         
         const IndexSpec& getSpec() const;
 
-        operator string() const {
+        string toString() const {
             return info.obj().toString();
         }
     };
@@ -300,6 +156,7 @@ namespace mongo {
     };
 
     class NamespaceDetails;
-    void getIndexChanges(vector<IndexChanges>& v, NamespaceDetails& d, BSONObj newObj, BSONObj oldObj);
+    // changedId should be initialized to false
+    void getIndexChanges(vector<IndexChanges>& v, NamespaceDetails& d, BSONObj newObj, BSONObj oldObj, bool &cangedId);
     void dupCheck(vector<IndexChanges>& v, NamespaceDetails& d, DiskLoc curObjLoc);
 } // namespace mongo
diff --git a/db/indexkey.cpp b/db/indexkey.cpp
new file mode 100644
index 0000000..70dd770
--- /dev/null
+++ b/db/indexkey.cpp
@@ -0,0 +1,238 @@
+// index_key.cpp
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "namespace.h"
+#include "index.h"
+#include "btree.h"
+#include "query.h"
+#include "background.h"
+
+namespace mongo {
+
+    map<string,IndexPlugin*> * IndexPlugin::_plugins;
+
+    IndexType::IndexType( const IndexPlugin * plugin , const IndexSpec * spec )
+        : _plugin( plugin ) , _spec( spec ){
+        
+    }
+
+    IndexType::~IndexType(){
+    }
+    
+    const BSONObj& IndexType::keyPattern() const { 
+        return _spec->keyPattern; 
+    }
+
+    IndexPlugin::IndexPlugin( const string& name )
+        : _name( name ){
+        if ( ! _plugins )
+            _plugins = new map<string,IndexPlugin*>();
+        (*_plugins)[name] = this;
+    }
+    
+    int IndexType::compare( const BSONObj& l , const BSONObj& r ) const {
+        return l.woCompare( r , _spec->keyPattern );
+    }
+
+    void IndexSpec::_init(){
+        assert( keyPattern.objsize() );
+        
+        string pluginName = "";
+
+        BSONObjIterator i( keyPattern );
+        BSONObjBuilder nullKeyB;
+        while( i.more() ) {
+            BSONElement e = i.next();
+            _fieldNames.push_back( e.fieldName() );
+            _fixed.push_back( BSONElement() );
+            nullKeyB.appendNull( "" );
+            if ( e.type() == String ){
+                uassert( 13007 , "can only have 1 index plugin / bad index key pattern" , pluginName.size() == 0 );
+                pluginName = e.valuestr();
+            }
+                
+        }
+        
+        _nullKey = nullKeyB.obj();
+
+        BSONObjBuilder b;
+        b.appendNull( "" );
+        _nullObj = b.obj();
+        _nullElt = _nullObj.firstElement();
+        
+        if ( pluginName.size() ){
+            IndexPlugin * plugin = IndexPlugin::get( pluginName );
+            if ( ! plugin ){
+                log() << "warning: can't find plugin [" << pluginName << "]" << endl;
+            }
+            else {
+                _indexType.reset( plugin->generate( this ) );
+            }
+        }
+        _finishedInit = true;
+    }
+
+    
+    void IndexSpec::getKeys( const BSONObj &obj, BSONObjSetDefaultOrder &keys ) const {
+        if ( _indexType.get() ){
+            _indexType->getKeys( obj , keys );
+            return;
+        }
+        vector<const char*> fieldNames( _fieldNames );
+        vector<BSONElement> fixed( _fixed );
+        _getKeys( fieldNames , fixed , obj, keys );
+        if ( keys.empty() )
+            keys.insert( _nullKey );
+    }
+
+    void IndexSpec::_getKeys( vector<const char*> fieldNames , vector<BSONElement> fixed , const BSONObj &obj, BSONObjSetDefaultOrder &keys ) const {
+        BSONElement arrElt;
+        unsigned arrIdx = ~0;
+        for( unsigned i = 0; i < fieldNames.size(); ++i ) {
+            if ( *fieldNames[ i ] == '\0' )
+                continue;
+            BSONElement e = obj.getFieldDottedOrArray( fieldNames[ i ] );
+            if ( e.eoo() )
+                e = _nullElt; // no matching field
+            if ( e.type() != Array )
+                fieldNames[ i ] = ""; // no matching field or non-array match
+            if ( *fieldNames[ i ] == '\0' )
+                fixed[ i ] = e; // no need for further object expansion (though array expansion still possible)
+            if ( e.type() == Array && arrElt.eoo() ) { // we only expand arrays on a single path -- track the path here
+                arrIdx = i;
+                arrElt = e;
+            }
+            // enforce single array path here
+            if ( e.type() == Array && e.rawdata() != arrElt.rawdata() ){
+                stringstream ss;
+                ss << "cannot index parallel arrays [" << e.fieldName() << "] [" << arrElt.fieldName() << "]";
+                uasserted( 10088 ,  ss.str() );
+            }
+        }
+
+        bool allFound = true; // have we found elements for all field names in the key spec?
+        for( vector<const char*>::const_iterator i = fieldNames.begin(); i != fieldNames.end(); ++i ){
+            if ( **i != '\0' ){
+                allFound = false;
+                break;
+            }
+        }
+
+        bool insertArrayNull = false;
+
+        if ( allFound ) {
+            if ( arrElt.eoo() ) {
+                // no terminal array element to expand
+                BSONObjBuilder b(_sizeTracker);
+                for( vector< BSONElement >::iterator i = fixed.begin(); i != fixed.end(); ++i )
+                    b.appendAs( *i, "" );
+                keys.insert( b.obj() );
+            } 
+            else {
+                // terminal array element to expand, so generate all keys
+                BSONObjIterator i( arrElt.embeddedObject() );
+                if ( i.more() ){
+                    while( i.more() ) {
+                        BSONObjBuilder b(_sizeTracker);
+                        for( unsigned j = 0; j < fixed.size(); ++j ) {
+                            if ( j == arrIdx )
+                                b.appendAs( i.next(), "" );
+                            else
+                                b.appendAs( fixed[ j ], "" );
+                        }
+                        keys.insert( b.obj() );
+                    }
+                }
+                else if ( fixed.size() > 1 ){
+                    insertArrayNull = true;
+                }
+            }
+        } else {
+            // nonterminal array element to expand, so recurse
+            assert( !arrElt.eoo() );
+            BSONObjIterator i( arrElt.embeddedObject() );
+            if ( i.more() ){
+                while( i.more() ) {
+                    BSONElement e = i.next();
+                    if ( e.type() == Object ){
+                        _getKeys( fieldNames, fixed, e.embeddedObject(), keys );
+                    }
+                }
+            }
+            else {
+                insertArrayNull = true;
+            }
+        }
+        
+        if ( insertArrayNull ) {
+            // x : [] - need to insert undefined
+            BSONObjBuilder b(_sizeTracker);
+            for( unsigned j = 0; j < fixed.size(); ++j ) {
+                if ( j == arrIdx ){
+                    b.appendUndefined( "" );
+                }
+                else {
+                    BSONElement e = fixed[j];
+                    if ( e.eoo() )
+                        b.appendNull( "" );
+                    else
+                        b.appendAs( e , "" );
+                }
+            }
+            keys.insert( b.obj() );
+        }
+    }
+
+    bool anyElementNamesMatch( const BSONObj& a , const BSONObj& b ){
+        BSONObjIterator x(a);
+        while ( x.more() ){
+            BSONElement e = x.next();
+            BSONObjIterator y(b);
+            while ( y.more() ){
+                BSONElement f = y.next();
+                FieldCompareResult res = compareDottedFieldNames( e.fieldName() , f.fieldName() );
+                if ( res == SAME || res == LEFT_SUBFIELD || res == RIGHT_SUBFIELD )
+                    return true;
+            }
+        }
+        return false;
+    }
+        
+    IndexSuitability IndexSpec::suitability( const BSONObj& query , const BSONObj& order ) const {
+        if ( _indexType.get() )
+            return _indexType->suitability( query , order );
+        return _suitability( query , order );
+    }
+    
+    IndexSuitability IndexSpec::_suitability( const BSONObj& query , const BSONObj& order ) const {
+        // TODO: optimize
+        if ( anyElementNamesMatch( keyPattern , query ) == 0 && anyElementNamesMatch( keyPattern , order ) == 0 )
+            return USELESS;
+        return HELPFUL;
+    }
+
+    IndexSuitability IndexType::suitability( const BSONObj& query , const BSONObj& order ) const {
+        return _spec->_suitability( query , order );
+    }
+
+    bool IndexType::scanAndOrderRequired( const BSONObj& query , const BSONObj& order ) const {
+        return ! order.isEmpty();
+    }
+
+}
diff --git a/db/indexkey.h b/db/indexkey.h
new file mode 100644
index 0000000..e73d9de
--- /dev/null
+++ b/db/indexkey.h
@@ -0,0 +1,174 @@
+// index_key.h
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include "../pch.h"
+#include "diskloc.h"
+#include "jsobj.h"
+#include <map>
+
+namespace mongo {
+
+    class Cursor;
+    class IndexSpec;
+    class IndexType; // TODO: this name sucks
+    class IndexPlugin;
+    class IndexDetails;
+
+    enum IndexSuitability { USELESS = 0 , HELPFUL = 1 , OPTIMAL = 2 };
+
+    /**
+     * this represents an instance of a index plugin
+     * done this way so parsing, etc... can be cached
+     * so if there is a FTS IndexPlugin, for each index using FTS
+     * there will be 1 of these, and it can have things pre-parsed, etc...
+     */
+    class IndexType : boost::noncopyable {
+    public:
+        IndexType( const IndexPlugin * plugin , const IndexSpec * spec );
+        virtual ~IndexType();
+
+        virtual void getKeys( const BSONObj &obj, BSONObjSetDefaultOrder &keys ) const = 0;
+        virtual shared_ptr<Cursor> newCursor( const BSONObj& query , const BSONObj& order , int numWanted ) const = 0;
+        
+        /** optional op : changes query to match what's in the index */
+        virtual BSONObj fixKey( const BSONObj& in ) { return in; }
+
+        /** optional op : compare 2 objects with regards to this index */
+        virtual int compare( const BSONObj& l , const BSONObj& r ) const;        
+
+        /** @return plugin */
+        const IndexPlugin * getPlugin() const { return _plugin; }
+        
+        const BSONObj& keyPattern() const;
+
+        virtual IndexSuitability suitability( const BSONObj& query , const BSONObj& order ) const ;
+
+        virtual bool scanAndOrderRequired( const BSONObj& query , const BSONObj& order ) const ;
+
+    protected:
+        const IndexPlugin * _plugin;
+        const IndexSpec * _spec;
+    };
+    
+    /**
+     * this represents a plugin
+     * a plugin could be something like full text search, sparse index, etc...
+     * 1 of these exists per type of index per server
+     * 1 IndexType is created per index using this plugin
+     */
+    class IndexPlugin : boost::noncopyable {
+    public:
+        IndexPlugin( const string& name );
+        virtual ~IndexPlugin(){}
+        
+        virtual IndexType* generate( const IndexSpec * spec ) const = 0;
+
+        static IndexPlugin* get( const string& name ){
+            if ( ! _plugins )
+                return 0;
+            map<string,IndexPlugin*>::iterator i = _plugins->find( name );
+            if ( i == _plugins->end() )
+                return 0;
+            return i->second;
+        }
+
+        string getName() const { return _name; }
+    private:
+        string _name;
+        static map<string,IndexPlugin*> * _plugins;
+    };
+
+    /* precomputed details about an index, used for inserting keys on updates
+       stored/cached in NamespaceDetailsTransient, or can be used standalone
+       */
+    class IndexSpec {
+    public:
+        BSONObj keyPattern; // e.g., { name : 1 }
+        BSONObj info; // this is the same as IndexDetails::info.obj()
+        
+        IndexSpec()
+            : _details(0) , _finishedInit(false){
+        }
+
+        IndexSpec( const BSONObj& k , const BSONObj& m = BSONObj() )
+            : keyPattern(k) , info(m) , _details(0) , _finishedInit(false){
+            _init();
+        }
+        
+        /**
+           this is a DiscLoc of an IndexDetails info
+           should have a key field 
+         */
+        IndexSpec( const DiskLoc& loc ){
+            reset( loc );
+        }
+        
+        void reset( const DiskLoc& loc );
+        void reset( const IndexDetails * details );
+        
+        void getKeys( const BSONObj &obj, BSONObjSetDefaultOrder &keys ) const;
+
+        BSONElement missingField() const { return _nullElt; }
+        
+        string getTypeName() const {
+            if ( _indexType.get() )
+                return _indexType->getPlugin()->getName();
+            return "";
+        }
+
+        IndexType* getType() const {
+            return _indexType.get();
+        }
+
+        const IndexDetails * getDetails() const {
+            return _details;
+        }
+
+        IndexSuitability suitability( const BSONObj& query , const BSONObj& order ) const ;
+
+    protected:
+
+        IndexSuitability _suitability( const BSONObj& query , const BSONObj& order ) const ;
+
+        void _getKeys( vector<const char*> fieldNames , vector<BSONElement> fixed , const BSONObj &obj, BSONObjSetDefaultOrder &keys ) const;
+        
+        BSONSizeTracker _sizeTracker;
+
+        vector<const char*> _fieldNames;
+        vector<BSONElement> _fixed;
+        BSONObj _nullKey;
+        
+        BSONObj _nullObj;
+        BSONElement _nullElt;
+        
+        shared_ptr<IndexType> _indexType;
+
+        const IndexDetails * _details;
+        
+        void _init();
+
+    public:
+        bool _finishedInit;
+
+        friend class IndexType;
+    };
+
+
+} // namespace mongo
diff --git a/db/instance.cpp b/db/instance.cpp
index d8a76cb..ec3b793 100644
--- a/db/instance.cpp
+++ b/db/instance.cpp
@@ -17,7 +17,7 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "db.h"
 #include "query.h"
 #include "introspect.h"
@@ -27,10 +27,11 @@
 #include "lasterror.h"
 #include "security.h"
 #include "json.h"
-#include "reccache.h"
-#include "replset.h"
+//#include "reccache.h"
+#include "replpair.h"
 #include "../s/d_logic.h"
 #include "../util/file_allocator.h"
+#include "../util/goodies.h"
 #include "cmdline.h"
 #if !defined(_WIN32)
 #include <sys/file.h>
@@ -40,6 +41,9 @@
 
 namespace mongo {
 
+    inline void opread(Message& m) { if( _diaglog.level & 2 ) _diaglog.readop((char *) m.singleData(), m.header()->len); }
+    inline void opwrite(Message& m) { if( _diaglog.level & 1 ) _diaglog.write((char *) m.singleData(), m.header()->len); }
+
     void receivedKillCursors(Message& m);
     void receivedUpdate(Message& m, CurOp& op);
     void receivedDelete(Message& m, CurOp& op);
@@ -51,9 +55,7 @@ namespace mongo {
 
     string dbExecCommand;
 
-    string bind_ip = "";
-
-    char *appsrvPath = null;
+    char *appsrvPath = NULL;
 
     DiagLog _diaglog;
 
@@ -76,7 +78,7 @@ namespace mongo {
 
     // see FSyncCommand:
     unsigned lockedForWriting; 
-    mongo::mutex lockedForWritingMutex;
+    mongo::mutex lockedForWritingMutex("lockedForWriting");
     bool unlockRequested = false;
 
     void inProgCmd( Message &m, DbResponse &dbresponse ) {
@@ -109,7 +111,7 @@ namespace mongo {
             unsigned x = lockedForWriting;
             if( x ) {
                 b.append("fsyncLock", x);
-                b.append("info", "use command {unlock:0} to terminate the fsync write/snapshot lock");
+                b.append("info", "use db.$cmd.sys.unlock.findOne() to terminate the fsync write/snapshot lock");
             }
         }
         
@@ -132,6 +134,7 @@ namespace mongo {
                 obj = fromjson("{\"err\":\"no op number field specified?\"}");
             }
             else { 
+                log() << "going to kill op: " << e << endl;
                 obj = fromjson("{\"info\":\"attempting to kill op\"}");
                 killCurrentOp.kill( (unsigned) e.number() );
             }
@@ -159,70 +162,73 @@ namespace mongo {
 
     static bool receivedQuery(Client& c, DbResponse& dbresponse, Message& m ){
         bool ok = true;
-        MSGID responseTo = m.data->id;
+        MSGID responseTo = m.header()->id;
 
         DbMessage d(m);
         QueryMessage q(d);
-        QueryResult* msgdata;
+        auto_ptr< Message > resp( new Message() );
 
         CurOp& op = *(c.curop());
         
         try {
-            msgdata = runQuery(m, q, op ).release();
+            dbresponse.exhaust = runQuery(m, q, op, *resp);
+            assert( !resp->empty() );
         }
         catch ( AssertionException& e ) {
             ok = false;
             op.debug().str << " exception ";
-            LOGSOME problem() << " Caught Assertion in runQuery ns:" << q.ns << ' ' << e.toString() << '\n';
-            log() << "  ntoskip:" << q.ntoskip << " ntoreturn:" << q.ntoreturn << '\n';
-            if ( q.query.valid() )
-                log() << "  query:" << q.query.toString() << endl;
-            else
-                log() << "  query object is not valid!" << endl;
+            LOGSOME { 
+                log() << "assertion " << e.toString() << " ns:" << q.ns << " query:" <<
+                    (q.query.valid() ? q.query.toString() : "query object is corrupt") << endl;
+                if( q.ntoskip || q.ntoreturn )
+                    log() << " ntoskip:" << q.ntoskip << " ntoreturn:" << q.ntoreturn << endl;
+            }
 
             BSONObjBuilder err;
-            err.append("$err", e.msg.empty() ? "assertion during query" : e.msg);
+            e.getInfo().append( err );
             BSONObj errObj = err.done();
 
             BufBuilder b;
             b.skip(sizeof(QueryResult));
-            b.append((void*) errObj.objdata(), errObj.objsize());
+            b.appendBuf((void*) errObj.objdata(), errObj.objsize());
 
             // todo: call replyToQuery() from here instead of this!!! see dbmessage.h
-            msgdata = (QueryResult *) b.buf();
+            QueryResult * msgdata = (QueryResult *) b.buf();
             b.decouple();
             QueryResult *qr = msgdata;
-            qr->_resultFlags() = QueryResult::ResultFlag_ErrSet;
+            qr->_resultFlags() = ResultFlag_ErrSet;
+            if ( e.getCode() == StaleConfigInContextCode )
+                qr->_resultFlags() |= ResultFlag_ShardConfigStale;
             qr->len = b.len();
             qr->setOperation(opReply);
             qr->cursorId = 0;
             qr->startingFrom = 0;
             qr->nReturned = 1;
+            resp.reset( new Message() );
+            resp->setData( msgdata, true );
+        }
 
+        if ( op.shouldDBProfile( 0 ) ){
+            op.debug().str << " bytes:" << resp->header()->dataLen();
         }
-        Message *resp = new Message();
-        resp->setData(msgdata, true); // transport will free
-        dbresponse.response = resp;
+        
+        dbresponse.response = resp.release();
         dbresponse.responseTo = responseTo;
         
-        if ( op.shouldDBProfile( 0 ) ){
-            op.debug().str << " bytes:" << resp->data->dataLen();
-        }
-
         return ok;
     }
 
     // Returns false when request includes 'end'
-    bool assembleResponse( Message &m, DbResponse &dbresponse, const sockaddr_in &client ) {
+    bool assembleResponse( Message &m, DbResponse &dbresponse, const SockAddr &client ) {
 
         // before we lock...
-        int op = m.data->operation();
+        int op = m.operation();
         bool isCommand = false;
-        const char *ns = m.data->_data + 4;
+        const char *ns = m.singleData()->_data + 4;
         if ( op == dbQuery ) {
             if( strstr(ns, ".$cmd") ) {
                 isCommand = true;
-                OPWRITE;
+                opwrite(m);
                 if( strstr(ns, ".$cmd.sys.") ) { 
                     if( strstr(ns, "$cmd.sys.inprog") ) {
                         inProgCmd(m, dbresponse);
@@ -237,28 +243,20 @@ namespace mongo {
                         return true;
                     }
                 }
-
             }
             else {
-                OPREAD;
+                opread(m);
             }
         }
         else if( op == dbGetMore ) {
-            OPREAD;
+            opread(m);
         }
         else {
-            OPWRITE;
+            opwrite(m);
         }
         
         globalOpCounters.gotOp( op , isCommand );
         
-        if ( handlePossibleShardedMessage( m , dbresponse ) ){
-            /* important to do this before we lock
-               so if a message has to be forwarded, doesn't block for that
-            */
-            return true;
-        }
-
         Client& c = cc();
         
         auto_ptr<CurOp> nestedOp;
@@ -278,22 +276,21 @@ namespace mongo {
         bool log = logLevel >= 1;
         
         if ( op == dbQuery ) {
-            if ( ! receivedQuery(c , dbresponse, m ) )
-                log = true;
+            if ( handlePossibleShardedMessage( m , &dbresponse ) )
+                return true;
+            receivedQuery(c , dbresponse, m );
         }
         else if ( op == dbGetMore ) {
-            DEV log = true;
             if ( ! receivedGetMore(dbresponse, m, currentOp) )
                 log = true;
         }
         else if ( op == dbMsg ) {
             // deprecated - replaced by commands
-            char *p = m.data->_data;
+            char *p = m.singleData()->_data;
             int len = strlen(p);
             if ( len > 400 )
                 out() << curTimeMillis() % 10000 <<
-                    " long msg received, len:" << len <<
-                    " ends with: " << p + len - 10 << endl;
+                    " long msg received, len:" << len << endl;
 
             Message *resp = new Message();
             if ( strcmp( "end" , p ) == 0 )
@@ -302,10 +299,10 @@ namespace mongo {
                 resp->setData( opReply , "i am fine - dbMsg deprecated");
 
             dbresponse.response = resp;
-            dbresponse.responseTo = m.data->id;
+            dbresponse.responseTo = m.header()->id;
         }
         else {
-            const char *ns = m.data->_data + 4;
+            const char *ns = m.singleData()->_data + 4;
             char cl[256];
             nsToDatabase(ns, cl);
             if( ! c.getAuthenticationInfo()->isAuthorized(cl) ) { 
@@ -329,13 +326,13 @@ namespace mongo {
                         receivedKillCursors(m);
                     }
                     else {
-                        out() << "    operation isn't supported: " << op << endl;
+                        mongo::log() << "    operation isn't supported: " << op << endl;
                         currentOp.done();
                         log = true;
                     }
                 }
                 catch ( AssertionException& e ) {
-                    problem() << " Caught Assertion in " << opToString(op) << " , continuing" << endl;
+                    tlog() << " Caught Assertion in " << opToString(op) << " , continuing" << endl;
                     ss << " exception " + e.toString();
                     log = true;
                 }
@@ -346,10 +343,14 @@ namespace mongo {
         int ms = currentOp.totalTimeMillis();
         
         log = log || (logLevel >= 2 && ++ctr % 512 == 0);
-        DEV log = true;
+        //DEV log = true; 
         if ( log || ms > logThreshold ) {
-            ss << ' ' << ms << "ms";
-            mongo::log() << ss.str() << endl;
+            if( logLevel < 3 && op == dbGetMore && strstr(ns, ".oplog.") && ms < 3000 && !log ) {
+                /* it's normal for getMore on the oplog to be slow because of use of awaitdata flag. */
+            } else {
+                ss << ' ' << ms << "ms";
+                mongo::tlog() << ss.str() << endl;
+            }
         }
         
         if ( currentOp.shouldDBProfile( ms ) ){
@@ -374,12 +375,12 @@ namespace mongo {
 
     void killCursors(int n, long long *ids);
     void receivedKillCursors(Message& m) {
-        int *x = (int *) m.data->_data;
+        int *x = (int *) m.singleData()->_data;
         x++; // reserved
         int n = *x++;
         uassert( 13004 , "sent 0 cursors to kill" , n >= 1 );
         if ( n > 2000 ) {
-            problem() << "Assertion failure, receivedKillCursors, n=" << n << endl;
+            log( n < 30000 ? LL_WARNING : LL_ERROR ) << "receivedKillCursors, n=" << n << endl;
             assert( n < 30000 );
         }
         killCursors(n, (long long *) x);
@@ -397,7 +398,7 @@ namespace mongo {
         Database *database = ctx->db();
         assert( database->name == db );
         
-        replCheckCloseDatabase( database );
+        oplogCheckCloseDatabase( database );
 
         if( BackgroundOperation::inProgForDb(db) ) { 
             log() << "warning: bg op in prog during close db? " << db << endl;
@@ -425,13 +426,14 @@ namespace mongo {
         BSONObj query = d.nextJsObj();
 
         assert( d.moreJSObjs() );
-        assert( query.objsize() < m.data->dataLen() );
+        assert( query.objsize() < m.header()->dataLen() );
         BSONObj toupdate = d.nextJsObj();
         uassert( 10055 , "update object too large", toupdate.objsize() <= MaxBSONObjectSize);
-        assert( toupdate.objsize() < m.data->dataLen() );
-        assert( query.objsize() + toupdate.objsize() < m.data->dataLen() );
+        assert( toupdate.objsize() < m.header()->dataLen() );
+        assert( query.objsize() + toupdate.objsize() < m.header()->dataLen() );
         bool upsert = flags & UpdateOption_Upsert;
         bool multi = flags & UpdateOption_Multi;
+        bool broadcast = flags & UpdateOption_Broadcast;
         {
             string s = query.toString();
             /* todo: we shouldn't do all this ss stuff when we don't need it, it will slow us down. 
@@ -443,10 +445,14 @@ namespace mongo {
         }        
 
         mongolock lk(1);
+
+        if ( ! broadcast && handlePossibleShardedMessage( m , 0 ) )
+            return;
+
         Client::Context ctx( ns );
 
         UpdateResult res = updateObjects(ns, toupdate, query, upsert, multi, true, op.debug() );
-        recordUpdate( res.existing , (int) res.num ); // for getlasterror
+        lastError.getSafe()->recordUpdate( res.existing , res.num , res.upserted ); // for getlasterror
     }
 
     void receivedDelete(Message& m, CurOp& op) {
@@ -455,7 +461,8 @@ namespace mongo {
         assert(*ns);
         uassert( 10056 ,  "not master", isMasterNs( ns ) );
         int flags = d.pullInt();
-        bool justOne = flags & 1;
+        bool justOne = flags & RemoveOption_JustOne;
+        bool broadcast = flags & RemoveOption_Broadcast;
         assert( d.moreJSObjs() );
         BSONObj pattern = d.nextJsObj();
         {
@@ -465,10 +472,12 @@ namespace mongo {
         }        
 
         writelock lk(ns);
+        if ( ! broadcast & handlePossibleShardedMessage( m , 0 ) )
+            return;
         Client::Context ctx(ns);
-
+        
         long long n = deleteObjects(ns, pattern, justOne, true);
-        recordDelete( (int) n );
+        lastError.getSafe()->recordDelete( n );
     }
     
     QueryResult* emptyMoreResult(long long);
@@ -483,26 +492,48 @@ namespace mongo {
         int ntoreturn = d.pullInt();
         long long cursorid = d.pullInt64();
         
-        ss << ns << " cid:" << cursorid << " ntoreturn:" << ntoreturn;;
+        ss << ns << " cid:" << cursorid;
+        if( ntoreturn ) 
+            ss << " ntoreturn:" << ntoreturn;
 
+        int pass = 0;        
+        bool exhaust = false;
         QueryResult* msgdata;
-        try {
-            mongolock lk(false);
-            Client::Context ctx(ns);
-            msgdata = getMore(ns, ntoreturn, cursorid, curop);
-        }
-        catch ( AssertionException& e ) {
-            ss << " exception " << e.toString();
-            msgdata = emptyMoreResult(cursorid);
-            ok = false;
-        }
+        while( 1 ) {
+            try {
+                mongolock lk(false);
+                Client::Context ctx(ns);
+                msgdata = processGetMore(ns, ntoreturn, cursorid, curop, pass, exhaust);
+            }
+            catch ( GetMoreWaitException& ) { 
+                exhaust = false;
+                massert(13073, "shutting down", !inShutdown() );
+                pass++;
+                DEV 
+                    sleepmillis(20);
+                else 
+                    sleepmillis(2);
+                continue;
+            }
+            catch ( AssertionException& e ) {
+                exhaust = false;
+                ss << " exception " << e.toString();
+                msgdata = emptyMoreResult(cursorid);
+                ok = false;
+            }
+            break;
+        };
+
         Message *resp = new Message();
         resp->setData(msgdata, true);
-        ss << " bytes:" << resp->data->dataLen();
+        ss << " bytes:" << resp->header()->dataLen();
         ss << " nreturned:" << msgdata->nReturned;
         dbresponse.response = resp;
-        dbresponse.responseTo = m.data->id;
-
+        dbresponse.responseTo = m.header()->id;
+        if( exhaust ) { 
+            ss << " exhaust "; 
+            dbresponse.exhaust = ns;
+        }
         return ok;
     }
 
@@ -514,12 +545,17 @@ namespace mongo {
         op.debug().str << ns;
 
         writelock lk(ns);
+
+        if ( handlePossibleShardedMessage( m , 0 ) )
+            return;
+
         Client::Context ctx(ns);		
         while ( d.moreJSObjs() ) {
             BSONObj js = d.nextJsObj();
             uassert( 10059 , "object to insert too large", js.objsize() <= MaxBSONObjectSize);
-            theDataFileMgr.insert(ns, js, false);
+            theDataFileMgr.insertWithObjMod(ns, js, false);
             logOp("i", ns, js);
+            globalOpCounters.gotInsert();
         }
     }
 
@@ -538,15 +574,15 @@ namespace mongo {
         Message & container;
     };
     
-    void getDatabaseNames( vector< string > &names ) {
-        boost::filesystem::path path( dbpath );
+    void getDatabaseNames( vector< string > &names , const string& usePath ) {
+        boost::filesystem::path path( usePath );
         for ( boost::filesystem::directory_iterator i( path );
                 i != boost::filesystem::directory_iterator(); ++i ) {
             if ( directoryperdb ) {
                 boost::filesystem::path p = *i;
                 string dbName = p.leaf();
                 p /= ( dbName + ".ns" );
-                if ( boost::filesystem::exists( p ) )
+                if ( MMF::exists( p ) )
                     names.push_back( dbName );
             } else {
                 string fileName = boost::filesystem::path(*i).leaf();
@@ -556,12 +592,34 @@ namespace mongo {
         }
     }
 
+    /* returns true if there is data on this server.  useful when starting replication. 
+       local database does NOT count except for rsoplog collection.
+    */
+    bool replHasDatabases() { 
+        vector<string> names;
+        getDatabaseNames(names);
+        if( names.size() >= 2 ) return true;
+        if( names.size() == 1 ){
+            if( names[0] != "local" )
+                return true;
+            // we have a local database.  return true if oplog isn't empty
+            {
+                readlock lk(rsoplog);
+                BSONObj o;
+                if( Helpers::getFirst(rsoplog, o) )
+                    return true;
+            }
+        }
+        return false;
+    }
+
     bool DBDirectClient::call( Message &toSend, Message &response, bool assertOk ) {
         if ( lastError._get() )
             lastError.startRequest( toSend, lastError._get() );
         DbResponse dbResponse;
         assembleResponse( toSend, dbResponse );
         assert( dbResponse.response );
+        dbResponse.response->concat(); // can get rid of this if we make response handling smarter
         response = *dbResponse.response;
         return true;
     }
@@ -583,14 +641,17 @@ namespace mongo {
         //throw UserException( (string)"yay:" + ns );
     }
 
+    void DBDirectClient::killCursor( long long id ){
+        ClientCursor::erase( id );
+    }
 
     DBClientBase * createDirectClient(){
         return new DBDirectClient();
     }
 
-    void recCacheCloseAll();
+    //void recCacheCloseAll();
 
-    mongo::mutex exitMutex;
+    mongo::mutex exitMutex("exit");
     int numExitCalls = 0;
     void shutdown();
 
@@ -642,6 +703,11 @@ namespace mongo {
         catch ( ... ){
             tryToOutputFatal( "shutdown failed with exception" );
         }
+
+        try { 
+            mutexDebugger.programEnding();
+        }
+        catch (...) { }
         
         tryToOutputFatal( "dbexit: really exiting now\n" );
         if ( c ) c->shutdown();
@@ -674,31 +740,43 @@ namespace mongo {
         rawOut( ss3.str() );
 
         // should we be locked here?  we aren't. might be ok as-is.
-        recCacheCloseAll();
+        //recCacheCloseAll();
         
 #if !defined(_WIN32) && !defined(__sunos__)
         if ( lockFile ){
             log() << "\t shutdown: removing fs lock..." << endl;
             if( ftruncate( lockFile , 0 ) ) 
-                log() << "\t couldn't remove fs lock " << OUTPUT_ERRNO << endl;
+                log() << "\t couldn't remove fs lock " << errnoWithDescription() << endl;
             flock( lockFile, LOCK_UN );
         }
 #endif
     }
 
-    void acquirePathLock() {
 #if !defined(_WIN32) && !defined(__sunos__)
-        string name = ( boost::filesystem::path( dbpath ) / "mongod.lock" ).native_file_string();
+    void writePid(int fd) {
+        stringstream ss;
+        ss << getpid() << endl;
+        string s = ss.str();
+        const char * data = s.c_str();
+        assert ( write( fd, data, strlen( data ) ) );
+    }
+
+    void acquirePathLock() {
+      string name = ( boost::filesystem::path( dbpath ) / "mongod.lock" ).native_file_string();
 
         bool oldFile = false;
 
         if ( boost::filesystem::exists( name ) && boost::filesystem::file_size( name ) > 0 ){
             oldFile = true;
         }
-        
-        lockFile = open( name.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO );
-        uassert( 10309 ,  "Unable to create / open lock file for dbpath: " + name, lockFile > 0 );
-        uassert( 10310 ,  "Unable to acquire lock for dbpath: " + name, flock( lockFile, LOCK_EX | LOCK_NB ) == 0 );
+
+        lockFile = open( name.c_str(), O_RDWR | O_CREAT , S_IRWXU | S_IRWXG | S_IRWXO );
+        uassert( 10309 ,  "Unable to create / open lock file for lockfilepath: " + name, lockFile > 0 );
+        if (flock( lockFile, LOCK_EX | LOCK_NB ) != 0) {
+            close ( lockFile );
+            lockFile = 0;
+            uassert( 10310 ,  "Unable to acquire lock for lockfilepath: " + name,  0 );
+        }
 
         if ( oldFile ){
             // we check this here because we want to see if we can get the lock
@@ -708,17 +786,19 @@ namespace mongo {
                  << "recommend removing file and running --repair\n" 
                  << "see: http://dochub.mongodb.org/core/repair for more information\n"
                  << "*************" << endl;
+            close ( lockFile );
+            lockFile = 0;
             uassert( 12596 , "old lock file" , 0 );
         }
 
-
-        stringstream ss;
-        ss << getpid() << endl;
-        string s = ss.str();
-        const char * data = s.c_str();
-        assert( write( lockFile , data , strlen( data ) ) );
+        uassert( 13342, "Unable to truncate lock file", ftruncate(lockFile, 0) == 0);
+        writePid( lockFile );
         fsync( lockFile );
-#endif        
     }
+#else
+    void acquirePathLock() {
+        // TODO - this is very bad
+    }
+#endif    
     
 } // namespace mongo
diff --git a/db/instance.h b/db/instance.h
index b545a78..1a4b855 100644
--- a/db/instance.h
+++ b/db/instance.h
@@ -19,6 +19,7 @@
 
 #pragma once
 
+
 #include "../client/dbclient.h"
 #include "curop.h"
 #include "security.h"
@@ -29,9 +30,6 @@ namespace mongo {
 
     extern string dbExecCommand;
 
-#define OPWRITE if( _diaglog.level & 1 ) _diaglog.write((char *) m.data, m.data->len);
-#define OPREAD if( _diaglog.level & 2 ) _diaglog.readop((char *) m.data, m.data->len);
-
     struct DiagLog {
         ofstream *f;
         /* 0 = off; 1 = writes, 2 = reads, 3 = both
@@ -40,7 +38,7 @@ namespace mongo {
         int level;
         mongo::mutex mutex;
 
-        DiagLog() : f(0) , level(0) { }
+        DiagLog() : f(0) , level(0), mutex("DiagLog") { }
         void init() {
             if ( ! f && level ){
                 log() << "diagLogging = " << level << endl;
@@ -96,21 +94,23 @@ namespace mongo {
     struct DbResponse {
         Message *response;
         MSGID responseTo;
-        DbResponse(Message *r, MSGID rt) : response(r), responseTo(rt) {
-        }
+        const char *exhaust; /* points to ns if exhaust mode. 0=normal mode*/
+        DbResponse(Message *r, MSGID rt) : response(r), responseTo(rt), exhaust(0) { }
         DbResponse() {
             response = 0;
+            exhaust = 0;
         }
-        ~DbResponse() {
-            delete response;
-        }
+        ~DbResponse() { delete response; }
     };
-
-    static SockAddr unknownAddress( "0.0.0.0", 0 );
     
-    bool assembleResponse( Message &m, DbResponse &dbresponse, const sockaddr_in &client = unknownAddress.sa );
+    bool assembleResponse( Message &m, DbResponse &dbresponse, const SockAddr &client = unknownAddress );
+
+    void getDatabaseNames( vector< string > &names , const string& usePath = dbpath );
 
-    void getDatabaseNames( vector< string > &names );
+    /* returns true if there is no data on this server.  useful when starting replication. 
+       local database does NOT count. 
+    */
+    bool replHasDatabases();
 
 // --- local client ---
     
@@ -119,7 +119,7 @@ namespace mongo {
     public:
         virtual auto_ptr<DBClientCursor> query(const string &ns, Query query, int nToReturn = 0, int nToSkip = 0,
                                                const BSONObj *fieldsToReturn = 0, int queryOptions = 0);
-
+        
         virtual bool isFailed() const {
             return false;
         }
@@ -135,9 +135,18 @@ namespace mongo {
             // don't need to piggy back when connected locally
             return say( toSend );
         }
+        
+        virtual void killCursor( long long cursorID );
+        
+        virtual bool callRead( Message& toSend , Message& response ){
+            return call( toSend , response );
+        }
+
+        virtual ConnectionString::ConnectionType type() const { return ConnectionString::MASTER; }  
     };
 
     extern int lockFile;
     void acquirePathLock();
+    void maybeCreatePidFile();
     
 } // namespace mongo
diff --git a/db/introspect.cpp b/db/introspect.cpp
index a041d48..d72bb3f 100644
--- a/db/introspect.cpp
+++ b/db/introspect.cpp
@@ -16,9 +16,9 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "introspect.h"
-#include "../util/builder.h"
+#include "../bson/util/builder.h"
 #include "../util/goodies.h"
 #include "pdfile.h"
 #include "jsobj.h"
diff --git a/db/introspect.h b/db/introspect.h
index 1c0fe92..3f6ef60 100644
--- a/db/introspect.h
+++ b/db/introspect.h
@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "jsobj.h"
 #include "pdfile.h"
 
diff --git a/db/jsobj.cpp b/db/jsobj.cpp
index 9f9a684..9f613c7 100644
--- a/db/jsobj.cpp
+++ b/db/jsobj.cpp
@@ -17,11 +17,10 @@
  *    limitations under the License.
  */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "jsobj.h"
 #include "nonce.h"
-#include "../util/atomic_int.h"
-#include "../util/goodies.h"
+#include "../bson/util/atomic_int.h"
 #include "../util/base64.h"
 #include "../util/md5.hpp"
 #include <limits>
@@ -31,9 +30,8 @@
 #include "jsobjmanipulator.h"
 #include "../util/optime.h"
 #include <boost/static_assert.hpp>
-#include <boost/any.hpp>
 #undef assert
-#define assert xassert
+#define assert MONGO_assert
 
 // make sure our assumptions are valid
 BOOST_STATIC_ASSERT( sizeof(int) == 4 );
@@ -46,114 +44,12 @@ namespace mongo {
 
     BSONElement nullElement;
 
-    ostream& operator<<( ostream &s, const OID &o ) {
-        s << o.str();
-        return s;
-    }
-
-    IDLabeler GENOID;
+    GENOIDLabeler GENOID;
 
     DateNowLabeler DATENOW;
 
-    string BSONElement::toString( bool includeFieldName ) const {
-        stringstream s;
-        if ( includeFieldName && type() != EOO )
-            s << fieldName() << ": ";
-        switch ( type() ) {
-        case EOO:
-            return "EOO";
-        case Date:
-            s << "new Date(" << date() << ')';
-            break;
-        case RegEx:
-            {
-                s << "/" << regex() << '/';
-                const char *p = regexFlags();
-                if ( p ) s << p;
-            }
-            break;
-        case NumberDouble:
-            {
-                stringstream tmp;
-                tmp.precision( 16 );
-                tmp << number();
-                string n = tmp.str();
-                s << n;
-                // indicate this is a double:
-                if( strchr(n.c_str(), '.') == 0 && strchr(n.c_str(), 'E') == 0 && strchr(n.c_str(), 'N') == 0 )
-                    s << ".0";
-            }
-            break;
-        case NumberLong:
-            s << _numberLong();
-            break;
-        case NumberInt:
-            s << _numberInt();
-            break;
-        case Bool:
-            s << ( boolean() ? "true" : "false" );
-            break;
-        case Object:
-        case Array:
-            s << embeddedObject().toString();
-            break;
-        case Undefined:
-            s << "undefined";
-            break;
-        case jstNULL:
-            s << "null";
-            break;
-        case MaxKey:
-            s << "MaxKey";
-            break;
-        case MinKey:
-            s << "MinKey";
-            break;
-        case CodeWScope:
-            s << "CodeWScope( "
-                << codeWScopeCode() << ", " << codeWScopeObject().toString() << ")";
-            break;
-        case Code:
-            if ( valuestrsize() > 80 )
-                s << string(valuestr()).substr(0, 70) << "...";
-            else {
-                s << valuestr();
-            }
-            break;
-        case Symbol:
-        case String:
-            if ( valuestrsize() > 80 )
-                s << '"' << string(valuestr()).substr(0, 70) << "...\"";
-            else {
-                s << '"' << valuestr() << '"';
-            }
-            break;
-        case DBRef:
-            s << "DBRef('" << valuestr() << "',";
-            {
-                OID *x = (OID *) (valuestr() + valuestrsize());
-                s << *x << ')';
-            }
-            break;
-        case jstOID:
-            s << "ObjId(";
-            s << __oid() << ')';
-            break;
-        case BinData:
-            s << "BinData";
-            break;
-        case Timestamp:
-            s << "Timestamp " << timestampTime() << "|" << timestampInc();
-            break;
-        default:
-            s << "?type=" << type();
-            break;
-        }
-        return s.str();
-    }
-
     string escape( string s , bool escape_slash=false) {
-        stringstream ret;
+        StringBuilder ret;
         for ( string::iterator i = s.begin(); i != s.end(); ++i ) {
             switch ( *i ) {
             case '"':
@@ -182,11 +78,9 @@ namespace mongo {
                 break;
             default:
                 if ( *i >= 0 && *i <= 0x1f ) {
-                    ret << "\\u";
-                    ret << hex;
-                    ret.width( 4 );
-                    ret.fill( '0' );
-                    ret << int( *i );
+                    //TODO: these should be utf16 code-units not bytes
+                    char c = *i;
+                    ret << "\\u00" << toHexLower(&c, 1);
                 } else {
                     ret << *i;
                 }
@@ -195,14 +89,18 @@ namespace mongo {
         return ret.str();
     }
 
-    string BSONElement::jsonString( JsonStringFormat format, bool includeFieldNames ) const {
+    string BSONElement::jsonString( JsonStringFormat format, bool includeFieldNames, int pretty ) const {
+        BSONType t = type();
+        if ( t == Undefined )
+            return "";
+
         stringstream s;
         if ( includeFieldNames )
             s << '"' << escape( fieldName() ) << "\" : ";
         switch ( type() ) {
-        case String:
+        case mongo::String:
         case Symbol:
-            s << '"' << escape( valuestr() ) << '"';
+            s << '"' << escape( string(valuestr(), valuestrsize()-1) ) << '"';
             break;
         case NumberLong:
             s << _numberLong();
@@ -214,22 +112,22 @@ namespace mongo {
                 s.precision( 16 );
                 s << number();
             } else {
-                stringstream ss;
+                StringBuilder ss;
                 ss << "Number " << number() << " cannot be represented in JSON";
                 string message = ss.str();
                 massert( 10311 ,  message.c_str(), false );
             }
             break;
-        case Bool:
+        case mongo::Bool:
             s << ( boolean() ? "true" : "false" );
             break;
         case jstNULL:
             s << "null";
             break;
         case Object:
-            s << embeddedObject().jsonString( format );
+            s << embeddedObject().jsonString( format, pretty );
             break;
-        case Array: {
+        case mongo::Array: {
             if ( embeddedObject().isEmpty() ) {
                 s << "[]";
                 break;
@@ -239,7 +137,12 @@ namespace mongo {
             BSONElement e = i.next();
             if ( !e.eoo() )
                 while ( 1 ) {
-                    s << e.jsonString( format, false );
+                    if( pretty ) {
+                        s << '\n';
+                        for( int x = 0; x < pretty; x++ )
+                            s << "  ";
+                    }
+                    s << e.jsonString( format, false, pretty?pretty+1:0 );
                     e = i.next();
                     if ( e.eoo() )
                         break;
@@ -249,7 +152,7 @@ namespace mongo {
             break;
         }
         case DBRef: {
-            OID *x = (OID *) (valuestr() + valuestrsize());
+            mongo::OID *x = (mongo::OID *) (valuestr() + valuestrsize());
             if ( format == TenGen )
                 s << "Dbref( ";
             else
@@ -290,12 +193,18 @@ namespace mongo {
             s << "\" }";
             break;
         }
-        case Date:
+        case mongo::Date:
             if ( format == Strict )
                 s << "{ \"$date\" : ";
             else
                 s << "Date( ";
-            s << date();
+            if( pretty ) {
+                Date_t d = date();
+                if( d == 0 ) s << '0';
+                else
+                    s << '"' << date().toString() << '"';
+            } else
+                s << date();
             if ( format == Strict )
                 s << " }";
             else
@@ -321,16 +230,34 @@ namespace mongo {
             }
             break;
 
+        case CodeWScope: {
+            BSONObj scope = codeWScopeObject();
+            if ( ! scope.isEmpty() ){
+                s << "{ \"$code\" : " << _asCode() << " , "
+                  << " \"$scope\" : " << scope.jsonString() << " }";
+                break;
+            }
+        }
+
+
         case Code:
-            s << ascode();
+            s << _asCode();
             break;
-
+            
         case Timestamp:
             s << "{ \"t\" : " << timestampTime() << " , \"i\" : " << timestampInc() << " }";
             break;
 
+        case MinKey:
+            s << "{ \"$minKey\" : 1 }";
+            break;
+
+        case MaxKey:
+            s << "{ \"$maxKey\" : 1 }";
+            break;
+
         default:
-            stringstream ss;
+            StringBuilder ss;
             ss << "Cannot create a properly formatted JSON string with "
             << "element: " << toString() << " of type: " << type();
             string message = ss.str();
@@ -339,82 +266,6 @@ namespace mongo {
         return s.str();
     }
 
-    int BSONElement::size( int maxLen ) const {
-        if ( totalSize >= 0 )
-            return totalSize;
-
-        int remain = maxLen - fieldNameSize() - 1;
-
-        int x = 0;
-        switch ( type() ) {
-        case EOO:
-        case Undefined:
-        case jstNULL:
-        case MaxKey:
-        case MinKey:
-            break;
-        case Bool:
-            x = 1;
-            break;
-        case NumberInt:
-            x = 4;
-            break;
-        case Timestamp:
-        case Date:
-        case NumberDouble:
-        case NumberLong:
-            x = 8;
-            break;
-        case jstOID:
-            x = 12;
-            break;
-        case Symbol:
-        case Code:
-        case String:
-            massert( 10313 ,  "Insufficient bytes to calculate element size", maxLen == -1 || remain > 3 );
-            x = valuestrsize() + 4;
-            break;
-        case CodeWScope:
-            massert( 10314 ,  "Insufficient bytes to calculate element size", maxLen == -1 || remain > 3 );
-            x = objsize();
-            break;
-
-        case DBRef:
-            massert( 10315 ,  "Insufficient bytes to calculate element size", maxLen == -1 || remain > 3 );
-            x = valuestrsize() + 4 + 12;
-            break;
-        case Object:
-        case Array:
-            massert( 10316 ,  "Insufficient bytes to calculate element size", maxLen == -1 || remain > 3 );
-            x = objsize();
-            break;
-        case BinData:
-            massert( 10317 ,  "Insufficient bytes to calculate element size", maxLen == -1 || remain > 3 );
-            x = valuestrsize() + 4 + 1/*subtype*/;
-            break;
-        case RegEx:
-        {
-            const char *p = value();
-            int len1 = ( maxLen == -1 ) ? strlen( p ) : strnlen( p, remain );
-            massert( 10318 ,  "Invalid regex string", len1 != -1 );
-            p = p + len1 + 1;
-            int len2 = ( maxLen == -1 ) ? strlen( p ) : strnlen( p, remain - len1 - 1 );
-            massert( 10319 ,  "Invalid regex options string", len2 != -1 );
-            x = len1 + 1 + len2 + 1;
-        }
-        break;
-        default: {
-            stringstream ss;
-            ss << "BSONElement: bad type " << (int) type();
-            string msg = ss.str();
-            massert( 10320 , msg.c_str(),false);
-        }
-        }
-        totalSize =  x + fieldNameSize() + 1; // BSONType
-
-        return totalSize;
-    }
-
     int BSONElement::getGtLtOp( int def ) const {
         const char *fn = fieldName();
         if ( fn[0] == '$' && fn[1] ) {
@@ -434,8 +285,12 @@ namespace mongo {
                 if ( fn[3] == 'a' && fn[4] == 'r' && fn[5] == 0 )
                     return BSONObj::opNEAR;
             }
-            else if ( fn[1] == 'm' && fn[2] == 'o' && fn[3] == 'd' && fn[4] == 0 )
-                return BSONObj::opMOD;
+            else if ( fn[1] == 'm' ){
+                if ( fn[2] == 'o' && fn[3] == 'd' && fn[4] == 0 )
+                    return BSONObj::opMOD;
+                if ( fn[2] == 'a' && fn[3] == 'x' && fn[4] == 'D' && fn[5] == 'i' && fn[6] == 's' && fn[7] == 't' && fn[8] == 'a' && fn[9] == 'n' && fn[10] == 'c' && fn[11] == 'e' && fn[12] == 0 )
+                    return BSONObj::opMAX_DISTANCE;
+            }
             else if ( fn[1] == 't' && fn[2] == 'y' && fn[3] == 'p' && fn[4] == 'e' && fn[5] == 0 )
                 return BSONObj::opTYPE;
             else if ( fn[1] == 'i' && fn[2] == 'n' && fn[3] == 0 )
@@ -579,40 +434,6 @@ namespace mongo {
         return -1;
     }
 
-    void BSONElement::validate() const {
-        switch( type() ) {
-        case DBRef:
-        case Code:
-        case Symbol:
-        case String: {
-            int x = valuestrsize();
-            if ( x > 0 && valuestr()[x-1] == 0 )
-                return;
-            StringBuilder buf;
-            buf <<  "Invalid dbref/code/string/symbol size: " << x << " strnlen:" << strnlen( valuestr() , x );
-            massert( 10321 , buf.str() , 0 );
-            break;
-        }
-        case CodeWScope: {
-            int totalSize = *( int * )( value() );
-            massert( 10322 ,  "Invalid CodeWScope size", totalSize >= 8 );
-            int strSizeWNull = *( int * )( value() + 4 );
-            massert( 10323 ,  "Invalid CodeWScope string size", totalSize >= strSizeWNull + 4 + 4 );
-            massert( 10324 ,  "Invalid CodeWScope string size",
-                     strSizeWNull > 0 &&
-                     strSizeWNull - 1 == strnlen( codeWScopeCode(), strSizeWNull ) );
-            massert( 10325 ,  "Invalid CodeWScope size", totalSize >= strSizeWNull + 4 + 4 + 4 );
-            int objSize = *( int * )( value() + 4 + 4 + strSizeWNull );
-            massert( 10326 ,  "Invalid CodeWScope object size", totalSize == 4 + 4 + strSizeWNull + objSize );
-            // Subobject validation handled elsewhere.
-        }
-        case Object:
-            // We expect Object size validation to be handled elsewhere.
-        default:
-            break;
-        }
-    }
-
     /* Matcher --------------------------------------*/
 
 // If the element is something like:
@@ -676,39 +497,6 @@ namespace mongo {
 
     /* BSONObj ------------------------------------------------------------*/
 
-    BSONObj::EmptyObject BSONObj::emptyObject;
-
-    string BSONObj::toString() const {
-        if ( isEmpty() ) return "{}";
-
-        stringstream s;
-        s << "{ ";
-        BSONObjIterator i(*this);
-        bool first = true;
-        while ( 1 ) {
-            massert( 10327 ,  "Object does not end with EOO", i.moreWithEOO() );
-            BSONElement e = i.next( true );
-            massert( 10328 ,  "Invalid element size", e.size() > 0 );
-            massert( 10329 ,  "Element too large", e.size() < ( 1 << 30 ) );
-            int offset = e.rawdata() - this->objdata();
-            massert( 10330 ,  "Element extends past end of object",
-                    e.size() + offset <= this->objsize() );
-            e.validate();
-            bool end = ( e.size() + offset == this->objsize() );
-            if ( e.eoo() ) {
-                massert( 10331 ,  "EOO Before end of object", end );
-                break;
-            }
-            if ( first )
-                first = false;
-            else
-                s << ", ";
-            s << e.toString();
-        }
-        s << " }";
-        return s.str();
-    }
-
     string BSONObj::md5() const {
         md5digest d;
         md5_state_t st;
@@ -718,21 +506,29 @@ namespace mongo {
         return digestToString( d );
     }
 
-    string BSONObj::jsonString( JsonStringFormat format ) const {
+    string BSONObj::jsonString( JsonStringFormat format, int pretty ) const {
 
         if ( isEmpty() ) return "{}";
 
-        stringstream s;
+        StringBuilder s;
         s << "{ ";
         BSONObjIterator i(*this);
         BSONElement e = i.next();
         if ( !e.eoo() )
             while ( 1 ) {
-                s << e.jsonString( format );
+                s << e.jsonString( format, true, pretty?pretty+1:0 );
                 e = i.next();
                 if ( e.eoo() )
                     break;
-                s << ", ";
+                s << ",";
+                if ( pretty ) {
+                    s << '\n';
+                    for( int x = 0; x < pretty; x++ )
+                        s << "  ";
+                }
+                else {
+                    s << " ";
+                }
             }
         s << " }";
         return s.str();
@@ -740,13 +536,60 @@ namespace mongo {
 
 // todo: can be a little faster if we don't use toString() here.
     bool BSONObj::valid() const {
-        try {
-            toString();
+        try{
+            BSONObjIterator it(*this);
+            while( it.moreWithEOO() ){
+                // both throw exception on failure
+                BSONElement e = it.next(true);
+                e.validate();
+
+                if (e.eoo()){
+                    if (it.moreWithEOO())
+                        return false;
+                    return true;
+                }else if (e.isABSONObj()){
+                    if(!e.embeddedObject().valid())
+                        return false;
+                }else if (e.type() == CodeWScope){
+                    if(!e.codeWScopeObject().valid())
+                        return false;
+                }
+            }
+        } catch (...) {
         }
-        catch (...) {
-            return false;
+        return false;
+    }
+
+    int BSONObj::woCompare(const BSONObj& r, const Ordering &o, bool considerFieldName) const { 
+        if ( isEmpty() )
+            return r.isEmpty() ? 0 : -1;
+        if ( r.isEmpty() )
+            return 1;
+
+        BSONObjIterator i(*this);
+        BSONObjIterator j(r);
+        unsigned mask = 1;
+        while ( 1 ) {
+            // so far, equal...
+
+            BSONElement l = i.next();
+            BSONElement r = j.next();
+            if ( l.eoo() )
+                return r.eoo() ? 0 : -1;
+            if ( r.eoo() )
+                return 1;
+
+            int x;
+            {
+                x = l.woCompare( r, considerFieldName );
+                if( o.descending(mask) )
+                    x = -x;
+            }
+            if ( x != 0 )
+                return x;
+            mask <<= 1;
         }
-        return true;
+        return -1;
     }
 
     /* well ordered compare */
@@ -796,7 +639,7 @@ namespace mongo {
     BSONObj staticNull = fromjson( "{'':null}" );
 
     /* well ordered compare */
-    int BSONObj::woSortOrder(const BSONObj& other, const BSONObj& sortKey ) const{
+    int BSONObj::woSortOrder(const BSONObj& other, const BSONObj& sortKey , bool useDotted ) const{
         if ( isEmpty() )
             return other.isEmpty() ? 0 : -1;
         if ( other.isEmpty() )
@@ -810,10 +653,10 @@ namespace mongo {
             if ( f.eoo() )
                 return 0;
 
-            BSONElement l = getField( f.fieldName() );
+            BSONElement l = useDotted ? getFieldDotted( f.fieldName() ) : getField( f.fieldName() );
             if ( l.eoo() )
                 l = staticNull.firstElement();
-            BSONElement r = other.getField( f.fieldName() );
+            BSONElement r = useDotted ? other.getFieldDotted( f.fieldName() ) : other.getField( f.fieldName() );
             if ( r.eoo() )
                 r = staticNull.firstElement();
 
@@ -826,78 +669,46 @@ namespace mongo {
         return -1;
     }
 
-
-    /* return has eoo() true if no match
-       supports "." notation to reach into embedded objects
-    */
-    BSONElement BSONObj::getFieldDotted(const char *name) const {
+    void BSONObj::getFieldsDotted(const StringData& name, BSONElementSet &ret ) const {
         BSONElement e = getField( name );
         if ( e.eoo() ) {
-            const char *p = strchr(name, '.');
+            const char *p = strchr(name.data(), '.');
             if ( p ) {
-                string left(name, p-name);
-                BSONObj sub = getObjectField(left.c_str());
-                return sub.isEmpty() ? nullElement : sub.getFieldDotted(p+1);
-            }
-        }
-
-        return e;
-    }
-
-    void BSONObj::getFieldsDotted(const char *name, BSONElementSet &ret ) const {
-        BSONObjIterator i(*this);
-        while ( i.more() ){
-            BSONElement e = i.next();
-            FieldCompareResult cmp = compareDottedFieldNames( name , e.fieldName() );
-            switch ( cmp ){
-
-            case LEFT_BEFORE: 
-            case RIGHT_BEFORE:
-                break;
-
-            case RIGHT_SUBFIELD: 
-                assert(0); 
-                break;
-
-            case LEFT_SUBFIELD: {
-                const char * next = name + strlen( e.fieldName() ) + 1;
-                bool allDigits = false;
-                if ( isdigit( *next ) ){
-                    const char * temp = next + 1;
-                    while ( isdigit( *temp ) )
-                        temp++;
-                    allDigits = *temp == '.';
-                }
-
-                if ( e.type() == Object || allDigits ){
-                    e.embeddedObject().getFieldsDotted( next , ret );
-                }
-                else if ( e.type() == Array ){
-                    BSONObjIterator j( e.embeddedObject() );
-                    while ( j.more() ){
-                        BSONElement f = j.next();
-                        if ( f.type() == Object )
-                            f.embeddedObject().getFieldsDotted( next , ret );
+                string left(name.data(), p-name.data());
+                const char* next = p+1;
+                BSONElement e = getField( left.c_str() );
+
+                if (e.type() == Object){
+                    e.embeddedObject().getFieldsDotted(next, ret);
+                } else if (e.type() == Array) {
+                    bool allDigits = false;
+                    if ( isdigit( *next ) ){
+                        const char * temp = next + 1;
+                        while ( isdigit( *temp ) )
+                            temp++;
+                        allDigits = *temp == '.';
                     }
+                    if (allDigits) {
+                        e.embeddedObject().getFieldsDotted(next, ret);
+                    } else {
+                        BSONObjIterator i(e.embeddedObject());
+                        while ( i.more() ){
+                            BSONElement e2 = i.next();
+                            if (e2.type() == Object || e2.type() == Array)
+                                e2.embeddedObject().getFieldsDotted(next, ret);
+                        }
+                    }
+                } else {
+                    // do nothing: no match
                 }
-                else {
-                    // intentially left blank, this means no match
-                }
-                return;
-            }
-
-            case SAME: {
-                if ( e.type() == Array ){
-                    BSONObjIterator j( e.embeddedObject() );
-                    while ( j.more() )
-                        ret.insert( j.next() );
-                }
-                else {
-                    ret.insert( e );
-                }
-                return;
             }
-
+        } else {
+            if (e.type() == Array){
+                BSONObjIterator i(e.embeddedObject());
+                while ( i.more() )
+                    ret.insert(i.next());
+            } else {
+                ret.insert(e);
             }
         }
     }
@@ -915,7 +726,7 @@ namespace mongo {
         BSONElement sub = getField(left.c_str());
         if ( sub.eoo() )
             return nullElement;
-        else if ( sub.type() == Array || strlen( name ) == 0 )
+        else if ( sub.type() == Array || name[0] == '\0')
             return sub;
         else if ( sub.type() == Object )
             return sub.embeddedObject().getFieldDottedOrArray( name );
@@ -923,31 +734,6 @@ namespace mongo {
             return nullElement;
     }
 
-    /* makes a new BSONObj with the fields specified in pattern.
-       fields returned in the order they appear in pattern.
-       if any field missing or undefined in the original object, that field
-       in the output will be null.
-
-       n^2 implementation bad if pattern and object have lots
-       of fields - normally pattern doesn't so should be fine.
-    */
-    BSONObj BSONObj::extractFieldsDotted(BSONObj pattern) const {
-        BSONObjBuilder b;
-        BSONObjIterator i(pattern);
-        while (i.more()) {
-            BSONElement e = i.next();
-            const char *name = e.fieldName();
-
-            BSONElement x = getFieldDotted( name );
-            if ( x.eoo() || x.type() == Undefined ) {
-                b.appendNull(name);
-            } else {
-                b.appendAs(x, name);
-            }
-        }
-        return b.done();
-    }
-
     /**
      sets element field names to empty string
      If a field in pattern is missing, it is omitted from the returned
@@ -1037,24 +823,6 @@ namespace mongo {
         return e.type() == String ? e.valuestr() : "";
     }
 
-    BSONObj BSONObj::getObjectField(const char *name) const {
-        BSONElement e = getField(name);
-        BSONType t = e.type();
-        return t == Object || t == Array ? e.embeddedObject() : BSONObj();
-    }
-
-    int BSONObj::nFields() const {
-        int n = 0;
-        BSONObjIterator i(*this);
-        while ( i.moreWithEOO() ) {
-            BSONElement e = i.next();
-            if ( e.eoo() )
-                break;
-            n++;
-        }
-        return n;
-    }
-
     /* grab names of all the fields in this object */
     int BSONObj::getFieldNames(set<string>& fields) const {
         int n = 0;
@@ -1186,6 +954,18 @@ namespace mongo {
         return true;
     }
 
+    void BSONObj::dump() const {
+        out() << hex;
+        const char *p = objdata();
+        for ( int i = 0; i < objsize(); i++ ) {
+            out() << i << '\t' << ( 0xff & ( (unsigned) *p ) );
+            if ( *p >= 'A' && *p <= 'z' )
+                out() << '\t' << *p;
+            out() << endl;
+            p++;
+        }
+    }
+
     string BSONObj::hexDump() const {
         stringstream ss;
         const char *d = objdata();
@@ -1202,14 +982,6 @@ namespace mongo {
         return ss.str();
     }
 
-    ostream& operator<<( ostream &s, const BSONObj &o ) {
-        return s << o.toString();
-    }
-
-    ostream& operator<<( ostream &s, const BSONElement &e ) {
-        return s << e.toString();
-    }
-
     void nested2dotted(BSONObjBuilder& b, const BSONObj& obj, const string& base){
         BSONObjIterator it(obj);
         while (it.more()){
@@ -1219,7 +991,7 @@ namespace mongo {
                 nested2dotted(b, e.embeddedObject(), newbase);
             }else{
                 string newbase = base + e.fieldName();
-                b.appendAs(e, newbase.c_str());
+                b.appendAs(e, newbase);
             }
         }
     }
@@ -1265,6 +1037,7 @@ namespace mongo {
     } minkeydata;
     BSONObj minKey((const char *) &minkeydata);
 
+/*
     struct JSObj0 {
         JSObj0() {
             totsize = 5;
@@ -1273,14 +1046,9 @@ namespace mongo {
         int totsize;
         char eoo;
     } js0;
+*/
 #pragma pack()
 
-    BSONElement::BSONElement() {
-        data = &js0.eoo;
-        fieldNameSize_ = 0;
-        totalSize = 1;
-    }
-
     struct BsonUnitTest : public UnitTest {
         void testRegex() {
 
@@ -1425,14 +1193,8 @@ namespace mongo {
     }
 */
 
-    unsigned OID::_machine = (unsigned) security.getNonceInitSafe();
-    void OID::newState(){
-        // using fresh Security object to avoid buffered devrandom
-        _machine = (unsigned) Security().getNonce();
-    }
-    
     void OID::init() {
-        static AtomicUInt inc = (unsigned) security.getNonce();
+        static AtomicUInt inc = getRandomNumber();
         unsigned t = (unsigned) time(0);
         char *T = (char *) &t;
         data[0] = T[3];
@@ -1451,31 +1213,45 @@ namespace mongo {
         raw[3] = T[0];
     }
 
+    unsigned OID::_machine = (unsigned) security.getNonceInitSafe();
+    void OID::newState(){
+        unsigned before = _machine;
+        // using fresh Security object to avoid buffered devrandom
+        _machine = (unsigned)security.getNonce();
+        assert( _machine != before );
+    }
+    
     void OID::init( string s ){
         assert( s.size() == 24 );
         const char *p = s.c_str();
-        char buf[3];
-        buf[2] = 0;
         for( int i = 0; i < 12; i++ ) {
-            buf[0] = p[0];
-            buf[1] = p[1];
+            data[i] = fromHex(p);
             p += 2;
-            stringstream ss(buf);
-            unsigned z;
-            ss >> hex >> z;
-            data[i] = z;
         }
+    }
 
-/*
-        string as = s.substr( 0 , 16 );
-        string bs = s.substr( 16 );
+    void OID::init(Date_t date, bool max){
+        int time = (int) (date / 1000);
+        char* T = (char *) &time;
+        data[0] = T[3];
+        data[1] = T[2];
+        data[2] = T[1];
+        data[3] = T[0];
 
-        stringstream ssa(as);
-        ssa >> hex >> a;
+        if (max)
+            *(long long*)(data + 4) = 0xFFFFFFFFFFFFFFFFll;
+        else
+            *(long long*)(data + 4) = 0x0000000000000000ll;
+    }
 
-        stringstream ssb(bs);
-        ssb >> hex >> b;
-*/
+    time_t OID::asTimeT(){
+        int time;
+        char* T = (char *) &time;
+        T[0] = data[3];
+        T[1] = data[2];
+        T[2] = data[1];
+        T[3] = data[0];
+        return time;
     }
 
     Labeler::Label GT( "$gt" );
@@ -1492,84 +1268,83 @@ namespace mongo {
             timestamp = OpTime::now().asDate();
     }
 
-
-    void BSONObjBuilder::appendMinForType( const string& field , int t ){
+    void BSONObjBuilder::appendMinForType( const StringData& fieldName , int t ){
         switch ( t ){
-        case MinKey: appendMinKey( field.c_str() ); return;
-        case MaxKey: appendMinKey( field.c_str() ); return;
+        case MinKey: appendMinKey( fieldName ); return;
+        case MaxKey: appendMinKey( fieldName ); return;
         case NumberInt:
         case NumberDouble:
         case NumberLong:
-            append( field.c_str() , - numeric_limits<double>::max() ); return;
+            append( fieldName , - numeric_limits<double>::max() ); return;
         case jstOID:
             {
                 OID o;
                 memset(&o, 0, sizeof(o));
-                appendOID( field.c_str() , &o);
+                appendOID( fieldName , &o);
                 return;
             }
-        case Bool: appendBool( field.c_str() , false); return;
-        case Date: appendDate( field.c_str() , 0); return;
-        case jstNULL: appendNull( field.c_str() ); return;
+        case Bool: appendBool( fieldName , false); return;
+        case Date: appendDate( fieldName , 0); return;
+        case jstNULL: appendNull( fieldName ); return;
         case Symbol:
-        case String: append( field.c_str() , "" ); return;
-        case Object: append( field.c_str() , BSONObj() ); return;
+        case String: append( fieldName , "" ); return;
+        case Object: append( fieldName , BSONObj() ); return;
         case Array:
-            appendArray( field.c_str() , BSONObj() ); return;
+            appendArray( fieldName , BSONObj() ); return;
         case BinData:
-            appendBinData( field.c_str() , 0 , Function , (const char *) 0 ); return;
+            appendBinData( fieldName , 0 , Function , (const char *) 0 ); return;
         case Undefined:
-            appendUndefined( field.c_str() ); return;
-        case RegEx: appendRegex( field.c_str() , "" ); return;
+            appendUndefined( fieldName ); return;
+        case RegEx: appendRegex( fieldName , "" ); return;
         case DBRef:
             {
                 OID o;
                 memset(&o, 0, sizeof(o));
-                appendDBRef( field.c_str() , "" , o );
+                appendDBRef( fieldName , "" , o );
                 return;
             }
-        case Code: appendCode( field.c_str() , "" ); return;
-        case CodeWScope: appendCodeWScope( field.c_str() , "" , BSONObj() ); return;
-        case Timestamp: appendTimestamp( field.c_str() , 0); return;
+        case Code: appendCode( fieldName , "" ); return;
+        case CodeWScope: appendCodeWScope( fieldName , "" , BSONObj() ); return;
+        case Timestamp: appendTimestamp( fieldName , 0); return;
 
         };
         log() << "type not support for appendMinElementForType: " << t << endl;
         uassert( 10061 ,  "type not supported for appendMinElementForType" , false );
     }
 
-    void BSONObjBuilder::appendMaxForType( const string& field , int t ){
+    void BSONObjBuilder::appendMaxForType( const StringData& fieldName , int t ){
         switch ( t ){
-        case MinKey: appendMaxKey( field.c_str() );  break;
-        case MaxKey: appendMaxKey( field.c_str() ); break;
+        case MinKey: appendMaxKey( fieldName );  break;
+        case MaxKey: appendMaxKey( fieldName ); break;
         case NumberInt:
         case NumberDouble:
         case NumberLong:
-            append( field.c_str() , numeric_limits<double>::max() );
+            append( fieldName , numeric_limits<double>::max() );
             break;
         case BinData:
-            appendMinForType( field , jstOID );
+            appendMinForType( fieldName , jstOID );
             break;
         case jstOID:
             {
                 OID o;
                 memset(&o, 0xFF, sizeof(o));
-                appendOID( field.c_str() , &o);
+                appendOID( fieldName , &o);
                 break;
             }
         case Undefined:
         case jstNULL:
-            appendMinForType( field , NumberInt );
-        case Bool: appendBool( field.c_str() , true); break;
-        case Date: appendDate( field.c_str() , 0xFFFFFFFFFFFFFFFFLL ); break;
+            appendMinForType( fieldName , NumberInt );
+        case Bool: appendBool( fieldName , true); break;
+        case Date: appendDate( fieldName , 0xFFFFFFFFFFFFFFFFLL ); break;
         case Symbol:
-        case String: append( field.c_str() , BSONObj() ); break;
+        case String: append( fieldName , BSONObj() ); break;
         case Code:
         case CodeWScope:
-            appendCodeWScope( field.c_str() , "ZZZ" , BSONObj() ); break;
+            appendCodeWScope( fieldName , "ZZZ" , BSONObj() ); break;
         case Timestamp:
-            appendTimestamp( field.c_str() , numeric_limits<unsigned long long>::max() ); break;
+            appendTimestamp( fieldName , numeric_limits<unsigned long long>::max() ); break;
         default:
-            appendMinForType( field , t + 1 );
+            appendMinForType( fieldName , t + 1 );
         }
     }
 
@@ -1586,8 +1361,8 @@ namespace mongo {
         "90", "91", "92", "93", "94", "95", "96", "97", "98", "99",
     };
 
-    bool BSONObjBuilder::appendAsNumber( const string& fieldName , const string& data ){
-        if ( data.size() == 0 )
+    bool BSONObjBuilder::appendAsNumber( const StringData& fieldName , const string& data ){
+        if ( data.size() == 0 || data == "-")
             return false;
         
         unsigned int pos=0;
@@ -1612,7 +1387,7 @@ namespace mongo {
         
         if ( hasDec ){
             double d = atof( data.c_str() );
-            append( fieldName.c_str() , d );
+            append( fieldName , d );
             return true;
         }
         
@@ -1665,5 +1440,27 @@ namespace mongo {
         _cur = 0;
     }
 
+    /** transform a BSON array into a vector of BSONElements.
+        we match array # positions with their vector position, and ignore 
+        any non-numeric fields. 
+        */
+    vector<BSONElement> BSONElement::Array() const { 
+        chk(mongo::Array);
+        vector<BSONElement> v;
+        BSONObjIterator i(Obj());
+        while( i.more() ) {
+            BSONElement e = i.next();
+            const char *f = e.fieldName();
+            try {
+                unsigned u = stringToNum(f);
+                assert( u < 4096 );
+                if( u >= v.size() )
+                    v.resize(u+1);
+                v[u] = e;
+            }
+            catch(unsigned) { }
+        }
+        return v;
+    }
 
 } // namespace mongo
diff --git a/db/jsobj.h b/db/jsobj.h
index aaf059b..258a952 100644
--- a/db/jsobj.h
+++ b/db/jsobj.h
@@ -23,2024 +23,25 @@
    "BSON" stands for "binary JSON" -- ie a binary way to represent objects that would be
    represented in JSON (plus a few extensions useful for databases & other languages).
 
-   http://www.mongodb.org/display/DOCS/BSON
+   http://www.bsonspec.org/
 */
 
 #pragma once
 
-#include "../stdafx.h"
-#include "../util/builder.h"
+#include "../pch.h"
+#include "../bson/util/builder.h"
 #include "../util/optime.h"
 #include "boost/utility.hpp"
-
 #include <set>
-
-namespace mongo {
-
-    class BSONObj;
-    struct BSONArray; // empty subclass of BSONObj useful for overloading
-    class BSONElement;
-    class Record;
-    class BSONObjBuilder;
-    class BSONArrayBuilder;
-    class BSONObjBuilderValueStream;
-
-#pragma pack(1)
-
-    /** 
-        the complete list of valid BSON types
-    */
-    enum BSONType {
-        /** smaller than all other types */
-        MinKey=-1,
-        /** end of object */
-        EOO=0,
-        /** double precision floating point value */
-        NumberDouble=1,
-        /** character string, stored in utf8 */
-        String=2,
-        /** an embedded object */
-        Object=3,
-        /** an embedded array */
-        Array=4,
-        /** binary data */
-        BinData=5,
-        /** Undefined type */
-        Undefined=6,
-        /** ObjectId */
-        jstOID=7,
-        /** boolean type */
-        Bool=8,
-        /** date type */
-        Date=9,
-        /** null type */
-        jstNULL=10,
-        /** regular expression, a pattern with options */
-        RegEx=11,
-        /** deprecated / will be redesigned */
-        DBRef=12,
-        /** deprecated / use CodeWScope */
-        Code=13,
-        /** a programming language (e.g., Python) symbol */
-        Symbol=14,
-        /** javascript code that can execute on the database server, with SavedContext */
-        CodeWScope=15,
-        /** 32 bit signed integer */
-        NumberInt = 16,
-        /** Updated to a Date with value next OpTime on insert */
-        Timestamp = 17,
-        /** 64 bit integer */
-        NumberLong = 18,
-        /** max type that is not MaxKey */
-        JSTypeMax=18,
-        /** larger than all other types */
-        MaxKey=127
-    };
-
-    /* subtypes of BinData.
-       bdtCustom and above are ones that the JS compiler understands, but are
-       opaque to the database.
-    */
-    enum BinDataType { Function=1, ByteArray=2, bdtUUID = 3, MD5Type=5, bdtCustom=128 };
-    
-    /**	Object ID type.
-        BSON objects typically have an _id field for the object id.  This field should be the first 
-        member of the object when present.  class OID is a special type that is a 12 byte id which 
-        is likely to be unique to the system.  You may also use other types for _id's.
-        When _id field is missing from a BSON object, on an insert the database may insert one 
-        automatically in certain circumstances.
-
-        Warning: You must call OID::newState() after a fork().
-    */
-    class OID {
-        union {
-            struct{
-                long long a;
-                unsigned b;
-            };
-            unsigned char data[12];
-        };
-        static unsigned _machine;
-    public:
-        /** call this after a fork */
-        static void newState();
-
-		/** initialize to 'null' */
-		void clear() { a = 0; b = 0; }
-
-        const unsigned char *getData() const { return data; }
-
-        bool operator==(const OID& r) {
-            return a==r.a&&b==r.b;
-        }
-        bool operator!=(const OID& r) {
-            return a!=r.a||b!=r.b;
-        }
-
-        /** The object ID output as 24 hex digits. */
-        string str() const {
-            stringstream s;
-            s << hex;
-            //            s.fill( '0' );
-            //            s.width( 2 );
-            // fill wasn't working so doing manually...
-            for( int i = 0; i < 8; i++ ) {
-                unsigned u = data[i];
-                if( u < 16 ) s << '0';
-                s << u;
-            }
-            const unsigned char * raw = (const unsigned char*)&b;
-            for( int i = 0; i < 4; i++ ) {
-                unsigned u = raw[i];
-                if( u < 16 ) s << '0';
-                s << u;
-            }
-            /*
-            s.width( 16 );
-            s << a;
-            s.width( 8 );
-            s << b;
-            s << dec;
-            */
-            return s.str();
-        }
-        
-        /**
-           sets the contents to a new oid / randomized value
-         */
-        void init();
-
-        /** Set to the hex string value specified. */
-        void init( string s );
-        
-    };
-    ostream& operator<<( ostream &s, const OID &o );
-
-    /** Formatting mode for generating JSON from BSON.
-        See <http://mongodb.onconfluence.com/display/DOCS/Mongo+Extended+JSON>
-        for details.
-     */
-    enum JsonStringFormat {
-        /** strict RFC format */
-        Strict,
-        /** 10gen format, which is close to JS format.  This form is understandable by
-        javascript running inside the Mongo server via eval() */
-        TenGen,
-        /** Javascript JSON compatible */
-        JS
-    };
-
-    /* l and r MUST have same type when called: check that first. */
-    int compareElementValues(const BSONElement& l, const BSONElement& r);
-
-#pragma pack()
-
-    /* internals
-       <type><fieldName    ><value>
-       -------- size() ------------
-             -fieldNameSize-
-                            value()
-       type()
-    */
-    /** BSONElement represents an "element" in a BSONObj.  So for the object { a : 3, b : "abc" },
-       'a : 3' is the first element (key+value).
-       
-       The BSONElement object points into the BSONObj's data.  Thus the BSONObj must stay in scope
-       for the life of the BSONElement.
-    */
-    class BSONElement {
-        friend class BSONObjIterator;
-        friend class BSONObj;
-    public:
-        string toString( bool includeFieldName = true ) const;
-        operator string() const { return toString(); }
-        string jsonString( JsonStringFormat format, bool includeFieldNames = true ) const;
-
-        /** Returns the type of the element */
-        BSONType type() const {
-            return (BSONType) *data;
-        }
-        
-        /** returns the tyoe of the element fixed for the main type
-            the main purpose is numbers.  any numeric type will return NumberDouble
-            Note: if the order changes, indexes have to be re-built or than can be corruption
-         */
-        int canonicalType() const {
-            BSONType t = type();
-            switch ( t ){
-            case MinKey:
-            case MaxKey:
-                return t;
-            case EOO:
-            case Undefined:
-                return 0;
-            case jstNULL:
-                return 5;
-            case NumberDouble:
-            case NumberInt:
-            case NumberLong:
-                return 10;
-            case String:
-            case Symbol:
-                return 15;
-            case Object:
-                return 20;
-            case Array:
-                return 25;
-            case BinData:
-                return 30;
-            case jstOID:
-                return 35;
-            case Bool:
-                return 40;
-            case Date:
-            case Timestamp:
-                return 45;
-            case RegEx:
-                return 50;
-            case DBRef:
-                return 55;
-            case Code:
-                return 60;
-            case CodeWScope:
-                return 65;
-            default:
-                assert(0);
-                return -1;
-            }
-        }
-
-        /** Indicates if it is the end-of-object element, which is present at the end of 
-            every BSON object. 
-        */
-        bool eoo() const {
-            return type() == EOO;
-        }
-
-        /** Size of the element.
-            @param maxLen If maxLen is specified, don't scan more than maxLen bytes to calculate size. 
-        */
-        int size( int maxLen = -1 ) const;
-
-        /** Wrap this element up as a singleton object. */
-        BSONObj wrap() const;
-
-        /** Wrap this element up as a singleton object with a new name. */
-        BSONObj wrap( const char* newName) const;
-
-        /** field name of the element.  e.g., for 
-           name : "Joe"
-           "name" is the fieldname
-        */
-        const char * fieldName() const {
-            if ( eoo() ) return ""; // no fieldname for it.
-            return data + 1;
-        }
-
-        /** raw data of the element's value (so be careful). */
-        const char * value() const {
-            return (data + fieldNameSize() + 1);
-        }
-        /** size in bytes of the element's value (when applicable). */
-        int valuesize() const {
-            return size() - fieldNameSize() - 1;
-        }
-
-        bool isBoolean() const {
-            return type() == Bool;
-        }
-
-        /** @return value of a boolean element.  
-            You must assure element is a boolean before 
-            calling. */
-        bool boolean() const {
-            return *value() ? true : false;
-        }
-
-        /** Retrieve a java style date value from the element. 
-            Ensure element is of type Date before calling.
-        */
-        Date_t date() const {
-            return *reinterpret_cast< const Date_t* >( value() );
-        }
-
-        /** Convert the value to boolean, regardless of its type, in a javascript-like fashion 
-            (i.e., treat zero and null as false).
-            */
-        bool trueValue() const {
-            switch( type() ) {
-                case NumberLong:
-                    return *reinterpret_cast< const long long* >( value() ) != 0;
-                case NumberDouble:
-                    return *reinterpret_cast< const double* >( value() ) != 0;
-                case NumberInt:
-                    return *reinterpret_cast< const int* >( value() ) != 0;
-                case Bool:
-                    return boolean();
-                case EOO:
-                case jstNULL:
-                case Undefined:
-                    return false;
-                
-                default:
-                    ;
-            }
-            return true;
-        }
-
-        /** True if element is of a numeric type. */
-        bool isNumber() const {
-            switch( type() ) {
-                case NumberLong:
-                case NumberDouble:
-                case NumberInt:
-                    return true;
-                default: 
-                    return false;
-            }
-        }
-
-        bool isSimpleType() const {
-            switch( type() ){
-            case NumberLong:
-            case NumberDouble:
-            case NumberInt:
-            case String:
-            case Bool:
-            case Date:
-            case jstOID:
-                return true;
-            default: 
-                return false;
-            }
-        }
-
-        /** Return double value for this field. MUST be NumberDouble type. */
-        double _numberDouble() const {return *reinterpret_cast< const double* >( value() ); }
-        /** Return double value for this field. MUST be NumberInt type. */
-        int _numberInt() const {return *reinterpret_cast< const int* >( value() ); }
-        /** Return double value for this field. MUST be NumberLong type. */
-        long long _numberLong() const {return *reinterpret_cast< const long long* >( value() ); }
-
-        /** Retrieve int value for the element safely.  Zero returned if not a number. */
-        int numberInt() const { 
-            switch( type() ) {
-                case NumberDouble:
-                    return (int) _numberDouble();
-                case NumberInt:
-                    return _numberInt();
-                case NumberLong:
-                    return (int) _numberLong();
-                default:
-                    return 0;
-            }
-        }
-
-        /** Retrieve long value for the element safely.  Zero returned if not a number. */
-        long long numberLong() const { 
-            switch( type() ) {
-                case NumberDouble:
-                    return (long long) _numberDouble();
-                case NumberInt:
-                    return _numberInt();
-                case NumberLong:
-                    return _numberLong();
-                default:
-                    return 0;
-            }
-        }
-
-        /** Retrieve the numeric value of the element.  If not of a numeric type, returns 0. 
-            NOTE: casts to double, data loss may occur with large (>52 bit) NumberLong values.
-        */
-        double numberDouble() const {
-            switch( type() ) {
-                case NumberDouble:
-                    return _numberDouble();
-                case NumberInt:
-                    return *reinterpret_cast< const int* >( value() );
-                case NumberLong:
-                    return (double) *reinterpret_cast< const long long* >( value() );
-                default:
-                    return 0;
-            }
-        }
-        /** Retrieve the numeric value of the element.  If not of a numeric type, returns 0. 
-            NOTE: casts to double, data loss may occur with large (>52 bit) NumberLong values.
-        */
-        double number() const { return numberDouble(); }
-
-        /** Retrieve the object ID stored in the object. 
-            You must ensure the element is of type jstOID first. */
-        const OID &__oid() const {
-            return *reinterpret_cast< const OID* >( value() );
-        }
-
-        /** True if element is null. */
-        bool isNull() const {
-            return type() == jstNULL;
-        }
-        
-        /** Size (length) of a string element.  
-            You must assure of type String first.  */
-        int valuestrsize() const {
-            return *reinterpret_cast< const int* >( value() );
-        }
-
-        // for objects the size *includes* the size of the size field
-        int objsize() const {
-            return *reinterpret_cast< const int* >( value() );
-        }
-
-        /** Get a string's value.  Also gives you start of the real data for an embedded object. 
-            You must assure data is of an appropriate type first -- see also valuestrsafe().
-        */
-        const char * valuestr() const {
-            return value() + 4;
-        }
-
-        /** Get the string value of the element.  If not a string returns "". */
-        const char *valuestrsafe() const {
-            return type() == String ? valuestr() : "";
-        }
-        /** Get the string value of the element.  If not a string returns "". */
-        string str() const { return valuestrsafe(); }
-
-        /** Get javascript code of a CodeWScope data element. */
-        const char * codeWScopeCode() const {
-            return value() + 8;
-        }
-        /** Get the scope SavedContext of a CodeWScope data element. */
-        const char * codeWScopeScopeData() const {
-            // TODO fix
-            return codeWScopeCode() + strlen( codeWScopeCode() ) + 1;
-        }
-
-        /** Get the embedded object this element holds. */
-        BSONObj embeddedObject() const;
-
-        /* uasserts if not an object */
-        BSONObj embeddedObjectUserCheck() const;
-
-        BSONObj codeWScopeObject() const;
-
-        string ascode() const {
-            switch( type() ){
-            case String:
-            case Code:
-                return valuestr();
-            case CodeWScope:
-                return codeWScopeCode();
-            default:
-                log() << "can't convert type: " << (int)(type()) << " to code" << endl;
-            }
-            uassert( 10062 ,  "not code" , 0 );
-            return "";
-        }
-
-        /** Get binary data.  Element must be of type BinData */
-        const char *binData(int& len) const { 
-            // BinData: <int len> <byte subtype> <byte[len] data>
-            assert( type() == BinData );
-            len = valuestrsize();
-            return value() + 5;
-        }
-        
-        BinDataType binDataType() const {
-            // BinData: <int len> <byte subtype> <byte[len] data>
-            assert( type() == BinData );
-            unsigned char c = (value() + 4)[0];
-            return (BinDataType)c;
-        }
-
-        /** Retrieve the regex string for a Regex element */
-        const char *regex() const {
-            assert(type() == RegEx);
-            return value();
-        }
-
-        /** Retrieve the regex flags (options) for a Regex element */
-        const char *regexFlags() const {
-            const char *p = regex();
-            return p + strlen(p) + 1;
-        }
-
-        /** like operator== but doesn't check the fieldname,
-           just the value.
-        */
-        bool valuesEqual(const BSONElement& r) const {
-            switch( type() ) {
-                case NumberLong:
-                    return _numberLong() == r.numberLong() && r.isNumber();
-                case NumberDouble:
-                    return _numberDouble() == r.number() && r.isNumber();
-                case NumberInt:
-                    return _numberInt() == r.numberInt() && r.isNumber();
-                default:
-                    ;
-            }
-            bool match= valuesize() == r.valuesize() &&
-                        memcmp(value(),r.value(),valuesize()) == 0;
-            return match && canonicalType() == r.canonicalType();
-        }
-
-        /** Returns true if elements are equal. */
-        bool operator==(const BSONElement& r) const {
-            if ( strcmp(fieldName(), r.fieldName()) != 0 )
-                return false;
-            return valuesEqual(r);
-        }
-
-
-        /** Well ordered comparison.
-         @return <0: l<r. 0:l==r. >0:l>r
-         order by type, field name, and field value.
-         If considerFieldName is true, pay attention to the field name.
-         */
-        int woCompare( const BSONElement &e, bool considerFieldName = true ) const;
-
-        const char * rawdata() const {
-            return data;
-        }
-        
-        /** 0 == Equality, just not defined yet */
-        int getGtLtOp( int def = 0 ) const;
-
-        /** Constructs an empty element */
-        BSONElement();
-        
-        /** Check that data is internally consistent. */
-        void validate() const;
-
-        /** True if this element may contain subobjects. */
-        bool mayEncapsulate() const {
-            switch ( type() ){
-            case Object:
-            case Array:
-            case CodeWScope:
-                return true;
-            default:
-                return false;
-            }
-        }
-
-        /** True if this element can be a BSONObj */
-        bool isABSONObj() const {
-            switch( type() ){
-            case Object:
-            case Array:
-                return true;
-            default:
-                return false;
-            }
-        }
-
-        Date_t timestampTime() const{
-            unsigned long long t = ((unsigned int*)(value() + 4 ))[0];
-            return t * 1000;
-        }
-        unsigned int timestampInc() const{
-            return ((unsigned int*)(value() ))[0];
-        }
-
-        const char * dbrefNS() const {
-            uassert( 10063 ,  "not a dbref" , type() == DBRef );
-            return value() + 4;
-        }
-
-        const OID& dbrefOID() const {
-            uassert( 10064 ,  "not a dbref" , type() == DBRef );
-            const char * start = value();
-            start += 4 + *reinterpret_cast< const int* >( start );
-            return *reinterpret_cast< const OID* >( start );
-        }
-
-        bool operator<( const BSONElement& other ) const {
-            int x = (int)canonicalType() - (int)other.canonicalType();
-            if ( x < 0 ) return true;
-            else if ( x > 0 ) return false;
-            return compareElementValues(*this,other) < 0;
-        }
-        
-        // If maxLen is specified, don't scan more than maxLen bytes.
-        BSONElement(const char *d, int maxLen = -1) : data(d) {
-            fieldNameSize_ = -1;
-            if ( eoo() )
-                fieldNameSize_ = 0;
-            else {
-                if ( maxLen != -1 ) {
-                    int size = strnlen( fieldName(), maxLen - 1 );
-                    massert( 10333 ,  "Invalid field name", size != -1 );
-                    fieldNameSize_ = size + 1;
-                }
-            }
-            totalSize = -1;
-        }
-    private:
-        const char *data;
-        mutable int fieldNameSize_; // cached value
-        int fieldNameSize() const {
-            if ( fieldNameSize_ == -1 )
-                fieldNameSize_ = (int)strlen( fieldName() ) + 1;
-            return fieldNameSize_;
-        }
-        mutable int totalSize; /* caches the computed size */
-    };
-    
-    int getGtLtOp(const BSONElement& e);
-
-    struct BSONElementCmpWithoutField {
-        bool operator()( const BSONElement &l, const BSONElement &r ) const {
-            return l.woCompare( r, false ) < 0;
-        }
-    };
-    
-    typedef set< BSONElement, BSONElementCmpWithoutField > BSONElementSet;
-    
-    /**
-	   C++ representation of a "BSON" object -- that is, an extended JSON-style 
-       object in a binary representation.
-
-       Note that BSONObj's have a smart pointer capability built in -- so you can 
-       pass them around by value.  The reference counts used to implement this
-       do not use locking, so copying and destroying BSONObj's are not thread-safe
-       operations.
-
-     BSON object format:
-     
-     \code
-     <unsigned totalSize> {<byte BSONType><cstring FieldName><Data>}* EOO
-     
-     totalSize includes itself.
-     
-     Data:
-     Bool:      <byte>
-     EOO:       nothing follows
-     Undefined: nothing follows
-     OID:       an OID object
-     NumberDouble: <double>
-     NumberInt: <int32>
-     String:    <unsigned32 strsizewithnull><cstring>
-     Date:      <8bytes>
-     Regex:     <cstring regex><cstring options>
-     Object:    a nested object, leading with its entire size, which terminates with EOO.
-     Array:     same as object
-     DBRef:     <strlen> <cstring ns> <oid>
-     DBRef:     a database reference: basically a collection name plus an Object ID
-     BinData:   <int len> <byte subtype> <byte[len] data>
-     Code:      a function (not a closure): same format as String.
-     Symbol:    a language symbol (say a python symbol).  same format as String.
-     Code With Scope: <total size><String><Object>
-     \endcode
-     */
-    class BSONObj {
-        friend class BSONObjIterator;
-        class Holder {
-        public:
-            Holder( const char *objdata ) :
-            _objdata( objdata ) {
-            }
-            ~Holder() {
-                free((void *)_objdata);
-                _objdata = 0;
-            }
-        private:
-            const char *_objdata;
-        };
-        const char *_objdata;
-        boost::shared_ptr< Holder > _holder;
-        void init(const char *data, bool ifree) {
-            if ( ifree )
-                _holder.reset( new Holder( data ) );
-            _objdata = data;
-            if ( ! isValid() ){
-                stringstream ss;
-                ss << "Invalid BSONObj spec size: " << objsize();
-                try {
-                    BSONElement e = firstElement();
-                    ss << " first element:" << e.toString() << " ";
-                }
-                catch ( ... ){}
-                string s = ss.str();
-                massert( 10334 ,  s , 0 );
-            }
-        }
-#pragma pack(1)
-        static struct EmptyObject {
-            EmptyObject() {
-                len = 5;
-                jstype = EOO;
-            }
-            int len;
-            char jstype;
-        } emptyObject;
-#pragma pack()
-    public:
-        /** Construct a BSONObj from data in the proper format. 
-            @param ifree true if the BSONObj should free() the msgdata when 
-            it destructs. 
-            */
-        explicit BSONObj(const char *msgdata, bool ifree = false) {
-            init(msgdata, ifree);
-        }
-        BSONObj(const Record *r);
-        /** Construct an empty BSONObj -- that is, {}. */
-        BSONObj() : _objdata( reinterpret_cast< const char * >( &emptyObject ) ) { }
-        // defensive
-        ~BSONObj() { _objdata = 0; }
-
-        void appendSelfToBufBuilder(BufBuilder& b) const {
-            assert( objsize() );
-            b.append(reinterpret_cast<const void *>( objdata() ), objsize());
-        }
-
-        /** Readable representation of a BSON object in an extended JSON-style notation. 
-            This is an abbreviated representation which might be used for logging.
-        */
-        string toString() const;
-        operator string() const { return toString(); }
-        
-        /** Properly formatted JSON string. */
-        string jsonString( JsonStringFormat format = Strict ) const;
-
-        /** note: addFields always adds _id even if not specified */
-        int addFields(BSONObj& from, set<string>& fields); /* returns n added */
-
-        /** returns # of top level fields in the object
-           note: iterates to count the fields
-        */
-        int nFields() const;
-
-        /** adds the field names to the fields set.  does NOT clear it (appends). */
-        int getFieldNames(set<string>& fields) const;
-
-        /** return has eoo() true if no match
-           supports "." notation to reach into embedded objects
-        */
-        BSONElement getFieldDotted(const char *name) const;
-        /** Like getFieldDotted(), but expands multikey arrays and returns all matching objects
-         */
-        void getFieldsDotted(const char *name, BSONElementSet &ret ) const;
-        /** Like getFieldDotted(), but returns first array encountered while traversing the
-            dotted fields of name.  The name variable is updated to represent field
-            names with respect to the returned element. */
-        BSONElement getFieldDottedOrArray(const char *&name) const;
-
-        /** Get the field of the specified name. eoo() is true on the returned 
-            element if not found. 
-        */
-        BSONElement getField(const char *name) const;
-
-        /** Get the field of the specified name. eoo() is true on the returned 
-            element if not found. 
-        */
-        BSONElement getField(const string name) const {
-            return getField( name.c_str() );
-        };
-
-        /** Get the field of the specified name. eoo() is true on the returned 
-            element if not found. 
-        */
-        BSONElement operator[] (const char *field) const { 
-            return getField(field);
-        }
-
-        BSONElement operator[] (const string& field) const { 
-            return getField(field);
-        }
-
-        BSONElement operator[] (int field) const { 
-            stringstream ss;
-            ss << field;
-            string s = ss.str();
-            return getField(s.c_str());
-        }
-
-		/** @return true if field exists */
-        bool hasField( const char * name )const {
-            return ! getField( name ).eoo();
-        }
-
-        /** @return "" if DNE or wrong type */
-        const char * getStringField(const char *name) const;
-
-		/** @return subobject of the given name */
-        BSONObj getObjectField(const char *name) const;
-
-        /** @return INT_MIN if not present - does some type conversions */
-        int getIntField(const char *name) const;
-
-        /** @return false if not present */
-        bool getBoolField(const char *name) const;
-
-        /** makes a new BSONObj with the fields specified in pattern.
-           fields returned in the order they appear in pattern.
-           if any field is missing or undefined in the object, that field in the
-           output will be null.
-
-           sets output field names to match pattern field names.
-           If an array is encountered while scanning the dotted names in pattern,
-           that field is treated as missing.
-        */
-        BSONObj extractFieldsDotted(BSONObj pattern) const;
-        
-        /**
-           sets element field names to empty string
-           If a field in pattern is missing, it is omitted from the returned
-           object.
-        */
-        BSONObj extractFieldsUnDotted(BSONObj pattern) const;
-        
-        /** extract items from object which match a pattern object.
-			e.g., if pattern is { x : 1, y : 1 }, builds an object with 
-			x and y elements of this object, if they are present.
-           returns elements with original field names
-        */
-        BSONObj extractFields(const BSONObj &pattern , bool fillWithNull=false) const;
-        
-        BSONObj filterFieldsUndotted(const BSONObj &filter, bool inFilter) const;
-
-        BSONElement getFieldUsingIndexNames(const char *fieldName, const BSONObj &indexKey) const;
-        
-        /** @return the raw data of the object */
-        const char *objdata() const {
-            return _objdata;
-        }
-        /** @return total size of the BSON object in bytes */
-        int objsize() const {
-            return *(reinterpret_cast<const int*>(objdata()));
-        }
-
-        bool isValid();
-
-        /** @return if the user is a valid user doc
-            criter: isValid() no . or $ field names
-         */
-        bool okForStorage() const;
-
-		/** @return true if object is empty -- i.e.,  {} */
-        bool isEmpty() const {
-            return objsize() <= 5;
-        }
-
-        void dump() const {
-            out() << hex;
-            const char *p = objdata();
-            for ( int i = 0; i < objsize(); i++ ) {
-                out() << i << '\t' << ( 0xff & ( (unsigned) *p ) );
-                if ( *p >= 'A' && *p <= 'z' )
-                    out() << '\t' << *p;
-                out() << endl;
-                p++;
-            }
-        }
-
-        // Alternative output format
-        string hexDump() const;
-        
-        /**wo='well ordered'.  fields must be in same order in each object.
-           Ordering is with respect to the signs of the elements in idxKey.
-		   @return  <0 if l<r. 0 if l==r. >0 if l>r
-        */
-        int woCompare(const BSONObj& r, const BSONObj &idxKey = BSONObj(),
-                      bool considerFieldName=true) const;
-        
-        int woSortOrder( const BSONObj& r , const BSONObj& sortKey ) const;
-
-        /** This is "shallow equality" -- ints and doubles won't match.  for a
-           deep equality test use woCompare (which is slower).
-        */
-        bool woEqual(const BSONObj& r) const {
-            int os = objsize();
-            if ( os == r.objsize() ) {
-                return (os == 0 || memcmp(objdata(),r.objdata(),os)==0);
-            }
-            return false;
-        }
-
-		/** @return first field of the object */
-        BSONElement firstElement() const {
-            return BSONElement(objdata() + 4);
-        }
-
-		/** use getField() instead. */
-        //BSONElement getField(const char *name) const;
-        //BSONElement getField(string name) const {
-
-		/** @return true if field exists in the object */
-        bool hasElement(const char *name) const;
-
-		/** Get the _id field from the object.  For good performance drivers should 
-            assure that _id is the first element of the object; however, correct operation 
-            is assured regardless.
-            @return true if found
-		*/
-		bool getObjectID(BSONElement& e) const;
-
-        /** makes a copy of the object. 
-        */
-        BSONObj copy() const;
-
-        /* make sure the data buffer is under the control of BSONObj's and not a remote buffer */
-        BSONObj getOwned() const{
-            if ( !isOwned() )
-                return copy();
-            return *this;
-        }
-        bool isOwned() const { return _holder.get() != 0; }
-
-        /** @return A hash code for the object */
-        int hash() const {
-            unsigned x = 0;
-            const char *p = objdata();
-            for ( int i = 0; i < objsize(); i++ )
-                x = x * 131 + p[i];
-            return (x & 0x7fffffff) | 0x8000000; // must be > 0
-        }
-
-        // Return a version of this object where top level elements of types
-        // that are not part of the bson wire protocol are replaced with
-        // string identifier equivalents.
-        // TODO Support conversion of element types other than min and max.
-        BSONObj clientReadable() const;
-        
-        /** Return new object with the field names replaced by those in the
-            passed object. */
-        BSONObj replaceFieldNames( const BSONObj &obj ) const;
-        
-        /** true unless corrupt */
-        bool valid() const;
-        
-        string md5() const;
-        
-        bool operator==( const BSONObj& other ){
-            return woCompare( other ) == 0;
-        }
-
-        enum MatchType {
-            Equality = 0,
-            LT = 0x1,
-            LTE = 0x3,
-            GTE = 0x6,
-            GT = 0x4,
-            opIN = 0x8, // { x : { $in : [1,2,3] } }
-            NE = 0x9,
-            opSIZE = 0x0A,
-            opALL = 0x0B,
-            NIN = 0x0C,
-            opEXISTS = 0x0D,
-            opMOD = 0x0E,
-            opTYPE = 0x0F,
-            opREGEX = 0x10,
-            opOPTIONS = 0x11,
-            opELEM_MATCH = 0x12,
-            opNEAR = 0x13,
-            opWITHIN = 0x14,
-        };        
-    };
-    ostream& operator<<( ostream &s, const BSONObj &o );
-    ostream& operator<<( ostream &s, const BSONElement &e );
-
-    struct BSONArray: BSONObj {
-        // Don't add anything other than forwarding constructors!!!
-        BSONArray(): BSONObj() {}
-        explicit BSONArray(const BSONObj& obj): BSONObj(obj) {}
-    };
-
-    class BSONObjCmp {
-    public:
-        BSONObjCmp( const BSONObj &_order = BSONObj() ) : order( _order ) {}
-        bool operator()( const BSONObj &l, const BSONObj &r ) const {
-            return l.woCompare( r, order ) < 0;
-        }
-    private:
-        BSONObj order;
-    };
-
-    class BSONObjCmpDefaultOrder : public BSONObjCmp {
-    public:
-        BSONObjCmpDefaultOrder() : BSONObjCmp( BSONObj() ) {}
-    };
-
-    typedef set< BSONObj, BSONObjCmpDefaultOrder > BSONObjSetDefaultOrder;
-
-    enum FieldCompareResult {
-        LEFT_SUBFIELD = -2,
-        LEFT_BEFORE = -1,
-        SAME = 0,
-        RIGHT_BEFORE = 1 ,
-        RIGHT_SUBFIELD = 2
-    };
-
-    FieldCompareResult compareDottedFieldNames( const string& l , const string& r );
-
-/** Use BSON macro to build a BSONObj from a stream 
-
-    e.g., 
-       BSON( "name" << "joe" << "age" << 33 )
-
-    with auto-generated object id:
-       BSON( GENOID << "name" << "joe" << "age" << 33 )
- 
-    The labels GT, GTE, LT, LTE, NE can be helpful for stream-oriented construction
-    of a BSONObj, particularly when assembling a Query.  For example,
-    BSON( "a" << GT << 23.4 << NE << 30 << "b" << 2 ) produces the object
-    { a: { \$gt: 23.4, \$ne: 30 }, b: 2 }.
-*/
-#define BSON(x) (( mongo::BSONObjBuilder(64) << x ).obj())
-
-/** Use BSON_ARRAY macro like BSON macro, but without keys
-
-    BSONArray arr = BSON_ARRAY( "hello" << 1 << BSON( "foo" << BSON_ARRAY( "bar" << "baz" << "qux" ) ) );
-
- */
-#define BSON_ARRAY(x) (( mongo::BSONArrayBuilder() << x ).arr())
-
-    /* Utility class to auto assign object IDs.
-       Example:
-         cout << BSON( GENOID << "z" << 3 ); // { _id : ..., z : 3 }
-    */
-    extern struct IDLabeler { } GENOID;
-
-    /* Utility class to add a Date element with the current time
-       Example: 
-         cout << BSON( "created" << DATENOW ); // { created : "2009-10-09 11:41:42" }
-    */
-    extern struct DateNowLabeler { } DATENOW;
-
-    // Utility class to implement GT, GTE, etc as described above.
-    class Labeler {
-    public:
-        struct Label {
-            Label( const char *l ) : l_( l ) {}
-            const char *l_;
-        };
-        Labeler( const Label &l, BSONObjBuilderValueStream *s ) : l_( l ), s_( s ) {}
-        template<class T>
-        BSONObjBuilder& operator<<( T value );
-
-        /* the value of the element e is appended i.e. for 
-             "age" << GT << someElement
-           one gets 
-             { age : { $gt : someElement's value } } 
-        */
-        BSONObjBuilder& operator<<( const BSONElement& e );
-    private:
-        const Label &l_;
-        BSONObjBuilderValueStream *s_;
-    };
-    
-    extern Labeler::Label GT;
-    extern Labeler::Label GTE;
-    extern Labeler::Label LT;
-    extern Labeler::Label LTE;
-    extern Labeler::Label NE;
-    extern Labeler::Label SIZE;
-    
-    // Utility class to implement BSON( key << val ) as described above.
-    class BSONObjBuilderValueStream : public boost::noncopyable {
-    public:
-        friend class Labeler;
-        BSONObjBuilderValueStream( BSONObjBuilder * builder );
-
-        BSONObjBuilder& operator<<( const BSONElement& e );
-        
-        template<class T> 
-        BSONObjBuilder& operator<<( T value );
-
-        BSONObjBuilder& operator<<(DateNowLabeler& id);
-        
-        Labeler operator<<( const Labeler::Label &l );
-
-        void endField( const char *nextFieldName = 0 );
-        bool subobjStarted() const { return _fieldName != 0; }
-        
-    private:
-        const char * _fieldName;
-        BSONObjBuilder * _builder;
-
-        bool haveSubobj() const { return _subobj.get() != 0; }
-        BSONObjBuilder *subobj();
-        auto_ptr< BSONObjBuilder > _subobj;
-    };
-    
-    /**
-       used in conjuction with BSONObjBuilder, allows for proper buffer size to prevent crazy memory usage
-     */
-    class BSONSizeTracker {
-    public:
-#define BSONSizeTrackerSize 10
-
-        BSONSizeTracker(){
-            _pos = 0;
-            for ( int i=0; i<BSONSizeTrackerSize; i++ )
-                _sizes[i] = 512; // this is the default, so just be consistent
-        }
-        
-        ~BSONSizeTracker(){
-        }
-        
-        void got( int size ){
-            _sizes[_pos++] = size;
-            if ( _pos >= BSONSizeTrackerSize )
-                _pos = 0;
-        }
-        
-        /**
-         * right now choosing largest size
-         */
-        int getSize() const {
-            int x = 16; // sane min
-            for ( int i=0; i<BSONSizeTrackerSize; i++ ){
-                if ( _sizes[i] > x )
-                    x = _sizes[i];
-            }
-            return x;
-        }
-        
-    private:
-        int _pos;
-        int _sizes[BSONSizeTrackerSize];
-    };
-    
-    /**
-       utility for creating a BSONObj
-     */
-    class BSONObjBuilder : boost::noncopyable {
-    public:
-        /** @param initsize this is just a hint as to the final size of the object */
-        BSONObjBuilder(int initsize=512) : b(buf_), buf_(initsize), offset_( 0 ), s_( this ) , _tracker(0) {
-            b.skip(4); /*leave room for size field*/
-        }
-
-        /** @param baseBuilder construct a BSONObjBuilder using an existing BufBuilder */
-        BSONObjBuilder( BufBuilder &baseBuilder ) : b( baseBuilder ), buf_( 0 ), offset_( baseBuilder.len() ), s_( this ) , _tracker(0) {
-            b.skip( 4 );
-        }
-        
-        BSONObjBuilder( const BSONSizeTracker & tracker ) : b(buf_) , buf_(tracker.getSize() ), offset_(0), s_( this ) , _tracker( (BSONSizeTracker*)(&tracker) ){
-            b.skip( 4 );
-        }
-
-        /** add all the fields from the object specified to this object */
-        BSONObjBuilder& appendElements(BSONObj x);
-
-        /** append element to the object we are building */
-        void append( const BSONElement& e) {
-            assert( !e.eoo() ); // do not append eoo, that would corrupt us. the builder auto appends when done() is called.
-            b.append((void*) e.rawdata(), e.size());
-        }
-
-        /** append an element but with a new name */
-        void appendAs(const BSONElement& e, const char *as) {
-            assert( !e.eoo() ); // do not append eoo, that would corrupt us. the builder auto appends when done() is called.
-            b.append((char) e.type());
-            b.append(as);
-            b.append((void *) e.value(), e.valuesize());
-        }
-
-        void appendAs(const BSONElement& e, const string& as) {
-            appendAs( e , as.c_str() );
-        }
-
-
-        /** add a subobject as a member */
-        void append(const char *fieldName, BSONObj subObj) {
-            b.append((char) Object);
-            b.append(fieldName);
-            b.append((void *) subObj.objdata(), subObj.objsize());
-        }
-
-        void append(const string& fieldName , BSONObj subObj) {
-            append( fieldName.c_str() , subObj );
-        }
-
-        /** add header for a new subobject and return bufbuilder for writing to
-            the subobject's body */
-        BufBuilder &subobjStart(const char *fieldName) {
-            b.append((char) Object);
-            b.append(fieldName);
-            return b;
-        }
-        
-        /** add a subobject as a member with type Array.  Thus arr object should have "0", "1", ...
-           style fields in it.
-        */
-        void appendArray(const char *fieldName, BSONObj subObj) {
-            b.append((char) Array);
-            b.append(fieldName);
-            b.append((void *) subObj.objdata(), subObj.objsize());
-        }
-        void append(const char *fieldName, BSONArray arr) { appendArray(fieldName, arr); }
-        
-
-        /** add header for a new subarray and return bufbuilder for writing to
-            the subarray's body */
-        BufBuilder &subarrayStart(const char *fieldName) {
-            b.append((char) Array);
-            b.append(fieldName);
-            return b;
-        }
-        
-        /** Append a boolean element */
-        void appendBool(const char *fieldName, int val) {
-            b.append((char) Bool);
-            b.append(fieldName);
-            b.append((char) (val?1:0));
-        }
-
-        /** Append a boolean element */
-        void append(const char *fieldName, bool val) {
-            b.append((char) Bool);
-            b.append(fieldName);
-            b.append((char) (val?1:0));            
-        }
-        
-        /** Append a 32 bit integer element */
-        void append(const char *fieldName, int n) {
-            b.append((char) NumberInt);
-            b.append(fieldName);
-            b.append(n);
-        }
-        /** Append a 32 bit integer element */
-        void append(const string &fieldName, int n) {
-            append( fieldName.c_str(), n );
-        }
-
-        /** Append a 32 bit unsigned element - cast to a signed int. */
-        void append(const char *fieldName, unsigned n) { append(fieldName, (int) n); }
-
-        /** Append a NumberLong */
-        void append(const char *fieldName, long long n) { 
-            b.append((char) NumberLong);
-            b.append(fieldName);
-            b.append(n);
-        }
-
-        /** Append a NumberLong */
-        void append(const string& fieldName, long long n) { 
-            append( fieldName.c_str() , n );
-        }
-
-        /** appends a number.  if n < max(int)/2 then uses int, otherwise long long */
-        void appendIntOrLL( const string& fieldName , long long n ){
-            long long x = n;
-            if ( x < 0 )
-                x = x * -1;
-            if ( x < ( numeric_limits<int>::max() / 2 ) )
-                append( fieldName.c_str() , (int)n );
-            else
-                append( fieldName.c_str() , n );
-        }
-
-
-        /**
-         * appendNumber is a series of method for appending the smallest sensible type
-         * mostly for JS
-         */
-        void appendNumber( const string& fieldName , int n ){
-            append( fieldName.c_str() , n );
-        }
-
-        void appendNumber( const string& fieldName , double d ){
-            append( fieldName.c_str() , d );
-        }
-
-        void appendNumber( const string& fieldName , long long l ){
-            static long long maxInt = (int)pow( 2.0 , 30.0 );
-            static long long maxDouble = (long long)pow( 2.0 , 40.0 );
-
-            if ( l < maxInt )
-                append( fieldName.c_str() , (int)l );
-            else if ( l < maxDouble )
-                append( fieldName.c_str() , (double)l );
-            else
-                append( fieldName.c_str() , l );
-        }
-        
-        /** Append a double element */
-        BSONObjBuilder& append(const char *fieldName, double n) {
-            b.append((char) NumberDouble);
-            b.append(fieldName);
-            b.append(n);
-            return *this;
-        }
-
-        /** tries to append the data as a number
-         * @return true if the data was able to be converted to a number
-         */
-        bool appendAsNumber( const string& fieldName , const string& data );
-
-        /** Append a BSON Object ID (OID type). */
-        void appendOID(const char *fieldName, OID *oid = 0 , bool generateIfBlank = false ) {
-            b.append((char) jstOID);
-            b.append(fieldName);
-            if ( oid )
-                b.append( (void *) oid, 12 );
-            else {
-                OID tmp;
-                if ( generateIfBlank )
-                    tmp.init();
-                else
-                    tmp.clear();
-                b.append( (void *) &tmp, 12 );
-            }
-        }
-        void append( const char *fieldName, OID oid ) {
-            appendOID( fieldName, &oid );
-        }
-        /** Append a time_t date.
-            @param dt a C-style 32 bit date value, that is
-                      the number of seconds since January 1, 1970, 00:00:00 GMT
-        */
-        void appendTimeT(const char *fieldName, time_t dt) {
-            b.append((char) Date);
-            b.append(fieldName);
-            b.append(static_cast<unsigned long long>(dt) * 1000);
-        }
-        /** Append a date.  
-            @param dt a Java-style 64 bit date value, that is 
-                      the number of milliseconds since January 1, 1970, 00:00:00 GMT
-        */
-        void appendDate(const char *fieldName, Date_t dt) {
-            b.append((char) Date);
-            b.append(fieldName);
-            b.append(dt);
-        }
-        void append(const char *fieldName, Date_t dt) {
-            appendDate(fieldName, dt);
-        }
-
-        /** Append a regular expression value
-            @param regex the regular expression pattern
-            @param regex options such as "i" or "g"
-        */
-        void appendRegex(const char *fieldName, const char *regex, const char *options = "") {
-            b.append((char) RegEx);
-            b.append(fieldName);
-            b.append(regex);
-            b.append(options);
-        }
-        /** Append a regular expression value
-            @param regex the regular expression pattern
-            @param regex options such as "i" or "g"
-        */
-        void appendRegex(string fieldName, string regex, string options = "") {
-            appendRegex(fieldName.c_str(), regex.c_str(), options.c_str());
-        }
-        void appendCode(const char *fieldName, const char *code) {
-            b.append((char) Code);
-            b.append(fieldName);
-            b.append((int) strlen(code)+1);
-            b.append(code);
-        }
-        /** Append a string element */
-        BSONObjBuilder& append(const char *fieldName, const char *str) {
-            b.append((char) String);
-            b.append(fieldName);
-            b.append((int) strlen(str)+1);
-            b.append(str);
-            return *this;
-        }
-        /** Append a string element */
-        void append(const char *fieldName, string str) {
-            append(fieldName, str.c_str());
-        }
-        void appendSymbol(const char *fieldName, const char *symbol) {
-            b.append((char) Symbol);
-            b.append(fieldName);
-            b.append((int) strlen(symbol)+1);
-            b.append(symbol);
-        }
-
-        /** Append a Null element to the object */
-        void appendNull( const char *fieldName ) {
-            b.append( (char) jstNULL );
-            b.append( fieldName );
-        }
-
-        // Append an element that is less than all other keys.
-        void appendMinKey( const char *fieldName ) {
-            b.append( (char) MinKey );
-            b.append( fieldName );
-        }
-        // Append an element that is greater than all other keys.
-        void appendMaxKey( const char *fieldName ) {
-            b.append( (char) MaxKey );
-            b.append( fieldName );
-        }
-        
-        // Append a Timestamp field -- will be updated to next OpTime on db insert.
-        void appendTimestamp( const char *fieldName ) {
-            b.append( (char) Timestamp );
-            b.append( fieldName );
-            b.append( (unsigned long long) 0 );
-        }
-
-        void appendTimestamp( const char *fieldName , unsigned long long val ) {
-            b.append( (char) Timestamp );
-            b.append( fieldName );
-            b.append( val );
-        }
-
-        /**
-         * @param time - in millis (but stored in seconds)
-         */
-        void appendTimestamp( const char *fieldName , unsigned long long time , unsigned int inc ){
-            OpTime t( (unsigned) (time / 1000) , inc );
-            appendTimestamp( fieldName , t.asDate() );
-        }
-        
-        /* Deprecated (but supported) */
-        void appendDBRef( const char *fieldName, const char *ns, const OID &oid ) {
-            b.append( (char) DBRef );
-            b.append( fieldName );
-            b.append( (int) strlen( ns ) + 1 );
-            b.append( ns );
-            b.append( (void *) &oid, 12 );
-        }
-
-        /** Append a binary data element 
-            @param fieldName name of the field
-            @param len length of the binary data in bytes
-            @param type type information for the data. @see BinDataType.  Use ByteArray if you 
-                   don't care about the type.
-            @param data the byte array
-        */
-        void appendBinData( const char *fieldName, int len, BinDataType type, const char *data ) {
-            b.append( (char) BinData );
-            b.append( fieldName );
-            b.append( len );
-            b.append( (char) type );
-            b.append( (void *) data, len );
-        }
-        void appendBinData( const char *fieldName, int len, BinDataType type, const unsigned char *data ) {
-            appendBinData(fieldName, len, type, (const char *) data);
-        }
-        
-        /**
-           @param len the length of data
-         */
-        void appendBinDataArray( const char * fieldName , const char * data , int len ){
-            b.append( (char) BinData );
-            b.append( fieldName );
-            b.append( len + 4 );
-            b.append( (char)0x2 );
-            b.append( len );
-            b.append( (void *) data, len );            
-        }
-
-        /** Append to the BSON object a field of type CodeWScope.  This is a javascript code 
-            fragment accompanied by some scope that goes with it.
-            */
-        void appendCodeWScope( const char *fieldName, const char *code, const BSONObj &scope ) {
-            b.append( (char) CodeWScope );
-            b.append( fieldName );
-            b.append( ( int )( 4 + 4 + strlen( code ) + 1 + scope.objsize() ) );
-            b.append( ( int ) strlen( code ) + 1 );
-            b.append( code );
-            b.append( ( void * )scope.objdata(), scope.objsize() );
-        }
-
-        void appendUndefined( const char *fieldName ) {
-            b.append( (char) Undefined );
-            b.append( fieldName );
-        }
-        
-        /* helper function -- see Query::where() for primary way to do this. */
-        void appendWhere( const char *code, const BSONObj &scope ){
-            appendCodeWScope( "$where" , code , scope );
-        }
-        void appendWhere( const string &code, const BSONObj &scope ){
-            appendWhere( code.c_str(), scope );
-        }
-        
-        /**
-           these are the min/max when comparing, not strict min/max elements for a given type
-         */
-        void appendMinForType( const string& field , int type );
-        void appendMaxForType( const string& field , int type );
-
-        /** Append an array of values. */
-        template < class T >
-        void append( const char *fieldName, const vector< T >& vals ) {
-            BSONObjBuilder arrBuilder;
-            for ( unsigned int i = 0; i < vals.size(); ++i )
-                arrBuilder.append( numStr( i ).c_str(), vals[ i ] );
-            marshalArray( fieldName, arrBuilder.done() );
-        }
-
-        /* Append an array of ints 
-        void appendArray( const char *fieldName, const vector< int >& vals ) {
-            BSONObjBuilder arrBuilder;
-            for ( unsigned i = 0; i < vals.size(); ++i )
-                arrBuilder.append( numStr( i ).c_str(), vals[ i ] );
-            marshalArray( fieldName, arrBuilder.done() );
-        }*/
-
-        /** The returned BSONObj will free the buffer when it is finished. */
-        BSONObj obj() {
-            massert( 10335 ,  "builder does not own memory", owned() );
-            int l;
-            return BSONObj(decouple(l), true);
-        }
-
-        /** Fetch the object we have built.
-			BSONObjBuilder still frees the object when the builder goes out of 
-			scope -- very important to keep in mind.  Use obj() if you 
-			would like the BSONObj to last longer than the builder.
-        */
-        BSONObj done() {
-            return BSONObj(_done());
-        }
-
-        /** Peek at what is in the builder, but leave the builder ready for more appends.
-            The returned object is only valid until the next modification or destruction of the builder.
-            Intended use case: append a field if not already there.
-        */
-        BSONObj asTempObj() {
-            BSONObj temp(_done());
-            b.setlen(b.len()-1); //next append should overwrite the EOO
-            return temp;
-        }
-
-        /* assume ownership of the buffer - you must then free it (with free()) */
-        char* decouple(int& l) {
-            char *x = _done();
-            assert( x );
-            l = b.len();
-            b.decouple();
-            return x;
-        }
-        void decouple() {
-            b.decouple();    // post done() call version.  be sure jsobj frees...
-        }
-
-        void appendKeys( const BSONObj& keyPattern , const BSONObj& values );
-
-    private:
-        static const string numStrs[100]; // cache of 0 to 99 inclusive
-    public:
-        static string numStr( int i ) {
-            if (i>=0 && i<100)
-                return numStrs[i];
-
-            stringstream o;
-            o << i;
-            return o.str();
-        }
-
-        /** Stream oriented way to add field names and values. */
-        BSONObjBuilderValueStream &operator<<(const char * name ) {
-            s_.endField( name );
-            return s_;
-        }
-
-        /** Stream oriented way to add field names and values. */
-        BSONObjBuilder& operator<<( IDLabeler ) {
-            OID oid;
-            oid.init();
-            appendOID("_id", &oid);
-            return *this;
-        }
-
-        // prevent implicit string conversions which would allow bad things like BSON( BSON( "foo" << 1 ) << 2 )
-        struct ForceExplicitString {
-        ForceExplicitString( const string &str ) : str_( str ) {}
-            string str_;
-        };
-
-        /** Stream oriented way to add field names and values. */
-        BSONObjBuilderValueStream &operator<<( const ForceExplicitString& name ) {
-            return operator<<( name.str_.c_str() );
-        }
-
-        Labeler operator<<( const Labeler::Label &l ) {
-            massert( 10336 ,  "No subobject started", s_.subobjStarted() );
-            return s_ << l;
-        }
-
-        bool owned() const {
-            return &b == &buf_;
-        }
-        
-    private:
-        // Append the provided arr object as an array.
-        void marshalArray( const char *fieldName, const BSONObj &arr ) {
-            b.append( (char) Array );
-            b.append( fieldName );
-            b.append( (void *) arr.objdata(), arr.objsize() );
-        }
-        
-        char* _done() {
-            s_.endField();
-            b.append((char) EOO);
-            char *data = b.buf() + offset_;
-            int size = b.len() - offset_;
-            *((int*)data) = size;
-            if ( _tracker )
-                _tracker->got( size );
-            return data;
-        }
-
-        BufBuilder &b;
-        BufBuilder buf_;
-        int offset_;
-        BSONObjBuilderValueStream s_;
-        BSONSizeTracker * _tracker;
-    };
-
-    class BSONArrayBuilder : boost::noncopyable{
-    public:
-        BSONArrayBuilder() : _i(0), _b() {}
-        BSONArrayBuilder( BufBuilder &b ) : _i(0), _b(b) {}
-
-        template <typename T>
-        BSONArrayBuilder& append(const T& x){
-            _b.append(num().c_str(), x);
-            return *this;
-        }
-
-        BSONArrayBuilder& append(const BSONElement& e){
-            _b.appendAs(e, num().c_str());
-            return *this;
-        }
-        
-        template <typename T>
-        BSONArrayBuilder& operator<<(const T& x){
-            return append(x);
-        }
-        
-        void appendNull() {
-            _b.appendNull(num().c_str());
-        }
-
-        BSONArray arr(){ return BSONArray(_b.obj()); }
-        
-        BSONObj done() { return _b.done(); }
-        
-        template <typename T>
-        BSONArrayBuilder& append(const char *name, const T& x){
-            fill( name );
-            append( x );
-            return *this;
-        }
-        
-        BufBuilder &subobjStart( const char *name ) {
-            fill( name );
-            return _b.subobjStart( num().c_str() );
-        }
-
-        BufBuilder &subarrayStart( const char *name ) {
-            fill( name );
-            return _b.subarrayStart( num().c_str() );
-        }
-        
-        void appendArray( const char *name, BSONObj subObj ) {
-            fill( name );
-            _b.appendArray( num().c_str(), subObj );
-        }
-        
-        void appendAs( const BSONElement &e, const char *name ) {
-            fill( name );
-            append( e );
-        }
-        
-    private:
-        void fill( const char *name ) {
-            char *r;
-            int n = strtol( name, &r, 10 );
-            uassert( 13048, "can't append to array using string field name", !*r );
-            while( _i < n )
-                append( nullElt() );
-        }
-        
-        static BSONElement nullElt() {
-            static BSONObj n = nullObj();
-            return n.firstElement();
-        }
-        
-        static BSONObj nullObj() {
-            BSONObjBuilder b;
-            b.appendNull( "" );
-            return b.obj();
-        }
-        
-        string num(){ return _b.numStr(_i++); }
-        int _i;
-        BSONObjBuilder _b;
-    };
-
-
-    /** iterator for a BSONObj
-
-       Note each BSONObj ends with an EOO element: so you will get more() on an empty
-       object, although next().eoo() will be true.
-
-       todo: we may want to make a more stl-like iterator interface for this
-             with things like begin() and end()
-    */
-    class BSONObjIterator {
-    public:
-        /** Create an iterator for a BSON object. 
-        */
-        BSONObjIterator(const BSONObj& jso) {
-            int sz = jso.objsize();
-            if ( sz == 0 ) {
-                pos = theend = 0;
-                return;
-            }
-            pos = jso.objdata() + 4;
-            theend = jso.objdata() + sz;
-        }
-        /** @return true if more elements exist to be enumerated. */
-        bool moreWithEOO() {
-            return pos < theend;
-        }
-        bool more(){
-            return pos < theend && pos[0];
-        }
-        /** @return the next element in the object. For the final element, element.eoo() will be true. */
-        BSONElement next( bool checkEnd = false ) {
-            assert( pos < theend );
-            BSONElement e( pos, checkEnd ? (int)(theend - pos) : -1 );
-            pos += e.size( checkEnd ? (int)(theend - pos) : -1 );
-            return e;
-        }
-    private:
-        const char *pos;
-        const char *theend;
-    };
-
-    /* iterator a BSONObj which is an array, in array order.
-    class JSArrayIter {
-    public:
-    	BSONObjIterator(const BSONObj& jso) {
-    ...
-    	}
-    	bool more() { return ... }
-    	BSONElement next() {
-    ...
-    	}
-    };
-    */
-
-    extern BSONObj maxKey;
-    extern BSONObj minKey;
-	
-    // a BoundList contains intervals specified by inclusive start
-    // and end bounds.  The intervals should be nonoverlapping and occur in
-    // the specified direction of traversal.  For example, given a simple index {i:1}
-    // and direction +1, one valid BoundList is: (1, 2); (4, 6).  The same BoundList
-    // would be valid for index {i:-1} with direction -1.
-    typedef vector< pair< BSONObj, BSONObj > > BoundList;	
-
-    /*- just for testing -- */
-
-#pragma pack(1)
-    struct JSObj1 {
-        JSObj1() {
-            totsize=sizeof(JSObj1);
-            n = NumberDouble;
-            strcpy_s(nname, 5, "abcd");
-            N = 3.1;
-            s = String;
-            strcpy_s(sname, 7, "abcdef");
-            slen = 10;
-            strcpy_s(sval, 10, "123456789");
-            eoo = EOO;
-        }
-        unsigned totsize;
-
-        char n;
-        char nname[5];
-        double N;
-
-        char s;
-        char sname[7];
-        unsigned slen;
-        char sval[10];
-
-        char eoo;
-    };
-#pragma pack()
-    extern JSObj1 js1;
-
-#ifdef _DEBUG
-#define CHECK_OBJECT( o , msg ) massert( 10337 ,  (string)"object not valid" + (msg) , (o).isValid() )
-#else
-#define CHECK_OBJECT( o , msg )
-#endif
-
-    inline BSONObj BSONElement::embeddedObjectUserCheck() const {
-        uassert( 10065 ,  "invalid parameter: expected an object", isABSONObj() );
-        return BSONObj(value());
-    }
-
-    inline BSONObj BSONElement::embeddedObject() const {
-        assert( isABSONObj() );
-        return BSONObj(value());
-    }
-
-    inline BSONObj BSONElement::codeWScopeObject() const {
-        assert( type() == CodeWScope );
-        int strSizeWNull = *(int *)( value() + 4 );
-        return BSONObj( value() + 4 + 4 + strSizeWNull );
-    }
-    
-    inline BSONObj BSONObj::copy() const {
-        char *p = (char*) malloc(objsize());
-        memcpy(p, objdata(), objsize());
-        return BSONObj(p, true);
-    }
-
-// wrap this element up as a singleton object.
-    inline BSONObj BSONElement::wrap() const {
-        BSONObjBuilder b(size()+6);
-        b.append(*this);
-        return b.obj();
-    }
-
-    inline BSONObj BSONElement::wrap( const char * newName ) const {
-        BSONObjBuilder b(size()+6+strlen(newName));
-        b.appendAs(*this,newName);
-        return b.obj();
-    }
-
-
-    inline bool BSONObj::hasElement(const char *name) const {
-        if ( !isEmpty() ) {
-            BSONObjIterator it(*this);
-            while ( it.moreWithEOO() ) {
-                BSONElement e = it.next();
-                if ( strcmp(name, e.fieldName()) == 0 )
-                    return true;
-            }
-        }
-        return false;
-    }
-
-    inline BSONElement BSONObj::getField(const char *name) const {
-        BSONObjIterator i(*this);
-        while ( i.more() ) {
-            BSONElement e = i.next();
-            if ( strcmp(e.fieldName(), name) == 0 )
-                return e;
-        }
-        return BSONElement();
-    }
-
-    /* add all the fields from the object specified to this object */
-    inline BSONObjBuilder& BSONObjBuilder::appendElements(BSONObj x) {
-        BSONObjIterator it(x);
-        while ( it.moreWithEOO() ) {
-            BSONElement e = it.next();
-            if ( e.eoo() ) break;
-            append(e);
-        }
-        return *this;
-    }
-
-    inline bool BSONObj::isValid(){
-        return objsize() > 0 && objsize() <= 1024 * 1024 * 8;
-    }
-
-    inline bool BSONObj::getObjectID(BSONElement& e) const { 
-        BSONElement f = getField("_id");
-        if( !f.eoo() ) { 
-            e = f;
-            return true;
-        }
-        return false;
-    }
-
-    inline BSONObjBuilderValueStream::BSONObjBuilderValueStream( BSONObjBuilder * builder ) {
-        _fieldName = 0;
-        _builder = builder;
-    }
-    
-    template<class T> 
-    inline BSONObjBuilder& BSONObjBuilderValueStream::operator<<( T value ) { 
-        _builder->append(_fieldName, value);
-        _fieldName = 0;
-        return *_builder;
-    }
-
-    inline BSONObjBuilder& BSONObjBuilderValueStream::operator<<( const BSONElement& e ) { 
-        _builder->appendAs( e , _fieldName );
-        _fieldName = 0;
-        return *_builder;
-    }
-
-    inline BSONObjBuilder& BSONObjBuilderValueStream::operator<<(DateNowLabeler& id){
-        _builder->appendDate(_fieldName, jsTime());
-        _fieldName = 0;
-        return *_builder;
-    }
-
-    inline Labeler BSONObjBuilderValueStream::operator<<( const Labeler::Label &l ) { 
-        return Labeler( l, this );
-    }
-
-    inline void BSONObjBuilderValueStream::endField( const char *nextFieldName ) {
-        if ( _fieldName && haveSubobj() ) {
-            _builder->append( _fieldName, subobj()->done() );
-        }
-        _subobj.reset();
-        _fieldName = nextFieldName;
-    }    
-
-    inline BSONObjBuilder *BSONObjBuilderValueStream::subobj() {
-        if ( !haveSubobj() )
-            _subobj.reset( new BSONObjBuilder() );
-        return _subobj.get();
-    }
-    
-    template<class T> inline
-    BSONObjBuilder& Labeler::operator<<( T value ) {
-        s_->subobj()->append( l_.l_, value );
-        return *s_->_builder;
-    }    
-
-    inline
-    BSONObjBuilder& Labeler::operator<<( const BSONElement& e ) {
-        s_->subobj()->appendAs( e, l_.l_ );
-        return *s_->_builder;
-    }    
-
-    // {a: {b:1}} -> {a.b:1}
-    void nested2dotted(BSONObjBuilder& b, const BSONObj& obj, const string& base="");
-    inline BSONObj nested2dotted(const BSONObj& obj){
-        BSONObjBuilder b;
-        nested2dotted(b, obj);
-        return b.obj();
-    }
-
-    // {a.b:1} -> {a: {b:1}}
-    void dotted2nested(BSONObjBuilder& b, const BSONObj& obj);
-    inline BSONObj dotted2nested(const BSONObj& obj){
-        BSONObjBuilder b;
-        dotted2nested(b, obj);
-        return b.obj();
-    }
-    
-    /* WARNING: nested/dotted conversions are not 100% reversible
-     * nested2dotted(dotted2nested({a.b: {c:1}})) -> {a.b.c: 1}
-     * also, dotted2nested ignores order
-     */
-
-    typedef map<string, BSONElement> BSONMap;
-    inline BSONMap bson2map(const BSONObj& obj){
-        BSONMap m;
-        BSONObjIterator it(obj);
-        while (it.more()){
-            BSONElement e = it.next();
-            m[e.fieldName()] = e;
-        }
-        return m;
-    }
-
-    struct BSONElementFieldNameCmp {
-        bool operator()( const BSONElement &l, const BSONElement &r ) const {
-            return strcmp( l.fieldName() , r.fieldName() ) <= 0;
-        }
-    };
-
-
-    typedef set<BSONElement, BSONElementFieldNameCmp> BSONSortedElements;
-    inline BSONSortedElements bson2set( const BSONObj& obj ){
-        BSONSortedElements s;
-        BSONObjIterator it(obj);
-        while ( it.more() )
-            s.insert( it.next() );
-        return s;
-    }
-    
-    class BSONObjIteratorSorted {
-    public:
-        BSONObjIteratorSorted( const BSONObj& o );
-        
-        ~BSONObjIteratorSorted(){
-            assert( _fields );
-            delete[] _fields;
-            _fields = 0;
-        }
-
-        bool more(){
-            return _cur < _nfields;
-        }
-        
-        BSONElement next(){
-            assert( _fields );
-            if ( _cur < _nfields )
-                return BSONElement( _fields[_cur++] );
-            return BSONElement();
-        }
-
-    private:
-        const char ** _fields;
-        int _nfields;
-        int _cur;
-    };
-
-} // namespace mongo
+#include "../bson/bsontypes.h"
+#include "../bson/oid.h"
+#include "../bson/bsonelement.h"
+#include "../bson/bsonobj.h"
+#include "../bson/bsonmisc.h"
+#include "../bson/bsonobjbuilder.h"
+#include "../bson/bsonobjiterator.h"
+#include "../bson/bsoninlines.h"
+#include "../bson/ordering.h"
+#include "../bson/stringdata.h"
+
+#include "../bson/bson_db.h"
diff --git a/db/jsobjmanipulator.h b/db/jsobjmanipulator.h
index 1771bff..c43e876 100644
--- a/db/jsobjmanipulator.h
+++ b/db/jsobjmanipulator.h
@@ -40,9 +40,11 @@ namespace mongo {
         void setNumber(double d) {
             if ( _element.type() == NumberDouble ) *reinterpret_cast< double * >( value() )  = d;
             else if ( _element.type() == NumberInt ) *reinterpret_cast< int * >( value() ) = (int) d;
+            else assert(0);
         }
         void setLong(long long n) { 
-            if( _element.type() == NumberLong ) *reinterpret_cast< long long * >( value() ) = n;
+            assert( _element.type() == NumberLong );
+            *reinterpret_cast< long long * >( value() ) = n;
         }
         void setInt(int n) { 
             assert( _element.type() == NumberInt );
diff --git a/db/json.cpp b/db/json.cpp
index 7645b6b..185a8ca 100644
--- a/db/json.cpp
+++ b/db/json.cpp
@@ -16,17 +16,44 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
+
+#define BOOST_SPIRIT_THREADSAFE
+#if BOOST_VERSION >= 103800
+#define BOOST_SPIRIT_USE_OLD_NAMESPACE
+#include <boost/spirit/include/classic_core.hpp>
+#include <boost/spirit/include/classic_loops.hpp>
+#include <boost/spirit/include/classic_lists.hpp>
+#else
+#include <boost/spirit/core.hpp>
+#include <boost/spirit/utility/loops.hpp>
+#include <boost/spirit/utility/lists.hpp>
+#endif
+#undef assert
+#define assert MONGO_assert
+
 #include "json.h"
-#include "../util/builder.h"
+#include "../bson/util/builder.h"
 #include "../util/base64.h"
 #include "../util/hex.h"
 
+
 using namespace boost::spirit;
 
 namespace mongo {
 
-    struct ObjectBuilder {
+    struct ObjectBuilder : boost::noncopyable {
+        ~ObjectBuilder(){
+            unsigned i = builders.size();
+            if ( i ){
+                i--;
+                for ( ; i>=1; i-- ){
+                    if ( builders[i] ){
+                        builders[i]->done();
+                    }
+                }
+            }
+        }
         BSONObjBuilder *back() {
             return builders.back().get();
         }
@@ -426,20 +453,20 @@ public:
                 array = ch_p( '[' )[ arrayStart( self.b ) ] >> !elements >> ']';
                 elements = list_p(value, ch_p(',')[arrayNext( self.b )]);
                 value =
-                    oid[ oidEnd( self.b ) ] |
-                    dbref[ dbrefEnd( self.b ) ] |
-                    bindata[ binDataEnd( self.b ) ] |
-                    date[ dateEnd( self.b ) ] |
-                    regex[ regexEnd( self.b ) ] |
                     str[ stringEnd( self.b ) ] |
-                    singleQuoteStr[ stringEnd( self.b ) ] |
                     number |
                     integer |
-                    object[ subobjectEnd( self.b ) ] |
                     array[ arrayEnd( self.b ) ] |
                     lexeme_d[ str_p( "true" ) ][ trueValue( self.b ) ] |
                     lexeme_d[ str_p( "false" ) ][ falseValue( self.b ) ] |
-                    lexeme_d[ str_p( "null" ) ][ nullValue( self.b ) ];
+                    lexeme_d[ str_p( "null" ) ][ nullValue( self.b ) ] |
+                    singleQuoteStr[ stringEnd( self.b ) ] |
+                    date[ dateEnd( self.b ) ] |
+                    oid[ oidEnd( self.b ) ] |
+                    bindata[ binDataEnd( self.b ) ] |
+                    dbref[ dbrefEnd( self.b ) ] |
+                    regex[ regexEnd( self.b ) ] |
+                    object[ subobjectEnd( self.b ) ] ;
                 // NOTE lexeme_d and rules don't mix well, so we have this mess.
                 // NOTE We use range_p rather than cntrl_p, because the latter is locale dependent.
                 str = lexeme_d[ ch_p( '"' )[ chClear( self.b ) ] >>
@@ -530,21 +557,25 @@ public:
         ObjectBuilder &b;
     };
 
-    BSONObj fromjson( const char *str ) {
-        if ( ! strlen(str) )
+    BSONObj fromjson( const char *str , int* len) {
+        if ( str[0] == '\0' ){
+            if (len) *len = 0;
             return BSONObj();
+        }
+
         ObjectBuilder b;
         JsonGrammar parser( b );
         parse_info<> result = parse( str, parser, space_p );
-        if ( !result.full ) {
-            int len = strlen( result.stop );
-            if ( len > 10 )
-                len = 10;
-            stringstream ss;
-            ss << "Failure parsing JSON string near: " << string( result.stop, len );
-            massert( 10340 ,  ss.str(), false );
-        }
-        return b.pop();
+        if (len) {
+            *len = result.stop - str;
+        } else if ( !result.full ) {
+            int limit = strnlen(result.stop , 10);
+            if (limit == -1) limit = 10;
+            msgasserted(10340, "Failure parsing JSON string near: " + string( result.stop, limit ));
+        }
+        BSONObj ret = b.pop();
+        assert( b.empty() );
+        return ret;
     }
 
     BSONObj fromjson( const string &str ) {
diff --git a/db/json.h b/db/json.h
index c65785a..68dae04 100644
--- a/db/json.h
+++ b/db/json.h
@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "jsobj.h"
 
 namespace mongo {
@@ -35,6 +35,7 @@ namespace mongo {
     */
     BSONObj fromjson(const string &str);
 
-    BSONObj fromjson(const char *str);
+    /** len will be size of JSON object in text chars. */
+    BSONObj fromjson(const char *str, int* len=NULL);
 
 } // namespace mongo
diff --git a/db/lasterror.cpp b/db/lasterror.cpp
index 53042e7..9fc5512 100644
--- a/db/lasterror.cpp
+++ b/db/lasterror.cpp
@@ -15,7 +15,7 @@
  *    limitations under the License.
  */
 
-#include "stdafx.h"
+#include "pch.h"
 
 #include "../util/unittest.h"
 #include "../util/message.h"
@@ -28,8 +28,21 @@ namespace mongo {
 
     LastError LastError::noError;
     LastErrorHolder lastError;
-    mongo::mutex LastErrorHolder::_idsmutex;
-
+    mongo::mutex LastErrorHolder::_idsmutex("LastErrorHolder");
+
+    bool isShell = false;
+    void raiseError(int code , const char *msg) {
+        LastError *le = lastError.get();
+        if ( le == 0 ) {
+            /* might be intentional (non-user thread) */            
+            OCCASIONALLY DEV if( !isShell ) log() << "warning dev: lastError==0 won't report:" << msg << endl;
+        } else if ( le->disabled ) {
+            log() << "lastError disabled, can't report: " << code << ":" << msg << endl;
+        } else {
+            le->raiseError(code, msg);
+        }
+    }
+    
     void LastError::appendSelf( BSONObjBuilder &b ) {
         if ( !valid ) {
             b.appendNull( "err" );
@@ -44,7 +57,11 @@ namespace mongo {
             b.append( "code" , code );
         if ( updatedExisting != NotUpdate )
             b.appendBool( "updatedExisting", updatedExisting == True );
-        b.append( "n", nObjects );
+        if ( upsertedId.isSet() )
+            b.append( "upserted" , upsertedId );
+        if ( writebackId.isSet() )
+            b.append( "writeback" , writebackId );
+        b.appendNumber( "n", nObjects );
     }
 
     void LastErrorHolder::setID( int id ){
@@ -119,6 +136,13 @@ namespace mongo {
         
         remove( id );
     }
+
+    /** ok to call more than once. */
+    void LastErrorHolder::initThread() { 
+        if( _tl.get() ) return;
+        assert( _id.get() == 0 );
+        _tl.reset( new LastError() );
+    }
     
     void LastErrorHolder::reset( LastError * le ){
         int id = _id.get();
@@ -132,10 +156,10 @@ namespace mongo {
         status.time = time(0);
         status.lerr = le;
     }
-
+    
     void prepareErrForNewRequest( Message &m, LastError * err ) {
         // a killCursors message shouldn't affect last error
-        if ( m.data->operation() == dbKillCursors ) {
+        if ( m.operation() == dbKillCursors ) {
             err->disabled = true;
         } else {
             err->disabled = false;
@@ -143,11 +167,15 @@ namespace mongo {
         }        
     }
     
-    void LastErrorHolder::startRequest( Message& m ) {
-        int id = m.data->id & 0xFFFF0000;
-        setID( id );
+    LastError * LastErrorHolder::startRequest( Message& m , int clientId ) {
+
+        if ( clientId == 0 )
+            clientId = m.header()->id & 0xFFFF0000;
+        setID( clientId );
+
         LastError * le = _get( true );
         prepareErrForNewRequest( m, le );
+        return le;
     }
 
     void LastErrorHolder::startRequest( Message& m , LastError * connectionOwned ) {
@@ -158,6 +186,11 @@ namespace mongo {
         startRequest(m);
     }
 
+    void LastErrorHolder::disconnect( int clientId ){
+        if ( clientId )
+            remove(clientId);
+    }
+
     struct LastErrorHolderTest : public UnitTest {
     public:
         
diff --git a/db/lasterror.h b/db/lasterror.h
index 78160eb..5900208 100644
--- a/db/lasterror.h
+++ b/db/lasterror.h
@@ -17,9 +17,7 @@
 
 #pragma once
 
-#include <boost/thread/tss.hpp>
-#undef assert
-#define assert xassert
+#include "../bson/oid.h"
 
 namespace mongo {
     class BSONObjBuilder;
@@ -29,21 +27,29 @@ namespace mongo {
         int code;
         string msg;
         enum UpdatedExistingType { NotUpdate, True, False } updatedExisting;
-        /* todo: nObjects should be 64 bit */
+        OID upsertedId;
+        OID writebackId;
         long long nObjects;
         int nPrev;
         bool valid;
         bool overridenById;
         bool disabled;
+        void writeback( OID& oid ){
+            reset( true );
+            writebackId = oid;
+        }
         void raiseError(int _code , const char *_msg) {
             reset( true );
             code = _code;
             msg = _msg;
         }
-        void recordUpdate( bool _updatedExisting, long long nChanged ) {
+        void recordUpdate( bool _updateObjects , long long _nObjects , OID _upsertedId ){
             reset( true );
-            nObjects = nChanged;
-            updatedExisting = _updatedExisting ? True : False;
+            nObjects = _nObjects;
+            updatedExisting = _updateObjects ? True : False;
+            if ( _upsertedId.isSet() )
+                upsertedId = _upsertedId;
+                
         }
         void recordDelete( long long nDeleted ) {
             reset( true );
@@ -61,8 +67,31 @@ namespace mongo {
             nPrev = 1;
             valid = _valid;
             disabled = false;
+            upsertedId.clear();
+            writebackId.clear();
         }
         void appendSelf( BSONObjBuilder &b );
+
+        struct Disabled : boost::noncopyable {
+            Disabled( LastError * le ){
+                _le = le;
+                if ( _le ){
+                    _prev = _le->disabled;
+                    _le->disabled = true;
+                } else {
+                    _prev = false;
+                }
+            }
+            
+            ~Disabled(){
+                if ( _le )
+                    _le->disabled = _prev;
+            }
+
+            LastError * _le;
+            bool _prev;
+        };
+        
         static LastError noError;
     };
 
@@ -71,11 +100,22 @@ namespace mongo {
         LastErrorHolder() : _id( 0 ) {}
 
         LastError * get( bool create = false );
+        LastError * getSafe(){
+            LastError * le = get(false);
+            if ( ! le ){
+                log( LL_ERROR ) << " no LastError!  id: " << getID() << endl;
+                assert( le );
+            }
+            return le;
+        }
 
         LastError * _get( bool create = false ); // may return a disabled LastError
 
         void reset( LastError * le );
-        
+
+        /** ok to call more than once. */
+        void initThread();
+
         /**
          * id of 0 means should use thread local management
          */
@@ -87,8 +127,10 @@ namespace mongo {
         
         /** when db receives a message/request, call this */
         void startRequest( Message& m , LastError * connectionOwned );
-        void startRequest( Message& m );
+        LastError * startRequest( Message& m , int clientId = 0 );
         
+        void disconnect( int clientId );
+
         // used to disable lastError reporting while processing a killCursors message
         // disable causes get() to return 0.
         LastError *disableForCommand(); // only call once per command invocation!
@@ -103,28 +145,7 @@ namespace mongo {
         static mongo::mutex _idsmutex;
         map<int,Status> _ids;    
     } lastError;
-    
-    inline void raiseError(int code , const char *msg) {
-        LastError *le = lastError.get();
-        if ( le == 0 ) {
-            DEV log() << "warning: lastError==0 can't report:" << msg << '\n';
-        } else if ( le->disabled ) {
-            log() << "lastError disabled, can't report: " << msg << endl;
-        } else {
-            le->raiseError(code, msg);
-        }
-    }
-    
-    inline void recordUpdate( bool updatedExisting, int nChanged ) {
-        LastError *le = lastError.get();
-        if ( le )
-            le->recordUpdate( updatedExisting, nChanged );        
-    }
-
-    inline void recordDelete( int nDeleted ) {
-        LastError *le = lastError.get();
-        if ( le )
-            le->recordDelete( nDeleted );        
-    }
+
+    void raiseError(int code , const char *msg);
 
 } // namespace mongo
diff --git a/db/matcher.cpp b/db/matcher.cpp
index 8c904e3..681a6dc 100644
--- a/db/matcher.cpp
+++ b/db/matcher.cpp
@@ -18,7 +18,7 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "matcher.h"
 #include "../util/goodies.h"
 #include "../util/unittest.h"
@@ -50,7 +50,9 @@ namespace {
 #define DEBUGMATCHER(x)
 
 namespace mongo {
-    
+
+    extern BSONObj staticNull;
+        
     class Where {
     public:
         Where() {
@@ -141,39 +143,6 @@ namespace mongo {
         
     }
     
-    CoveredIndexMatcher::CoveredIndexMatcher(const BSONObj &jsobj, const BSONObj &indexKeyPattern) :
-        _keyMatcher(jsobj.filterFieldsUndotted(indexKeyPattern, true), 
-        indexKeyPattern),
-        _docMatcher(jsobj) 
-    {
-        _needRecord = ! ( 
-                         _docMatcher.keyMatch() && 
-                         _keyMatcher.jsobj.nFields() == _docMatcher.jsobj.nFields() &&
-                         ! _keyMatcher.hasType( BSONObj::opEXISTS )
-                          );
-
-    }
-    
-    bool CoveredIndexMatcher::matches(const BSONObj &key, const DiskLoc &recLoc , MatchDetails * details ) {
-        if ( details )
-            details->reset();
-        
-        if ( _keyMatcher.keyMatch() ) {
-            if ( !_keyMatcher.matches(key, details ) ){
-                return false;
-            }
-        }
-        
-        if ( ! _needRecord ){
-            return true;
-        }
-
-        if ( details )
-            details->loadedObject = true;
-
-        return _docMatcher.matches(recLoc.rec() , details );
-    }
-    
     
     void Matcher::addRegex(const char *fieldName, const char *regex, const char *flags, bool isNot){
 
@@ -230,9 +199,11 @@ namespace mongo {
             case BSONObj::opALL:
                 all = true;
             case BSONObj::opIN:
+                uassert( 13276 , "$in needs an array" , fe.isABSONObj() );
                 basics.push_back( ElementMatcher( e , op , fe.embeddedObject(), isNot ) );
                 break;
             case BSONObj::NIN:
+                uassert( 13277 , "$nin needs an array" , fe.isABSONObj() );
                 haveNeg = true;
                 basics.push_back( ElementMatcher( e , op , fe.embeddedObject(), isNot ) );
                 break;
@@ -279,6 +250,7 @@ namespace mongo {
             }
             case BSONObj::opNEAR:
             case BSONObj::opWITHIN:
+            case BSONObj::opMAX_DISTANCE:
                 break;
             default:
                 uassert( 10069 ,  (string)"BUG - can't operator for: " + fn , 0 );
@@ -286,19 +258,50 @@ namespace mongo {
         return true;
     }
     
+    void Matcher::parseOr( const BSONElement &e, bool subMatcher, list< shared_ptr< Matcher > > &matchers ) {
+        uassert( 13090, "recursive $or/$nor not allowed", !subMatcher );
+        uassert( 13086, "$or/$nor must be a nonempty array", e.type() == Array && e.embeddedObject().nFields() > 0 );
+        BSONObjIterator j( e.embeddedObject() );
+        while( j.more() ) {
+            BSONElement f = j.next();
+            uassert( 13087, "$or/$nor match element must be an object", f.type() == Object );
+            // until SERVER-109 this is never a covered index match, so don't constrain index key for $or matchers
+            matchers.push_back( shared_ptr< Matcher >( new Matcher( f.embeddedObject(), true ) ) );
+        }
+    }
+
+    bool Matcher::parseOrNor( const BSONElement &e, bool subMatcher ) {
+        const char *ef = e.fieldName();
+        if ( ef[ 0 ] != '$' )
+            return false;
+        if ( ef[ 1 ] == 'o' && ef[ 2 ] == 'r' && ef[ 3 ] == 0 ) {
+            parseOr( e, subMatcher, _orMatchers );
+        } else if ( ef[ 1 ] == 'n' && ef[ 2 ] == 'o' && ef[ 3 ] == 'r' && ef[ 4 ] == 0 ) {
+            parseOr( e, subMatcher, _norMatchers );
+        } else {
+            return false;
+        }
+        return true;
+    }
+    
     /* _jsobj          - the query pattern
     */
-    Matcher::Matcher(const BSONObj &_jsobj, const BSONObj &constrainIndexKey) :
+    Matcher::Matcher(const BSONObj &_jsobj, bool subMatcher) :
         where(0), jsobj(_jsobj), haveSize(), all(), hasArray(0), haveNeg(), _atomic(false), nRegex(0) {
 
         BSONObjIterator i(jsobj);
         while ( i.more() ) {
             BSONElement e = i.next();
+            
+            if ( parseOrNor( e, subMatcher ) ) {
+                continue;
+            }
 
             if ( ( e.type() == CodeWScope || e.type() == Code || e.type() == String ) && strcmp(e.fieldName(), "$where")==0 ) {
                 // $where: function()...
-                uassert( 10066 ,  "$where occurs twice?", where == 0 );
-                uassert( 10067 ,  "$where query, but no script engine", globalScriptEngine );
+                uassert( 10066 , "$where occurs twice?", where == 0 );
+                uassert( 10067 , "$where query, but no script engine", globalScriptEngine );
+                massert( 13089 , "no current client needed for $where" , haveClient() ); 
                 where = new Where();
                 where->scope = globalScriptEngine->getPooledScope( cc().ns() );
                 where->scope->localConnect( cc().database()->name.c_str() );
@@ -348,7 +351,7 @@ namespace mongo {
                                     BSONObjIterator k( fe.embeddedObject() );
                                     uassert( 13030, "$not cannot be empty", k.more() );
                                     while( k.more() ) {
-                                        addOp( e, k.next(), true, regex, flags );
+                                        addOp( e, k.next(), true, regex, flags );   
                                     }
                                     break;
                                 }
@@ -388,8 +391,35 @@ namespace mongo {
             // normal, simple case e.g. { a : "foo" }
             addBasic(e, BSONObj::Equality, false);
         }
-        
-        constrainIndexKey_ = constrainIndexKey;
+    }
+    
+    Matcher::Matcher( const Matcher &other, const BSONObj &key ) :
+    where(0), constrainIndexKey_( key ), haveSize(), all(), hasArray(0), haveNeg(), _atomic(false), nRegex(0) {
+        // do not include fields which would make keyMatch() false
+        for( vector< ElementMatcher >::const_iterator i = other.basics.begin(); i != other.basics.end(); ++i ) {
+            if ( key.hasField( i->toMatch.fieldName() ) ) {
+                switch( i->compareOp ) {
+                    case BSONObj::opSIZE:
+                    case BSONObj::opALL:
+                    case BSONObj::NE:
+                    case BSONObj::NIN:
+                        break;
+                    default: {
+                        if ( !i->isNot && i->toMatch.type() != Array ) {
+                            basics.push_back( *i );                            
+                        }
+                    }
+                }
+            }
+        }
+        for( int i = 0; i < other.nRegex; ++i ) {
+            if ( !other.regexs[ i ].isNot && key.hasField( other.regexs[ i ].fieldName ) ) {
+                regexs[ nRegex++ ] = other.regexs[ i ];
+            }
+        }
+        for( list< shared_ptr< Matcher > >::const_iterator i = other._orMatchers.begin(); i != other._orMatchers.end(); ++i ) {
+            _orMatchers.push_back( shared_ptr< Matcher >( new Matcher( **i, key ) ) );
+        }
     }
     
     inline bool regexMatches(const RegexMatcher& rm, const BSONElement& e) {
@@ -711,7 +741,7 @@ namespace mongo {
                 return false;
             if ( cmp == 0 ) {
                 /* missing is ok iff we were looking for null */
-                if ( m.type() == jstNULL || m.type() == Undefined ) {
+                if ( m.type() == jstNULL || m.type() == Undefined || ( bm.compareOp == BSONObj::opIN && bm.myset->count( staticNull.firstElement() ) > 0 ) ) {
                     if ( ( bm.compareOp == BSONObj::NE ) ^ bm.isNot ) {
                         return false;
                     }
@@ -741,6 +771,42 @@ namespace mongo {
                 return false;
         }
         
+        if ( _orMatchers.size() > 0 ) {
+            bool match = false;
+            for( list< shared_ptr< Matcher > >::const_iterator i = _orMatchers.begin();
+                i != _orMatchers.end(); ++i ) {
+                // SERVER-205 don't submit details - we don't want to track field
+                // matched within $or, and at this point we've already loaded the
+                // whole document
+                if ( (*i)->matches( jsobj ) ) {
+                    match = true;
+                    break;
+                }
+            }
+            if ( !match ) {
+                return false;
+            }
+        }
+        
+        if ( _norMatchers.size() > 0 ) {
+            for( list< shared_ptr< Matcher > >::const_iterator i = _norMatchers.begin();
+                i != _norMatchers.end(); ++i ) {
+                // SERVER-205 don't submit details - we don't want to track field
+                // matched within $nor, and at this point we've already loaded the
+                // whole document
+                if ( (*i)->matches( jsobj ) ) {
+                    return false;
+                }
+            }            
+        }
+        
+        for( vector< shared_ptr< FieldRangeVector > >::const_iterator i = _orConstraints.begin();
+            i != _orConstraints.end(); ++i ) {
+            if ( (*i)->matches( jsobj ) ) {
+                return false;
+            }
+        }
+                
         if ( where ) {
             if ( where->func == 0 ) {
                 uassert( 10070 , "$where compile error", false);
@@ -769,7 +835,7 @@ namespace mongo {
             return where->scope->getBoolean( "return" ) != 0;
 
         }
-
+        
         return true;
     }
 
@@ -780,6 +846,72 @@ namespace mongo {
         return false;
     }
 
+    bool Matcher::sameCriteriaCount( const Matcher &other ) const {
+        if ( !( basics.size() == other.basics.size() && nRegex == other.nRegex && !where == !other.where ) ) {
+            return false;
+        }
+        if ( _norMatchers.size() != other._norMatchers.size() ) {
+            return false;
+        }
+        if ( _orMatchers.size() != other._orMatchers.size() ) {
+            return false;
+        }
+        if ( _orConstraints.size() != other._orConstraints.size() ) {
+            return false;
+        }
+        {
+            list< shared_ptr< Matcher > >::const_iterator i = _norMatchers.begin();
+            list< shared_ptr< Matcher > >::const_iterator j = other._norMatchers.begin();
+            while( i != _norMatchers.end() ) {
+                if ( !(*i)->sameCriteriaCount( **j ) ) {
+                    return false;
+                }
+                ++i; ++j;
+            }
+        }
+        {
+            list< shared_ptr< Matcher > >::const_iterator i = _orMatchers.begin();
+            list< shared_ptr< Matcher > >::const_iterator j = other._orMatchers.begin();
+            while( i != _orMatchers.end() ) {
+                if ( !(*i)->sameCriteriaCount( **j ) ) {
+                    return false;
+                }
+                ++i; ++j;
+            }
+        }
+        return true;
+    }        
+        
+    
+    /*- just for testing -- */
+#pragma pack(1)
+    struct JSObj1 {
+        JSObj1() {
+            totsize=sizeof(JSObj1);
+            n = NumberDouble;
+            strcpy_s(nname, 5, "abcd");
+            N = 3.1;
+            s = String;
+            strcpy_s(sname, 7, "abcdef");
+            slen = 10;
+            strcpy_s(sval, 10, "123456789");
+            eoo = EOO;
+        }
+        unsigned totsize;
+
+        char n;
+        char nname[5];
+        double N;
+
+        char s;
+        char sname[7];
+        unsigned slen;
+        char sval[10];
+
+        char eoo;
+    };
+#pragma pack()
+
     struct JSObj1 js1;
 
 #pragma pack(1)
diff --git a/db/matcher.h b/db/matcher.h
index 3839b68..a4e1667 100644
--- a/db/matcher.h
+++ b/db/matcher.h
@@ -24,9 +24,11 @@
 #include <pcrecpp.h>
 
 namespace mongo {
-
+    
+    class Cursor;
     class CoveredIndexMatcher;
     class Matcher;
+    class FieldRangeVector;
 
     class RegexMatcher {
     public:
@@ -133,20 +135,40 @@ namespace mongo {
             return op <= BSONObj::LTE ? -1 : 1;
         }
 
-        // Only specify constrainIndexKey if matches() will be called with
-        // index keys having empty string field names.
-        Matcher(const BSONObj &pattern, const BSONObj &constrainIndexKey = BSONObj());
+        Matcher(const BSONObj &pattern, bool subMatcher = false);
 
         ~Matcher();
 
         bool matches(const BSONObj& j, MatchDetails * details = 0 );
         
+        // fast rough check to see if we must load the real doc - we also
+        // compare field counts against covereed index matcher; for $or clauses
+        // we just compare field counts
         bool keyMatch() const { return !all && !haveSize && !hasArray && !haveNeg; }
 
         bool atomic() const { return _atomic; }
-
+        
         bool hasType( BSONObj::MatchType type ) const;
+
+        string toString() const {
+            return jsobj.toString();
+        }
+
+        void addOrConstraint( const shared_ptr< FieldRangeVector > &frv ) {
+            _orConstraints.push_back( frv );
+        }
+        
+        void popOrClause() {
+            _orMatchers.pop_front();
+        }
+        
+        bool sameCriteriaCount( const Matcher &other ) const;
+        
     private:
+        // Only specify constrainIndexKey if matches() will be called with
+        // index keys having empty string field names.
+        Matcher( const Matcher &other, const BSONObj &constrainIndexKey );
+        
         void addBasic(const BSONElement &e, int c, bool isNot) {
             // TODO May want to selectively ignore these element types based on op type.
             if ( e.type() == MinKey || e.type() == MaxKey )
@@ -159,6 +181,9 @@ namespace mongo {
         
         int valuesMatch(const BSONElement& l, const BSONElement& r, int op, const ElementMatcher& bm);
 
+        bool parseOrNor( const BSONElement &e, bool subMatcher );
+        void parseOr( const BSONElement &e, bool subMatcher, list< shared_ptr< Matcher > > &matchers );
+
         Where *where;                    // set if query uses $where
         BSONObj jsobj;                  // the query pattern.  e.g., { name: "joe" }
         BSONObj constrainIndexKey_;
@@ -180,6 +205,9 @@ namespace mongo {
 
         // so we delete the mem when we're done:
         vector< shared_ptr< BSONObjBuilder > > _builders;
+        list< shared_ptr< Matcher > > _orMatchers;
+        list< shared_ptr< Matcher > > _norMatchers;
+        vector< shared_ptr< FieldRangeVector > > _orConstraints;
 
         friend class CoveredIndexMatcher;
     };
@@ -187,15 +215,30 @@ namespace mongo {
     // If match succeeds on index key, then attempt to match full document.
     class CoveredIndexMatcher : boost::noncopyable {
     public:
-        CoveredIndexMatcher(const BSONObj &pattern, const BSONObj &indexKeyPattern);
-        bool matches(const BSONObj &o){ return _docMatcher.matches( o ); }
+        CoveredIndexMatcher(const BSONObj &pattern, const BSONObj &indexKeyPattern , bool alwaysUseRecord=false );
+        bool matches(const BSONObj &o){ return _docMatcher->matches( o ); }
         bool matches(const BSONObj &key, const DiskLoc &recLoc , MatchDetails * details = 0 );
+        bool matchesCurrent( Cursor * cursor , MatchDetails * details = 0 );
         bool needRecord(){ return _needRecord; }
-
-        Matcher& docMatcher() { return _docMatcher; }
+        
+        Matcher& docMatcher() { return *_docMatcher; }
+
+        // once this is called, shouldn't use this matcher for matching any more
+        void advanceOrClause( const shared_ptr< FieldRangeVector > &frv ) {
+            _docMatcher->addOrConstraint( frv );
+            // TODO this is not an optimal optimization, since we could skip an entire
+            // or clause (if a match is impossible) between calls to advanceOrClause()
+            _docMatcher->popOrClause();
+        }
+        
+        CoveredIndexMatcher *nextClauseMatcher( const BSONObj &indexKeyPattern, bool alwaysUseRecord=false ) {
+            return new CoveredIndexMatcher( _docMatcher, indexKeyPattern, alwaysUseRecord );
+        }
     private:
+        CoveredIndexMatcher(const shared_ptr< Matcher > &docMatcher, const BSONObj &indexKeyPattern , bool alwaysUseRecord=false );
+        void init( bool alwaysUseRecord );
+        shared_ptr< Matcher > _docMatcher;
         Matcher _keyMatcher;
-        Matcher _docMatcher;
         bool _needRecord;
     };
     
diff --git a/db/matcher_covered.cpp b/db/matcher_covered.cpp
new file mode 100644
index 0000000..5866505
--- /dev/null
+++ b/db/matcher_covered.cpp
@@ -0,0 +1,80 @@
+// matcher_covered.cpp
+
+/* Matcher is our boolean expression evaluator for "where" clauses */
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "matcher.h"
+#include "../util/goodies.h"
+#include "../util/unittest.h"
+#include "diskloc.h"
+#include "../scripting/engine.h"
+#include "db.h"
+#include "client.h"
+
+#include "pdfile.h"
+
+namespace mongo {
+
+    CoveredIndexMatcher::CoveredIndexMatcher( const BSONObj &jsobj, const BSONObj &indexKeyPattern, bool alwaysUseRecord) :
+        _docMatcher( new Matcher( jsobj ) ),
+        _keyMatcher( *_docMatcher, indexKeyPattern )
+    {
+        init( alwaysUseRecord );
+    }
+ 
+    CoveredIndexMatcher::CoveredIndexMatcher( const shared_ptr< Matcher > &docMatcher, const BSONObj &indexKeyPattern , bool alwaysUseRecord ) :
+        _docMatcher( docMatcher ),
+        _keyMatcher( *_docMatcher, indexKeyPattern )
+    {
+        init( alwaysUseRecord );
+    }
+
+    void CoveredIndexMatcher::init( bool alwaysUseRecord ) {
+        _needRecord = 
+        alwaysUseRecord || 
+        ! ( _docMatcher->keyMatch() && 
+           _keyMatcher.sameCriteriaCount( *_docMatcher ) &&
+           ! _keyMatcher.hasType( BSONObj::opEXISTS ) );
+        ;        
+    }
+    
+    bool CoveredIndexMatcher::matchesCurrent( Cursor * cursor , MatchDetails * details ){
+        return matches( cursor->currKey() , cursor->currLoc() , details );
+    }
+    
+    bool CoveredIndexMatcher::matches(const BSONObj &key, const DiskLoc &recLoc , MatchDetails * details ) {
+        if ( details )
+            details->reset();
+        
+        if ( !_keyMatcher.matches(key, details ) ){
+            return false;
+        }
+        
+        if ( ! _needRecord ){
+            return true;
+        }
+
+        if ( details )
+            details->loadedObject = true;
+
+        return _docMatcher->matches(recLoc.rec() , details );
+    }
+    
+
+}
diff --git a/db/module.cpp b/db/module.cpp
index 78f8f79..1e4f511 100644
--- a/db/module.cpp
+++ b/db/module.cpp
@@ -16,7 +16,7 @@
  */
 
 
-#include "stdafx.h"
+#include "pch.h"
 #include "module.h"
 
 namespace mongo {
diff --git a/db/module.h b/db/module.h
index 728e861..d4939dd 100644
--- a/db/module.h
+++ b/db/module.h
@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include <boost/program_options.hpp>
 #include <list>
 
diff --git a/db/modules/mms.cpp b/db/modules/mms.cpp
index 248a4e4..40e9001 100644
--- a/db/modules/mms.cpp
+++ b/db/modules/mms.cpp
@@ -1,4 +1,4 @@
-// mms.cpp
+// @file mms.cpp
 /*
  *    Copyright (C) 2010 10gen Inc.
  *
@@ -16,7 +16,7 @@
  */
 
 
-#include "stdafx.h"
+#include "pch.h"
 #include "../db.h"
 #include "../instance.h"
 #include "../module.h"
diff --git a/db/mongo.ico b/db/mongo.ico
new file mode 100755
index 0000000..5258b6e
--- /dev/null
+++ b/db/mongo.ico
diff --git a/db/mr.cpp b/db/mr.cpp
index 210dfca..8fa8d50 100644
--- a/db/mr.cpp
+++ b/db/mr.cpp
@@ -15,7 +15,7 @@
  *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "db.h"
 #include "instance.h"
 #include "commands.h"
@@ -23,6 +23,9 @@
 #include "../client/dbclient.h"
 #include "../client/connpool.h"
 #include "../client/parallel.h"
+#include "queryoptimizer.h"
+#include "matcher.h"
+#include "clientcursor.h"
 
 namespace mongo {
 
@@ -49,7 +52,7 @@ namespace mongo {
             BSONObj key;
 
             BSONObjBuilder reduceArgs( sizeEstimate );
-            BSONArrayBuilder * valueBuilder = 0;
+            boost::scoped_ptr<BSONArrayBuilder>  valueBuilder;
             
             int sizeSoFar = 0;
             unsigned n = 0;
@@ -59,8 +62,8 @@ namespace mongo {
                 if ( n == 0 ){
                     reduceArgs.append( keyE );
                     key = keyE.wrap();
-                    valueBuilder = new BSONArrayBuilder( reduceArgs.subarrayStart( "values" ) );
                     sizeSoFar = 5 + keyE.size();
+                    valueBuilder.reset(new BSONArrayBuilder( reduceArgs.subarrayStart( "values" ) ));
                 }
                 
                 BSONElement ee = j.next();
@@ -77,7 +80,6 @@ namespace mongo {
             }
             assert(valueBuilder);
             valueBuilder->done();
-            delete valueBuilder;
             BSONObj args = reduceArgs.obj();
 
             s->invokeSafe( reduce , args );
@@ -152,10 +154,10 @@ namespace mongo {
                 }
              
                 { // code
-                    mapCode = cmdObj["map"].ascode();
-                    reduceCode = cmdObj["reduce"].ascode();
+                    mapCode = cmdObj["map"]._asCode();
+                    reduceCode = cmdObj["reduce"]._asCode();
                     if ( cmdObj["finalize"].type() ){
-                        finalizeCode = cmdObj["finalize"].ascode();
+                        finalizeCode = cmdObj["finalize"]._asCode();
                     }
                     checkCodeWScope( "map" , cmdObj );
                     checkCodeWScope( "reduce" , cmdObj );
@@ -287,7 +289,7 @@ namespace mongo {
                 if ( setup.replicate )
                     theDataFileMgr.insertAndLog( setup.tempLong.c_str() , res , false );
                 else
-                    theDataFileMgr.insert( setup.tempLong.c_str() , res , false );
+                    theDataFileMgr.insertWithObjMod( setup.tempLong.c_str() , res , false );
             }
 
             
@@ -303,21 +305,17 @@ namespace mongo {
         
         class MRTL {
         public:
-            MRTL( MRState& state ) : _state( state ){
-                _temp = new InMemory();
+            MRTL( MRState& state ) 
+                : _state( state )
+                , _temp(new InMemory())
+            {
                 _size = 0;
                 numEmits = 0;
             }
-            ~MRTL(){
-                delete _temp;
-            }
-            
             
             void reduceInMemory(){
-                
-                InMemory * old = _temp;
-                InMemory * n = new InMemory();
-                _temp = n;
+                boost::shared_ptr<InMemory> old = _temp;
+                _temp.reset(new InMemory());
                 _size = 0;
                 
                 for ( InMemory::iterator i=old->begin(); i!=old->end(); i++ ){
@@ -327,6 +325,7 @@ namespace mongo {
                     if ( all.size() == 1 ){
                         // this key has low cardinality, so just write to db
                         writelock l(_state.setup.incLong);
+                        Client::Context ctx(_state.setup.incLong.c_str());
                         write( *(all.begin()) );
                     }
                     else if ( all.size() > 1 ){
@@ -334,9 +333,6 @@ namespace mongo {
                         insert( res );
                     }
                 }
-                
-                delete( old );
-
             }
 
             void dump(){
@@ -379,12 +375,12 @@ namespace mongo {
 
         private:
             void write( BSONObj& o ){
-                theDataFileMgr.insert( _state.setup.incLong.c_str() , o , true );
+                theDataFileMgr.insertWithObjMod( _state.setup.incLong.c_str() , o , true );
             }
             
             MRState& _state;
         
-            InMemory * _temp;
+            boost::shared_ptr<InMemory> _temp;
             long _size;
             
         public:
@@ -403,20 +399,22 @@ namespace mongo {
 
         class MapReduceCommand : public Command {
         public:
-            MapReduceCommand() : Command("mapreduce"){}
-            virtual bool slaveOk() { return true; }
+            MapReduceCommand() : Command("mapReduce", false, "mapreduce"){}
+            virtual bool slaveOk() const { return true; }
         
             virtual void help( stringstream &help ) const {
-                help << "see http://www.mongodb.org/display/DOCS/MapReduce";
+                help << "Run a map/reduce operation on the server.\n";
+                help << "Note this is used for aggregation, not querying, in MongoDB.\n";
+                help << "http://www.mongodb.org/display/DOCS/MapReduce";
             }
-            virtual LockType locktype(){ return WRITE; } // TODO, READ?
-            bool run(const char *dbname, BSONObj& cmd, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
+            virtual LockType locktype() const { return NONE; } 
+            bool run(const string& dbname , BSONObj& cmd, string& errmsg, BSONObjBuilder& result, bool fromRepl ){
                 Timer t;
                 Client::GodScope cg;
                 Client& client = cc();
                 CurOp * op = client.curop();
 
-                MRSetup mr( client.database()->name , cmd );
+                MRSetup mr( dbname , cmd );
 
                 log(1) << "mr ns: " << mr.ns << endl;
                 
@@ -440,36 +438,58 @@ namespace mongo {
                     MRTL * mrtl = new MRTL( state );
                     _tlmr.reset( mrtl );
 
-                    ProgressMeter & pm = op->setMessage( "m/r: (1/3) emit phase" , db.count( mr.ns , mr.filter ) );
-                    auto_ptr<DBClientCursor> cursor = db.query( mr.ns , mr.q );
+                    ProgressMeterHolder pm( op->setMessage( "m/r: (1/3) emit phase" , db.count( mr.ns , mr.filter ) ) );
                     long long mapTime = 0;
-                    Timer mt;
-                    while ( cursor->more() ){
-                        BSONObj o = cursor->next(); 
-                    
-                        if ( mr.verbose ) mt.reset();
-                        
-                        state.scope->setThis( &o );
-                        if ( state.scope->invoke( state.map , state.setup.mapparams , 0 , true ) )
-                            throw UserException( 9014, (string)"map invoke failed: " + state.scope->getError() );
+                    {
+                        readlock lock( mr.ns );
+                        Client::Context ctx( mr.ns );
                         
-                        if ( mr.verbose ) mapTime += mt.micros();
-                        
-                        num++;
-                        if ( num % 100 == 0 ){
-                            Timer t;
-                            mrtl->checkSize();
-                            inReduce += t.micros();
-                            killCurrentOp.checkForInterrupt();
-                            dbtemprelease temprlease;
+                        shared_ptr<Cursor> temp = bestGuessCursor( mr.ns.c_str(), mr.filter, BSONObj() );
+                        auto_ptr<ClientCursor> cursor( new ClientCursor( QueryOption_NoCursorTimeout , temp , mr.ns.c_str() ) );
+
+                        Timer mt;
+                        while ( cursor->ok() ){
+                            
+                            if ( ! cursor->currentMatches() ){
+                                cursor->advance();
+                                continue;
+                            }
+                            
+                            BSONObj o = cursor->current(); 
+                            cursor->advance();
+                            
+                            if ( mr.verbose ) mt.reset();
+                            
+                            state.scope->setThis( &o );
+                            if ( state.scope->invoke( state.map , state.setup.mapparams , 0 , true ) )
+                                throw UserException( 9014, (string)"map invoke failed: " + state.scope->getError() );
+                            
+                            if ( mr.verbose ) mapTime += mt.micros();
+                            
+                            num++;
+                            if ( num % 100 == 0 ){
+                                ClientCursor::YieldLock yield (cursor.get());
+                                Timer t;
+                                mrtl->checkSize();
+                                inReduce += t.micros();
+                                
+                                if ( ! yield.stillOk() ){
+                                    cursor.release();
+                                    break;
+                                }
+
+                                killCurrentOp.checkForInterrupt();
+                            }
+                            pm.hit();
+                            
+                            if ( mr.limit && num >= mr.limit )
+                                break;
                         }
-                        pm.hit();
-
-                        if ( mr.limit && num >= mr.limit )
-                            break;
                     }
                     pm.finished();
                     
+                    killCurrentOp.checkForInterrupt();
+
                     countsBuilder.appendNumber( "input" , num );
                     countsBuilder.appendNumber( "emit" , mrtl->numEmits );
                     if ( mrtl->numEmits )
@@ -486,36 +506,68 @@ namespace mongo {
                     BSONObj sortKey = BSON( "0" << 1 );
                     db.ensureIndex( mr.incLong , sortKey );
                     
-                    BSONObj prev;
-                    BSONList all;
-                    
-                    assert( userCreateNS( mr.tempLong.c_str() , BSONObj() , errmsg , mr.replicate ) );
+                    {
+                        writelock lock( mr.tempLong.c_str() );
+                        Client::Context ctx( mr.tempLong.c_str() );
+                        assert( userCreateNS( mr.tempLong.c_str() , BSONObj() , errmsg , mr.replicate ) );
+                    }
 
-                    pm = op->setMessage( "m/r: (3/3) final reduce to collection" , db.count( mr.incLong ) );
-                    cursor = db.query( mr.incLong, Query().sort( sortKey ) );
 
-                    while ( cursor->more() ){
-                        BSONObj o = cursor->next().getOwned();
-                        pm.hit();
+                    {
+                        readlock rl(mr.incLong.c_str());
+                        Client::Context ctx( mr.incLong );
+                        
+                        BSONObj prev;
+                        BSONList all;
+                        
+                        assert( pm == op->setMessage( "m/r: (3/3) final reduce to collection" , db.count( mr.incLong ) ) );
 
-                        if ( o.woSortOrder( prev , sortKey ) == 0 ){
+                        shared_ptr<Cursor> temp = bestGuessCursor( mr.incLong.c_str() , BSONObj() , sortKey );
+                        auto_ptr<ClientCursor> cursor( new ClientCursor( QueryOption_NoCursorTimeout , temp , mr.incLong.c_str() ) );
+                        
+                        while ( cursor->ok() ){
+                            BSONObj o = cursor->current().getOwned();
+                            cursor->advance();
+                            
+                            pm.hit();
+                            
+                            if ( o.woSortOrder( prev , sortKey ) == 0 ){
+                                all.push_back( o );
+                                if ( pm->hits() % 1000 == 0 ){
+                                    if ( ! cursor->yield() ){
+                                        cursor.release();
+                                        break;
+                                    } 
+                                    killCurrentOp.checkForInterrupt();
+                                }
+                                continue;
+                            }
+                        
+                            ClientCursor::YieldLock yield (cursor.get());
+                            state.finalReduce( all );
+                            
+                            all.clear();
+                            prev = o;
                             all.push_back( o );
-                            if ( pm.hits() % 1000 == 0 ){
-                                dbtemprelease tl;
+
+                            if ( ! yield.stillOk() ){
+                                cursor.release();
+                                break;
                             }
-                            continue;
+                            
+                            killCurrentOp.checkForInterrupt();
                         }
-                        
-                        state.finalReduce( all );
-                        
-                        all.clear();
-                        prev = o;
-                        all.push_back( o );
-                        killCurrentOp.checkForInterrupt();
-                        dbtemprelease tl;
+
+                        {
+                            dbtempreleasecond tl;
+                            if ( ! tl.unlocked() )
+                                log( LL_WARNING ) << "map/reduce can't temp release" << endl;
+                            state.finalReduce( all );
+                        }
+
+                        pm.finished();
                     }
-                    state.finalReduce( all );
-                    pm.finished();
+
                     _tlmr.reset( 0 );
                 }
                 catch ( ... ){
@@ -525,9 +577,13 @@ namespace mongo {
                     throw;
                 }
                 
-                db.dropCollection( mr.incLong );
+                long long finalCount = 0;
+                {
+                    dblock lock;
+                    db.dropCollection( mr.incLong );
                 
-                long long finalCount = mr.renameIfNeeded( db );
+                    finalCount = mr.renameIfNeeded( db );
+                }
 
                 timingBuilder.append( "total" , t.millis() );
                 
@@ -554,13 +610,10 @@ namespace mongo {
         class MapReduceFinishCommand : public Command {
         public:
             MapReduceFinishCommand() : Command( "mapreduce.shardedfinish" ){}
-            virtual bool slaveOk() { return true; }
+            virtual bool slaveOk() const { return true; }
             
-            virtual LockType locktype(){ return WRITE; } 
-            bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool){
-                string dbname = cc().database()->name; // this has to come before dbtemprelease
-                dbtemprelease temprelease; // we don't touch the db directly
-
+            virtual LockType locktype() const { return NONE; } 
+            bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool){
                 string shardedOutputCollection = cmdObj["shardedOutputCollection"].valuestrsafe();
 
                 MRSetup mr( dbname , cmdObj.firstElement().embeddedObjectUserCheck() , false );
@@ -572,71 +625,80 @@ namespace mongo {
                 
                 BSONObj shards = cmdObj["shards"].embeddedObjectUserCheck();
                 vector< auto_ptr<DBClientCursor> > shardCursors;
-                BSONObjIterator i( shards );
-                while ( i.more() ){
-                    BSONElement e = i.next();
-                    string shard = e.fieldName();
 
-                    BSONObj res = e.embeddedObjectUserCheck();
-                    
-                    uassert( 10078 ,  "something bad happened" , shardedOutputCollection == res["result"].valuestrsafe() );
-                    servers.insert( shard );
-                    shardCounts.appendAs( res["counts"] , shard.c_str() );
-
-                    BSONObjIterator j( res["counts"].embeddedObjectUserCheck() );
-                    while ( j.more() ){
-                        BSONElement temp = j.next();
-                        counts[temp.fieldName()] += temp.numberLong();
+                { // parse per shard results 
+                    BSONObjIterator i( shards );
+                    while ( i.more() ){
+                        BSONElement e = i.next();
+                        string shard = e.fieldName();
+                        
+                        BSONObj res = e.embeddedObjectUserCheck();
+                        
+                        uassert( 10078 ,  "something bad happened" , shardedOutputCollection == res["result"].valuestrsafe() );
+                        servers.insert( shard );
+                        shardCounts.appendAs( res["counts"] , shard.c_str() );
+                        
+                        BSONObjIterator j( res["counts"].embeddedObjectUserCheck() );
+                        while ( j.more() ){
+                            BSONElement temp = j.next();
+                            counts[temp.fieldName()] += temp.numberLong();
+                        }
+                        
                     }
-
+                    
                 }
-
-                BSONObj sortKey = BSON( "_id" << 1 );
-
-                ParallelSortClusteredCursor cursor( servers , dbname + "." + shardedOutputCollection ,
-                                                    Query().sort( sortKey ) );
                 
-                
-                auto_ptr<Scope> s = globalScriptEngine->getPooledScope( ns );
-                ScriptingFunction reduceFunction = s->createFunction( mr.reduceCode.c_str() );
-                ScriptingFunction finalizeFunction = 0;
-                if ( mr.finalizeCode.size() )
-                    finalizeFunction = s->createFunction( mr.finalizeCode.c_str() );
-
-                BSONList values;
-
-                result.append( "result" , mr.finalShort );
-
                 DBDirectClient db;
-                
-                while ( cursor.more() ){
-                    BSONObj t = cursor.next().getOwned();
-                                        
-                    if ( values.size() == 0 ){
-                        values.push_back( t );
-                        continue;
-                    }
                     
-                    if ( t.woSortOrder( *(values.begin()) , sortKey ) == 0 ){
+                { // reduce from each stream
+                    
+                    BSONObj sortKey = BSON( "_id" << 1 );
+                    
+                    ParallelSortClusteredCursor cursor( servers , dbname + "." + shardedOutputCollection ,
+                                                        Query().sort( sortKey ) );
+                    cursor.init();
+                    
+                    auto_ptr<Scope> s = globalScriptEngine->getPooledScope( dbname );
+                    s->localConnect( dbname.c_str() );
+                    ScriptingFunction reduceFunction = s->createFunction( mr.reduceCode.c_str() );
+                    ScriptingFunction finalizeFunction = 0;
+                    if ( mr.finalizeCode.size() )
+                        finalizeFunction = s->createFunction( mr.finalizeCode.c_str() );
+                    
+                    BSONList values;
+                    
+                    result.append( "result" , mr.finalShort );
+                    
+                    while ( cursor.more() ){
+                        BSONObj t = cursor.next().getOwned();
+                        
+                        if ( values.size() == 0 ){
+                            values.push_back( t );
+                            continue;
+                        }
+                        
+                        if ( t.woSortOrder( *(values.begin()) , sortKey ) == 0 ){
+                            values.push_back( t );
+                            continue;
+                        }
+                        
+                        
+                        db.insert( mr.tempLong , reduceValues( values , s.get() , reduceFunction , 1 , finalizeFunction ) );
+                        values.clear();
                         values.push_back( t );
-                        continue;
                     }
                     
-
-                    db.insert( mr.tempLong , reduceValues( values , s.get() , reduceFunction , 1 , finalizeFunction ) );
-                    values.clear();
-                    values.push_back( t );
+                    if ( values.size() )
+                        db.insert( mr.tempLong , reduceValues( values , s.get() , reduceFunction , 1 , finalizeFunction ) );
                 }
                 
-                if ( values.size() )
-                    db.insert( mr.tempLong , reduceValues( values , s.get() , reduceFunction , 1 , finalizeFunction ) );
-                
                 long long finalCount = mr.renameIfNeeded( db );
                 log(0) << " mapreducefinishcommand " << mr.finalLong << " " << finalCount << endl;
 
                 for ( set<ServerAndQuery>::iterator i=servers.begin(); i!=servers.end(); i++ ){
                     ScopedDbConnection conn( i->_server );
                     conn->dropCollection( dbname + "." + shardedOutputCollection );
+                    conn.done();
                 }
                 
                 result.append( "shardCounts" , shardCounts.obj() );
diff --git a/db/namespace.cpp b/db/namespace.cpp
index 210efb6..8be6655 100644
--- a/db/namespace.cpp
+++ b/db/namespace.cpp
@@ -16,7 +16,7 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "pdfile.h"
 #include "db.h"
 #include "../util/mmap.h"
@@ -42,8 +42,34 @@ namespace mongo {
         0x400000, 0x800000
     };
 
+    NamespaceDetails::NamespaceDetails( const DiskLoc &loc, bool _capped ) {
+        /* be sure to initialize new fields here -- doesn't default to zeroes the way we use it */
+        firstExtent = lastExtent = capExtent = loc;
+        datasize = nrecords = 0;
+        lastExtentSize = 0;
+        nIndexes = 0;
+        capped = _capped;
+        max = 0x7fffffff;
+        paddingFactor = 1.0;
+        flags = 0;
+        capFirstNewRecord = DiskLoc();
+        // Signal that we are on first allocation iteration through extents.
+        capFirstNewRecord.setInvalid();
+        // For capped case, signal that we are doing initial extent allocation.
+        if ( capped )
+            cappedLastDelRecLastExtent().setInvalid();
+		assert( sizeof(dataFileVersion) == 2 );
+		dataFileVersion = 0;
+		indexFileVersion = 0;
+        multiKeyIndexBits = 0;
+        reservedA = 0;
+        extraOffset = 0;
+        backgroundIndexBuildInProgress = 0;
+        memset(reserved, 0, sizeof(reserved));
+    }
+
     bool NamespaceIndex::exists() const {
-        return !boost::filesystem::exists(path());
+        return !MMF::exists(path());
     }
     
     boost::filesystem::path NamespaceIndex::path() const {
@@ -78,7 +104,7 @@ namespace mongo {
         }
     }
 
-    static void callback(const Namespace& k, NamespaceDetails& v) { 
+    static void namespaceOnLoadCallback(const Namespace& k, NamespaceDetails& v) { 
         v.onLoad(k);
     }
 
@@ -100,10 +126,10 @@ namespace mongo {
 		int len = -1;
         boost::filesystem::path nsPath = path();
         string pathString = nsPath.string();
-		void *p;
-        if( boost::filesystem::exists(nsPath) ) { 
+        MMF::Pointer p;
+        if( MMF::exists(nsPath) ) { 
 			p = f.map(pathString.c_str());
-            if( p ) {
+            if( !p.isNull() ) {
                 len = f.length();
                 if ( len % (1024*1024) != 0 ){
                     log() << "bad .ns file: " << pathString << endl;
@@ -117,22 +143,38 @@ namespace mongo {
             maybeMkdir();
 			long l = lenForNewNsFiles;
 			p = f.map(pathString.c_str(), l);
-            if( p ) { 
+            if( !p.isNull() ) {
                 len = (int) l;
                 assert( len == lenForNewNsFiles );
             }
 		}
 
-        if ( p == 0 ) {
+        if ( p.isNull() ) {
             problem() << "couldn't open file " << pathString << " terminating" << endl;
             dbexit( EXIT_FS );
         }
-        ht = new HashTable<Namespace,NamespaceDetails>(p, len, "namespace index");
+
+        ht = new HashTable<Namespace,NamespaceDetails,MMF::Pointer>(p, len, "namespace index");
         if( checkNsFilesOnLoad )
-            ht->iterAll(callback);
+            ht->iterAll(namespaceOnLoadCallback);
+    }
+    
+    static void namespaceGetNamespacesCallback( const Namespace& k , NamespaceDetails& v , void * extra ) {
+        list<string> * l = (list<string>*)extra;
+        if ( ! k.hasDollarSign() )
+            l->push_back( (string)k );
+    }
+
+    void NamespaceIndex::getNamespaces( list<string>& tofill , bool onlyCollections ) const {
+        assert( onlyCollections ); // TODO: need to implement this
+        //                                  need boost::bind or something to make this less ugly
+        
+        if ( ht )
+            ht->iterAll( namespaceGetNamespacesCallback , (void*)&tofill );
     }
 
     void NamespaceDetails::addDeletedRec(DeletedRecord *d, DiskLoc dloc) {
+		BOOST_STATIC_ASSERT( sizeof(NamespaceDetails::Extra) <= sizeof(NamespaceDetails) );
         {
             // defensive code: try to make us notice if we reference a deleted record
             (unsigned&) (((Record *) d)->data) = 0xeeeeeeee;
@@ -140,19 +182,20 @@ namespace mongo {
         dassert( dloc.drec() == d );
         DEBUGGING out() << "TEMP: add deleted rec " << dloc.toString() << ' ' << hex << d->extentOfs << endl;
         if ( capped ) {
-            if ( !deletedList[ 1 ].isValid() ) {
+            if ( !cappedLastDelRecLastExtent().isValid() ) {
                 // Initial extent allocation.  Insert at end.
                 d->nextDeleted = DiskLoc();
-                if ( deletedList[ 0 ].isNull() )
-                    deletedList[ 0 ] = dloc;
+                if ( cappedListOfAllDeletedRecords().isNull() )
+                    cappedListOfAllDeletedRecords() = dloc;
                 else {
-                    DiskLoc i = deletedList[ 0 ];
+                    DiskLoc i = cappedListOfAllDeletedRecords();
                     for (; !i.drec()->nextDeleted.isNull(); i = i.drec()->nextDeleted );
                     i.drec()->nextDeleted = dloc;
                 }
             } else {
-                d->nextDeleted = firstDeletedInCapExtent();
-                firstDeletedInCapExtent() = dloc;
+                d->nextDeleted = cappedFirstDeletedInCurExtent();
+                cappedFirstDeletedInCurExtent() = dloc;
+                // always compact() after this so order doesn't matter
             }
         } else {
             int b = bucket(d->lengthWithHeaders);
@@ -186,15 +229,17 @@ namespace mongo {
         if ( capped == 0 ) {
             if ( left < 24 || left < (lenToAlloc >> 3) ) {
                 // you get the whole thing.
+				DataFileMgr::grow(loc, regionlen);
                 return loc;
             }
         }
 
         /* split off some for further use. */
         r->lengthWithHeaders = lenToAlloc;
+		DataFileMgr::grow(loc, lenToAlloc);
         DiskLoc newDelLoc = loc;
         newDelLoc.inc(lenToAlloc);
-        DeletedRecord *newDel = newDelLoc.drec();
+        DeletedRecord *newDel = DataFileMgr::makeDeletedRecord(newDelLoc, left);
         newDel->extentOfs = r->extentOfs;
         newDel->lengthWithHeaders = left;
         newDel->nextDeleted.Null();
@@ -298,53 +343,6 @@ namespace mongo {
         }
     }
 
-    /* combine adjacent deleted records
-
-       this is O(n^2) but we call it for capped tables where typically n==1 or 2!
-       (or 3...there will be a little unused sliver at the end of the extent.)
-    */
-    void NamespaceDetails::compact() {
-        assert(capped);
-
-        list<DiskLoc> drecs;
-
-        // Pull out capExtent's DRs from deletedList
-        DiskLoc i = firstDeletedInCapExtent();
-        for (; !i.isNull() && inCapExtent( i ); i = i.drec()->nextDeleted )
-            drecs.push_back( i );
-        firstDeletedInCapExtent() = i;
-
-        // This is the O(n^2) part.
-        drecs.sort();
-
-        list<DiskLoc>::iterator j = drecs.begin();
-        assert( j != drecs.end() );
-        DiskLoc a = *j;
-        while ( 1 ) {
-            j++;
-            if ( j == drecs.end() ) {
-                DEBUGGING out() << "TEMP: compact adddelrec\n";
-                addDeletedRec(a.drec(), a);
-                break;
-            }
-            DiskLoc b = *j;
-            while ( a.a() == b.a() && a.getOfs() + a.drec()->lengthWithHeaders == b.getOfs() ) {
-                // a & b are adjacent.  merge.
-                a.drec()->lengthWithHeaders += b.drec()->lengthWithHeaders;
-                j++;
-                if ( j == drecs.end() ) {
-                    DEBUGGING out() << "temp: compact adddelrec2\n";
-                    addDeletedRec(a.drec(), a);
-                    return;
-                }
-                b = *j;
-            }
-            DEBUGGING out() << "temp: compact adddelrec3\n";
-            addDeletedRec(a.drec(), a);
-            a = b;
-        }
-    }
-
     DiskLoc NamespaceDetails::firstRecord( const DiskLoc &startExtent ) const {
         for (DiskLoc i = startExtent.isNull() ? firstExtent : startExtent;
                 !i.isNull(); i = i.ext()->xnext ) {
@@ -363,47 +361,6 @@ namespace mongo {
         return DiskLoc();
     }
 
-    DiskLoc &NamespaceDetails::firstDeletedInCapExtent() {
-        if ( deletedList[ 1 ].isNull() )
-            return deletedList[ 0 ];
-        else
-            return deletedList[ 1 ].drec()->nextDeleted;
-    }
-
-    bool NamespaceDetails::inCapExtent( const DiskLoc &dl ) const {
-        assert( !dl.isNull() );
-        // We could have a rec or drec, doesn't matter.
-        return dl.drec()->myExtent( dl ) == capExtent.ext();
-    }
-
-    bool NamespaceDetails::nextIsInCapExtent( const DiskLoc &dl ) const {
-        assert( !dl.isNull() );
-        DiskLoc next = dl.drec()->nextDeleted;
-        if ( next.isNull() )
-            return false;
-        return inCapExtent( next );
-    }
-
-    void NamespaceDetails::advanceCapExtent( const char *ns ) {
-        // We want deletedList[ 1 ] to be the last DeletedRecord of the prev cap extent
-        // (or DiskLoc() if new capExtent == firstExtent)
-        if ( capExtent == lastExtent )
-            deletedList[ 1 ] = DiskLoc();
-        else {
-            DiskLoc i = firstDeletedInCapExtent();
-            for (; !i.isNull() && nextIsInCapExtent( i ); i = i.drec()->nextDeleted );
-            deletedList[ 1 ] = i;
-        }
-
-        capExtent = theCapExtent()->xnext.isNull() ? firstExtent : theCapExtent()->xnext;
-
-        /* this isn't true if a collection has been renamed...that is ok just used for diagnostics */
-        //dassert( theCapExtent()->ns == ns );
-
-        theCapExtent()->assertOk();
-        capFirstNewRecord = DiskLoc();
-    }
-
     int n_complaints_cap = 0;
     void NamespaceDetails::maybeComplain( const char *ns, int len ) const {
         if ( ++n_complaints_cap < 8 ) {
@@ -422,157 +379,84 @@ namespace mongo {
         }
     }
 
-    DiskLoc NamespaceDetails::__capAlloc( int len ) {
-        DiskLoc prev = deletedList[ 1 ];
-        DiskLoc i = firstDeletedInCapExtent();
-        DiskLoc ret;
-        for (; !i.isNull() && inCapExtent( i ); prev = i, i = i.drec()->nextDeleted ) {
-            // We need to keep at least one DR per extent in deletedList[ 0 ],
-            // so make sure there's space to create a DR at the end.
-            if ( i.drec()->lengthWithHeaders >= len + 24 ) {
-                ret = i;
-                break;
-            }
-        }
-
-        /* unlink ourself from the deleted list */
-        if ( !ret.isNull() ) {
-            if ( prev.isNull() )
-                deletedList[ 0 ] = ret.drec()->nextDeleted;
-            else
-                prev.drec()->nextDeleted = ret.drec()->nextDeleted;
-            ret.drec()->nextDeleted.setInvalid(); // defensive.
-            assert( ret.drec()->extentOfs < ret.getOfs() );
-        }
-
-        return ret;
-    }
-
-    void NamespaceDetails::checkMigrate() {
-        // migrate old NamespaceDetails format
-        if ( capped && capExtent.a() == 0 && capExtent.getOfs() == 0 ) {
-            capFirstNewRecord = DiskLoc();
-            capFirstNewRecord.setInvalid();
-            // put all the DeletedRecords in deletedList[ 0 ]
-            for ( int i = 1; i < Buckets; ++i ) {
-                DiskLoc first = deletedList[ i ];
-                if ( first.isNull() )
-                    continue;
-                DiskLoc last = first;
-                for (; !last.drec()->nextDeleted.isNull(); last = last.drec()->nextDeleted );
-                last.drec()->nextDeleted = deletedList[ 0 ];
-                deletedList[ 0 ] = first;
-                deletedList[ i ] = DiskLoc();
-            }
-            // NOTE deletedList[ 1 ] set to DiskLoc() in above
-
-            // Last, in case we're killed before getting here
-            capExtent = firstExtent;
-        }
-    }
-
     /* alloc with capped table handling. */
     DiskLoc NamespaceDetails::_alloc(const char *ns, int len) {
         if ( !capped )
             return __stdAlloc(len);
 
-        // capped.
+        return cappedAlloc(ns,len);
+    }
 
-        // signal done allocating new extents.
-        if ( !deletedList[ 1 ].isValid() )
-            deletedList[ 1 ] = DiskLoc();
+    /* extra space for indexes when more than 10 */
+    NamespaceDetails::Extra* NamespaceIndex::newExtra(const char *ns, int i, NamespaceDetails *d) {
+        assert( i >= 0 && i <= 1 );
+        Namespace n(ns);
+        Namespace extra(n.extraName(i).c_str()); // throws userexception if ns name too long
         
-        assert( len < 400000000 );
-        int passes = 0;
-        int maxPasses = ( len / 30 ) + 2; // 30 is about the smallest entry that could go in the oplog
-        if ( maxPasses < 5000 ){
-            // this is for bacwards safety since 5000 was the old value
-            maxPasses = 5000;
+        massert( 10350 ,  "allocExtra: base ns missing?", d );
+        massert( 10351 ,  "allocExtra: extra already exists", ht->get(extra) == 0 );
+
+        NamespaceDetails::Extra temp;
+        temp.init();
+        uassert( 10082 ,  "allocExtra: too many namespaces/collections", ht->put(extra, (NamespaceDetails&) temp));
+        NamespaceDetails::Extra *e = (NamespaceDetails::Extra *) ht->get(extra);
+        return e;
+    }
+    NamespaceDetails::Extra* NamespaceDetails::allocExtra(const char *ns, int nindexessofar) {
+        NamespaceIndex *ni = nsindex(ns);
+        int i = (nindexessofar - NIndexesBase) / NIndexesExtra;
+        Extra *e = ni->newExtra(ns, i, this);
+        long ofs = e->ofsFrom(this);
+        if( i == 0 ) {
+            assert( extraOffset == 0 );
+            extraOffset = ofs;
+            assert( extra() == e );
         }
-        DiskLoc loc;
-
-        // delete records until we have room and the max # objects limit achieved.
-
-        /* this fails on a rename -- that is ok but must keep commented out */
-        //assert( theCapExtent()->ns == ns );
-
-        theCapExtent()->assertOk();
-        DiskLoc firstEmptyExtent;
-        while ( 1 ) {
-            if ( nrecords < max ) {
-                loc = __capAlloc( len );
-                if ( !loc.isNull() )
-                    break;
-            }
-
-            // If on first iteration through extents, don't delete anything.
-            if ( !capFirstNewRecord.isValid() ) {
-                advanceCapExtent( ns );
-                if ( capExtent != firstExtent )
-                    capFirstNewRecord.setInvalid();
-                // else signal done with first iteration through extents.
-                continue;
-            }
-
-            if ( !capFirstNewRecord.isNull() &&
-                    theCapExtent()->firstRecord == capFirstNewRecord ) {
-                // We've deleted all records that were allocated on the previous
-                // iteration through this extent.
-                advanceCapExtent( ns );
-                continue;
-            }
-
-            if ( theCapExtent()->firstRecord.isNull() ) {
-                if ( firstEmptyExtent.isNull() )
-                    firstEmptyExtent = capExtent;
-                advanceCapExtent( ns );
-                if ( firstEmptyExtent == capExtent ) {
-                    maybeComplain( ns, len );
-                    return DiskLoc();
-                }
-                continue;
-            }
-
-            massert( 10344 ,  "Capped collection full and delete not allowed", cappedMayDelete() );
-            DiskLoc fr = theCapExtent()->firstRecord;
-            theDataFileMgr.deleteRecord(ns, fr.rec(), fr, true);
-            compact();
-            if( ++passes > maxPasses ) {
-                log() << "passes ns:" << ns << " len:" << len << " maxPasses: " << maxPasses << '\n';
-                log() << "passes max:" << max << " nrecords:" << nrecords << " datasize: " << datasize << endl;
-                massert( 10345 ,  "passes >= maxPasses in capped collection alloc", false );
-            }
+        else { 
+            Extra *hd = extra();
+            assert( hd->next(this) == 0 );
+            hd->setNext(ofs);
         }
-
-        // Remember first record allocated on this iteration through capExtent.
-        if ( capFirstNewRecord.isValid() && capFirstNewRecord.isNull() )
-            capFirstNewRecord = loc;
-
-        return loc;
+        return e;
     }
 
     /* you MUST call when adding an index.  see pdfile.cpp */
     IndexDetails& NamespaceDetails::addIndex(const char *thisns, bool resetTransient) {
         assert( nsdetails(thisns) == this );
 
-        if( nIndexes == NIndexesBase && extraOffset == 0 ) { 
-            nsindex(thisns)->allocExtra(thisns);
+        IndexDetails *id;
+        try {
+            id = &idx(nIndexes,true);
+        }
+        catch(DBException&) { 
+            allocExtra(thisns, nIndexes);
+            id = &idx(nIndexes,false);
         }
 
-        IndexDetails& id = idx(nIndexes);
         nIndexes++;
         if ( resetTransient )
             NamespaceDetailsTransient::get_w(thisns).addedIndex();
-        return id;
+        return *id;
     }
 
     // must be called when renaming a NS to fix up extra
     void NamespaceDetails::copyingFrom(const char *thisns, NamespaceDetails *src) { 
-        if( extraOffset ) {
-            extraOffset = 0; // so allocExtra() doesn't assert.
-            Extra *e = nsindex(thisns)->allocExtra(thisns);
-            memcpy(e, src->extra(), sizeof(Extra));
-        } 
+        extraOffset = 0; // we are a copy -- the old value is wrong.  fixing it up below.
+        Extra *se = src->extra();
+        int n = NIndexesBase;
+        if( se ) {
+            Extra *e = allocExtra(thisns, n);
+            while( 1 ) {
+                n += NIndexesExtra;
+                e->copy(this, *se);
+                se = se->next(src);
+                if( se == 0 ) break;
+                Extra *nxt = allocExtra(thisns, n);
+                e->setNext( nxt->ofsFrom(this) );
+                e = nxt;
+            } 
+            assert( extraOffset );
+        }
     }
 
     /* returns index of the first index in which the field is present. -1 if not present.
@@ -610,8 +494,8 @@ namespace mongo {
     
     /* ------------------------------------------------------------------------- */
 
-    mongo::mutex NamespaceDetailsTransient::_qcMutex;
-    mongo::mutex NamespaceDetailsTransient::_isMutex;
+    mongo::mutex NamespaceDetailsTransient::_qcMutex("qc");
+    mongo::mutex NamespaceDetailsTransient::_isMutex("is");
     map< string, shared_ptr< NamespaceDetailsTransient > > NamespaceDetailsTransient::_map;
     typedef map< string, shared_ptr< NamespaceDetailsTransient > >::iterator ouriter;
 
@@ -651,42 +535,6 @@ namespace mongo {
             i.next().keyPattern().getFieldNames(_indexKeys);
     }
 
-    void NamespaceDetailsTransient::cllStart( int logSizeMb ) {
-        assertInWriteLock();
-        _cll_ns = "local.temp.oplog." + _ns;
-        _cll_enabled = true;
-        stringstream spec;
-        // 128MB
-        spec << "{size:" << logSizeMb * 1024 * 1024 << ",capped:true,autoIndexId:false}";
-        Client::Context ct( _cll_ns );
-        string err;
-        massert( 10347 ,  "Could not create log ns", userCreateNS( _cll_ns.c_str(), fromjson( spec.str() ), err, false ) );
-        NamespaceDetails *d = nsdetails( _cll_ns.c_str() );
-        d->cappedDisallowDelete();
-    }
-
-    void NamespaceDetailsTransient::cllInvalidate() {
-        assertInWriteLock();
-        cllDrop();
-        _cll_enabled = false;
-    }
-    
-    bool NamespaceDetailsTransient::cllValidateComplete() {
-        assertInWriteLock();
-        cllDrop();
-        bool ret = _cll_enabled;
-        _cll_enabled = false;
-        _cll_ns = "";
-        return ret;
-    }
-    
-    void NamespaceDetailsTransient::cllDrop() {
-        assertInWriteLock();
-        if ( !_cll_enabled )
-            return;
-        Client::Context ctx( _cll_ns );
-        dropNS( _cll_ns );
-    }
 
     /* ------------------------------------------------------------------------- */
 
diff --git a/db/namespace.h b/db/namespace.h
index 1b1a954..abc35bb 100644
--- a/db/namespace.h
+++ b/db/namespace.h
@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "jsobj.h"
 #include "queryutil.h"
 #include "diskloc.h"
@@ -27,10 +27,6 @@
 
 namespace mongo {
 
-    class Cursor;
-
-#pragma pack(1)
-
 	/* in the mongo source code, "client" means "database". */
 
     const int MaxDatabaseLen = 256; // max str len for the db name, including null char
@@ -55,6 +51,12 @@ namespace mongo {
         nsToDatabase(ns, buf);
         return buf;
     }
+    inline string nsToDatabase(const string& ns) {
+        size_t i = ns.find( '.' );
+        if ( i == string::npos )
+            return ns;
+        return ns.substr( 0 , i );
+    }
 
 	/* e.g.
 	   NamespaceString ns("acme.orders");
@@ -84,6 +86,7 @@ namespace mongo {
         }
     };
 
+#pragma pack(1)
 	/* This helper class is used to make the HashMap below in NamespaceDetails */
     class Namespace {
     public:
@@ -99,26 +102,21 @@ namespace mongo {
         }
 
         /* for more than 10 indexes -- see NamespaceDetails::Extra */
-        string extraName() { 
-            string s = string(buf) + "$extra";
-            massert( 10348 , "ns name too long", s.size() < MaxNsLen);
+        string extraName(int i) {
+            char ex[] = "$extra";
+            ex[5] += i;
+            string s = string(buf) + ex;
+            massert( 10348 , "$extra: ns name too long", s.size() < MaxNsLen);
             return s;
         }
         bool isExtra() const { 
-            const char *p = strstr(buf, "$extra");
-            return p && p[6] == 0; //==0 important in case an index uses name "$extra_1" for example
-        }
-
-        void kill() {
-            buf[0] = 0x7f;
-        }
-
-        bool operator==(const char *r) {
-            return strcmp(buf, r) == 0;
-        }
-        bool operator==(const Namespace& r) {
-            return strcmp(buf, r.buf) == 0;
+            const char *p = strstr(buf, "$extr");
+            return p && p[5] && p[6] == 0; //==0 important in case an index uses name "$extra_1" for example
         }
+        bool hasDollarSign() const { return strchr( buf , '$' ) > 0;  }
+        void kill() { buf[0] = 0x7f; }
+        bool operator==(const char *r) const { return strcmp(buf, r) == 0; }
+        bool operator==(const Namespace& r) const { return strcmp(buf, r.buf) == 0; }
         int hash() const {
             unsigned x = 0;
             const char *p = buf;
@@ -141,26 +139,28 @@ namespace mongo {
             return old + "." + local;
         }
 
+        string toString() const {
+            return (string)buf;
+        }
+
         operator string() const {
             return (string)buf;
         }
 
         char buf[MaxNsLen];
     };
+#pragma pack()
 
-}
+} // namespace mongo
 
 #include "index.h"
 
 namespace mongo {
 
-    /**
-       @return true if a client can modify this namespace
-       things like *.system.users
-     */
+    /** @return true if a client can modify this namespace
+        things like *.system.users */
     bool legalClientSystemNS( const string& ns , bool write );
 
-
     /* deleted lists -- linked lists of deleted records -- are placed in 'buckets' of various sizes
        so you can look for a deleterecord about the right size.
     */
@@ -169,6 +169,7 @@ namespace mongo {
 
     extern int bucketSizes[];
 
+#pragma pack(1)
     /* this is the "header" for a collection that has all its details.  in the .ns file.
     */
     class NamespaceDetails {
@@ -176,75 +177,108 @@ namespace mongo {
         enum { NIndexesExtra = 30,
                NIndexesBase  = 10
         };
-        struct Extra { 
+    public:
+        struct ExtraOld {
             // note we could use this field for more chaining later, so don't waste it:
-            unsigned long long reserved1; 
+            unsigned long long reserved1;
             IndexDetails details[NIndexesExtra];
             unsigned reserved2;
             unsigned reserved3;
         };
+        class Extra { 
+            long long _next;
+		public:
+            IndexDetails details[NIndexesExtra];
+		private:
+            unsigned reserved2;
+            unsigned reserved3;
+			Extra(const Extra&) { assert(false); }
+			Extra& operator=(const Extra& r) { assert(false); return *this; }
+        public:
+            Extra() { }
+            long ofsFrom(NamespaceDetails *d) { 
+                return ((char *) this) - ((char *) d);
+            }
+            void init() { memset(this, 0, sizeof(Extra)); }
+            Extra* next(NamespaceDetails *d) { 
+                if( _next == 0 ) return 0;
+                return (Extra*) (((char *) d) + _next);
+            }
+            void setNext(long ofs) { _next = ofs;  }
+            void copy(NamespaceDetails *d, const Extra& e) { 
+                memcpy(this, &e, sizeof(Extra));
+                _next = 0;
+            }
+        }; // Extra
+
         Extra* extra() { 
-            assert( extraOffset );
+            if( extraOffset == 0 ) return 0;
             return (Extra *) (((char *) this) + extraOffset);
         }
+
     public:
+        /* add extra space for indexes when more than 10 */
+        Extra* allocExtra(const char *ns, int nindexessofar);
+
         void copyingFrom(const char *thisns, NamespaceDetails *src); // must be called when renaming a NS to fix up extra
 
-        enum { NIndexesMax = 40 };
+        enum { NIndexesMax = 64 };
 
-        BOOST_STATIC_ASSERT( NIndexesMax == NIndexesBase + NIndexesExtra );
+        BOOST_STATIC_ASSERT( NIndexesMax <= NIndexesBase + NIndexesExtra*2 );
+        BOOST_STATIC_ASSERT( NIndexesMax <= 64 ); // multiKey bits
+		BOOST_STATIC_ASSERT( sizeof(NamespaceDetails::ExtraOld) == 496 );
+		BOOST_STATIC_ASSERT( sizeof(NamespaceDetails::Extra) == 496 );
 
         /* called when loaded from disk */
         void onLoad(const Namespace& k);
 
-        NamespaceDetails( const DiskLoc &loc, bool _capped ) {
-            /* be sure to initialize new fields here -- doesn't default to zeroes the way we use it */
-            firstExtent = lastExtent = capExtent = loc;
-            datasize = nrecords = 0;
-            lastExtentSize = 0;
-            nIndexes = 0;
-            capped = _capped;
-            max = 0x7fffffff;
-            paddingFactor = 1.0;
-            flags = 0;
-            capFirstNewRecord = DiskLoc();
-            // Signal that we are on first allocation iteration through extents.
-            capFirstNewRecord.setInvalid();
-            // For capped case, signal that we are doing initial extent allocation.
-            if ( capped )
-                deletedList[ 1 ].setInvalid();
-			assert( sizeof(dataFileVersion) == 2 );
-			dataFileVersion = 0;
-			indexFileVersion = 0;
-            multiKeyIndexBits = 0;
-            reservedA = 0;
-            extraOffset = 0;
-            backgroundIndexBuildInProgress = 0;
-            memset(reserved, 0, sizeof(reserved));
-        }
+        NamespaceDetails( const DiskLoc &loc, bool _capped );
+
         DiskLoc firstExtent;
         DiskLoc lastExtent;
 
         /* NOTE: capped collections override the meaning of deleted list.  
                  deletedList[0] points to a list of free records (DeletedRecord's) for all extents in
-                 the namespace.
+                 the capped namespace.
                  deletedList[1] points to the last record in the prev extent.  When the "current extent" 
                  changes, this value is updated.  !deletedList[1].isValid() when this value is not 
                  yet computed.
         */
         DiskLoc deletedList[Buckets];
 
+        void dumpExtents();
+
         long long datasize;
         long long nrecords;
         int lastExtentSize;
         int nIndexes;
+
     private:
         IndexDetails _indexes[NIndexesBase];
+
+    private:
+        Extent *theCapExtent() const { return capExtent.ext(); }
+        void advanceCapExtent( const char *ns );
+        DiskLoc __capAlloc(int len);
+        DiskLoc cappedAlloc(const char *ns, int len);
+        DiskLoc &cappedFirstDeletedInCurExtent();
+        bool nextIsInCapExtent( const DiskLoc &dl ) const;
     public:
+        DiskLoc& cappedListOfAllDeletedRecords() { return deletedList[0]; }
+        DiskLoc& cappedLastDelRecLastExtent()    { return deletedList[1]; }
+        void cappedDumpDelInfo();
+        bool capLooped() const { return capped && capFirstNewRecord.isValid();  }
+        bool inCapExtent( const DiskLoc &dl ) const;
+        void cappedCheckMigrate();
+        void cappedTruncateAfter(const char *ns, DiskLoc after, bool inclusive); /** remove rest of the capped collection from this point onward */
+        void emptyCappedCollection(const char *ns);
+        
         int capped;
-        int max; // max # of objects for a capped table.
+
+        int max; // max # of objects for a capped table.  TODO: should this be 64 bit? 
         double paddingFactor; // 1.0 = no padding.
         int flags;
+
         DiskLoc capExtent;
         DiskLoc capFirstNewRecord;
 
@@ -265,22 +299,35 @@ namespace mongo {
         /* when a background index build is in progress, we don't count the index in nIndexes until 
            complete, yet need to still use it in _indexRecord() - thus we use this function for that.
         */
-        int nIndexesBeingBuilt() const {
-            return nIndexes + backgroundIndexBuildInProgress;
-        }
+        int nIndexesBeingBuilt() const { return nIndexes + backgroundIndexBuildInProgress; }
 
         /* NOTE: be careful with flags.  are we manipulating them in read locks?  if so, 
                  this isn't thread safe.  TODO
         */
         enum NamespaceFlags {
-            Flag_HaveIdIndex = 1 << 0, // set when we have _id index (ONLY if ensureIdIndex was called -- 0 if that has never been called)
-            Flag_CappedDisallowDelete = 1 << 1 // set when deletes not allowed during capped table allocation.
+            Flag_HaveIdIndex = 1 << 0 // set when we have _id index (ONLY if ensureIdIndex was called -- 0 if that has never been called)
         };
 
-        IndexDetails& idx(int idxNo) {
+        IndexDetails& idx(int idxNo, bool missingExpected = false ) {
             if( idxNo < NIndexesBase ) 
                 return _indexes[idxNo];
-            return extra()->details[idxNo-NIndexesBase];
+            Extra *e = extra();
+            if ( ! e ){
+                if ( missingExpected )
+                    throw MsgAssertionException( 13283 , "Missing Extra" );
+                massert(13282, "missing Extra", e);
+            }
+            int i = idxNo - NIndexesBase;
+            if( i >= NIndexesExtra ) {
+                e = e->next(this);
+                if ( ! e ){
+                    if ( missingExpected )
+                        throw MsgAssertionException( 13283 , "missing extra" );
+                    massert(13283, "missing Extra", e);
+                }
+                i -= NIndexesExtra;
+            }
+            return e->details[i];
         }
         IndexDetails& backgroundIdx() { 
             DEV assert(backgroundIndexBuildInProgress);
@@ -292,28 +339,18 @@ namespace mongo {
             int i;
             int n;
             NamespaceDetails *d;
-            Extra *e;
             IndexIterator(NamespaceDetails *_d) { 
                 d = _d;
                 i = 0;
                 n = d->nIndexes;
-                if( n > NIndexesBase )
-                    e = d->extra();
             }
         public:
             int pos() { return i; } // note this is the next one to come
             bool more() { return i < n; }
-            IndexDetails& next() { 
-                int k = i;
-                i++;
-                return k < NIndexesBase ? d->_indexes[k] : 
-                    e->details[k-10];
-            }
-        };
+            IndexDetails& next() { return d->idx(i++); }
+        }; // IndexIterator
 
-        IndexIterator ii() { 
-            return IndexIterator(this);
-        }
+        IndexIterator ii() { return IndexIterator(this); }
 
         /* hackish - find our index # in the indexes array
         */
@@ -348,14 +385,8 @@ namespace mongo {
          */
         IndexDetails& addIndex(const char *thisns, bool resetTransient=true);
 
-        void aboutToDeleteAnIndex() {
-            flags &= ~Flag_HaveIdIndex;
-        }
+        void aboutToDeleteAnIndex() { flags &= ~Flag_HaveIdIndex;  }
 
-        void cappedDisallowDelete() {
-            flags |= Flag_CappedDisallowDelete;
-        }
-        
         /* returns index of the first index in which the field is present. -1 if not present. */
         int fieldIsIndexed(const char *fieldName);
 
@@ -389,6 +420,14 @@ namespace mongo {
             }
             return -1;
         }
+        
+        void findIndexByType( const string& name , vector<int>& matches ) {
+            IndexIterator i = ii();
+            while ( i.more() ){
+                if ( i.next().getSpec().getTypeName() == name )
+                    matches.push_back( i.pos() - 1 );
+            }
+        }
 
         /* @return -1 = not found 
            generally id is first index, so not that expensive an operation (assuming present).
@@ -418,43 +457,25 @@ namespace mongo {
 
         void dumpDeleted(set<DiskLoc> *extents = 0);
 
-        bool capLooped() const {
-            return capped && capFirstNewRecord.isValid();
-        }
-
         // Start from firstExtent by default.
         DiskLoc firstRecord( const DiskLoc &startExtent = DiskLoc() ) const;
 
         // Start from lastExtent by default.
         DiskLoc lastRecord( const DiskLoc &startExtent = DiskLoc() ) const;
 
-        bool inCapExtent( const DiskLoc &dl ) const;
-
-        void checkMigrate();
-
         long long storageSize( int * numExtents = 0 );
-
+        
     private:
-        bool cappedMayDelete() const {
-            return !( flags & Flag_CappedDisallowDelete );
-        }
-        Extent *theCapExtent() const {
-            return capExtent.ext();
-        }
-        void advanceCapExtent( const char *ns );
+        DiskLoc _alloc(const char *ns, int len);
         void maybeComplain( const char *ns, int len ) const;
         DiskLoc __stdAlloc(int len);
-        DiskLoc __capAlloc(int len);
-        DiskLoc _alloc(const char *ns, int len);
         void compact(); // combine adjacent deleted records
-
-        DiskLoc &firstDeletedInCapExtent();
-        bool nextIsInCapExtent( const DiskLoc &dl ) const;
-    };
-
+    }; // NamespaceDetails
 #pragma pack()
 
-    /* these are things we know / compute about a namespace that are transient -- things
+    /* NamespaceDetailsTransient
+
+       these are things we know / compute about a namespace that are transient -- things
        we don't actually store in the .ns file.  so mainly caching of frequently used
        information.
 
@@ -465,13 +486,15 @@ namespace mongo {
        todo: cleanup code, need abstractions and separation
     */
     class NamespaceDetailsTransient : boost::noncopyable {
+		BOOST_STATIC_ASSERT( sizeof(NamespaceDetails) == 496 );
+
         /* general ------------------------------------------------------------- */
     private:
         string _ns;
         void reset();
         static std::map< string, shared_ptr< NamespaceDetailsTransient > > _map;
     public:
-        NamespaceDetailsTransient(const char *ns) : _ns(ns), _keysComputed(false), _qcWriteCount(), _cll_enabled() { }
+        NamespaceDetailsTransient(const char *ns) : _ns(ns), _keysComputed(false), _qcWriteCount(){ }
         /* _get() is not threadsafe -- see get_inlock() comments */
         static NamespaceDetailsTransient& _get(const char *ns);
         /* use get_w() when doing write operations */
@@ -551,19 +574,6 @@ namespace mongo {
             _qcCache[ pattern ] = make_pair( indexKey, nScanned );
         }
 
-        /* for collection-level logging -- see CmdLogCollection ----------------- */ 
-        /* assumed to be in write lock for this */
-    private:
-        string _cll_ns; // "local.temp.oplog." + _ns;
-        bool _cll_enabled;
-        void cllDrop(); // drop _cll_ns
-    public:
-        string cllNS() const { return _cll_ns; }
-        bool cllEnabled() const { return _cll_enabled; }
-        void cllStart( int logSizeMb = 256 ); // begin collection level logging
-        void cllInvalidate();
-        bool cllValidateComplete();
-
     }; /* NamespaceDetailsTransient */
 
     inline NamespaceDetailsTransient& NamespaceDetailsTransient::_get(const char *ns) {
@@ -578,23 +588,20 @@ namespace mongo {
     */
     class NamespaceIndex {
         friend class NamespaceCursor;
-        BOOST_STATIC_ASSERT( sizeof(NamespaceDetails::Extra) <= sizeof(NamespaceDetails) );
+
     public:
         NamespaceIndex(const string &dir, const string &database) :
-          ht( 0 ),
-          dir_( dir ),
-          database_( database ) {}
+          ht( 0 ), dir_( dir ), database_( database ) {}
 
         /* returns true if new db will be created if we init lazily */
         bool exists() const;
-        
+
         void init();
 
         void add_ns(const char *ns, DiskLoc& loc, bool capped) {
             NamespaceDetails details( loc, capped );
 			add_ns( ns, details );
         }
-
 		void add_ns( const char *ns, const NamespaceDetails &details ) {
             init();
             Namespace n(ns);
@@ -602,36 +609,19 @@ namespace mongo {
 		}
 
         /* just for diagnostics */
-        size_t detailsOffset(NamespaceDetails *d) {
+        /*size_t detailsOffset(NamespaceDetails *d) {
             if ( !ht )
                 return -1;
             return ((char *) d) -  (char *) ht->nodes;
-        }
-
-        /* extra space for indexes when more than 10 */
-        NamespaceDetails::Extra* allocExtra(const char *ns) { 
-            Namespace n(ns);
-            Namespace extra(n.extraName().c_str()); // throws userexception if ns name too long
-            NamespaceDetails *d = details(ns);
-            massert( 10350 ,  "allocExtra: base ns missing?", d );
-            assert( d->extraOffset == 0 );
-            massert( 10351 ,  "allocExtra: extra already exists", ht->get(extra) == 0 );
-            NamespaceDetails::Extra temp;
-            memset(&temp, 0, sizeof(temp));
-            uassert( 10082 ,  "allocExtra: too many namespaces/collections", ht->put(extra, (NamespaceDetails&) temp));
-            NamespaceDetails::Extra *e = (NamespaceDetails::Extra *) ht->get(extra);
-            d->extraOffset = ((char *) e) - ((char *) d);
-            assert( d->extra() == e );
-            return e;
-        }
+        }*/
 
         NamespaceDetails* details(const char *ns) {
             if ( !ht )
                 return 0;
             Namespace n(ns);
             NamespaceDetails *d = ht->get(n);
-            if ( d )
-                d->checkMigrate();
+            if ( d && d->capped )
+                d->cappedCheckMigrate();
             return d;
         }
 
@@ -641,11 +631,13 @@ namespace mongo {
             Namespace n(ns);
             ht->kill(n);
 
-            try {
-                Namespace extra(n.extraName().c_str());
-                ht->kill(extra);
+            for( int i = 0; i<=1; i++ ) {
+                try {
+                    Namespace extra(n.extraName(i).c_str());
+                    ht->kill(extra);
+                }
+                catch(DBException&) { }
             }
-            catch(DBException&) { }
         }
 
         bool find(const char *ns, DiskLoc& loc) {
@@ -661,12 +653,17 @@ namespace mongo {
             return ht != 0;
         }
 
-    private:
+        void getNamespaces( list<string>& tofill , bool onlyCollections = true ) const;
+
+        NamespaceDetails::Extra* newExtra(const char *ns, int n, NamespaceDetails *d);
+
         boost::filesystem::path path() const;
+    private:
+
         void maybeMkdir() const;
         
-        MemoryMappedFile f;
-        HashTable<Namespace,NamespaceDetails> *ht;
+        MMF f;
+        HashTable<Namespace,NamespaceDetails,MMF::Pointer> *ht;
         string dir_;
         string database_;
     };
diff --git a/db/nonce.cpp b/db/nonce.cpp
index d8db58d..519cfaa 100644
--- a/db/nonce.cpp
+++ b/db/nonce.cpp
@@ -15,7 +15,7 @@
  *    limitations under the License.
  */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "nonce.h"
 
 extern int do_md5_test(void);
@@ -32,7 +32,7 @@ namespace mongo {
 		if( _initialized ) return;
 		_initialized = true;
 
-#if defined(__linux__)
+#if defined(__linux__) || defined(__sunos__)
         _devrandom = new ifstream("/dev/urandom", ios::binary|ios::in);
         massert( 10353 ,  "can't open dev/urandom", _devrandom->is_open() );
 #elif defined(_WIN32)
@@ -49,7 +49,7 @@ namespace mongo {
     }
     
     nonce Security::getNonce(){
-        static mongo::mutex m;
+        static mongo::mutex m("getNonce");
         scoped_lock lk(m);
 
 		/* question/todo: /dev/random works on OS X.  is it better 
@@ -57,7 +57,7 @@ namespace mongo {
 		*/
 
         nonce n;
-#if defined(__linux__)
+#if defined(__linux__) || defined(__sunos__)
         _devrandom->read((char*)&n, sizeof(n));
         massert( 10355 , "devrandom failed", !_devrandom->fail());
 #elif defined(_WIN32)
@@ -67,6 +67,7 @@ namespace mongo {
 #endif
         return n;
     }
+    unsigned getRandomNumber() { return (unsigned) security.getNonce(); }
     
 	bool Security::_initialized;
     Security security;
diff --git a/db/oplog.cpp b/db/oplog.cpp
new file mode 100644
index 0000000..4ad4ca9
--- /dev/null
+++ b/db/oplog.cpp
@@ -0,0 +1,601 @@
+// @file oplog.cpp
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "oplog.h"
+#include "repl_block.h"
+#include "repl.h"
+#include "commands.h"
+#include "repl/rs.h"
+
+namespace mongo {
+
+    void logOpForSharding( const char * opstr , const char * ns , const BSONObj& obj , BSONObj * patt );
+
+    int __findingStartInitialTimeout = 5; // configurable for testing    
+
+    // cached copies of these...so don't rename them, drop them, etc.!!!
+    static NamespaceDetails *localOplogMainDetails = 0;
+    static Database *localDB = 0;
+    static NamespaceDetails *rsOplogDetails = 0;
+    void oplogCheckCloseDatabase( Database * db ){
+        localDB = 0;
+        localOplogMainDetails = 0;
+        rsOplogDetails = 0;
+        resetSlaveCache();
+    }
+
+    static void _logOpUninitialized(const char *opstr, const char *ns, const char *logNS, const BSONObj& obj, BSONObj *o2, bool *bb ) {
+        uassert(13288, "replSet error write op to db before replSet initialized", str::startsWith(ns, "local.") || *opstr == 'n');
+    }
+
+    /** write an op to the oplog that is already built. 
+        todo : make _logOpRS() call this so we don't repeat ourself?
+        */
+    void _logOpObjRS(const BSONObj& op) { 
+        DEV assertInWriteLock();
+
+        const OpTime ts = op["ts"]._opTime();
+        long long h = op["h"].numberLong();
+
+        {
+            const char *logns = rsoplog;
+            if ( rsOplogDetails == 0 ) {
+                Client::Context ctx( logns , dbpath, 0, false);
+                localDB = ctx.db();
+                assert( localDB );
+                rsOplogDetails = nsdetails(logns);
+                massert(13389, "local.oplog.rs missing. did you drop it? if so restart server", rsOplogDetails);
+            }
+            Client::Context ctx( "" , localDB, false );
+            {
+                int len = op.objsize();
+                Record *r = theDataFileMgr.fast_oplog_insert(rsOplogDetails, logns, len);
+                memcpy(r->data, op.objdata(), len);
+            }
+            /* todo: now() has code to handle clock skew.  but if the skew server to server is large it will get unhappy.
+                     this code (or code in now() maybe) should be improved.
+                     */
+            if( theReplSet ) {
+                if( !(theReplSet->lastOpTimeWritten<ts) ) {
+                    log() << "replSet error possible failover clock skew issue? " << theReplSet->lastOpTimeWritten.toString() << ' ' << endl;
+                }
+                theReplSet->lastOpTimeWritten = ts;
+                theReplSet->lastH = h;
+                ctx.getClient()->setLastOp( ts.asDate() );
+            }
+        }
+    }
+
+    static void _logOpRS(const char *opstr, const char *ns, const char *logNS, const BSONObj& obj, BSONObj *o2, bool *bb ) {
+        DEV assertInWriteLock();
+        static BufBuilder bufbuilder(8*1024);
+        
+        if ( strncmp(ns, "local.", 6) == 0 ){
+            if ( strncmp(ns, "local.slaves", 12) == 0 )
+                resetSlaveCache();
+            return;
+        }
+
+        const OpTime ts = OpTime::now();
+
+        long long hNew;
+        if( theReplSet ) { 
+            massert(13312, "replSet error : logOp() but not primary?", theReplSet->box.getState().primary());
+            hNew = (theReplSet->lastH * 131 + ts.asLL()) * 17 + theReplSet->selfId();
+        }
+        else {
+            // must be initiation
+            assert( *ns == 0 );
+            hNew = 0;
+        }
+
+        /* we jump through a bunch of hoops here to avoid copying the obj buffer twice --
+           instead we do a single copy to the destination position in the memory mapped file.
+        */
+
+        bufbuilder.reset();
+        BSONObjBuilder b(bufbuilder);
+
+        b.appendTimestamp("ts", ts.asDate());
+        b.append("h", hNew);
+
+        b.append("op", opstr);
+        b.append("ns", ns);
+        if ( bb )
+            b.appendBool("b", *bb);
+        if ( o2 )
+            b.append("o2", *o2);
+        BSONObj partial = b.done();
+        int posz = partial.objsize();
+        int len = posz + obj.objsize() + 1 + 2 /*o:*/;
+
+        Record *r;
+        DEV assert( logNS == 0 );
+        {
+            const char *logns = rsoplog;
+            if ( rsOplogDetails == 0 ) {
+                Client::Context ctx( logns , dbpath, 0, false);
+                localDB = ctx.db();
+                assert( localDB );
+                rsOplogDetails = nsdetails(logns);
+                massert(13347, "local.oplog.rs missing. did you drop it? if so restart server", rsOplogDetails);
+            }
+            Client::Context ctx( "" , localDB, false );
+            r = theDataFileMgr.fast_oplog_insert(rsOplogDetails, logns, len);
+            /* todo: now() has code to handle clock skew.  but if the skew server to server is large it will get unhappy.
+                     this code (or code in now() maybe) should be improved.
+                     */
+            if( theReplSet ) {
+                if( !(theReplSet->lastOpTimeWritten<ts) ) 
+                    log() << "replSet ERROR possible failover clock skew issue? " << theReplSet->lastOpTimeWritten << ' ' << ts << endl;
+                theReplSet->lastOpTimeWritten = ts;
+                theReplSet->lastH = hNew;
+                ctx.getClient()->setLastOp( ts.asDate() );
+            }
+        }
+
+        char *p = r->data;
+        memcpy(p, partial.objdata(), posz);
+        *((unsigned *)p) += obj.objsize() + 1 + 2;
+        p += posz - 1;
+        *p++ = (char) Object;
+        *p++ = 'o';
+        *p++ = 0;
+        memcpy(p, obj.objdata(), obj.objsize());
+        p += obj.objsize();
+        *p = EOO;
+        
+        if ( logLevel >= 6 ) {
+            BSONObj temp(r);
+            log( 6 ) << "logOp:" << temp << endl;
+        }
+    }
+
+    /* we write to local.opload.$main:
+         { ts : ..., op: ..., ns: ..., o: ... }
+       ts: an OpTime timestamp
+       op:
+        "i" insert
+        "u" update
+        "d" delete
+        "c" db cmd
+        "db" declares presence of a database (ns is set to the db name + '.')
+        "n" no op
+       logNS - where to log it.  0/null means "local.oplog.$main".
+       bb:
+         if not null, specifies a boolean to pass along to the other side as b: param.
+         used for "justOne" or "upsert" flags on 'd', 'u'
+       first: true
+         when set, indicates this is the first thing we have logged for this database.
+         thus, the slave does not need to copy down all the data when it sees this.
+
+       note this is used for single collection logging even when --replSet is enabled.
+    */
+    static void _logOpOld(const char *opstr, const char *ns, const char *logNS, const BSONObj& obj, BSONObj *o2, bool *bb ) {
+        DEV assertInWriteLock();
+        static BufBuilder bufbuilder(8*1024);
+        
+        if ( strncmp(ns, "local.", 6) == 0 ){
+            if ( strncmp(ns, "local.slaves", 12) == 0 ){
+                resetSlaveCache();
+            }
+            return;
+        }
+
+        const OpTime ts = OpTime::now();
+        Client::Context context;
+        
+        /* we jump through a bunch of hoops here to avoid copying the obj buffer twice --
+           instead we do a single copy to the destination position in the memory mapped file.
+        */
+
+        bufbuilder.reset();
+        BSONObjBuilder b(bufbuilder);
+        b.appendTimestamp("ts", ts.asDate());
+        b.append("op", opstr);
+        b.append("ns", ns);
+        if ( bb )
+            b.appendBool("b", *bb);
+        if ( o2 )
+            b.append("o2", *o2);
+        BSONObj partial = b.done();
+        int posz = partial.objsize();
+        int len = posz + obj.objsize() + 1 + 2 /*o:*/;
+
+        Record *r;
+        if( logNS == 0 ) {
+            logNS = "local.oplog.$main";
+            if ( localOplogMainDetails == 0 ) {
+                Client::Context ctx( logNS , dbpath, 0, false);
+                localDB = ctx.db();
+                assert( localDB );
+                localOplogMainDetails = nsdetails(logNS);
+                assert( localOplogMainDetails );
+            }
+            Client::Context ctx( "" , localDB, false );
+            r = theDataFileMgr.fast_oplog_insert(localOplogMainDetails, logNS, len);
+        } else {
+            Client::Context ctx( logNS, dbpath, 0, false );
+            assert( nsdetails( logNS ) );
+            r = theDataFileMgr.fast_oplog_insert( nsdetails( logNS ), logNS, len);
+        }
+
+        char *p = r->data;
+        memcpy(p, partial.objdata(), posz);
+        *((unsigned *)p) += obj.objsize() + 1 + 2;
+        p += posz - 1;
+        *p++ = (char) Object;
+        *p++ = 'o';
+        *p++ = 0;
+        memcpy(p, obj.objdata(), obj.objsize());
+        p += obj.objsize();
+        *p = EOO;
+        
+        context.getClient()->setLastOp( ts.asDate() );
+
+        if ( logLevel >= 6 ) {
+            BSONObj temp(r);
+            log( 6 ) << "logging op:" << temp << endl;
+        }
+
+    }
+
+    static void (*_logOp)(const char *opstr, const char *ns, const char *logNS, const BSONObj& obj, BSONObj *o2, bool *bb ) = _logOpOld;
+    void newReplUp() { 
+        replSettings.master = true;
+        _logOp = _logOpRS; 
+    }
+    void newRepl() { 
+        replSettings.master = true;
+        _logOp = _logOpUninitialized; 
+    }
+    void oldRepl() { _logOp = _logOpOld; }
+
+    void logKeepalive() { 
+        _logOp("n", "", 0, BSONObj(), 0, 0);
+    }
+    void logOpComment(const BSONObj& obj) {
+        _logOp("n", "", 0, obj, 0, 0);
+    }
+    void logOpInitiate(const BSONObj& obj) {
+        _logOpRS("n", "", 0, obj, 0, 0);
+    }
+
+    /*@ @param opstr:
+          c userCreateNS
+          i insert
+          n no-op / keepalive
+          d delete / remove
+          u update
+    */
+    void logOp(const char *opstr, const char *ns, const BSONObj& obj, BSONObj *patt, bool *b) {
+        if ( replSettings.master ) {
+            _logOp(opstr, ns, 0, obj, patt, b);
+            // why? :
+            //char cl[ 256 ];
+            //nsToDatabase( ns, cl );
+        }
+        
+        logOpForSharding( opstr , ns , obj , patt );
+    }    
+
+    void createOplog() {
+        dblock lk;
+
+        const char * ns = "local.oplog.$main";
+
+        bool rs = !cmdLine._replSet.empty();
+        if( rs )
+            ns = rsoplog;
+
+        Client::Context ctx(ns);
+        
+        NamespaceDetails * nsd = nsdetails( ns );
+
+        if ( nsd ) {
+            
+            if ( cmdLine.oplogSize != 0 ){
+                int o = (int)(nsd->storageSize() / ( 1024 * 1024 ) );
+                int n = (int)(cmdLine.oplogSize / ( 1024 * 1024 ) );
+                if ( n != o ){
+                    stringstream ss;
+                    ss << "cmdline oplogsize (" << n << ") different than existing (" << o << ") see: http://dochub.mongodb.org/core/increase-oplog";
+                    log() << ss.str() << endl;
+                    throw UserException( 13257 , ss.str() );
+                }
+            }
+
+            if( rs ) return;
+
+            DBDirectClient c;
+            BSONObj lastOp = c.findOne( ns, Query().sort(reverseNaturalObj) );
+            if ( !lastOp.isEmpty() ) {
+                OpTime::setLast( lastOp[ "ts" ].date() );
+            }
+            return;
+        }
+        
+        /* create an oplog collection, if it doesn't yet exist. */
+        BSONObjBuilder b;
+        double sz;
+        if ( cmdLine.oplogSize != 0 )
+            sz = (double)cmdLine.oplogSize;
+        else {
+			/* not specified. pick a default size */
+            sz = 50.0 * 1000 * 1000;
+            if ( sizeof(int *) >= 8 ) {
+#if defined(__APPLE__)
+				// typically these are desktops (dev machines), so keep it smallish
+				sz = (256-64) * 1000 * 1000;
+#else
+                sz = 990.0 * 1000 * 1000;
+                boost::intmax_t free = freeSpace(); //-1 if call not supported.
+                double fivePct = free * 0.05;
+                if ( fivePct > sz )
+                    sz = fivePct;
+#endif
+            }
+        }
+
+        log() << "******" << endl;
+        log() << "creating replication oplog of size: " << (int)( sz / ( 1024 * 1024 ) ) << "MB... (use --oplogSize to change)" << endl;
+
+        b.append("size", sz);
+        b.appendBool("capped", 1);
+        b.appendBool("autoIndexId", false);
+
+        string err;
+        BSONObj o = b.done();
+        userCreateNS(ns, o, err, false);
+        if( !rs )
+            logOp( "n", "dummy", BSONObj() );
+
+        /* sync here so we don't get any surprising lag later when we try to sync */
+        MemoryMappedFile::flushAll(true);
+        log() << "******" << endl;
+    }
+
+    // -------------------------------------
+
+    struct TestOpTime {
+        TestOpTime() {
+            OpTime t;
+            for ( int i = 0; i < 10; i++ ) {
+                OpTime s = OpTime::now();
+                assert( s != t );
+                t = s;
+            }
+            OpTime q = t;
+            assert( q == t );
+            assert( !(q != t) );
+        }
+    } testoptime;
+
+    int _dummy_z;
+
+    void pretouchN(vector<BSONObj>& v, unsigned a, unsigned b) {
+        DEV assert( !dbMutex.isWriteLocked() );
+
+        Client *c = &cc();
+        if( c == 0 ) { 
+            Client::initThread("pretouchN");
+            c = &cc();
+        }
+
+        readlock lk("");
+        for( unsigned i = a; i <= b; i++ ) {
+            const BSONObj& op = v[i];
+            const char *which = "o";
+            const char *opType = op.getStringField("op");
+            if ( *opType == 'i' )
+                ;
+            else if( *opType == 'u' )
+                which = "o2";
+            else
+                continue;
+            /* todo : other operations */
+
+            try { 
+                BSONObj o = op.getObjectField(which);
+                BSONElement _id;
+                if( o.getObjectID(_id) ) {
+                    const char *ns = op.getStringField("ns");
+                    BSONObjBuilder b;
+                    b.append(_id);
+                    BSONObj result;
+                    Client::Context ctx( ns );
+                    if( Helpers::findById(cc(), ns, b.done(), result) )
+                        _dummy_z += result.objsize(); // touch
+                }
+            }
+            catch( DBException& e ) { 
+                log() << "ignoring assertion in pretouchN() " << a << ' ' << b << ' ' << i << ' ' << e.toString() << endl;
+            }
+        }
+    }
+
+    void pretouchOperation(const BSONObj& op) {
+
+        if( dbMutex.isWriteLocked() )
+            return; // no point pretouching if write locked. not sure if this will ever fire, but just in case.
+
+        const char *which = "o";
+        const char *opType = op.getStringField("op");
+        if ( *opType == 'i' )
+            ;
+        else if( *opType == 'u' )
+            which = "o2";
+        else
+            return;
+        /* todo : other operations */
+
+        try { 
+            BSONObj o = op.getObjectField(which);
+            BSONElement _id;
+            if( o.getObjectID(_id) ) {
+                const char *ns = op.getStringField("ns");
+                BSONObjBuilder b;
+                b.append(_id);
+                BSONObj result;
+                readlock lk(ns);
+                Client::Context ctx( ns );
+                if( Helpers::findById(cc(), ns, b.done(), result) )
+                    _dummy_z += result.objsize(); // touch
+            }
+        }
+        catch( DBException& ) { 
+            log() << "ignoring assertion in pretouchOperation()" << endl;
+        }
+    }
+
+    void applyOperation_inlock(const BSONObj& op){
+        if( logLevel >= 6 ) 
+            log() << "applying op: " << op << endl;
+        
+        assertInWriteLock();
+
+        OpDebug debug;
+        BSONObj o = op.getObjectField("o");
+        const char *ns = op.getStringField("ns");
+        // operation type -- see logOp() comments for types
+        const char *opType = op.getStringField("op");
+
+        if ( *opType == 'i' ) {
+            const char *p = strchr(ns, '.');
+            if ( p && strcmp(p, ".system.indexes") == 0 ) {
+                // updates aren't allowed for indexes -- so we will do a regular insert. if index already
+                // exists, that is ok.
+                theDataFileMgr.insert(ns, (void*) o.objdata(), o.objsize());
+            }
+            else {
+                // do upserts for inserts as we might get replayed more than once
+                BSONElement _id;
+                if( !o.getObjectID(_id) ) {
+                    /* No _id.  This will be very slow. */
+                    Timer t;
+                    updateObjects(ns, o, o, true, false, false , debug );
+                    if( t.millis() >= 2 ) {
+                        RARELY OCCASIONALLY log() << "warning, repl doing slow updates (no _id field) for " << ns << endl;
+                    }
+                }
+                else {
+                    BSONObjBuilder b;
+                    b.append(_id);
+                    
+                    /* erh 10/16/2009 - this is probably not relevant any more since its auto-created, but not worth removing */
+                    RARELY ensureHaveIdIndex(ns); // otherwise updates will be slow 
+
+                    /* todo : it may be better to do an insert here, and then catch the dup key exception and do update 
+                              then.  very few upserts will not be inserts...
+                              */
+                    updateObjects(ns, o, b.done(), true, false, false , debug );
+                }
+            }
+        }
+        else if ( *opType == 'u' ) {
+            RARELY ensureHaveIdIndex(ns); // otherwise updates will be super slow
+            updateObjects(ns, o, op.getObjectField("o2"), /*upsert*/ op.getBoolField("b"), /*multi*/ false, /*logop*/ false , debug );
+        }
+        else if ( *opType == 'd' ) {
+            if ( opType[1] == 0 )
+                deleteObjects(ns, o, op.getBoolField("b"));
+            else
+                assert( opType[1] == 'b' ); // "db" advertisement
+        }
+        else if ( *opType == 'n' ) {
+            // no op
+        }
+        else if ( *opType == 'c' ){
+            BufBuilder bb;
+            BSONObjBuilder ob;
+            _runCommands(ns, o, bb, ob, true, 0);
+        }
+        else {
+            stringstream ss;
+            ss << "unknown opType [" << opType << "]";
+            throw MsgAssertionException( 13141 , ss.str() );
+        }
+        
+    }
+    
+    class ApplyOpsCmd : public Command {
+    public:
+        virtual bool slaveOk() const { return false; }
+        virtual LockType locktype() const { return WRITE; }
+        ApplyOpsCmd() : Command( "applyOps" ) {}
+        virtual void help( stringstream &help ) const {
+            help << "examples: { applyOps : [ ] , preCondition : [ { ns : ... , q : ... , res : ... } ] }";
+        }
+        virtual bool run(const string& dbname, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            
+            if ( cmdObj.firstElement().type() != Array ){
+                errmsg = "ops has to be an array";
+                return false;
+            }
+            
+            BSONObj ops = cmdObj.firstElement().Obj();
+            
+            { // check input
+                BSONObjIterator i( ops );
+                while ( i.more() ){
+                    BSONElement e = i.next();
+                    if ( e.type() == Object )
+                        continue;
+                    errmsg = "op not an object: ";
+                    errmsg += e.fieldName();
+                    return false;
+                }
+            }
+            
+            if ( cmdObj["preCondition"].type() == Array ){
+                BSONObjIterator i( cmdObj["preCondition"].Obj() );
+                while ( i.more() ){
+                    BSONObj f = i.next().Obj();
+                    
+                    BSONObj realres = db.findOne( f["ns"].String() , f["q"].Obj() );
+                    
+                    Matcher m( f["res"].Obj() );
+                    if ( ! m.matches( realres ) ){
+                        result.append( "got" , realres );
+                        result.append( "whatFailed" , f );
+                        errmsg = "pre-condition failed";
+                        return false;
+                    }
+                }
+            }
+            
+            // apply
+            int num = 0;
+            BSONObjIterator i( ops );
+            while ( i.more() ){
+                BSONElement e = i.next();
+                applyOperation_inlock( e.Obj() );
+                num++;
+            }
+
+            result.append( "applied" , num );
+
+            return true;
+        }
+
+        DBDirectClient db;
+        
+    } applyOpsCmd;
+
+}
diff --git a/db/oplog.h b/db/oplog.h
new file mode 100644
index 0000000..d1e4990
--- /dev/null
+++ b/db/oplog.h
@@ -0,0 +1,214 @@
+// oplog.h - writing to and reading from oplog
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+/* 
+
+     local.oplog.$main is the default
+*/
+
+#pragma once
+
+#include "pdfile.h"
+#include "db.h"
+#include "dbhelpers.h"
+#include "query.h"
+#include "queryoptimizer.h"
+#include "../client/dbclient.h"
+#include "../util/optime.h"
+
+namespace mongo {
+
+    void createOplog();
+
+    void _logOpObjRS(const BSONObj& op);
+
+    /** Write operation to the log (local.oplog.$main)
+      
+       @param opstr
+        "i" insert
+        "u" update
+        "d" delete
+        "c" db cmd
+        "n" no-op
+        "db" declares presence of a database (ns is set to the db name + '.')
+
+       See _logOp() in oplog.cpp for more details.   
+    */
+    void logOp(const char *opstr, const char *ns, const BSONObj& obj, BSONObj *patt = 0, bool *b = 0);
+
+    void logKeepalive();
+
+    /** puts obj in the oplog as a comment (a no-op).  Just for diags. 
+        convention is 
+          { msg : "text", ... }
+    */
+    void logOpComment(const BSONObj& obj);
+
+    void oplogCheckCloseDatabase( Database * db );
+    
+    extern int __findingStartInitialTimeout; // configurable for testing    
+
+    class FindingStartCursor {
+    public:
+        FindingStartCursor( const QueryPlan & qp ) : 
+        _qp( qp ),
+        _findingStart( true ),
+        _findingStartMode(),
+        _findingStartTimer( 0 ),
+        _findingStartCursor( 0 )
+        { init(); }
+        bool done() const { return !_findingStart; }
+        shared_ptr<Cursor> cRelease() { return _c; }
+        void next() {
+            if ( !_findingStartCursor || !_findingStartCursor->c->ok() ) {
+                _findingStart = false;
+                _c = _qp.newCursor(); // on error, start from beginning
+                destroyClientCursor();
+                return;
+            }
+            switch( _findingStartMode ) {
+                case Initial: {
+                    if ( !_matcher->matches( _findingStartCursor->c->currKey(), _findingStartCursor->c->currLoc() ) ) {
+                        _findingStart = false; // found first record out of query range, so scan normally
+                        _c = _qp.newCursor( _findingStartCursor->c->currLoc() );
+                        destroyClientCursor();
+                        return;
+                    }
+                    _findingStartCursor->c->advance();
+                    RARELY {
+                        if ( _findingStartTimer.seconds() >= __findingStartInitialTimeout ) {
+                            createClientCursor( startLoc( _findingStartCursor->c->currLoc() ) );
+                            _findingStartMode = FindExtent;
+                            return;
+                        }
+                    }
+                    return;
+                }
+                case FindExtent: {
+                    if ( !_matcher->matches( _findingStartCursor->c->currKey(), _findingStartCursor->c->currLoc() ) ) {
+                        _findingStartMode = InExtent;
+                        return;
+                    }
+                    DiskLoc prev = prevLoc( _findingStartCursor->c->currLoc() );
+                    if ( prev.isNull() ) { // hit beginning, so start scanning from here
+                        createClientCursor();
+                        _findingStartMode = InExtent;
+                        return;
+                    }
+                    // There might be a more efficient implementation than creating new cursor & client cursor each time,
+                    // not worrying about that for now
+                    createClientCursor( prev );
+                    return;
+                }
+                case InExtent: {
+                    if ( _matcher->matches( _findingStartCursor->c->currKey(), _findingStartCursor->c->currLoc() ) ) {
+                        _findingStart = false; // found first record in query range, so scan normally
+                        _c = _qp.newCursor( _findingStartCursor->c->currLoc() );
+                        destroyClientCursor();
+                        return;
+                    }
+                    _findingStartCursor->c->advance();
+                    return;
+                }
+                default: {
+                    massert( 12600, "invalid _findingStartMode", false );
+                }
+            }                
+        }     
+        bool prepareToYield() {
+            if ( _findingStartCursor ) {
+                return _findingStartCursor->prepareToYield( _yieldData );
+            }
+            return true;
+        }
+        void recoverFromYield() {
+            if ( _findingStartCursor ) {
+                if ( !ClientCursor::recoverFromYield( _yieldData ) ) {
+                    _findingStartCursor = 0;
+                }
+            }
+        }        
+    private:
+        enum FindingStartMode { Initial, FindExtent, InExtent };
+        const QueryPlan &_qp;
+        bool _findingStart;
+        FindingStartMode _findingStartMode;
+        auto_ptr< CoveredIndexMatcher > _matcher;
+        Timer _findingStartTimer;
+        ClientCursor * _findingStartCursor;
+        shared_ptr<Cursor> _c;
+        ClientCursor::YieldData _yieldData;
+        DiskLoc startLoc( const DiskLoc &rec ) {
+            Extent *e = rec.rec()->myExtent( rec );
+            if ( !_qp.nsd()->capLooped() || ( e->myLoc != _qp.nsd()->capExtent ) )
+                return e->firstRecord;
+            // Likely we are on the fresh side of capExtent, so return first fresh record.
+            // If we are on the stale side of capExtent, then the collection is small and it
+            // doesn't matter if we start the extent scan with capFirstNewRecord.
+            return _qp.nsd()->capFirstNewRecord;
+        }
+        
+        // should never have an empty extent in the oplog, so don't worry about that case
+        DiskLoc prevLoc( const DiskLoc &rec ) {
+            Extent *e = rec.rec()->myExtent( rec );
+            if ( _qp.nsd()->capLooped() ) {
+                if ( e->xprev.isNull() )
+                    e = _qp.nsd()->lastExtent.ext();
+                else
+                    e = e->xprev.ext();
+                if ( e->myLoc != _qp.nsd()->capExtent )
+                    return e->firstRecord;
+            } else {
+                if ( !e->xprev.isNull() ) {
+                    e = e->xprev.ext();
+                    return e->firstRecord;
+                }
+            }
+            return DiskLoc(); // reached beginning of collection
+        }
+        void createClientCursor( const DiskLoc &startLoc = DiskLoc() ) {
+            shared_ptr<Cursor> c = _qp.newCursor( startLoc );
+            _findingStartCursor = new ClientCursor(QueryOption_NoCursorTimeout, c, _qp.ns());
+        }
+        void destroyClientCursor() {
+            if ( _findingStartCursor ) {
+                ClientCursor::erase( _findingStartCursor->cursorid );
+                _findingStartCursor = 0;
+            }
+        }
+        void init() {
+            // Use a ClientCursor here so we can release db mutex while scanning
+            // oplog (can take quite a while with large oplogs).
+            shared_ptr<Cursor> c = _qp.newReverseCursor();
+            _findingStartCursor = new ClientCursor(QueryOption_NoCursorTimeout, c, _qp.ns());
+            _findingStartTimer.reset();
+            _findingStartMode = Initial;
+            BSONElement tsElt = _qp.originalQuery()[ "ts" ];
+            massert( 13044, "no ts field in query", !tsElt.eoo() );
+            BSONObjBuilder b;
+            b.append( tsElt );
+            BSONObj tsQuery = b.obj();
+            _matcher.reset(new CoveredIndexMatcher(tsQuery, _qp.indexKey()));
+        }
+    };
+
+    void pretouchOperation(const BSONObj& op);
+    void pretouchN(vector<BSONObj>&, unsigned a, unsigned b);
+
+    void applyOperation_inlock(const BSONObj& op);
+}
diff --git a/db/oplogreader.h b/db/oplogreader.h
new file mode 100644
index 0000000..5c2881b
--- /dev/null
+++ b/db/oplogreader.h
@@ -0,0 +1,109 @@
+/** @file oplogreader.h */
+
+#pragma once
+
+#include "../client/dbclient.h"
+#include "../client/constants.h"
+#include "dbhelpers.h"
+
+namespace mongo {
+
+    /* started abstracting out the querying of the primary/master's oplog 
+       still fairly awkward but a start.
+    */
+    class OplogReader {
+        auto_ptr<DBClientConnection> _conn;
+        auto_ptr<DBClientCursor> cursor;
+    public:
+
+        OplogReader() { 
+            DEV log() << "TEMP *** OplogReader()" << endl;
+        }
+        ~OplogReader() { 
+            DEV log() << "TEMP *** ~OplogReader()" << endl;
+        }
+
+        void resetCursor() {
+            DEV log() << "TEMP *** OplogReader::resetCursor" << endl;
+            cursor.reset();
+        }
+        void resetConnection() {
+            DEV log() << "TEMP *** OplogReader::resetConnection" << endl;
+            cursor.reset();
+            _conn.reset();
+        }
+        DBClientConnection* conn() { return _conn.get(); }
+        BSONObj findOne(const char *ns, const Query& q) { 
+            return conn()->findOne(ns, q);
+        }
+
+        BSONObj getLastOp(const char *ns) { 
+            return findOne(ns, Query().sort(reverseNaturalObj));
+        }
+
+        /* ok to call if already connected */
+        bool connect(string hostname);
+
+        void tailCheck() {
+            if( cursor.get() && cursor->isDead() ) { 
+                log() << "repl: old cursor isDead, will initiate a new one" << endl;
+                resetCursor();
+            }
+        }
+
+        bool haveCursor() { return cursor.get() != 0; }
+
+        void query(const char *ns, const BSONObj& query) { 
+            assert( !haveCursor() );
+            cursor = _conn->query(ns, query, 0, 0, 0, QueryOption_SlaveOk);
+        }
+
+        void tailingQuery(const char *ns, const BSONObj& query) { 
+            assert( !haveCursor() );
+            log(2) << "repl: " << ns << ".find(" << query.toString() << ')' << endl;
+            cursor = _conn->query( ns, query, 0, 0, 0, 
+                                  QueryOption_CursorTailable | QueryOption_SlaveOk | QueryOption_OplogReplay |
+                                  /* TODO: slaveok maybe shouldn't use? */
+                                  QueryOption_AwaitData
+                                  );
+        }
+
+        void tailingQueryGTE(const char *ns, OpTime t) {
+            BSONObjBuilder q;
+            q.appendDate("$gte", t.asDate());
+            BSONObjBuilder query;
+            query.append("ts", q.done());
+            tailingQuery(ns, query.done());
+        }
+
+        bool more() { 
+            assert( cursor.get() );
+            return cursor->more();
+        }
+        bool moreInCurrentBatch() { 
+            assert( cursor.get() );
+            return cursor->moreInCurrentBatch();
+        }
+
+        /* old mongod's can't do the await flag... */
+        bool awaitCapable() { 
+            return cursor->hasResultFlag(ResultFlag_AwaitCapable);
+        }
+
+        void peek(vector<BSONObj>& v, int n) { 
+            if( cursor.get() )
+                cursor->peek(v,n);
+        }
+
+        BSONObj nextSafe() { return cursor->nextSafe(); }
+
+        BSONObj next() { 
+            return cursor->next();
+        }
+
+        void putBack(BSONObj op) { 
+            cursor->putBack(op);
+        }
+    };
+    
+}
diff --git a/db/pdfile.cpp b/db/pdfile.cpp
index 95bdb17..787e070 100644
--- a/db/pdfile.cpp
+++ b/db/pdfile.cpp
@@ -24,7 +24,7 @@ _ coalesce deleted
 _ disallow system* manipulations from the database.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "pdfile.h"
 #include "db.h"
 #include "../util/mmap.h"
@@ -45,6 +45,17 @@ _ disallow system* manipulations from the database.
 
 namespace mongo {
 
+    bool inDBRepair = false;
+    struct doingRepair {
+        doingRepair(){
+            assert( ! inDBRepair );
+            inDBRepair = true;
+        }
+        ~doingRepair(){
+            inDBRepair = false;
+        }
+    };
+
     const int MaxExtentSize = 0x7ff00000;
 
     map<string, unsigned> BackgroundOperation::dbsInProg;
@@ -100,6 +111,7 @@ namespace mongo {
     string dbpath = "/data/db/";
     bool directoryperdb = false;
     string repairpath;
+    string pidfilepath;
 
     DataFileMgr theDataFileMgr;
     DatabaseHolder dbHolder;
@@ -111,7 +123,7 @@ namespace mongo {
     void ensureIdIndexForNewNs(const char *ns) {
         if ( ( strstr( ns, ".system." ) == 0 || legalClientSystemNS( ns , false ) ) &&
              strstr( ns, ".$freelist" ) == 0 ){
-            log( 1 ) << "adding _id index for new collection" << endl;
+            log( 1 ) << "adding _id index for collection " << ns << endl;
             ensureHaveIdIndex( ns );
         }        
     }
@@ -145,29 +157,29 @@ namespace mongo {
             sz = 1000000000;
         int z = ((int)sz) & 0xffffff00;
         assert( z > len );
-        DEV log() << "initialExtentSize(" << len << ") returns " << z << endl;
+        DEV tlog() << "initialExtentSize(" << len << ") returns " << z << endl;
         return z;
     }
 
-    bool _userCreateNS(const char *ns, const BSONObj& j, string& err) {
+    bool _userCreateNS(const char *ns, const BSONObj& options, string& err, bool *deferIdIndex) {
         if ( nsdetails(ns) ) {
             err = "collection already exists";
             return false;
         }
 
-        log(1) << "create collection " << ns << ' ' << j << '\n';
+        log(1) << "create collection " << ns << ' ' << options << '\n';
 
         /* todo: do this only when we have allocated space successfully? or we could insert with a { ok: 0 } field
            and then go back and set to ok : 1 after we are done.
         */
         bool isFreeList = strstr(ns, ".$freelist") != 0;
         if( !isFreeList )
-            addNewNamespaceToCatalog(ns, j.isEmpty() ? 0 : &j);
+            addNewNamespaceToCatalog(ns, options.isEmpty() ? 0 : &options);
 
         long long size = initialExtentSize(128);
-        BSONElement e = j.getField("size");
+        BSONElement e = options.getField("size");
         if ( e.isNumber() ) {
-            size = (long long) e.number();
+            size = e.numberLong();
             size += 256;
             size &= 0xffffffffffffff00LL;
         }
@@ -176,18 +188,18 @@ namespace mongo {
 
         bool newCapped = false;
         int mx = 0;
-        e = j.getField("capped");
+        e = options.getField("capped");
         if ( e.type() == Bool && e.boolean() ) {
             newCapped = true;
-            e = j.getField("max");
+            e = options.getField("max");
             if ( e.isNumber() ) {
-                mx = (int) e.number();
+                mx = e.numberInt();
             }
         }
 
         // $nExtents just for debug/testing.  We create '$nExtents' extents,
         // each of size 'size'.
-        e = j.getField( "$nExtents" );
+        e = options.getField( "$nExtents" );
         int nExtents = int( e.number() );
         Database *database = cc().database();
         if ( nExtents > 0 ) {
@@ -201,7 +213,7 @@ namespace mongo {
             }
         } else {
             while ( size > 0 ) {
-                int max = MongoDataFile::maxSize() - MDFHeader::headerSize();
+                int max = MongoDataFile::maxSize() - DataFileHeader::HeaderSize;
                 int desiredExtentSize = (int) (size > max ? max : size);
                 Extent *e = database->allocExtent( ns, desiredExtentSize, newCapped );
                 size -= e->length;
@@ -211,15 +223,22 @@ namespace mongo {
         NamespaceDetails *d = nsdetails(ns);
         assert(d);
 
-        if ( j.getField( "autoIndexId" ).type() ) {
-            if ( j["autoIndexId"].trueValue() ){
-                ensureIdIndexForNewNs( ns );
+        bool ensure = false;
+        if ( options.getField( "autoIndexId" ).type() ) {
+            if ( options["autoIndexId"].trueValue() ){
+                ensure = true;
             }
         } else {
             if ( !newCapped ) {
-                ensureIdIndexForNewNs( ns );
+                ensure=true;
             }
         }
+        if( ensure ) { 
+            if( deferIdIndex )
+                *deferIdIndex = true;
+            else
+                ensureIdIndexForNewNs( ns );
+        }
 
         if ( mx > 0 )
             d->max = mx;
@@ -227,23 +246,25 @@ namespace mongo {
         return true;
     }
 
-    // { ..., capped: true, size: ..., max: ... }
-    // returns true if successful
-    bool userCreateNS(const char *ns, BSONObj j, string& err, bool logForReplication) {
+    /** { ..., capped: true, size: ..., max: ... }
+        @param deferIdIndex - if not not, defers id index creation.  sets the bool value to true if we wanted to create the id index.
+        @return true if successful
+    */
+    bool userCreateNS(const char *ns, BSONObj options, string& err, bool logForReplication, bool *deferIdIndex) {
         const char *coll = strchr( ns, '.' ) + 1;
         massert( 10356 ,  "invalid ns", coll && *coll );
         char cl[ 256 ];
         nsToDatabase( ns, cl );
-        bool ok = _userCreateNS(ns, j, err);
+        bool ok = _userCreateNS(ns, options, err, deferIdIndex);
         if ( logForReplication && ok ) {
-            if ( j.getField( "create" ).eoo() ) {
+            if ( options.getField( "create" ).eoo() ) {
                 BSONObjBuilder b;
                 b << "create" << coll;
-                b.appendElements( j );
-                j = b.obj();
+                b.appendElements( options );
+                options = b.obj();
             }
             string logNs = string( cl ) + ".$cmd";
-            logOp("c", logNs.c_str(), j);
+            logOp("c", logNs.c_str(), options);
         }
         return ok;
     }
@@ -286,7 +307,7 @@ namespace mongo {
                very simple temporary implementation - we will in future look up
                the quota from the grid database
             */
-            if ( cmdLine.quota && fileNo > cmdLine.quotaFiles && !boost::filesystem::exists(filename) ) {
+            if ( cmdLine.quota && fileNo > cmdLine.quotaFiles && !MMF::exists(filename) ) {
                 /* todo: if we were adding / changing keys in an index did we do some
                    work previously that needs cleaning up?  Possible.  We should
                    check code like that and have it catch the exception and do
@@ -322,7 +343,8 @@ namespace mongo {
             return;
         }
         
-        header = (MDFHeader *) mmf.map(filename, size);
+        _p = mmf.map(filename, size);
+        header = (DataFileHeader *) _p.at(0, DataFileHeader::HeaderSize);
         if( sizeof(char *) == 4 ) 
             uassert( 10084 , "can't map file memory - mongo requires 64 bit build for larger datasets", header);
         else
@@ -330,6 +352,10 @@ namespace mongo {
         header->init(fileNo, size);
     }
 
+    void MongoDataFile::flush( bool sync ){
+        mmf.flush( sync );
+    }
+
     void addNewExtentToNamespace(const char *ns, Extent *e, DiskLoc eloc, DiskLoc emptyLoc, bool capped) { 
         DiskLoc oldExtentLoc;
         NamespaceIndex *ni = nsindex(ns);
@@ -377,8 +403,8 @@ namespace mongo {
 
         addNewExtentToNamespace(ns, e, loc, emptyLoc, newCapped);
 
-        DEV log() << "new extent " << ns << " size: 0x" << hex << ExtentSize << " loc: 0x" << hex << offset
-                  << " emptyLoc:" << hex << emptyLoc.getOfs() << dec << endl;
+        DEV tlog() << "new extent " << ns << " size: 0x" << hex << ExtentSize << " loc: 0x" << hex << offset
+                   << " emptyLoc:" << hex << emptyLoc.getOfs() << dec << endl;
         return e;
     }
 
@@ -447,6 +473,7 @@ namespace mongo {
     /*---------------------------------------------------------------------*/
 
     DiskLoc Extent::reuse(const char *nsname) { 
+		/*TODOMMF - work to do when extent is freed. */
         log(3) << "reset extent was:" << nsDiagnostic.buf << " now:" << nsname << '\n';
         massert( 10360 ,  "Extent::reset bad magic value", magic == 0x41424344 );
         xnext.Null();
@@ -456,13 +483,14 @@ namespace mongo {
         lastRecord.Null();
 
         DiskLoc emptyLoc = myLoc;
-        emptyLoc.inc( (extentData-(char*)this) );
+        emptyLoc.inc( (int) (_extentData-(char*)this) );
 
-        int delRecLength = length - (extentData - (char *) this);
-        DeletedRecord *empty1 = (DeletedRecord *) extentData;
-        DeletedRecord *empty = (DeletedRecord *) getRecord(emptyLoc);
-        assert( empty == empty1 );
-        memset(empty, delRecLength, 1);
+        int delRecLength = length - (_extentData - (char *) this);
+        //DeletedRecord *empty1 = (DeletedRecord *) extentData;
+        DeletedRecord *empty = DataFileMgr::makeDeletedRecord(emptyLoc, delRecLength);//(DeletedRecord *) getRecord(emptyLoc);
+        //assert( empty == empty1 );
+
+        // do we want to zero the record? memset(empty, ...)
 
         empty->lengthWithHeaders = delRecLength;
         empty->extentOfs = myLoc.getOfs();
@@ -483,19 +511,20 @@ namespace mongo {
         lastRecord.Null();
 
         DiskLoc emptyLoc = myLoc;
-        emptyLoc.inc( (extentData-(char*)this) );
+        emptyLoc.inc( (int) (_extentData-(char*)this) );
 
-        DeletedRecord *empty1 = (DeletedRecord *) extentData;
-        DeletedRecord *empty = (DeletedRecord *) getRecord(emptyLoc);
-        assert( empty == empty1 );
-        empty->lengthWithHeaders = _length - (extentData - (char *) this);
+        int l = _length - (_extentData - (char *) this);
+        //DeletedRecord *empty1 = (DeletedRecord *) extentData;
+        DeletedRecord *empty = DataFileMgr::makeDeletedRecord(emptyLoc, l);
+        //assert( empty == empty1 );
+        empty->lengthWithHeaders = l;
         empty->extentOfs = myLoc.getOfs();
         return emptyLoc;
     }
 
     /*
       Record* Extent::newRecord(int len) {
-      if( firstEmptyRegion.isNull() )
+      if( firstEmptyRegion.isNull() )8
       return 0;
 
       assert(len > 0);
@@ -541,10 +570,10 @@ namespace mongo {
 
     /*---------------------------------------------------------------------*/
 
-    auto_ptr<Cursor> DataFileMgr::findAll(const char *ns, const DiskLoc &startLoc) {
+    shared_ptr<Cursor> DataFileMgr::findAll(const char *ns, const DiskLoc &startLoc) {
         NamespaceDetails * d = nsdetails( ns );
         if ( ! d )
-            return auto_ptr<Cursor>(new BasicCursor(DiskLoc()));
+            return shared_ptr<Cursor>(new BasicCursor(DiskLoc()));
 
         DiskLoc loc = d->firstExtent;
         Extent *e = getExtent(loc);
@@ -569,10 +598,10 @@ namespace mongo {
         }
 
         if ( d->capped ) 
-            return auto_ptr< Cursor >( new ForwardCappedCursor( d , startLoc ) );
+            return shared_ptr<Cursor>( new ForwardCappedCursor( d , startLoc ) );
         
         if ( !startLoc.isNull() )
-            return auto_ptr<Cursor>(new BasicCursor( startLoc ));                
+            return shared_ptr<Cursor>(new BasicCursor( startLoc ));                
         
         while ( e->firstRecord.isNull() && !e->xnext.isNull() ) {
             /* todo: if extent is empty, free it for reuse elsewhere.
@@ -583,13 +612,13 @@ namespace mongo {
             // it might be nice to free the whole extent here!  but have to clean up free recs then.
             e = e->getNextExtent();
         }
-        return auto_ptr<Cursor>(new BasicCursor( e->firstRecord ));
+        return shared_ptr<Cursor>(new BasicCursor( e->firstRecord ));
     }
 
     /* get a table scan cursor, but can be forward or reverse direction.
        order.$natural - if set, > 0 means forward (asc), < 0 backward (desc).
     */
-    auto_ptr<Cursor> findTableScan(const char *ns, const BSONObj& order, const DiskLoc &startLoc) {
+    shared_ptr<Cursor> findTableScan(const char *ns, const BSONObj& order, const DiskLoc &startLoc) {
         BSONElement el = order.getField("$natural"); // e.g., { $natural : -1 }
 
         if ( el.number() >= 0 )
@@ -599,19 +628,19 @@ namespace mongo {
         NamespaceDetails *d = nsdetails(ns);
         
         if ( !d )
-            return auto_ptr<Cursor>(new BasicCursor(DiskLoc()));
+            return shared_ptr<Cursor>(new BasicCursor(DiskLoc()));
         
         if ( !d->capped ) {
             if ( !startLoc.isNull() )
-                return auto_ptr<Cursor>(new ReverseCursor( startLoc ));                
+                return shared_ptr<Cursor>(new ReverseCursor( startLoc ));                
             Extent *e = d->lastExtent.ext();
             while ( e->lastRecord.isNull() && !e->xprev.isNull() ) {
                 OCCASIONALLY out() << "  findTableScan: extent empty, skipping ahead" << endl;
                 e = e->getPrevExtent();
             }
-            return auto_ptr<Cursor>(new ReverseCursor( e->lastRecord ));
+            return shared_ptr<Cursor>(new ReverseCursor( e->lastRecord ));
         } else {
-            return auto_ptr< Cursor >( new ReverseCappedCursor( d, startLoc ) );
+            return shared_ptr<Cursor>( new ReverseCappedCursor( d, startLoc ) );
         }
     }
 
@@ -663,7 +692,7 @@ namespace mongo {
             NamespaceDetails *freeExtents = nsdetails(s.c_str());
             if( freeExtents == 0 ) { 
                 string err;
-                _userCreateNS(s.c_str(), BSONObj(), err);
+                _userCreateNS(s.c_str(), BSONObj(), err, 0);
                 freeExtents = nsdetails(s.c_str());
                 massert( 10361 , "can't create .$freelist", freeExtents);
             }
@@ -690,7 +719,8 @@ namespace mongo {
     void dropCollection( const string &name, string &errmsg, BSONObjBuilder &result ) {
         log(1) << "dropCollection: " << name << endl;
         NamespaceDetails *d = nsdetails(name.c_str());
-        assert( d );
+        if( d == 0 )
+            return;
 
         BackgroundOperation::assertNoBgOpInProgForNs(name.c_str());
 
@@ -706,6 +736,7 @@ namespace mongo {
         log(1) << "\t dropIndexes done" << endl;
         result.append("ns", name.c_str());
         ClientCursor::invalidate(name.c_str());
+        Client::invalidateNS( name );
         Top::global.collectionDropped( name );
         dropNS(name);        
     }
@@ -831,7 +862,7 @@ namespace mongo {
         NamespaceDetails *d,
         NamespaceDetailsTransient *nsdt,
         Record *toupdate, const DiskLoc& dl,
-        const char *_buf, int _len, OpDebug& debug)
+        const char *_buf, int _len, OpDebug& debug, bool &changedId, bool god)
     {
         StringBuilder& ss = debug.str;
         dassert( toupdate == dl.rec() );
@@ -858,7 +889,7 @@ namespace mongo {
            below.  that is suboptimal, but it's pretty complicated to do it the other way without rollbacks...
         */
         vector<IndexChanges> changes;
-        getIndexChanges(changes, *d, objNew, objOld);
+        getIndexChanges(changes, *d, objNew, objOld, changedId);
         dupCheck(changes, *d, dl);
 
         if ( toupdate->netLength() < objNew.objsize() ) {
@@ -868,7 +899,7 @@ namespace mongo {
             if ( cc().database()->profile )
                 ss << " moved ";
             deleteRecord(ns, toupdate, dl);
-            return insert(ns, objNew.objdata(), objNew.objsize(), false);
+            return insert(ns, objNew.objdata(), objNew.objsize(), god);
         }
 
         nsdt->notifyOfWriteOp();
@@ -891,13 +922,14 @@ namespace mongo {
                 }
                 assert( !dl.isNull() );
                 BSONObj idxKey = idx.info.obj().getObjectField("key");
+                Ordering ordering = Ordering::make(idxKey);
                 keyUpdates += changes[x].added.size();
                 for ( unsigned i = 0; i < changes[x].added.size(); i++ ) {
                     try {
                         /* we did the dupCheck() above.  so we don't have to worry about it here. */
                         idx.head.btree()->bt_insert(
                                                     idx.head,
-                                                    dl, *changes[x].added[i], idxKey, /*dupsAllowed*/true, idx);
+                                                    dl, *changes[x].added[i], ordering, /*dupsAllowed*/true, idx);
                     }
                     catch (AssertionException& e) {
                         ss << " exception update index ";
@@ -937,6 +969,7 @@ namespace mongo {
         BSONObjSetDefaultOrder keys;
         idx.getKeysFromObject(obj, keys);
         BSONObj order = idx.keyPattern();
+        Ordering ordering = Ordering::make(order);
         int n = 0;
         for ( BSONObjSetDefaultOrder::iterator i=keys.begin(); i != keys.end(); i++ ) {
             if( ++n == 2 ) { 
@@ -945,10 +978,10 @@ namespace mongo {
             assert( !recordLoc.isNull() );
             try {
                 idx.head.btree()->bt_insert(idx.head, recordLoc,
-                                            *i, order, dupsAllowed, idx);
+                                            *i, ordering, dupsAllowed, idx);
             }
             catch (AssertionException& e) {
-                if( e.code == 10287 && idxNo == d->nIndexes ) { 
+                if( e.getCode() == 10287 && idxNo == d->nIndexes ) { 
                     DEV log() << "info: caught key already in index on bg indexing (ok)" << endl;
                     continue;
                 }
@@ -980,9 +1013,9 @@ namespace mongo {
         auto_ptr<BSONObjExternalSorter::Iterator> i = sorter.iterator();
         while( i->more() ) { 
             BSONObjExternalSorter::Data d = i->next();
-            cout << d.second.toString() << endl;
+            /*cout << d.second.toString() << endl;
             cout << d.first.objsize() << endl;
-            cout<<"SORTER next:" << d.first.toString() << endl;
+            cout<<"SORTER next:" << d.first.toString() << endl;*/
         }
     }
 
@@ -993,10 +1026,10 @@ namespace mongo {
 
         Timer t;
 
-        log() << "Buildindex " << ns << " idxNo:" << idxNo << ' ' << idx.info.obj().toString() << endl;
+        tlog() << "Buildindex " << ns << " idxNo:" << idxNo << ' ' << idx.info.obj().toString() << endl;
 
         bool dupsAllowed = !idx.unique();
-        bool dropDups = idx.dropDups();
+        bool dropDups = idx.dropDups() || inDBRepair;
         BSONObj order = idx.keyPattern();
 
         idx.head.Null();
@@ -1005,11 +1038,11 @@ namespace mongo {
 
         /* get and sort all the keys ----- */
         unsigned long long n = 0;
-        auto_ptr<Cursor> c = theDataFileMgr.findAll(ns);
+        shared_ptr<Cursor> c = theDataFileMgr.findAll(ns);
         BSONObjExternalSorter sorter(order);
         sorter.hintNumObjects( d->nrecords );
         unsigned long long nkeys = 0;
-        ProgressMeter & pm = op->setMessage( "index: (1/3) external sort" , d->nrecords , 10 );
+        ProgressMeterHolder pm( op->setMessage( "index: (1/3) external sort" , d->nrecords , 10 ) );
         while ( c->ok() ) {
             BSONObj o = c->current();
             DiskLoc loc = c->currLoc();
@@ -1048,7 +1081,7 @@ namespace mongo {
             BtreeBuilder btBuilder(dupsAllowed, idx);
             BSONObj keyLast;
             auto_ptr<BSONObjExternalSorter::Iterator> i = sorter.iterator();
-            pm = op->setMessage( "index: (2/3) btree bottom up" , nkeys , 10 );
+            assert( pm == op->setMessage( "index: (2/3) btree bottom up" , nkeys , 10 ) );
             while( i->more() ) { 
                 RARELY killCurrentOp.checkForInterrupt();
                 BSONObjExternalSorter::Data d = i->next();
@@ -1102,8 +1135,8 @@ namespace mongo {
             unsigned long long n = 0;
             auto_ptr<ClientCursor> cc;
             {
-                auto_ptr<Cursor> c = theDataFileMgr.findAll(ns);
-                cc.reset( new ClientCursor(c, ns, false) );
+                shared_ptr<Cursor> c = theDataFileMgr.findAll(ns);
+                cc.reset( new ClientCursor(QueryOption_NoCursorTimeout, c, ns) );
             }
             CursorId id = cc->cursorid;
 
@@ -1155,6 +1188,7 @@ namespace mongo {
 
         void prep(const char *ns, NamespaceDetails *d) {
             assertInWriteLock();
+            uassert( 13130 , "can't start bg index b/c in recursive lock (db.eval?)" , dbMutex.getState() == 1 );
             bgJobsInProgress.insert(d);
             d->backgroundIndexBuildInProgress = 1;
             d->nIndexes--;
@@ -1196,7 +1230,7 @@ namespace mongo {
 
     // throws DBException
     static void buildAnIndex(string ns, NamespaceDetails *d, IndexDetails& idx, int idxNo, bool background) { 
-        log() << "building new index on " << idx.keyPattern() << " for " << ns << ( background ? " background" : "" ) << endl;
+        tlog() << "building new index on " << idx.keyPattern() << " for " << ns << ( background ? " background" : "" ) << endl;
         Timer t;
 		unsigned long long n;
 
@@ -1205,7 +1239,7 @@ namespace mongo {
         }
 
         assert( !BackgroundOperation::inProgForNs(ns.c_str()) ); // should have been checked earlier, better not be...
-        if( !background ) {
+        if( inDBRepair || !background ) {
 			n = fastBuildIndex(ns.c_str(), d, idx, idxNo);
 			assert( !idx.head.isNull() );
 		}
@@ -1213,7 +1247,7 @@ namespace mongo {
             BackgroundIndexBuildJob j(ns.c_str());
             n = j.go(ns, d, idx, idxNo);
 		}
-        log() << "done for " << n << " records " << t.millis() / 1000.0 << "secs" << endl;
+        tlog() << "done for " << n << " records " << t.millis() / 1000.0 << "secs" << endl;
     }
 
     /* add keys to indexes for a new record */
@@ -1289,17 +1323,21 @@ namespace mongo {
     
     void DataFileMgr::insertAndLog( const char *ns, const BSONObj &o, bool god ) {
         BSONObj tmp = o;
-        insert( ns, tmp, god );
+        insertWithObjMod( ns, tmp, god );
         logOp( "i", ns, tmp );
     }
     
-    DiskLoc DataFileMgr::insert(const char *ns, BSONObj &o, bool god) {
+    DiskLoc DataFileMgr::insertWithObjMod(const char *ns, BSONObj &o, bool god) {
         DiskLoc loc = insert( ns, o.objdata(), o.objsize(), god );
         if ( !loc.isNull() )
             o = BSONObj( loc.rec() );
         return loc;
     }
 
+    void DataFileMgr::insertNoReturnVal(const char *ns,  BSONObj o, bool god) {
+        insert( ns, o.objdata(), o.objsize(), god );
+    }
+
     bool prepareToBuildIndex(const BSONObj& io, bool god, string& sourceNS, NamespaceDetails *&sourceCollection);
 
     // We are now doing two btree scans for all unique indexes (one here, and one when we've
@@ -1320,13 +1358,13 @@ namespace mongo {
             }
         }        
     }
-    
+
     /* note: if god==true, you may pass in obuf of NULL and then populate the returned DiskLoc 
              after the call -- that will prevent a double buffer copy in some cases (btree.cpp).
     */
     DiskLoc DataFileMgr::insert(const char *ns, const void *obuf, int len, bool god, const BSONElement &writeId, bool mayAddIndex) {
         bool wouldAddIndex = false;
-        massert( 10093 , "cannot insert into reserved $ collection", god || strchr(ns, '$') == 0 );
+        massert( 10093 , "cannot insert into reserved $ collection", god || nsDollarCheck( ns ) );
         uassert( 10094 , "invalid ns", strchr( ns , '.' ) > 0 );
         const char *sys = strstr(ns, "system.");
         if ( sys ) {
@@ -1366,6 +1404,7 @@ namespace mongo {
 
         string tabletoidxns;
         if ( addIndex ) {
+            assert( obuf );
             BSONObj io((const char *) obuf);
             if( !prepareToBuildIndex(io, god, tabletoidxns, tableToIndex) )
                 return DiskLoc();
@@ -1428,7 +1467,7 @@ namespace mongo {
                 }
             }
             if ( loc.isNull() ) {
-                log() << "out of space in datafile " << ns << " capped:" << d->capped << endl;
+                log() << "insert: couldn't alloc space for object ns:" << ns << " capped:" << d->capped << endl;
                 assert(d->capped);
                 return DiskLoc();
             }
@@ -1468,6 +1507,8 @@ namespace mongo {
             NamespaceDetailsTransient::get_w( ns ).notifyOfWriteOp();
         
         if ( tableToIndex ) {
+            uassert( 13143 , "can't create index on system.indexes" , tabletoidxns.find( ".system.indexes" ) == string::npos );
+
             BSONObj info = loc.obj();
             bool background = info["background"].trueValue();
 
@@ -1476,7 +1517,7 @@ namespace mongo {
             idx.info = loc;
             try {
                 buildAnIndex(tabletoidxns, tableToIndex, idx, idxNo, background);
-            } catch( DBException& ) {
+            } catch( DBException& e ) {
                 // save our error msg string as an exception or dropIndexes will overwrite our message
                 LastError *le = lastError.get();
                 int savecode = 0;
@@ -1485,6 +1526,10 @@ namespace mongo {
                     savecode = le->code;
                     saveerrmsg = le->msg;
                 }
+                else {
+                    savecode = e.getCode();
+                    saveerrmsg = e.what();
+                }
 
                 // roll back this index
                 string name = idx.indexName();
@@ -1494,7 +1539,7 @@ namespace mongo {
                 if( !ok ) {
                     log() << "failed to drop index after a unique key error building it: " << errmsg << ' ' << tabletoidxns << ' ' << name << endl;
                 }
-
+                
                 assert( le && !saveerrmsg.empty() );
                 raiseError(savecode,saveerrmsg.c_str());
                 throw;
@@ -1571,21 +1616,46 @@ namespace mongo {
 
 namespace mongo {
 
-    void dropDatabase(const char *ns) {
-        // ns is of the form "<dbname>.$cmd"
-        char db[256];
-        nsToDatabase(ns, db);
+    void dropAllDatabasesExceptLocal() { 
+        writelock lk("");
+
+        vector<string> n;
+        getDatabaseNames(n);
+        if( n.size() == 0 ) return;
+        log() << "dropAllDatabasesExceptLocal " << n.size() << endl;
+        for( vector<string>::iterator i = n.begin(); i != n.end(); i++ ) {
+            if( *i != "local" ) {
+                Client::Context ctx(*i);
+                dropDatabase(*i);
+            }
+        }
+    }
+
+    void dropDatabase(string db) {
         log(1) << "dropDatabase " << db << endl;
+        assert( cc().database() );
         assert( cc().database()->name == db );
 
-        BackgroundOperation::assertNoBgOpInProgForDb(db);
+        BackgroundOperation::assertNoBgOpInProgForDb(db.c_str());
+
+        Client::invalidateDB( db );
 
-        closeDatabase( db );
-        _deleteDataFiles(db);
+        closeDatabase( db.c_str() );
+        _deleteDataFiles( db.c_str() );
     }
 
     typedef boost::filesystem::path Path;
 
+    void boostRenameWrapper( const Path &from, const Path &to ) {
+        try {
+            boost::filesystem::rename( from, to );
+        } catch ( const boost::filesystem::filesystem_error & ) {
+            // boost rename doesn't work across partitions
+            boost::filesystem::copy_file( from, to);
+            boost::filesystem::remove( from );
+        }
+    }
+    
     // back up original database files to 'temp' dir
     void _renameForBackup( const char *database, const Path &reservedPath ) {
         Path newPath( reservedPath );
@@ -1599,7 +1669,7 @@ namespace mongo {
             virtual bool apply( const Path &p ) {
                 if ( !boost::filesystem::exists( p ) )
                     return false;
-                boost::filesystem::rename( p, newPath_ / ( p.leaf() + ".bak" ) );
+                boostRenameWrapper( p, newPath_ / ( p.leaf() + ".bak" ) );
                 return true;
             }
             virtual const char * op() const {
@@ -1622,7 +1692,7 @@ namespace mongo {
             virtual bool apply( const Path &p ) {
                 if ( !boost::filesystem::exists( p ) )
                     return false;
-                boost::filesystem::rename( p, newPath_ / p.leaf() );
+                boostRenameWrapper( p, newPath_ / p.leaf() );
                 return true;
             }
             virtual const char * op() const {
@@ -1676,32 +1746,33 @@ namespace mongo {
 #include <sys/statvfs.h>
 namespace mongo {
 #endif
-    boost::intmax_t freeSpace() {
+    boost::intmax_t freeSpace ( const string &path ) {
 #if !defined(_WIN32)
         struct statvfs info;
-        assert( !statvfs( dbpath.c_str() , &info ) );
+        assert( !statvfs( path.c_str() , &info ) );
         return boost::intmax_t( info.f_bavail ) * info.f_frsize;
 #else
         return -1;
 #endif
     }
 
-    bool repairDatabase( const char *ns, string &errmsg,
+    bool repairDatabase( string dbNameS , string &errmsg,
                          bool preserveClonedFilesOnFailure, bool backupOriginalFiles ) {
+        doingRepair dr;
+        dbNameS = nsToDatabase( dbNameS );
+        const char * dbName = dbNameS.c_str();
+
         stringstream ss;
         ss << "localhost:" << cmdLine.port;
         string localhost = ss.str();
-
-        // ns is of the form "<dbname>.$cmd"
-        char dbName[256];
-        nsToDatabase(ns, dbName);
+        
         problem() << "repairDatabase " << dbName << endl;
         assert( cc().database()->name == dbName );
 
         BackgroundOperation::assertNoBgOpInProgForDb(dbName);
 
         boost::intmax_t totalSize = dbSize( dbName );
-        boost::intmax_t freeSize = freeSpace();
+        boost::intmax_t freeSize = freeSpace( repairpath );
         if ( freeSize > -1 && freeSize < totalSize ) {
             stringstream ss;
             ss << "Cannot repair database " << dbName << " having size: " << totalSize
@@ -1800,6 +1871,8 @@ namespace mongo {
             dbs.insert( i->first );
         }
         
+        currentClient.get()->getContext()->clear();
+
         BSONObjBuilder bb( result.subarrayStart( "dbs" ) );
         int n = 0;
         int nNotClosed = 0;
@@ -1813,7 +1886,7 @@ namespace mongo {
             }
             else {
                 closeDatabase( name.c_str() , path );
-                bb.append( bb.numStr( n++ ).c_str() , name );
+                bb.append( bb.numStr( n++ ) , name );
             }
         }
         bb.done();
diff --git a/db/pdfile.h b/db/pdfile.h
index 85dc191..084a542 100644
--- a/db/pdfile.h
+++ b/db/pdfile.h
@@ -25,7 +25,7 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "../util/mmap.h"
 #include "diskloc.h"
 #include "jsobjmanipulator.h"
@@ -34,29 +34,28 @@
 
 namespace mongo {
 
-    class MDFHeader;
+    class DataFileHeader;
     class Extent;
     class Record;
     class Cursor;
     class OpDebug;
 
-    void dropDatabase(const char *ns);
-    bool repairDatabase(const char *ns, string &errmsg, bool preserveClonedFilesOnFailure = false, bool backupOriginalFiles = false);
+    void dropDatabase(string db);
+    bool repairDatabase(string db, string &errmsg, bool preserveClonedFilesOnFailure = false, bool backupOriginalFiles = false);
 
     /* low level - only drops this ns */
     void dropNS(const string& dropNs);
     
     /* deletes this ns, indexes and cursors */
     void dropCollection( const string &name, string &errmsg, BSONObjBuilder &result ); 
-    bool userCreateNS(const char *ns, BSONObj j, string& err, bool logForReplication);
-    auto_ptr<Cursor> findTableScan(const char *ns, const BSONObj& order, const DiskLoc &startLoc=DiskLoc());
+    bool userCreateNS(const char *ns, BSONObj j, string& err, bool logForReplication, bool *deferIdIndex = 0);
+    shared_ptr<Cursor> findTableScan(const char *ns, const BSONObj& order, const DiskLoc &startLoc=DiskLoc());
 
 // -1 if library unavailable.
-    boost::intmax_t freeSpace();
+    boost::intmax_t freeSpace( const string &path = dbpath );
 
     /*---------------------------------------------------------------------*/
 
-    class MDFHeader;
     class MongoDataFile {
         friend class DataFileMgr;
         friend class BasicCursor;
@@ -70,22 +69,27 @@ namespace mongo {
         */
         Extent* createExtent(const char *ns, int approxSize, bool capped = false, int loops = 0);
 
-        MDFHeader *getHeader() {
+        DataFileHeader *getHeader() {
             return header;
         }
 
         /* return max size an extent may be */
         static int maxSize();
-
+        
+        void flush( bool sync );
+        
     private:
         int defaultSize( const char *filename ) const;
 
         Extent* getExtent(DiskLoc loc);
         Extent* _getExtent(DiskLoc loc);
         Record* recordAt(DiskLoc dl);
+        Record* makeRecord(DiskLoc dl, int size);
+		void grow(DiskLoc dl, int size);
 
-        MemoryMappedFile mmf;
-        MDFHeader *header;
+        MMF mmf;
+        MMF::Pointer _p;
+        DataFileHeader *header;
         int fileNo;
     };
 
@@ -98,18 +102,26 @@ namespace mongo {
         static Extent* allocFromFreeList(const char *ns, int approxSize, bool capped = false);
 
         /** @return DiskLoc where item ends up */
+        // changedId should be initialized to false
         const DiskLoc updateRecord(
             const char *ns,
             NamespaceDetails *d,
             NamespaceDetailsTransient *nsdt,
             Record *toupdate, const DiskLoc& dl,
-            const char *buf, int len, OpDebug& debug);
+            const char *buf, int len, OpDebug& debug, bool &changedId, bool god=false);
+
         // The object o may be updated if modified on insert.                                
         void insertAndLog( const char *ns, const BSONObj &o, bool god = false );
-        DiskLoc insert(const char *ns, BSONObj &o, bool god = false);
+
+        /** @param obj both and in and out param -- insert can sometimes modify an object (such as add _id). */
+        DiskLoc insertWithObjMod(const char *ns, BSONObj &o, bool god = false);
+
+        /** @param obj in value only for this version. */
+        void insertNoReturnVal(const char *ns, BSONObj o, bool god = false);
+
         DiskLoc insert(const char *ns, const void *buf, int len, bool god = false, const BSONElement &writeId = BSONElement(), bool mayAddIndex = true);
         void deleteRecord(const char *ns, Record *todelete, const DiskLoc& dl, bool cappedOK = false, bool noWarn = false);
-        static auto_ptr<Cursor> findAll(const char *ns, const DiskLoc &startLoc = DiskLoc());
+        static shared_ptr<Cursor> findAll(const char *ns, const DiskLoc &startLoc = DiskLoc());
 
         /* special version of insert for transaction logging -- streamlined a bit.
            assumes ns is capped and no indexes
@@ -119,6 +131,8 @@ namespace mongo {
 
         static Extent* getExtent(const DiskLoc& dl);
         static Record* getRecord(const DiskLoc& dl);
+        static DeletedRecord* makeDeletedRecord(const DiskLoc& dl, int len);
+		static void grow(const DiskLoc& dl, int len);
 
         /* does not clean up indexes, etc. : just deletes the record in the pdfile. */
         void _deleteRecord(NamespaceDetails *d, const char *ns, Record *todelete, const DiskLoc& dl);
@@ -197,7 +211,9 @@ namespace mongo {
 
         int length;   /* size of the extent, including these fields */
         DiskLoc firstRecord, lastRecord;
-        char extentData[4];
+        char _extentData[4];
+
+        static int HeaderSize() { return sizeof(Extent)-4; }
 
         bool validates() {
             return !(firstRecord.isNull() ^ lastRecord.isNull()) &&
@@ -254,8 +270,7 @@ namespace mongo {
           ----------------------
     */
 
-    /* data file header */
-    class MDFHeader {
+    class DataFileHeader {
     public:
         int version;
         int versionMinor;
@@ -266,9 +281,7 @@ namespace mongo {
 
         char data[4];
 
-        static int headerSize() {
-            return sizeof(MDFHeader) - 4;
-        }
+        enum { HeaderSize = 8192 };
 
         bool currentVersion() const {
             return ( version == VERSION ) && ( versionMinor == VERSION_MINOR );
@@ -279,28 +292,28 @@ namespace mongo {
             return false;
         }
 
-        Record* getRecord(DiskLoc dl) {
+        /*Record* __getRecord(DiskLoc dl) {
             int ofs = dl.getOfs();
-            assert( ofs >= headerSize() );
+            assert( ofs >= HeaderSize );
             return (Record*) (((char *) this) + ofs);
-        }
+        }*/
 
         void init(int fileno, int filelength) {
             if ( uninitialized() ) {
                 assert(filelength > 32768 );
-                assert( headerSize() == 8192 );
+                assert( HeaderSize == 8192 );
                 fileLength = filelength;
                 version = VERSION;
                 versionMinor = VERSION_MINOR;
-                unused.setOfs( fileno, headerSize() );
-                assert( (data-(char*)this) == headerSize() );
-                unusedLength = fileLength - headerSize() - 16;
-                memcpy(data+unusedLength, "      \nthe end\n", 16);
+                unused.setOfs( fileno, HeaderSize );
+                assert( (data-(char*)this) == HeaderSize );
+                unusedLength = fileLength - HeaderSize - 16;
+                //memcpy(data+unusedLength, "      \nthe end\n", 16);
             }
         }
         
         bool isEmpty() const {
-            return uninitialized() || ( unusedLength == fileLength - headerSize() - 16 );
+            return uninitialized() || ( unusedLength == fileLength - HeaderSize - 16 );
         }
     };
 
@@ -308,7 +321,7 @@ namespace mongo {
 
     inline Extent* MongoDataFile::_getExtent(DiskLoc loc) {
         loc.assertOk();
-        Extent *e = (Extent *) (((char *)header) + loc.getOfs());
+        Extent *e = (Extent *) _p.at(loc.getOfs(), Extent::HeaderSize());
         return e;
     }
 
@@ -325,7 +338,20 @@ namespace mongo {
 namespace mongo {
 
     inline Record* MongoDataFile::recordAt(DiskLoc dl) {
-        return header->getRecord(dl);
+        int ofs = dl.getOfs();
+        assert( ofs >= DataFileHeader::HeaderSize );
+        return (Record*) _p.at(ofs, -1);
+    }
+
+	inline void MongoDataFile::grow(DiskLoc dl, int size) { 
+        int ofs = dl.getOfs();
+        _p.grow(ofs, size);
+	}
+
+    inline Record* MongoDataFile::makeRecord(DiskLoc dl, int size) { 
+        int ofs = dl.getOfs();
+        assert( ofs >= DataFileHeader::HeaderSize );
+        return (Record*) _p.at(ofs, size);
     }
 
     inline DiskLoc Record::getNext(const DiskLoc& myLoc) {
@@ -446,9 +472,31 @@ namespace mongo {
         assert( dl.a() != -1 );
         return cc().database()->getFile(dl.a())->recordAt(dl);
     }
+
+	BOOST_STATIC_ASSERT( 16 == sizeof(DeletedRecord) );
+
+	inline void DataFileMgr::grow(const DiskLoc& dl, int len) { 
+        assert( dl.a() != -1 );
+        cc().database()->getFile(dl.a())->grow(dl, len);
+	}
+
+    inline DeletedRecord* DataFileMgr::makeDeletedRecord(const DiskLoc& dl, int len) { 
+        assert( dl.a() != -1 );
+        return (DeletedRecord*) cc().database()->getFile(dl.a())->makeRecord(dl, sizeof(DeletedRecord));
+    }
     
     void ensureHaveIdIndex(const char *ns);
     
     bool dropIndexes( NamespaceDetails *d, const char *ns, const char *name, string &errmsg, BSONObjBuilder &anObjBuilder, bool maydeleteIdIndex );
+
+
+    /**
+     * @return true if ns is ok
+     */
+    inline bool nsDollarCheck( const char* ns ){
+        if ( strchr( ns , '$' ) == 0 )
+            return true;
         
+        return strcmp( ns, "local.oplog.$main" ) == 0;
+    }
 } // namespace mongo
diff --git a/db/query.cpp b/db/query.cpp
index 761a312..3d251a0 100644
--- a/db/query.cpp
+++ b/db/query.cpp
@@ -16,24 +16,26 @@
  *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "query.h"
 #include "pdfile.h"
 #include "jsobjmanipulator.h"
-#include "../util/builder.h"
+#include "../bson/util/builder.h"
 #include <time.h>
 #include "introspect.h"
 #include "btree.h"
 #include "../util/lruishmap.h"
 #include "json.h"
 #include "repl.h"
-#include "replset.h"
+#include "replpair.h"
 #include "scanandorder.h"
 #include "security.h"
 #include "curop.h"
 #include "commands.h"
 #include "queryoptimizer.h"
 #include "lasterror.h"
+#include "../s/d_logic.h"
+#include "repl_block.h"
 
 namespace mongo {
 
@@ -49,7 +51,7 @@ namespace mongo {
     extern bool useHints;
 
     // Just try to identify best plan.
-    class DeleteOp : public QueryOp {
+    class DeleteOp : public MultiCursor::CursorOp {
     public:
         DeleteOp( bool justOne, int& bestCount ) :
             justOne_( justOne ),
@@ -57,9 +59,21 @@ namespace mongo {
             bestCount_( bestCount ),
             _nscanned() {
         }
-        virtual void init() {
+        virtual void _init() {
             c_ = qp().newCursor();
-            _matcher.reset( new CoveredIndexMatcher( qp().query(), qp().indexKey() ) );
+        }
+        virtual bool prepareToYield() {
+            if ( ! _cc ) {
+                _cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , c_ , qp().ns() ) );
+            }
+            return _cc->prepareToYield( _yieldData );
+        }        
+        virtual void recoverFromYield() {
+            if ( !ClientCursor::recoverFromYield( _yieldData ) ) {
+                _cc.reset();
+                c_.reset();
+                massert( 13340, "cursor dropped during delete", false );
+            }
         }
         virtual void next() {
             if ( !c_->ok() ) {
@@ -69,7 +83,7 @@ namespace mongo {
             
             DiskLoc rloc = c_->currLoc();
             
-            if ( _matcher->matches(c_->currKey(), rloc ) ) {
+            if ( matcher()->matches(c_->currKey(), rloc ) ) {
                 if ( !c_->getsetdup(rloc) )
                     ++count_;
             }
@@ -87,17 +101,19 @@ namespace mongo {
             }
         }
         virtual bool mayRecordPlan() const { return !justOne_; }
-        virtual QueryOp *clone() const {
+        virtual QueryOp *_createChild() const {
+            bestCount_ = 0; // should be safe to reset this in contexts where createChild() is called
             return new DeleteOp( justOne_, bestCount_ );
         }
-        auto_ptr< Cursor > newCursor() const { return qp().newCursor(); }
+        virtual shared_ptr<Cursor> newCursor() const { return qp().newCursor(); }
     private:
         bool justOne_;
         int count_;
         int &bestCount_;
         long long _nscanned;
-        auto_ptr< Cursor > c_;
-        auto_ptr< CoveredIndexMatcher > _matcher;
+        shared_ptr<Cursor> c_;
+        ClientCursor::CleanupPointer _cc;
+        ClientCursor::YieldData _yieldData;
     };
     
     /* ns:      namespace, e.g. <database>.<collection>
@@ -105,7 +121,7 @@ namespace mongo {
        justOne: stop after 1 match
        god:     allow access to system namespaces, and don't yield
     */
-    long long deleteObjects(const char *ns, BSONObj pattern, bool justOne, bool logop, bool god) {
+    long long deleteObjects(const char *ns, BSONObj pattern, bool justOneOrig, bool logop, bool god, RemoveSaver * rs ) {
         if( !god ) {
             if ( strstr(ns, ".system.") ) {
                 /* note a delete from system.indexes would corrupt the db 
@@ -126,52 +142,58 @@ namespace mongo {
         uassert( 10101 ,  "can't remove from a capped collection" , ! d->capped );
 
         long long nDeleted = 0;
-        QueryPlanSet s( ns, pattern, BSONObj() );
+
         int best = 0;
-        DeleteOp original( justOne, best );
-        shared_ptr< DeleteOp > bestOp = s.runOp( original );
-        auto_ptr< Cursor > creal = bestOp->newCursor();
+        shared_ptr< MultiCursor::CursorOp > opPtr( new DeleteOp( justOneOrig, best ) );
+        shared_ptr< MultiCursor > creal( new MultiCursor( ns, pattern, BSONObj(), opPtr, true ) );
         
         if( !creal->ok() )
             return nDeleted;
-
-        CoveredIndexMatcher matcher(pattern, creal->indexKeyPattern());
-
-        auto_ptr<ClientCursor> cc( new ClientCursor(creal, ns, false) );
+            
+        shared_ptr< Cursor > cPtr = creal;
+        auto_ptr<ClientCursor> cc( new ClientCursor( QueryOption_NoCursorTimeout, cPtr, ns) );
         cc->setDoingDeletes( true );
-
+            
         CursorId id = cc->cursorid;
-        
-        unsigned long long nScanned = 0;
+            
+        bool justOne = justOneOrig;
+        bool canYield = !god && !creal->matcher()->docMatcher().atomic();
         do {
-            if ( ++nScanned % 128 == 0 && !god && !matcher.docMatcher().atomic() ) {
-                if ( ! cc->yield() ){
-                    cc.release(); // has already been deleted elsewhere
-                    break;
-                }
+            if ( canYield && ! cc->yieldSometimes() ){
+                cc.release(); // has already been deleted elsewhere
+                // TODO should we assert or something?
+                break;
             }
-            
+            if ( !cc->c->ok() ) {
+                break; // if we yielded, could have hit the end
+            }
+                
             // this way we can avoid calling updateLocation() every time (expensive)
             // as well as some other nuances handled
             cc->setDoingDeletes( true );
-            
+                
             DiskLoc rloc = cc->c->currLoc();
             BSONObj key = cc->c->currKey();
+
+            // NOTE Calling advance() may change the matcher, so it's important 
+            // to try to match first.
+            bool match = creal->matcher()->matches( key , rloc );
             
-            cc->c->advance();
-            
-            if ( ! matcher.matches( key , rloc ) )
+            if ( ! cc->c->advance() )
+                justOne = true;
+                
+            if ( ! match )
                 continue;
-            
+                            
             assert( !cc->c->getsetdup(rloc) ); // can't be a dup, we deleted it!
-
+                
             if ( !justOne ) {
                 /* NOTE: this is SLOW.  this is not good, noteLocation() was designed to be called across getMore
-                   blocks.  here we might call millions of times which would be bad.
-                */
+                    blocks.  here we might call millions of times which would be bad.
+                    */
                 cc->c->noteLocation();
             }
-            
+                
             if ( logop ) {
                 BSONElement e;
                 if( BSONObj( rloc.rec() ).getObjectID( e ) ) {
@@ -184,18 +206,22 @@ namespace mongo {
                 }
             }
 
+            if ( rs )
+                rs->goingToDelete( rloc.obj() /*cc->c->current()*/ );
+
             theDataFileMgr.deleteRecord(ns, rloc.rec(), rloc);
             nDeleted++;
-            if ( justOne )
+            if ( justOne ) {
                 break;
+            }
             cc->c->checkLocation();
-            
+                
         } while ( cc->c->ok() );
 
         if ( cc.get() && ClientCursor::find( id , false ) == 0 ){
             cc.release();
         }
-
+              
         return nDeleted;
     }
 
@@ -208,14 +234,13 @@ namespace mongo {
             return _runCommands(ns, jsobj, b, anObjBuilder, fromRepl, queryOptions);
         }
         catch ( AssertionException& e ) {
-            if ( !e.msg.empty() )
-                anObjBuilder.append("assertion", e.msg);
+            e.getInfo().append( anObjBuilder , "assertion" , "assertionCode" );
         }
         curop.debug().str << " assertion ";
         anObjBuilder.append("errmsg", "db assertion failure");
         anObjBuilder.append("ok", 0.0);
         BSONObj x = anObjBuilder.done();
-        b.append((void*) x.objdata(), x.objsize());
+        b.appendBuf((void*) x.objdata(), x.objsize());
         return true;
     }
 
@@ -227,7 +252,9 @@ namespace mongo {
             if ( ClientCursor::erase(ids[i]) )
                 k++;
         }
-        log( k == n ) << "killcursors: found " << k << " of " << n << '\n';
+        if ( logLevel > 0 || k != n ){
+            log( k == n ) << "killcursors: found " << k << " of " << n << endl;
+        }
     }
 
     BSONObj id_obj = fromjson("{\"_id\":1}");
@@ -250,8 +277,9 @@ namespace mongo {
         return qr;
     }
 
-    QueryResult* getMore(const char *ns, int ntoreturn, long long cursorid , CurOp& curop ) {
-        StringBuilder& ss = curop.debug().str;
+    QueryResult* processGetMore(const char *ns, int ntoreturn, long long cursorid , CurOp& curop, int pass, bool& exhaust ) {
+//        log() << "TEMP GETMORE " << ns << ' ' << cursorid << ' ' << pass << endl;
+        exhaust = false;
         ClientCursor::Pointer p(cursorid);
         ClientCursor *cc = p._c;
         
@@ -260,30 +288,51 @@ namespace mongo {
             bufSize += sizeof( QueryResult );
             bufSize += ( ntoreturn ? 4 : 1 ) * 1024 * 1024;
         }
+
         BufBuilder b( bufSize );
 
         b.skip(sizeof(QueryResult));
-
-        int resultFlags = 0; //QueryResult::ResultFlag_AwaitCapable;
+        
+        int resultFlags = ResultFlag_AwaitCapable;
         int start = 0;
         int n = 0;
 
         if ( !cc ) {
             log() << "getMore: cursorid not found " << ns << " " << cursorid << endl;
             cursorid = 0;
-            resultFlags = QueryResult::ResultFlag_CursorNotFound;
+            resultFlags = ResultFlag_CursorNotFound;
         }
         else {
-            ss << " query: " << cc->query << " ";
+            if ( pass == 0 )
+                cc->updateSlaveLocation( curop );
+
+            int queryOptions = cc->_queryOptions;
+
+            if( pass == 0 ) {
+                StringBuilder& ss = curop.debug().str;
+                ss << " getMore: " << cc->query.toString() << " ";
+            }
+            
             start = cc->pos;
             Cursor *c = cc->c.get();
             c->checkLocation();
+            DiskLoc last;
+
             while ( 1 ) {
                 if ( !c->ok() ) {
+//                    log() << "TEMP Tailable : " << c->tailable() << ' ' << (queryOptions & QueryOption_AwaitData) << endl;
                     if ( c->tailable() ) {
-                        if ( c->advance() ) {
+                        /* when a tailable cursor hits "EOF", ok() goes false, and current() is null.  however 
+                           advance() can still be retries as a reactivation attempt.  when there is new data, it will 
+                           return true.  that's what we are doing here.
+                           */
+                        if ( c->advance() )
                             continue;
+
+                        if( n == 0 && (queryOptions & QueryOption_AwaitData) && pass < 1000 ) {
+                            throw GetMoreWaitException();
                         }
+
                         break;
                     }
                     p.release();
@@ -293,31 +342,42 @@ namespace mongo {
                     cc = 0;
                     break;
                 }
-                if ( !cc->matcher->matches(c->currKey(), c->currLoc() ) ) {
+                // in some cases (clone collection) there won't be a matcher
+                if ( c->matcher() && !c->matcher()->matches(c->currKey(), c->currLoc() ) ) {
                 }
+                /*
+                  TODO
+                else if ( _chunkMatcher && ! _chunkMatcher->belongsToMe( c->currKey(), c->currLoc() ) ){
+                    cout << "TEMP skipping un-owned chunk: " << c->current() << endl;
+                }
+                */
                 else {
-                    //out() << "matches " << c->currLoc().toString() << '\n';
                     if( c->getsetdup(c->currLoc()) ) {
                         //out() << "  but it's a dup \n";
                     }
                     else {
+                        last = c->currLoc();
                         BSONObj js = c->current();
-                        fillQueryResultFromObj(b, cc->fields.get(), js);
+
+                        // show disk loc should be part of the main query, not in an $or clause, so this should be ok
+                        fillQueryResultFromObj(b, cc->fields.get(), js, ( cc->pq.get() && cc->pq->showDiskLoc() ? &last : 0));
                         n++;
                         if ( (ntoreturn>0 && (n >= ntoreturn || b.len() > MaxBytesToReturnToClientAtOnce)) ||
                              (ntoreturn==0 && b.len()>1*1024*1024) ) {
                             c->advance();
                             cc->pos += n;
-                            //cc->updateLocation();
                             break;
                         }
                     }
                 }
                 c->advance();
             }
+            
             if ( cc ) {
                 cc->updateLocation();
                 cc->mayUpgradeStorage();
+                cc->storeOpForSlave( last );
+                exhaust = cc->_queryOptions & QueryOption_Exhaust;
             }
         }
 
@@ -335,26 +395,45 @@ namespace mongo {
 
     class CountOp : public QueryOp {
     public:
-        CountOp( const BSONObj &spec ) : spec_( spec ), count_(), bc_() {}
-        virtual void init() {
-            query_ = spec_.getObjectField( "query" );
+        CountOp( const string& ns , const BSONObj &spec ) :
+            _ns(ns), count_(),
+            skip_( spec["skip"].numberLong() ),
+            limit_( spec["limit"].numberLong() ),
+            bc_(){
+        }
+        
+        virtual void _init() {
             c_ = qp().newCursor();
-            _matcher.reset( new CoveredIndexMatcher( query_, c_->indexKeyPattern() ) );
-            if ( qp().exactKeyMatch() && ! _matcher->needRecord() ) {
+            
+            if ( qp().exactKeyMatch() && ! matcher()->needRecord() ) {
                 query_ = qp().simplifiedQuery( qp().indexKey() );
                 bc_ = dynamic_cast< BtreeCursor* >( c_.get() );
                 bc_->forgetEndKey();
             }
-            
-            skip_ = spec_["skip"].numberLong();
-            limit_ = spec_["limit"].numberLong();
         }
 
+        virtual bool prepareToYield() {
+            if ( ! _cc ) {
+                _cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , c_ , _ns.c_str() ) );
+            }
+            return _cc->prepareToYield( _yieldData );
+        }
+        
+        virtual void recoverFromYield() {
+            if ( !ClientCursor::recoverFromYield( _yieldData ) ) {
+                c_.reset();
+                _cc.reset();
+                massert( 13337, "cursor dropped during count", false );
+                // TODO maybe we want to prevent recording the winning plan as well?
+            }
+        }
+        
         virtual void next() {
             if ( !c_->ok() ) {
                 setComplete();
                 return;
             }
+
             if ( bc_ ) {
                 if ( firstMatch_.isEmpty() ) {
                     firstMatch_ = bc_->currKeyNode().key;
@@ -371,8 +450,9 @@ namespace mongo {
                     }
                     _gotOne();
                 }
-            } else {
-                if ( !_matcher->matches(c_->currKey(), c_->currLoc() ) ) {
+            } 
+            else {
+                if ( !matcher()->matches(c_->currKey(), c_->currLoc() ) ) {
                 }
                 else if( !c_->getsetdup(c_->currLoc()) ) {
                     _gotOne();
@@ -380,8 +460,12 @@ namespace mongo {
             }
             c_->advance();
         }
-        virtual QueryOp *clone() const {
-            return new CountOp( spec_ );
+        virtual QueryOp *_createChild() const {
+            CountOp *ret = new CountOp( _ns , BSONObj() );
+            ret->count_ = count_;
+            ret->skip_ = skip_;
+            ret->limit_ = limit_;
+            return ret;
         }
         long long count() const { return count_; }
         virtual bool mayRecordPlan() const { return true; }
@@ -394,95 +478,188 @@ namespace mongo {
             }
             
             if ( limit_ > 0 && count_ >= limit_ ){
-                setComplete();
+                setStop();
                 return;
             }
 
             count_++;
         }
 
-        BSONObj spec_;
+        string _ns;
+        
         long long count_;
         long long skip_;
         long long limit_;
-        auto_ptr< Cursor > c_;
+        shared_ptr<Cursor> c_;
         BSONObj query_;
         BtreeCursor *bc_;
-        auto_ptr< CoveredIndexMatcher > _matcher;
         BSONObj firstMatch_;
+
+        ClientCursor::CleanupPointer _cc;
+        ClientCursor::YieldData _yieldData;
     };
-    
+
     /* { count: "collectionname"[, query: <query>] }
        returns -1 on ns does not exist error.
     */    
     long long runCount( const char *ns, const BSONObj &cmd, string &err ) {
+        Client::Context cx(ns);
         NamespaceDetails *d = nsdetails( ns );
         if ( !d ) {
             err = "ns missing";
             return -1;
         }
         BSONObj query = cmd.getObjectField("query");
-
+        
         // count of all objects
         if ( query.isEmpty() ){
-            long long num = d->nrecords;
-            num = num - cmd["skip"].numberLong();
-            if ( num < 0 ) {
-                num = 0;
-            }
-            if ( cmd["limit"].isNumber() ){
-                long long limit = cmd["limit"].numberLong();
-                if ( limit < num ){
-                    num = limit;
-                }
-            }
-            return num;
+            return applySkipLimit( d->nrecords , cmd );
         }
-        QueryPlanSet qps( ns, query, BSONObj() );
-        CountOp original( cmd );
-        shared_ptr< CountOp > res = qps.runOp( original );
+        MultiPlanScanner mps( ns, query, BSONObj(), 0, true, BSONObj(), BSONObj(), false, true );
+        CountOp original( ns , cmd );
+        shared_ptr< CountOp > res = mps.runOp( original );
         if ( !res->complete() ) {
             log() << "Count with ns: " << ns << " and query: " << query
-                  << " failed with exception: " << res->exceptionMessage()
+                  << " failed with exception: " << res->exception()
                   << endl;
             return 0;
         }
         return res->count();
     }
+    
+    class ExplainBuilder {
+    public:
+        ExplainBuilder() : _i() {}
+        void ensureStartScan() {
+            if ( !_a.get() ) {
+                _a.reset( new BSONArrayBuilder() );
+            }
+        }
+        void noteCursor( Cursor *c ) {
+            BSONObjBuilder b( _a->subobjStart() );
+            b << "cursor" << c->toString() << "indexBounds" << c->prettyIndexBounds();
+            b.done();
+        }
+        void noteScan( Cursor *c, long long nscanned, long long nscannedObjects, int n, bool scanAndOrder, int millis, bool hint ) {
+            if ( _i == 1 ) {
+                _c.reset( new BSONArrayBuilder() );
+                *_c << _b->obj();
+            }
+            if ( _i == 0 ) {
+                _b.reset( new BSONObjBuilder() );
+            } else {
+                _b.reset( new BSONObjBuilder( _c->subobjStart() ) );
+            }
+            *_b << "cursor" << c->toString();
+            _b->appendNumber( "nscanned", nscanned );
+            _b->appendNumber( "nscannedObjects", nscannedObjects );
+            *_b << "n" << n;
+
+            if ( scanAndOrder )
+                *_b << "scanAndOrder" << true;
 
+            *_b << "millis" << millis;
+
+            *_b << "indexBounds" << c->prettyIndexBounds();
+
+            if ( !hint ) {
+                *_b << "allPlans" << _a->arr();
+            }
+            if ( _i != 0 ) {
+                _b->done();
+            }
+            _a.reset( 0 );
+            ++_i;
+        }
+        BSONObj finishWithSuffix( long long nscanned, long long nscannedObjects, int n, int millis, const BSONObj &suffix ) { 
+            if ( _i > 1 ) {
+                BSONObjBuilder b;
+                b << "clauses" << _c->arr();
+                b.appendNumber( "nscanned", nscanned );
+                b.appendNumber( "nscanneObjects", nscannedObjects );
+                b << "n" << n;
+                b << "millis" << millis;
+                b.appendElements( suffix );
+                return b.obj();
+            } else {
+                _b->appendElements( suffix );
+                return _b->obj();                
+            }
+        }
+    private:
+        auto_ptr< BSONArrayBuilder > _a;
+        auto_ptr< BSONObjBuilder > _b;
+        auto_ptr< BSONArrayBuilder > _c;
+        int _i;
+    };
+    
     // Implements database 'query' requests using the query optimizer's QueryOp interface
     class UserQueryOp : public QueryOp {
     public:
         
-        UserQueryOp( const ParsedQuery& pq ) :
-        //int ntoskip, int ntoreturn, const BSONObj &order, bool wantMore,
-        //                   bool explain, FieldMatcher *filter, int queryOptions ) :
+        UserQueryOp( const ParsedQuery& pq, Message &response, ExplainBuilder &eb, CurOp &curop ) :
             _buf( 32768 ) , // TODO be smarter here
             _pq( pq ) ,
             _ntoskip( pq.getSkip() ) ,
-            _nscanned(0), _nscannedObjects(0),
+            _nscanned(0), _oldNscanned(0), _nscannedObjects(0), _oldNscannedObjects(0),
             _n(0),
+            _oldN(0),
+            _chunkMatcher(shardingState.getChunkMatcher(pq.ns())),
             _inMemSort(false),
             _saveClientCursor(false),
-            _oplogReplay( pq.hasOption( QueryOption_OplogReplay) )
+            _wouldSaveClientCursor(false),
+            _oplogReplay( pq.hasOption( QueryOption_OplogReplay) ),
+            _response( response ),
+            _eb( eb ),
+            _curop( curop )
         {}
         
-        virtual void init() {
-            _buf.skip( sizeof( QueryResult ) );
+        virtual void _init() {
+            // only need to put the QueryResult fields there if we're building the first buffer in the message.
+            if ( _response.empty() ) {
+                _buf.skip( sizeof( QueryResult ) );
+            }
             
             if ( _oplogReplay ) {
                 _findingStartCursor.reset( new FindingStartCursor( qp() ) );
             } else {
                 _c = qp().newCursor( DiskLoc() , _pq.getNumToReturn() + _pq.getSkip() );
             }
-            _matcher.reset(new CoveredIndexMatcher( qp().query() , qp().indexKey()));
 
             if ( qp().scanAndOrderRequired() ) {
                 _inMemSort = true;
                 _so.reset( new ScanAndOrder( _pq.getSkip() , _pq.getNumToReturn() , _pq.getOrder() ) );
             }
+            
+            if ( _pq.isExplain() ) {
+                _eb.noteCursor( _c.get() );
+            }
         }
         
+        virtual bool prepareToYield() {
+            if ( _findingStartCursor.get() ) {
+                return _findingStartCursor->prepareToYield();
+            } else {
+                if ( ! _cc ) {
+                    _cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , _c , _pq.ns() ) );
+                }
+                return _cc->prepareToYield( _yieldData );
+            }
+        }
+        
+        virtual void recoverFromYield() {
+            if ( _findingStartCursor.get() ) {
+                _findingStartCursor->recoverFromYield();
+            } else {
+                if ( !ClientCursor::recoverFromYield( _yieldData ) ) {
+                    _c.reset();
+                    _cc.reset();
+                    massert( 13338, "cursor dropped during query", false );
+                    // TODO maybe we want to prevent recording the winning plan as well?
+                } 
+            }
+        }        
+        
         virtual void next() {
             if ( _findingStartCursor.get() ) {
                 if ( _findingStartCursor->done() ) {
@@ -495,18 +672,23 @@ namespace mongo {
             }
             
             if ( !_c->ok() ) {
-                finish();
+                finish( false );
                 return;
             }
-            
+
             bool mayCreateCursor1 = _pq.wantMore() && ! _inMemSort && _pq.getNumToReturn() != 1 && useCursors;
             
             if( 0 ) { 
                 cout << "SCANNING this: " << this << " key: " << _c->currKey() << " obj: " << _c->current() << endl;
             }
+            
+            if ( _pq.getMaxScan() && _nscanned >= _pq.getMaxScan() ){
+                finish( true ); //?
+                return;
+            }
 
             _nscanned++;
-            if ( !_matcher->matches(_c->currKey(), _c->currLoc() , &_details ) ) {
+            if ( !matcher()->matches(_c->currKey(), _c->currLoc() , &_details ) ) {
                 // not a match, continue onward
                 if ( _details.loadedObject )
                     _nscannedObjects++;
@@ -514,15 +696,18 @@ namespace mongo {
             else {
                 _nscannedObjects++;
                 DiskLoc cl = _c->currLoc();
-                if( !_c->getsetdup(cl) ) { 
+                if ( _chunkMatcher && ! _chunkMatcher->belongsToMe( _c->currKey(), _c->currLoc() ) ){
+                    // cout << "TEMP skipping un-owned chunk: " << _c->current() << endl;
+                }
+                else if( _c->getsetdup(cl) ) { 
+                    // dup
+                }
+                else {
                     // got a match.
                     
-                    BSONObj js = _pq.returnKey() ? _c->currKey() : _c->current();
-                    assert( js.objsize() >= 0 ); //defensive for segfaults
-
                     if ( _inMemSort ) {
                         // note: no cursors for non-indexed, ordered results.  results must be fairly small.
-                        _so->add(js);
+                        _so->add( _pq.returnKey() ? _c->currKey() : _c->current(), _pq.showDiskLoc() ? &cl : 0 );
                     }
                     else if ( _ntoskip > 0 ) {
                         _ntoskip--;
@@ -530,38 +715,48 @@ namespace mongo {
                     else {
                         if ( _pq.isExplain() ) {
                             _n++;
-                            if ( _n >= _pq.getNumToReturn() && !_pq.wantMore() ) {
+                            if ( n() >= _pq.getNumToReturn() && !_pq.wantMore() ) {
                                 // .limit() was used, show just that much.
-                                finish();
+                                finish( true ); //?
                                 return;
                             }
                         }
                         else {
+
                             if ( _pq.returnKey() ){
                                 BSONObjBuilder bb( _buf );
-                                bb.appendKeys( _c->indexKeyPattern() , js );
+                                bb.appendKeys( _c->indexKeyPattern() , _c->currKey() );
                                 bb.done();
                             }
                             else {
-                                fillQueryResultFromObj( _buf , _pq.getFields() , js );
+                                BSONObj js = _c->current();
+                                assert( js.isValid() );
+
+                                if ( _oplogReplay ){
+                                    BSONElement e = js["ts"];
+                                    if ( e.type() == Date || e.type() == Timestamp )
+                                        _slaveReadTill = e._opTime();
+                                }
+
+                                fillQueryResultFromObj( _buf , _pq.getFields() , js , (_pq.showDiskLoc() ? &cl : 0));
                             }
                             _n++;
                             if ( ! _c->supportGetMore() ){
-                                if ( _pq.enough( _n ) || _buf.len() >= MaxBytesToReturnToClientAtOnce ){
-                                    finish();
+                                if ( _pq.enough( n() ) || _buf.len() >= MaxBytesToReturnToClientAtOnce ){
+                                    finish( true );
                                     return;
                                 }
                             }
-                            else if ( _pq.enoughForFirstBatch( _n , _buf.len() ) ){
+                            else if ( _pq.enoughForFirstBatch( n() , _buf.len() ) ){
                                 /* if only 1 requested, no cursor saved for efficiency...we assume it is findOne() */
                                 if ( mayCreateCursor1 ) {
-                                    _c->advance();
-                                    if ( _c->ok() ) {
+                                    _wouldSaveClientCursor = true;
+                                    if ( _c->advance() ) {
                                         // more...so save a cursor
                                         _saveClientCursor = true;
                                     }
                                 }
-                                finish();
+                                finish( true );
                                 return;
                             }
                         }
@@ -571,7 +766,8 @@ namespace mongo {
             _c->advance();            
         }
 
-        void finish() {
+        // this plan won, so set data for response broadly
+        void finish( bool stop ) {
             if ( _pq.isExplain() ) {
                 _n = _inMemSort ? _so->size() : _n;
             } 
@@ -586,121 +782,174 @@ namespace mongo {
             if ( _c->tailable() )
                 _saveClientCursor = true;
 
-            setComplete();            
+            if ( _pq.isExplain()) {
+                _eb.noteScan( _c.get(), _nscanned, _nscannedObjects, _n, scanAndOrderRequired(), _curop.elapsedMillis(), useHints && !_pq.getHint().eoo() );
+            } else {
+                _response.appendData( _buf.buf(), _buf.len() );
+                _buf.decouple();
+            }
+            if ( stop ) {
+                setStop();
+            } else {
+                setComplete();
+            }
+
+        }
+        
+        void finishExplain( const BSONObj &suffix ) {
+            BSONObj obj = _eb.finishWithSuffix( nscanned(), nscannedObjects(), n(), _curop.elapsedMillis(), suffix);
+            fillQueryResultFromObj(_buf, 0, obj);
+            _n = 1;
+            _oldN = 0;
+            _response.appendData( _buf.buf(), _buf.len() );
+            _buf.decouple();
         }
         
         virtual bool mayRecordPlan() const { return _pq.getNumToReturn() != 1; }
         
-        virtual QueryOp *clone() const {
-            return new UserQueryOp( _pq );
+        virtual QueryOp *_createChild() const {
+            if ( _pq.isExplain() ) {
+                _eb.ensureStartScan();
+            }
+            UserQueryOp *ret = new UserQueryOp( _pq, _response, _eb, _curop );
+            ret->_oldN = n();
+            ret->_oldNscanned = nscanned();
+            ret->_oldNscannedObjects = nscannedObjects();
+            ret->_ntoskip = _ntoskip;
+            return ret;
         }
 
-        BufBuilder &builder() { return _buf; }
         bool scanAndOrderRequired() const { return _inMemSort; }
-        auto_ptr< Cursor > cursor() { return _c; }
-        auto_ptr< CoveredIndexMatcher > matcher() { return _matcher; }
-        int n() const { return _n; }
-        long long nscanned() const { return _nscanned; }
-        long long nscannedObjects() const { return _nscannedObjects; }
+        shared_ptr<Cursor> cursor() { return _c; }
+        int n() const { return _oldN + _n; }
+        long long nscanned() const { return _nscanned + _oldNscanned; }
+        long long nscannedObjects() const { return _nscannedObjects + _oldNscannedObjects; }
         bool saveClientCursor() const { return _saveClientCursor; }
+        bool wouldSaveClientCursor() const { return _wouldSaveClientCursor; }
+        
+        void finishForOplogReplay( ClientCursor * cc ){
+            if ( _oplogReplay && ! _slaveReadTill.isNull() )
+                cc->_slaveReadTill = _slaveReadTill;
 
+        }
     private:
         BufBuilder _buf;
         const ParsedQuery& _pq;
 
         long long _ntoskip;
         long long _nscanned;
+        long long _oldNscanned;
         long long _nscannedObjects;
+        long long _oldNscannedObjects;
         int _n; // found so far
+        int _oldN;
         
         MatchDetails _details;
 
+        ChunkMatcherPtr _chunkMatcher;
+        
         bool _inMemSort;
         auto_ptr< ScanAndOrder > _so;
         
-        auto_ptr< Cursor > _c;
-
-        auto_ptr< CoveredIndexMatcher > _matcher;
+        shared_ptr<Cursor> _c;
+        ClientCursor::CleanupPointer _cc;
+        ClientCursor::YieldData _yieldData;
 
         bool _saveClientCursor;
+        bool _wouldSaveClientCursor;
         bool _oplogReplay;
         auto_ptr< FindingStartCursor > _findingStartCursor;
+        
+        Message &_response;
+        ExplainBuilder &_eb;
+        CurOp &_curop;
+        OpTime _slaveReadTill;
     };
     
     /* run a query -- includes checking for and running a Command */
-    auto_ptr< QueryResult > runQuery(Message& m, QueryMessage& q, CurOp& curop ) {
+    const char *runQuery(Message& m, QueryMessage& q, CurOp& curop, Message &result) {
         StringBuilder& ss = curop.debug().str;
-        ParsedQuery pq( q );
-        const char *ns = q.ns;
+        shared_ptr<ParsedQuery> pq_shared( new ParsedQuery(q) );
+        ParsedQuery& pq( *pq_shared );
         int ntoskip = q.ntoskip;
         BSONObj jsobj = q.query;
         int queryOptions = q.queryOptions;
-        BSONObj snapshotHint;
+        const char *ns = q.ns;
         
         if( logLevel >= 2 )
-            log() << "runQuery: " << ns << jsobj << endl;
+            log() << "query: " << ns << jsobj << endl;
         
-        long long nscanned = 0;
-        ss << ns << " ntoreturn:" << pq.getNumToReturn();
+        ss << ns;
+        {
+            // only say ntoreturn if nonzero. 
+            int n =  pq.getNumToReturn();
+            if( n ) 
+                ss << " ntoreturn:" << n;
+        }
         curop.setQuery(jsobj);
         
-        BSONObjBuilder cmdResBuf;
-        long long cursorid = 0;
-        
-        auto_ptr< QueryResult > qr;
-        int n = 0;
-        
-        Client& c = cc();
-
-        if ( pq.couldBeCommand() ){
+        if ( pq.couldBeCommand() ) {
             BufBuilder bb;
             bb.skip(sizeof(QueryResult));
-
+            BSONObjBuilder cmdResBuf;
             if ( runCommands(ns, jsobj, curop, bb, cmdResBuf, false, queryOptions) ) {
-                ss << " command ";
+                ss << " command: " << jsobj.toString();
                 curop.markCommand();
-                n = 1;
+                auto_ptr< QueryResult > qr;
                 qr.reset( (QueryResult *) bb.buf() );
                 bb.decouple();
                 qr->setResultFlagsToOk();
                 qr->len = bb.len();
                 ss << " reslen:" << bb.len();
-                //	qr->channel = 0;
                 qr->setOperation(opReply);
-                qr->cursorId = cursorid;
+                qr->cursorId = 0;
                 qr->startingFrom = 0;
-                qr->nReturned = n;
+                qr->nReturned = 1;
+                result.setData( qr.release(), true );
             }
-            return qr;
+            return false;
         }
         
-        // regular query
-
-        mongolock lk(false); // read lock
-        Client::Context ctx( ns , dbpath , &lk );
-
-        /* we allow queries to SimpleSlave's -- but not to the slave (nonmaster) member of a replica pair 
-           so that queries to a pair are realtime consistent as much as possible.  use setSlaveOk() to 
-           query the nonmaster member of a replica pair.
-        */
-        uassert( 10107 , "not master" , isMaster() || pq.hasOption( QueryOption_SlaveOk ) || replSettings.slave == SimpleSlave );
+        /* --- regular query --- */
 
+        int n = 0;
         BSONElement hint = useHints ? pq.getHint() : BSONElement();
         bool explain = pq.isExplain();
         bool snapshot = pq.isSnapshot();
-        BSONObj query = pq.getFilter();
         BSONObj order = pq.getOrder();
+        BSONObj query = pq.getFilter();
+
+        /* The ElemIter will not be happy if this isn't really an object. So throw exception
+           here when that is true.
+           (Which may indicate bad data from client.)
+        */
+        if ( query.objsize() == 0 ) {
+            out() << "Bad query object?\n  jsobj:";
+            out() << jsobj.toString() << "\n  query:";
+            out() << query.toString() << endl;
+            uassert( 10110 , "bad query object", false);
+        }
+            
+        /* --- read lock --- */
+
+        mongolock lk(false);
+
+        Client::Context ctx( ns , dbpath , &lk );
+
+        replVerifyReadsOk(pq);
 
         if ( pq.hasOption( QueryOption_CursorTailable ) ) {
             NamespaceDetails *d = nsdetails( ns );
             uassert( 13051, "tailable cursor requested on non capped collection", d && d->capped );
+            const BSONObj nat1 = BSON( "$natural" << 1 );
             if ( order.isEmpty() ) {
-                order = BSON( "$natural" << 1 );
+                order = nat1;
             } else {
-                uassert( 13052, "only {$natural:1} order allowed for tailable cursor", order == BSON( "$natural" << 1 ) );
+                uassert( 13052, "only {$natural:1} order allowed for tailable cursor", order == nat1 );
             }
         }
         
+        BSONObj snapshotHint; // put here to keep the data in scope
         if( snapshot ) { 
             NamespaceDetails *d = nsdetails(ns);
             if ( d ){
@@ -722,25 +971,12 @@ namespace mongo {
             }
         }
             
-        /* The ElemIter will not be happy if this isn't really an object. So throw exception
-           here when that is true.
-           (Which may indicate bad data from client.)
-        */
-        if ( query.objsize() == 0 ) {
-            out() << "Bad query object?\n  jsobj:";
-            out() << jsobj.toString() << "\n  query:";
-            out() << query.toString() << endl;
-            uassert( 10110 , "bad query object", false);
-        }
-            
-
-        if ( ! explain && isSimpleIdQuery( query ) && !pq.hasOption( QueryOption_CursorTailable ) ) {
-            nscanned = 1;
-
+        if ( ! (explain || pq.showDiskLoc()) && isSimpleIdQuery( query ) && !pq.hasOption( QueryOption_CursorTailable ) ) {
             bool nsFound = false;
             bool indexFound = false;
 
             BSONObj resObject;
+            Client& c = cc();
             bool found = Helpers::findById( c, ns , query , resObject , &nsFound , &indexFound );
             if ( nsFound == false || indexFound == true ){
                 BufBuilder bb(sizeof(QueryResult)+resObject.objsize()+32);
@@ -751,16 +987,18 @@ namespace mongo {
                     n = 1;
                     fillQueryResultFromObj( bb , pq.getFields() , resObject );
                 }
+                auto_ptr< QueryResult > qr;
                 qr.reset( (QueryResult *) bb.buf() );
                 bb.decouple();
                 qr->setResultFlagsToOk();
                 qr->len = bb.len();
                 ss << " reslen:" << bb.len();
                 qr->setOperation(opReply);
-                qr->cursorId = cursorid;
+                qr->cursorId = 0;
                 qr->startingFrom = 0;
-                qr->nReturned = n;       
-                return qr;
+                qr->nReturned = n;      
+                result.setData( qr.release(), true );
+                return false;
             }     
         }
         
@@ -768,67 +1006,73 @@ namespace mongo {
         
         BSONObj oldPlan;
         if ( explain && ! pq.hasIndexSpecifier() ){
-            QueryPlanSet qps( ns, query, order );
-            if ( qps.usingPrerecordedPlan() )
-                oldPlan = qps.explain();
+            MultiPlanScanner mps( ns, query, order );
+            if ( mps.usingPrerecordedPlan() )
+                oldPlan = mps.oldExplain();
+        }
+        auto_ptr< MultiPlanScanner > mps( new MultiPlanScanner( ns, query, order, &hint, !explain, pq.getMin(), pq.getMax(), false, true ) );
+        BSONObj explainSuffix;
+        if ( explain ) {
+            BSONObjBuilder bb;
+            if ( !oldPlan.isEmpty() )
+                bb.append( "oldPlan", oldPlan.firstElement().embeddedObject().firstElement().embeddedObject() );
+            explainSuffix = bb.obj();
         }
-        QueryPlanSet qps( ns, query, order, &hint, !explain, pq.getMin(), pq.getMax() );
-        UserQueryOp original( pq );
-        shared_ptr< UserQueryOp > o = qps.runOp( original );
+        ExplainBuilder eb;
+        UserQueryOp original( pq, result, eb, curop );
+        shared_ptr< UserQueryOp > o = mps->runOp( original );
         UserQueryOp &dqo = *o;
-        massert( 10362 ,  dqo.exceptionMessage(), dqo.complete() );
+        if ( ! dqo.complete() )
+            throw MsgAssertionException( dqo.exception() );
+        if ( explain ) {
+            dqo.finishExplain( explainSuffix );
+        }
         n = dqo.n();
-        nscanned = dqo.nscanned();
+        long long nscanned = dqo.nscanned();
         if ( dqo.scanAndOrderRequired() )
             ss << " scanAndOrder ";
-        auto_ptr<Cursor> cursor = dqo.cursor();
-        log( 5 ) << "   used cursor: " << cursor.get() << endl;
-        if ( dqo.saveClientCursor() ) {
-            // the clientcursor now owns the Cursor* and 'c' is released:
-            ClientCursor *cc = new ClientCursor(cursor, ns, !(queryOptions & QueryOption_NoCursorTimeout));
+        shared_ptr<Cursor> cursor = dqo.cursor();
+        if( logLevel >= 5 )
+            log() << "   used cursor: " << cursor.get() << endl;
+        long long cursorid = 0;
+        const char * exhaust = 0;
+        if ( dqo.saveClientCursor() || ( dqo.wouldSaveClientCursor() && mps->mayRunMore() ) ) {
+            ClientCursor *cc;
+            bool moreClauses = mps->mayRunMore();
+            if ( moreClauses ) {
+                // this MultiCursor will use a dumb NoOp to advance(), so no need to specify mayYield
+                shared_ptr< Cursor > multi( new MultiCursor( mps, cursor, dqo.matcher(), dqo ) );
+                cc = new ClientCursor(queryOptions, multi, ns);
+            } else {
+                cursor->setMatcher( dqo.matcher() );
+                cc = new ClientCursor( queryOptions, cursor, ns );
+            }
             cursorid = cc->cursorid;
             cc->query = jsobj.getOwned();
-            DEV out() << "  query has more, cursorid: " << cursorid << endl;
-            cc->matcher = dqo.matcher();
+            DEV tlog() << "query has more, cursorid: " << cursorid << endl;
             cc->pos = n;
+            cc->pq = pq_shared;
             cc->fields = pq.getFieldPtr();
             cc->originalMessage = m;
             cc->updateLocation();
-            if ( !cc->c->ok() && cc->c->tailable() ) {
-                DEV out() << "  query has no more but tailable, cursorid: " << cursorid << endl;
-            } else {
-                DEV out() << "  query has more, cursorid: " << cursorid << endl;
+            if ( !cc->c->ok() && cc->c->tailable() )
+                DEV tlog() << "query has no more but tailable, cursorid: " << cursorid << endl;
+            if( queryOptions & QueryOption_Exhaust ) {
+                exhaust = ns;
+                ss << " exhaust ";
             }
+            dqo.finishForOplogReplay(cc);
         }
-        if ( explain ) {
-            BSONObjBuilder builder;
-            builder.append("cursor", cursor->toString());
-            builder.appendArray("indexBounds", cursor->prettyIndexBounds());
-            builder.appendNumber("nscanned", dqo.nscanned() );
-            builder.appendNumber("nscannedObjects", dqo.nscannedObjects() );
-            builder.append("n", n);
-            if ( dqo.scanAndOrderRequired() )
-                builder.append("scanAndOrder", true);
-            builder.append("millis", curop.elapsedMillis());
-            if ( !oldPlan.isEmpty() )
-                builder.append( "oldPlan", oldPlan.firstElement().embeddedObject().firstElement().embeddedObject() );
-            if ( hint.eoo() )
-                builder.appendElements(qps.explain());
-            BSONObj obj = builder.done();
-            fillQueryResultFromObj(dqo.builder(), 0, obj);
-            n = 1;
-        }
-        qr.reset( (QueryResult *) dqo.builder().buf() );
-        dqo.builder().decouple();
+
+        QueryResult *qr = (QueryResult *) result.header();
         qr->cursorId = cursorid;
         qr->setResultFlagsToOk();
-        qr->len = dqo.builder().len();
+        // qr->len is updated automatically by appendData()
         ss << " reslen:" << qr->len;
         qr->setOperation(opReply);
         qr->startingFrom = 0;
         qr->nReturned = n;
 
-        
         int duration = curop.elapsedMillis();
         bool dbprofile = curop.shouldDBProfile( duration );
         if ( dbprofile || duration >= cmdLine.slowMS ) {
@@ -837,10 +1081,10 @@ namespace mongo {
                 ss << " ntoskip:" << ntoskip;
             if ( dbprofile )
                 ss << " \nquery: ";
-            ss << jsobj << ' ';
+            ss << jsobj.toString() << ' ';
         }
         ss << " nreturned:" << n;
-        return qr;        
+        return exhaust;
     }    
     
 } // namespace mongo
diff --git a/db/query.h b/db/query.h
index fdc33c9..cc88e5c 100644
--- a/db/query.h
+++ b/db/query.h
@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "../util/message.h"
 #include "dbmessage.h"
 #include "jsobj.h"
@@ -74,44 +74,44 @@ namespace mongo {
     extern const int MaxBytesToReturnToClientAtOnce;
 
     // for an existing query (ie a ClientCursor), send back additional information.
-    QueryResult* getMore(const char *ns, int ntoreturn, long long cursorid , CurOp& op);
+    struct GetMoreWaitException { };
 
+    QueryResult* processGetMore(const char *ns, int ntoreturn, long long cursorid , CurOp& op, int pass, bool& exhaust);
+    
     struct UpdateResult {
-        bool existing;
-        bool mod;
-        long long num;
-
-        UpdateResult( bool e, bool m, unsigned long long n )
-            : existing(e) , mod(m), num(n ){}
-
-        int oldCode(){
-            if ( ! num )
-                return 0;
-            
-            if ( existing ){
-                if ( mod )
-                    return 2;
-                return 1;
+        bool existing; // if existing objects were modified
+        bool mod;      // was this a $ mod
+        long long num; // how many objects touched
+        OID upserted;  // if something was upserted, the new _id of the object
+
+        UpdateResult( bool e, bool m, unsigned long long n , const BSONObj& upsertedObject = BSONObj() )
+            : existing(e) , mod(m), num(n){
+            upserted.clear();
+
+            BSONElement id = upsertedObject["_id"];
+            if ( ! e && n == 1 && id.type() == jstOID ){
+                upserted = id.OID();
             }
-            
-            if ( mod )
-                return 3;
-            return 4;
         }
+        
     };
+
+    class RemoveSaver;
     
     /* returns true if an existing object was updated, false if no existing object was found.
        multi - update multiple objects - mostly useful with things like $set
-       god - allow access to system namespaces and don't yield
+       god - allow access to system namespaces
     */
     UpdateResult updateObjects(const char *ns, const BSONObj& updateobj, BSONObj pattern, bool upsert, bool multi , bool logop , OpDebug& debug );
+    UpdateResult _updateObjects(bool god, const char *ns, const BSONObj& updateobj, BSONObj pattern, 
+                                bool upsert, bool multi , bool logop , OpDebug& debug , RemoveSaver * rs = 0 );
 
     // If justOne is true, deletedId is set to the id of the deleted object.
-    long long deleteObjects(const char *ns, BSONObj pattern, bool justOne, bool logop = false, bool god=false);
+    long long deleteObjects(const char *ns, BSONObj pattern, bool justOne, bool logop = false, bool god=false, RemoveSaver * rs=0);
 
     long long runCount(const char *ns, const BSONObj& cmd, string& err);
-    
-    auto_ptr< QueryResult > runQuery(Message& m, QueryMessage& q, CurOp& curop );
+
+    const char * runQuery(Message& m, QueryMessage& q, CurOp& curop, Message &result);
     
     /* This is for languages whose "objects" are not well ordered (JSON is well ordered).
        [ { a : ... } , { b : ... } ] -> { a : ..., b : ... }
@@ -157,6 +157,7 @@ namespace mongo {
         ~ParsedQuery(){}
 
         const char * ns() const { return _ns; }
+        bool isLocalDB() const { return strncmp(_ns, "local.", 6) == 0; }
 
         const BSONObj& getFilter() const { return _filter; }
         FieldMatcher* getFields() const { return _fields.get(); }
@@ -172,12 +173,14 @@ namespace mongo {
         bool isExplain() const { return _explain; }
         bool isSnapshot() const { return _snapshot; }
         bool returnKey() const { return _returnKey; }
+        bool showDiskLoc() const { return _showDiskLoc; }
 
         const BSONObj& getMin() const { return _min; }
         const BSONObj& getMax() const { return _max; }
         const BSONObj& getOrder() const { return _order; }
         const BSONElement& getHint() const { return _hint; }
-
+        int getMaxScan() const { return _maxScan; }
+        
         bool couldBeCommand() const {
             /* we assume you are using findOne() for running a cmd... */
             return _ntoreturn == 1 && strstr( _ns , ".$cmd" );
@@ -239,6 +242,8 @@ namespace mongo {
             _explain = false;
             _snapshot = false;
             _returnKey = false;
+            _showDiskLoc = false;
+            _maxScan = 0;
         }
 
         void _initTop( const BSONObj& top ){
@@ -268,6 +273,11 @@ namespace mongo {
                     _hint = e;
                 else if ( strcmp( "$returnKey" , name ) == 0 )
                     _returnKey = e.trueValue();
+                else if ( strcmp( "$maxScan" , name ) == 0 )
+                    _maxScan = e.numberInt();
+                else if ( strcmp( "$showDiskLoc" , name ) == 0 )
+                    _showDiskLoc = e.trueValue();
+                
 
             }
 
@@ -302,12 +312,14 @@ namespace mongo {
         bool _explain;
         bool _snapshot;
         bool _returnKey;
+        bool _showDiskLoc;
         BSONObj _min;
         BSONObj _max;
         BSONElement _hint;
         BSONObj _order;
+        int _maxScan;
     };
-
+    
 
 } // namespace mongo
 
diff --git a/db/queryoptimizer.cpp b/db/queryoptimizer.cpp
index fa08323..3d4cbd0 100644
--- a/db/queryoptimizer.cpp
+++ b/db/queryoptimizer.cpp
@@ -16,13 +16,14 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 
 #include "db.h"
 #include "btree.h"
 #include "pdfile.h"
 #include "queryoptimizer.h"
 #include "cmdline.h"
+#include "clientcursor.h"
 
 //#define DEBUGQO(x) cout << x << endl;
 #define DEBUGQO(x)
@@ -51,9 +52,10 @@ namespace mongo {
     
     QueryPlan::QueryPlan( 
         NamespaceDetails *_d, int _idxNo,
-        const FieldRangeSet &fbs, const BSONObj &order, const BSONObj &startKey, const BSONObj &endKey , string special ) :
+        const FieldRangeSet &fbs, const BSONObj &originalQuery, const BSONObj &order, const BSONObj &startKey, const BSONObj &endKey , string special ) :
     d(_d), idxNo(_idxNo),
     fbs_( fbs ),
+    _originalQuery( originalQuery ),
     order_( order ),
     index_( 0 ),
     optimal_( false ),
@@ -63,7 +65,8 @@ namespace mongo {
     endKeyInclusive_( endKey.isEmpty() ),
     unhelpful_( false ),
     _special( special ),
-    _type(0){
+    _type(0),
+    _startOrEndSpec( !startKey.isEmpty() || !endKey.isEmpty() ){
 
         if ( !fbs_.matchPossible() ) {
             unhelpful_ = true;
@@ -84,7 +87,8 @@ namespace mongo {
             optimal_ = true;
             _type  = index_->getSpec().getType();
             massert( 13040 , (string)"no type for special: " + _special , _type );
-            scanAndOrderRequired_ = _type->scanAndOrderRequired( fbs.query() , order );
+            // hopefully safe to use original query in these contexts - don't think we can mix special with $or clause separation yet
+            scanAndOrderRequired_ = _type->scanAndOrderRequired( _originalQuery , order );
             return;
         }
 
@@ -153,38 +157,39 @@ namespace mongo {
         if ( exactIndexedQueryCount == fbs.nNontrivialRanges() &&
             orderFieldsUnindexed.size() == 0 &&
             exactIndexedQueryCount == index_->keyPattern().nFields() &&
-            exactIndexedQueryCount == fbs.query().nFields() ) {
+            exactIndexedQueryCount == _originalQuery.nFields() ) {
             exactKeyMatch_ = true;
         }
-        indexBounds_ = fbs.indexBounds( idxKey, direction_ );
-        if ( !startKey.isEmpty() || !endKey.isEmpty() ) {
+        _frv.reset( new FieldRangeVector( fbs, idxKey, direction_ ) );
+        if ( _startOrEndSpec ) {
             BSONObj newStart, newEnd;
             if ( !startKey.isEmpty() )
-                newStart = startKey;
+                _startKey = startKey;
             else
-                newStart = indexBounds_[ 0 ].first;
+                _startKey = _frv->startKey();
             if ( !endKey.isEmpty() )
-                newEnd = endKey;
+                _endKey = endKey;
             else
-                newEnd = indexBounds_[ indexBounds_.size() - 1 ].second;
-            BoundList newBounds;
-            newBounds.push_back( make_pair( newStart, newEnd ) );
-            indexBounds_ = newBounds;
+                _endKey = _frv->endKey();
         }
+
         if ( ( scanAndOrderRequired_ || order_.isEmpty() ) &&
-            !fbs.range( idxKey.firstElement().fieldName() ).nontrivial() )
+            !fbs.range( idxKey.firstElement().fieldName() ).nontrivial() ) {
             unhelpful_ = true;
+        }
     }
     
-    auto_ptr< Cursor > QueryPlan::newCursor( const DiskLoc &startLoc , int numWanted ) const {
+    shared_ptr<Cursor> QueryPlan::newCursor( const DiskLoc &startLoc , int numWanted ) const {
 
-        if ( _type )
-            return _type->newCursor( fbs_.query() , order_ , numWanted );
+        if ( _type ) {
+            // hopefully safe to use original query in these contexts - don't think we can mix type with $or clause separation yet   
+            return _type->newCursor( _originalQuery , order_ , numWanted );
+        }
         
         if ( !fbs_.matchPossible() ){
             if ( fbs_.nNontrivialRanges() )
                 checkTableScanAllowed( fbs_.ns() );
-            return auto_ptr< Cursor >( new BasicCursor( DiskLoc() ) );
+            return shared_ptr<Cursor>( new BasicCursor( DiskLoc() ) );
         }
         if ( !index_ ){
             if ( fbs_.nNontrivialRanges() )
@@ -194,17 +199,19 @@ namespace mongo {
 
         massert( 10363 ,  "newCursor() with start location not implemented for indexed plans", startLoc.isNull() );
         
-        if ( indexBounds_.size() < 2 ) {
+        if ( _startOrEndSpec ) {
             // we are sure to spec endKeyInclusive_
-            return auto_ptr< Cursor >( new BtreeCursor( d, idxNo, *index_, indexBounds_[ 0 ].first, indexBounds_[ 0 ].second, endKeyInclusive_, direction_ >= 0 ? 1 : -1 ) );
+            return shared_ptr<Cursor>( new BtreeCursor( d, idxNo, *index_, _startKey, _endKey, endKeyInclusive_, direction_ >= 0 ? 1 : -1 ) );
+        } else if ( index_->getSpec().getType() ) {
+            return shared_ptr<Cursor>( new BtreeCursor( d, idxNo, *index_, _frv->startKey(), _frv->endKey(), true, direction_ >= 0 ? 1 : -1 ) );            
         } else {
-            return auto_ptr< Cursor >( new BtreeCursor( d, idxNo, *index_, indexBounds_, direction_ >= 0 ? 1 : -1 ) );
+            return shared_ptr<Cursor>( new BtreeCursor( d, idxNo, *index_, _frv, direction_ >= 0 ? 1 : -1 ) );
         }
     }
     
-    auto_ptr< Cursor > QueryPlan::newReverseCursor() const {
+    shared_ptr<Cursor> QueryPlan::newReverseCursor() const {
         if ( !fbs_.matchPossible() )
-            return auto_ptr< Cursor >( new BasicCursor( DiskLoc() ) );
+            return shared_ptr<Cursor>( new BasicCursor( DiskLoc() ) );
         if ( !index_ ) {
             int orderSpec = order_.getIntField( "$natural" );
             if ( orderSpec == INT_MIN )
@@ -212,7 +219,7 @@ namespace mongo {
             return findTableScan( fbs_.ns(), BSON( "$natural" << -orderSpec ) );
         }
         massert( 10364 ,  "newReverseCursor() not implemented for indexed plans", false );
-        return auto_ptr< Cursor >( 0 );
+        return shared_ptr<Cursor>();
     }
     
     BSONObj QueryPlan::indexKey() const {
@@ -228,10 +235,10 @@ namespace mongo {
         }
     }
     
-    QueryPlanSet::QueryPlanSet( const char *_ns, const BSONObj &query, const BSONObj &order, const BSONElement *hint, bool honorRecordedPlan, const BSONObj &min, const BSONObj &max ) :
+    QueryPlanSet::QueryPlanSet( const char *_ns, auto_ptr< FieldRangeSet > frs, const BSONObj &originalQuery, const BSONObj &order, const BSONElement *hint, bool honorRecordedPlan, const BSONObj &min, const BSONObj &max, bool bestGuessOnly, bool mayYield ) :
     ns(_ns),
-    query_( query.getOwned() ),
-    fbs_( _ns, query ),
+    _originalQuery( originalQuery ),
+    fbs_( frs ),
     mayRecordPlan_( true ),
     usingPrerecordedPlan_( false ),
     hint_( BSONObj() ),
@@ -239,7 +246,10 @@ namespace mongo {
     oldNScanned_( 0 ),
     honorRecordedPlan_( honorRecordedPlan ),
     min_( min.getOwned() ),
-    max_( max.getOwned() ) {
+    max_( max.getOwned() ),
+    _bestGuessOnly( bestGuessOnly ),
+    _mayYield( mayYield ),
+    _yieldSometimesTracker( 256, 20 ){
         if ( hint && !hint->eoo() ) {
             hint_ = hint->wrap();
         }
@@ -251,59 +261,70 @@ namespace mongo {
             string errmsg;
             BSONObj keyPattern = id.keyPattern();
             // This reformats min_ and max_ to be used for index lookup.
-            massert( 10365 ,  errmsg, indexDetailsForRange( fbs_.ns(), errmsg, min_, max_, keyPattern ) );
+            massert( 10365 ,  errmsg, indexDetailsForRange( fbs_->ns(), errmsg, min_, max_, keyPattern ) );
         }
         NamespaceDetails *d = nsdetails(ns);
-        plans_.push_back( PlanPtr( new QueryPlan( d, d->idxNo(id), fbs_, order_, min_, max_ ) ) );
+        plans_.push_back( PlanPtr( new QueryPlan( d, d->idxNo(id), *fbs_, _originalQuery, order_, min_, max_ ) ) );
+    }
+    
+    // returns an IndexDetails * for a hint, 0 if hint is $natural.
+    // hint must not be eoo()
+    IndexDetails *parseHint( const BSONElement &hint, NamespaceDetails *d ) {
+        massert( 13292, "hint eoo", !hint.eoo() );
+        if( hint.type() == String ) {
+            string hintstr = hint.valuestr();
+            NamespaceDetails::IndexIterator i = d->ii();
+            while( i.more() ) {
+                IndexDetails& ii = i.next();
+                if ( ii.indexName() == hintstr ) {
+                    return &ii;
+                }
+            }
+        }
+        else if( hint.type() == Object ) { 
+            BSONObj hintobj = hint.embeddedObject();
+            uassert( 10112 ,  "bad hint", !hintobj.isEmpty() );
+            if ( !strcmp( hintobj.firstElement().fieldName(), "$natural" ) ) {
+                return 0;
+            }
+            NamespaceDetails::IndexIterator i = d->ii();
+            while( i.more() ) {
+                IndexDetails& ii = i.next();
+                if( ii.keyPattern().woCompare(hintobj) == 0 ) {
+                    return &ii;
+                }
+            }
+        }        
+        uassert( 10113 ,  "bad hint", false );
+        return 0;
     }
     
     void QueryPlanSet::init() {
-        DEBUGQO( "QueryPlanSet::init " << ns << "\t" << query_ );
+        DEBUGQO( "QueryPlanSet::init " << ns << "\t" << _originalQuery );
         plans_.clear();
         mayRecordPlan_ = true;
         usingPrerecordedPlan_ = false;
         
-        const char *ns = fbs_.ns();
+        const char *ns = fbs_->ns();
         NamespaceDetails *d = nsdetails( ns );
-        if ( !d || !fbs_.matchPossible() ) {
+        if ( !d || !fbs_->matchPossible() ) {
             // Table scan plan, when no matches are possible
-            plans_.push_back( PlanPtr( new QueryPlan( d, -1, fbs_, order_ ) ) );
+            plans_.push_back( PlanPtr( new QueryPlan( d, -1, *fbs_, _originalQuery, order_ ) ) );
             return;
         }
         
         BSONElement hint = hint_.firstElement();
         if ( !hint.eoo() ) {
             mayRecordPlan_ = false;
-            if( hint.type() == String ) {
-                string hintstr = hint.valuestr();
-                NamespaceDetails::IndexIterator i = d->ii();
-                while( i.more() ) {
-                    IndexDetails& ii = i.next();
-                    if ( ii.indexName() == hintstr ) {
-                        addHint( ii );
-                        return;
-                    }
-                }
-            }
-            else if( hint.type() == Object ) { 
-                BSONObj hintobj = hint.embeddedObject();
-                uassert( 10112 ,  "bad hint", !hintobj.isEmpty() );
-                if ( !strcmp( hintobj.firstElement().fieldName(), "$natural" ) ) {
-                    massert( 10366 ,  "natural order cannot be specified with $min/$max", min_.isEmpty() && max_.isEmpty() );
-                    // Table scan plan
-                    plans_.push_back( PlanPtr( new QueryPlan( d, -1, fbs_, order_ ) ) );
-                    return;
-                }
-                NamespaceDetails::IndexIterator i = d->ii();
-                while( i.more() ) {
-                    IndexDetails& ii = i.next();
-                    if( ii.keyPattern().woCompare(hintobj) == 0 ) {
-                        addHint( ii );
-                        return;
-                    }
-                }
+            IndexDetails *id = parseHint( hint, d );
+            if ( id ) {
+                addHint( *id );
+            } else {
+                massert( 10366 ,  "natural order cannot be specified with $min/$max", min_.isEmpty() && max_.isEmpty() );
+                // Table scan plan
+                plans_.push_back( PlanPtr( new QueryPlan( d, -1, *fbs_, _originalQuery, order_ ) ) );                
             }
-            uassert( 10113 ,  "bad hint", false );
+            return;
         }
         
         if ( !min_.isEmpty() || !max_.isEmpty() ) {
@@ -311,56 +332,54 @@ namespace mongo {
             BSONObj keyPattern;
             IndexDetails *idx = indexDetailsForRange( ns, errmsg, min_, max_, keyPattern );
             massert( 10367 ,  errmsg, idx );
-            plans_.push_back( PlanPtr( new QueryPlan( d, d->idxNo(*idx), fbs_, order_, min_, max_ ) ) );
+            plans_.push_back( PlanPtr( new QueryPlan( d, d->idxNo(*idx), *fbs_, _originalQuery, order_, min_, max_ ) ) );
             return;
         }
 
-        if ( isSimpleIdQuery( query_ ) ){
+        if ( isSimpleIdQuery( _originalQuery ) ){
             int idx = d->findIdIndex();
             if ( idx >= 0 ){
                 usingPrerecordedPlan_ = true;
                 mayRecordPlan_ = false;
-                plans_.push_back( PlanPtr( new QueryPlan( d , idx , fbs_ , order_ ) ) );
+                plans_.push_back( PlanPtr( new QueryPlan( d , idx , *fbs_ , _originalQuery, order_ ) ) );
                 return;
             }
         }
 
-        if ( query_.isEmpty() && order_.isEmpty() ){
-            plans_.push_back( PlanPtr( new QueryPlan( d, -1, fbs_, order_ ) ) );
+        if ( _originalQuery.isEmpty() && order_.isEmpty() ){
+            plans_.push_back( PlanPtr( new QueryPlan( d, -1, *fbs_, _originalQuery, order_ ) ) );
             return;
         }
 
-        DEBUGQO( "\t special : " << fbs_.getSpecial() );
-        if ( fbs_.getSpecial().size() ){
-            _special = fbs_.getSpecial();
+        DEBUGQO( "\t special : " << fbs_->getSpecial() );
+        if ( fbs_->getSpecial().size() ){
+            _special = fbs_->getSpecial();
             NamespaceDetails::IndexIterator i = d->ii();
             while( i.more() ) {
                 int j = i.pos();
                 IndexDetails& ii = i.next();
                 const IndexSpec& spec = ii.getSpec();
-                if ( spec.getTypeName() == _special && spec.suitability( query_ , order_ ) ){
+                if ( spec.getTypeName() == _special && spec.suitability( _originalQuery , order_ ) ){
                     usingPrerecordedPlan_ = true;
                     mayRecordPlan_ = true;
-                    plans_.push_back( PlanPtr( new QueryPlan( d , j , fbs_ , order_ , 
+                    plans_.push_back( PlanPtr( new QueryPlan( d , j , *fbs_ , _originalQuery, order_ , 
                                                               BSONObj() , BSONObj() , _special ) ) );
                     return;
                 }
             }
-            uassert( 13038 , (string)"can't find special index: " + _special + " for: " + query_.toString() , 0 );
+            uassert( 13038 , (string)"can't find special index: " + _special + " for: " + _originalQuery.toString() , 0 );
         }
 
         if ( honorRecordedPlan_ ) {
             scoped_lock lk(NamespaceDetailsTransient::_qcMutex);
             NamespaceDetailsTransient& nsd = NamespaceDetailsTransient::get_inlock( ns );
-            BSONObj bestIndex = nsd.indexForPattern( fbs_.pattern( order_ ) );
+            BSONObj bestIndex = nsd.indexForPattern( fbs_->pattern( order_ ) );
             if ( !bestIndex.isEmpty() ) {
-                usingPrerecordedPlan_ = true;
-                mayRecordPlan_ = false;
-                oldNScanned_ = nsd.nScannedForPattern( fbs_.pattern( order_ ) );
+                PlanPtr p;
+                oldNScanned_ = nsd.nScannedForPattern( fbs_->pattern( order_ ) );
                 if ( !strcmp( bestIndex.firstElement().fieldName(), "$natural" ) ) {
                     // Table scan plan
-                    plans_.push_back( PlanPtr( new QueryPlan( d, -1, fbs_, order_ ) ) );
-                    return;
+                    p.reset( new QueryPlan( d, -1, *fbs_, _originalQuery, order_ ) );
                 }
 
                 NamespaceDetails::IndexIterator i = d->ii();
@@ -368,11 +387,17 @@ namespace mongo {
                     int j = i.pos();
                     IndexDetails& ii = i.next();
                     if( ii.keyPattern().woCompare(bestIndex) == 0 ) {
-                        plans_.push_back( PlanPtr( new QueryPlan( d, j, fbs_, order_ ) ) );
-                        return;
+                        p.reset( new QueryPlan( d, j, *fbs_, _originalQuery, order_ ) );
                     }
                 }
-                massert( 10368 ,  "Unable to locate previously recorded index", false );
+
+                massert( 10368 ,  "Unable to locate previously recorded index", p.get() );
+                if ( !( _bestGuessOnly && p->scanAndOrderRequired() ) ) {
+                    usingPrerecordedPlan_ = true;
+                    mayRecordPlan_ = false;
+                    plans_.push_back( p );
+                    return;
+                }
             }
         }
         
@@ -380,16 +405,16 @@ namespace mongo {
     }
     
     void QueryPlanSet::addOtherPlans( bool checkFirst ) {
-        const char *ns = fbs_.ns();
+        const char *ns = fbs_->ns();
         NamespaceDetails *d = nsdetails( ns );
         if ( !d )
             return;
 
         // If table scan is optimal or natural order requested or tailable cursor requested
-        if ( !fbs_.matchPossible() || ( fbs_.nNontrivialRanges() == 0 && order_.isEmpty() ) ||
+        if ( !fbs_->matchPossible() || ( fbs_->nNontrivialRanges() == 0 && order_.isEmpty() ) ||
             ( !order_.isEmpty() && !strcmp( order_.firstElement().fieldName(), "$natural" ) ) ) {
             // Table scan plan
-            addPlan( PlanPtr( new QueryPlan( d, -1, fbs_, order_ ) ), checkFirst );
+            addPlan( PlanPtr( new QueryPlan( d, -1, *fbs_, _originalQuery, order_ ) ), checkFirst );
             return;
         }
         
@@ -401,12 +426,12 @@ namespace mongo {
             const IndexSpec& spec = id.getSpec();
             IndexSuitability suitability = HELPFUL;
             if ( normalQuery ){
-                suitability = spec.suitability( query_ , order_ );
+                suitability = spec.suitability( fbs_->simplifiedQuery() , order_ );
                 if ( suitability == USELESS )
                     continue;
             }
 
-            PlanPtr p( new QueryPlan( d, i, fbs_, order_ ) );
+            PlanPtr p( new QueryPlan( d, i, *fbs_, _originalQuery, order_ ) );
             if ( p->optimal() ) {
                 addPlan( p, checkFirst );
                 return;
@@ -418,7 +443,7 @@ namespace mongo {
             addPlan( *i, checkFirst );
 
         // Table scan plan
-        addPlan( PlanPtr( new QueryPlan( d, -1, fbs_, order_ ) ), checkFirst );
+        addPlan( PlanPtr( new QueryPlan( d, -1, *fbs_, _originalQuery, order_ ) ), checkFirst );
     }
     
     shared_ptr< QueryOp > QueryPlanSet::runOp( QueryOp &op ) {
@@ -426,11 +451,11 @@ namespace mongo {
             Runner r( *this, op );
             shared_ptr< QueryOp > res = r.run();
             // plans_.size() > 1 if addOtherPlans was called in Runner::run().
-            if ( res->complete() || plans_.size() > 1 )
+            if ( _bestGuessOnly || res->complete() || plans_.size() > 1 )
                 return res;
             {
                 scoped_lock lk(NamespaceDetailsTransient::_qcMutex);
-                NamespaceDetailsTransient::get_inlock( fbs_.ns() ).registerIndexForPattern( fbs_.pattern( order_ ), BSONObj(), 0 );
+                NamespaceDetailsTransient::get_inlock( fbs_->ns() ).registerIndexForPattern( fbs_->pattern( order_ ), BSONObj(), 0 );
             }
             init();
         }
@@ -441,33 +466,80 @@ namespace mongo {
     BSONObj QueryPlanSet::explain() const {
         vector< BSONObj > arr;
         for( PlanSet::const_iterator i = plans_.begin(); i != plans_.end(); ++i ) {
-            auto_ptr< Cursor > c = (*i)->newCursor();
+            shared_ptr<Cursor> c = (*i)->newCursor();
             BSONObjBuilder explain;
             explain.append( "cursor", c->toString() );
-            explain.appendArray( "indexBounds", c->prettyIndexBounds() );
+            explain.append( "indexBounds", c->prettyIndexBounds() );
             arr.push_back( explain.obj() );
         }
         BSONObjBuilder b;
         b.append( "allPlans", arr );
         return b.obj();
     }
+
+    QueryPlanSet::PlanPtr QueryPlanSet::getBestGuess() const {
+        assert( plans_.size() );
+        if ( plans_[ 0 ]->scanAndOrderRequired() ){
+            for ( unsigned i=1; i<plans_.size(); i++ ){
+                if ( ! plans_[i]->scanAndOrderRequired() )
+                    return plans_[i];
+            }
+            
+            stringstream ss;
+            ss << "best guess plan requested, but scan and order required:";
+            ss << " query: " << fbs_->simplifiedQuery();
+            ss << " order: " << order_;
+            ss << " choices: ";
+            for ( unsigned i=0; i<plans_.size(); i++ ){
+                ss << plans_[i]->indexKey() << " ";
+            }
+
+            string s = ss.str();
+            msgassertedNoTrace( 13284, s.c_str() );
+        }
+        return plans_[0];
+    }
     
     QueryPlanSet::Runner::Runner( QueryPlanSet &plans, QueryOp &op ) :
     op_( op ),
     plans_( plans ) {
     }
     
+    void QueryPlanSet::Runner::mayYield( const vector< shared_ptr< QueryOp > > &ops ) {
+        if ( plans_._mayYield ) {
+            if ( plans_._yieldSometimesTracker.ping() ) {
+                int micros = ClientCursor::yieldSuggest();
+                if ( micros > 0 ) {
+                    for( vector< shared_ptr< QueryOp > >::const_iterator i = ops.begin(); i != ops.end(); ++i ) {
+                        if ( !prepareToYield( **i ) ) {
+                            return;
+                        }
+                    }
+                    ClientCursor::staticYield( micros );
+                    for( vector< shared_ptr< QueryOp > >::const_iterator i = ops.begin(); i != ops.end(); ++i ) {
+                        recoverFromYield( **i );
+                    }                        
+                }
+            }
+        }        
+    }
+    
     shared_ptr< QueryOp > QueryPlanSet::Runner::run() {
         massert( 10369 ,  "no plans", plans_.plans_.size() > 0 );
         
-        if ( plans_.plans_.size() > 1 )
-            log(1) << "  running multiple plans" << endl;
-
         vector< shared_ptr< QueryOp > > ops;
-        for( PlanSet::iterator i = plans_.plans_.begin(); i != plans_.plans_.end(); ++i ) {
-            shared_ptr< QueryOp > op( op_.clone() );
-            op->setQueryPlan( i->get() );
-            ops.push_back( op );
+        if ( plans_._bestGuessOnly ) {
+            shared_ptr< QueryOp > op( op_.createChild() );
+            op->setQueryPlan( plans_.getBestGuess().get() );
+            ops.push_back( op );            
+        } else {
+            if ( plans_.plans_.size() > 1 )
+                log(1) << "  running multiple plans" << endl;            
+            for( PlanSet::iterator i = plans_.plans_.begin(); i != plans_.plans_.end(); ++i ) {
+                shared_ptr< QueryOp > op( op_.createChild() );
+                op->setQueryPlan( i->get() );
+                ops.push_back( op );
+            }
         }
 
         for( vector< shared_ptr< QueryOp > >::iterator i = ops.begin(); i != ops.end(); ++i ) {
@@ -483,13 +555,16 @@ namespace mongo {
             unsigned errCount = 0;
             bool first = true;
             for( vector< shared_ptr< QueryOp > >::iterator i = ops.begin(); i != ops.end(); ++i ) {
+                mayYield( ops );
                 QueryOp &op = **i;
                 nextOp( op );
                 if ( op.complete() ) {
-                    if ( first )
+                    if ( first ) {
                         nScanned += nScannedBackup;
-                    if ( plans_.mayRecordPlan_ && op.mayRecordPlan() )
+                    }
+                    if ( plans_.mayRecordPlan_ && op.mayRecordPlan() ) {
                         op.qp().registerSelf( nScanned );
+                    }
                     return *i;
                 }
                 if ( op.error() )
@@ -498,12 +573,12 @@ namespace mongo {
             }
             if ( errCount == ops.size() )
                 break;
-            if ( plans_.usingPrerecordedPlan_ && nScanned > plans_.oldNScanned_ * 10 && plans_._special.empty() ) {
+            if ( !plans_._bestGuessOnly && plans_.usingPrerecordedPlan_ && nScanned > plans_.oldNScanned_ * 10 && plans_._special.empty() ) {
                 plans_.addOtherPlans( true );
                 PlanSet::iterator i = plans_.plans_.begin();
                 ++i;
                 for( ; i != plans_.plans_.end(); ++i ) {
-                    shared_ptr< QueryOp > op( op_.clone() );
+                    shared_ptr< QueryOp > op( op_.createChild() );
                     op->setQueryPlan( i->get() );
                     ops.push_back( op );
                     initOp( *op );
@@ -519,27 +594,143 @@ namespace mongo {
         return ops[ 0 ];
     }
     
+#define GUARD_OP_EXCEPTION( op, expression ) \
+    try { \
+        expression; \
+    } \
+    catch ( DBException& e ) { \
+        op.setException( e.getInfo() ); \
+    } \
+    catch ( const std::exception &e ) { \
+        op.setException( ExceptionInfo( e.what() , 0 ) ); \
+    } \
+    catch ( ... ) { \
+        op.setException( ExceptionInfo( "Caught unknown exception" , 0 ) ); \
+    }
+        
+    
     void QueryPlanSet::Runner::initOp( QueryOp &op ) {
-        try {
-            op.init();
-        } catch ( const std::exception &e ) {
-            op.setExceptionMessage( e.what() );
-        } catch ( ... ) {
-            op.setExceptionMessage( "Caught unknown exception" );
-        }        
+        GUARD_OP_EXCEPTION( op, op.init() );
     }
 
     void QueryPlanSet::Runner::nextOp( QueryOp &op ) {
-        try {
-            if ( !op.error() )
-                op.next();
-        } catch ( const std::exception &e ) {
-            op.setExceptionMessage( e.what() );
-        } catch ( ... ) {
-            op.setExceptionMessage( "Caught unknown exception" );
-        }        
+        GUARD_OP_EXCEPTION( op, if ( !op.error() ) { op.next(); } );
+    }
+
+    bool QueryPlanSet::Runner::prepareToYield( QueryOp &op ) {
+        GUARD_OP_EXCEPTION( op,
+                           if ( op.error() ) {
+                               return true;
+                           } else {
+                               return op.prepareToYield();
+                           } );
+        return true;
     }
 
+    void QueryPlanSet::Runner::recoverFromYield( QueryOp &op ) {
+        GUARD_OP_EXCEPTION( op, if ( !op.error() ) { op.recoverFromYield(); } );
+    }
+    
+    
+    MultiPlanScanner::MultiPlanScanner( const char *ns,
+                                       const BSONObj &query,
+                                       const BSONObj &order,
+                                       const BSONElement *hint,
+                                       bool honorRecordedPlan,
+                                       const BSONObj &min,
+                                       const BSONObj &max,
+                                       bool bestGuessOnly,
+                                       bool mayYield ) :
+    _ns( ns ),
+    _or( !query.getField( "$or" ).eoo() ),
+    _query( query.getOwned() ),
+    _fros( ns, _query ),
+    _i(),
+    _honorRecordedPlan( honorRecordedPlan ),
+    _bestGuessOnly( bestGuessOnly ),
+    _hint( ( hint && !hint->eoo() ) ? hint->wrap() : BSONObj() ),
+    _mayYield( mayYield ),
+    _tableScanned()
+    {
+        if ( !order.isEmpty() || !min.isEmpty() || !max.isEmpty() || !_fros.getSpecial().empty() ) {
+            _or = false;
+        }
+        if ( _or && uselessOr( _hint.firstElement() ) ) {
+            _or = false;
+        }
+        // if _or == false, don't use or clauses for index selection
+        if ( !_or ) {
+            auto_ptr< FieldRangeSet > frs( new FieldRangeSet( ns, _query ) );
+            _currentQps.reset( new QueryPlanSet( ns, frs, _query, order, hint, honorRecordedPlan, min, max, _bestGuessOnly, _mayYield ) );
+        } else {
+            BSONElement e = _query.getField( "$or" );
+            massert( 13268, "invalid $or spec", e.type() == Array && e.embeddedObject().nFields() > 0 );
+        }
+    }
+
+    shared_ptr< QueryOp > MultiPlanScanner::runOpOnce( QueryOp &op ) {
+        massert( 13271, "can't run more ops", mayRunMore() );
+        if ( !_or ) {
+            ++_i;
+            return _currentQps->runOp( op );
+        }
+        ++_i;
+        auto_ptr< FieldRangeSet > frs( _fros.topFrs() );
+        BSONElement hintElt = _hint.firstElement();
+        _currentQps.reset( new QueryPlanSet( _ns, frs, _query, BSONObj(), &hintElt, _honorRecordedPlan, BSONObj(), BSONObj(), _bestGuessOnly, _mayYield ) );
+        shared_ptr< QueryOp > ret( _currentQps->runOp( op ) );
+        if ( ret->qp().willScanTable() ) {
+            _tableScanned = true;
+        }
+        _fros.popOrClause();
+        return ret;
+    }
+    
+    shared_ptr< QueryOp > MultiPlanScanner::runOp( QueryOp &op ) {
+        shared_ptr< QueryOp > ret = runOpOnce( op );
+        while( !ret->stopRequested() && mayRunMore() ) {
+            ret = runOpOnce( *ret );
+        }
+        return ret;
+    }
+    
+    bool MultiPlanScanner::uselessOr( const BSONElement &hint ) const {
+        NamespaceDetails *nsd = nsdetails( _ns );
+        if ( !nsd ) {
+            return true;
+        }
+        IndexDetails *id = 0;
+        if ( !hint.eoo() ) {
+            IndexDetails *id = parseHint( hint, nsd );
+            if ( !id ) {
+                return true;
+            }
+        }
+        vector< BSONObj > ret;
+        _fros.allClausesSimplified( ret );
+        for( vector< BSONObj >::const_iterator i = ret.begin(); i != ret.end(); ++i ) {
+            if ( id ) {
+                if ( id->getSpec().suitability( *i, BSONObj() ) == USELESS ) {
+                    return true;
+                }
+            } else {
+                bool useful = false;
+                NamespaceDetails::IndexIterator j = nsd->ii();
+                while( j.more() ) {
+                    IndexDetails &id = j.next();
+                    if ( id.getSpec().suitability( *i, BSONObj() ) != USELESS ) {
+                        useful = true;
+                        break;
+                    }
+                }
+                if ( !useful ) {
+                    return true;
+                }       
+            }
+        }
+        return false;
+    }
+    
     bool indexWorks( const BSONObj &idxPattern, const BSONObj &sampleKey, int direction, int firstSignificantField ) {
         BSONObjIterator p( idxPattern );
         BSONObjIterator k( sampleKey );
diff --git a/db/queryoptimizer.h b/db/queryoptimizer.h
index 1cb5052..8314bfa 100644
--- a/db/queryoptimizer.h
+++ b/db/queryoptimizer.h
@@ -21,6 +21,8 @@
 #include "cursor.h"
 #include "jsobj.h"
 #include "queryutil.h"
+#include "matcher.h"
+#include "../util/message.h"
 
 namespace mongo {
     
@@ -32,6 +34,7 @@ namespace mongo {
         QueryPlan(NamespaceDetails *_d, 
                   int _idxNo, // -1 = no index
                   const FieldRangeSet &fbs,
+                  const BSONObj &originalQuery,
                   const BSONObj &order,
                   const BSONObj &startKey = BSONObj(),
                   const BSONObj &endKey = BSONObj() ,
@@ -49,32 +52,36 @@ namespace mongo {
            requested sort order */
         bool unhelpful() const { return unhelpful_; }
         int direction() const { return direction_; }
-        auto_ptr< Cursor > newCursor( const DiskLoc &startLoc = DiskLoc() , int numWanted=0 ) const;
-        auto_ptr< Cursor > newReverseCursor() const;
+        shared_ptr<Cursor> newCursor( const DiskLoc &startLoc = DiskLoc() , int numWanted=0 ) const;
+        shared_ptr<Cursor> newReverseCursor() const;
         BSONObj indexKey() const;
+        bool willScanTable() const { return !index_ && fbs_.matchPossible(); }
         const char *ns() const { return fbs_.ns(); }
         NamespaceDetails *nsd() const { return d; }
-        BSONObj query() const { return fbs_.query(); }
+        BSONObj originalQuery() const { return _originalQuery; }
         BSONObj simplifiedQuery( const BSONObj& fields = BSONObj() ) const { return fbs_.simplifiedQuery( fields ); }
         const FieldRange &range( const char *fieldName ) const { return fbs_.range( fieldName ); }
         void registerSelf( long long nScanned ) const;
-        // just for testing
-        BoundList indexBounds() const { return indexBounds_; }
+        shared_ptr< FieldRangeVector > frv() const { return _frv; }
     private:
         NamespaceDetails *d;
         int idxNo;
         const FieldRangeSet &fbs_;
+        const BSONObj &_originalQuery;
         const BSONObj &order_;
         const IndexDetails *index_;
         bool optimal_;
         bool scanAndOrderRequired_;
         bool exactKeyMatch_;
         int direction_;
-        BoundList indexBounds_;
+        shared_ptr< FieldRangeVector > _frv;
+        BSONObj _startKey;
+        BSONObj _endKey;
         bool endKeyInclusive_;
         bool unhelpful_;
         string _special;
         IndexType * _type;
+        bool _startOrEndSpec;
     };
 
     // Inherit from this interface to implement a new query operation.
@@ -82,61 +89,113 @@ namespace mongo {
     // each clone its own query plan.
     class QueryOp {
     public:
-        QueryOp() : complete_(), qp_(), error_() {}
+        QueryOp() : _complete(), _stopRequested(), _qp(), _error() {}
+
+        // Used when handing off from one QueryOp type to another
+        QueryOp( const QueryOp &other ) :
+        _complete(), _stopRequested(), _qp(), _error(), _matcher( other._matcher ),
+        _orConstraint( other._orConstraint ) {}
+        
         virtual ~QueryOp() {}
         
-        /** this gets called after a query plan is set? ERH 2/16/10 */
-        virtual void init() = 0;
+        /** these gets called after a query plan is set */
+        void init() { 
+            if ( _oldMatcher.get() ) {
+                _matcher.reset( _oldMatcher->nextClauseMatcher( qp().indexKey() ) );
+            } else {
+                _matcher.reset( new CoveredIndexMatcher( qp().originalQuery(), qp().indexKey(), alwaysUseRecord() ) );
+            }
+            _init();
+        }
         virtual void next() = 0;
+
         virtual bool mayRecordPlan() const = 0;
         
+        virtual bool prepareToYield() { massert( 13335, "yield not supported", false ); return false; }
+        virtual void recoverFromYield() { massert( 13336, "yield not supported", false ); }
+        
         /** @return a copy of the inheriting class, which will be run with its own
-                    query plan.
+                    query plan.  If multiple plan sets are required for an $or query,
+                    the QueryOp of the winning plan from a given set will be cloned
+                    to generate QueryOps for the subsequent plan set.  This function
+                    should only be called after the query op has completed executing.
         */
-        virtual QueryOp *clone() const = 0;
-        bool complete() const { return complete_; }
-        bool error() const { return error_; }
-        string exceptionMessage() const { return exceptionMessage_; }
-        const QueryPlan &qp() const { return *qp_; }
+        QueryOp *createChild() {
+            if( _orConstraint.get() ) {
+                _matcher->advanceOrClause( _orConstraint );
+                _orConstraint.reset();
+            }
+            QueryOp *ret = _createChild();
+            ret->_oldMatcher = _matcher;
+            return ret;
+        }
+        bool complete() const { return _complete; }
+        bool error() const { return _error; }
+        bool stopRequested() const { return _stopRequested; }
+        ExceptionInfo exception() const { return _exception; }
+        const QueryPlan &qp() const { return *_qp; }
         // To be called by QueryPlanSet::Runner only.
-        void setQueryPlan( const QueryPlan *qp ) { qp_ = qp; }
-        void setExceptionMessage( const string &exceptionMessage ) {
-            error_ = true;
-            exceptionMessage_ = exceptionMessage;
+        void setQueryPlan( const QueryPlan *qp ) { _qp = qp; }
+        void setException( const DBException &e ) {
+            _error = true;
+            _exception = e.getInfo();
         }
+        shared_ptr< CoveredIndexMatcher > matcher() const { return _matcher; }
     protected:
-        void setComplete() { complete_ = true; }
+        void setComplete() {
+            _orConstraint = qp().frv();
+            _complete = true;
+        }
+        void setStop() { setComplete(); _stopRequested = true; }
+
+        virtual void _init() = 0;
+        
+        virtual QueryOp *_createChild() const = 0;
+        
+        virtual bool alwaysUseRecord() const { return false; }
+    
     private:
-        bool complete_;
-        string exceptionMessage_;
-        const QueryPlan *qp_;
-        bool error_;
+        bool _complete;
+        bool _stopRequested;
+        ExceptionInfo _exception;
+        const QueryPlan *_qp;
+        bool _error;
+        shared_ptr< CoveredIndexMatcher > _matcher;
+        shared_ptr< CoveredIndexMatcher > _oldMatcher;
+        shared_ptr< FieldRangeVector > _orConstraint;
     };
     
     // Set of candidate query plans for a particular query.  Used for running
     // a QueryOp on these plans.
     class QueryPlanSet {
     public:
+
+        typedef boost::shared_ptr< QueryPlan > PlanPtr;
+        typedef vector< PlanPtr > PlanSet;
+
         QueryPlanSet( const char *ns,
-                     const BSONObj &query,
+                     auto_ptr< FieldRangeSet > frs,
+                     const BSONObj &originalQuery,
                      const BSONObj &order,
                      const BSONElement *hint = 0,
                      bool honorRecordedPlan = true,
                      const BSONObj &min = BSONObj(),
-                     const BSONObj &max = BSONObj() );
+                     const BSONObj &max = BSONObj(),
+                     bool bestGuessOnly = false,
+                     bool mayYield = false);
         int nPlans() const { return plans_.size(); }
         shared_ptr< QueryOp > runOp( QueryOp &op );
         template< class T >
         shared_ptr< T > runOp( T &op ) {
             return dynamic_pointer_cast< T >( runOp( static_cast< QueryOp& >( op ) ) );
         }
-        const FieldRangeSet &fbs() const { return fbs_; }
         BSONObj explain() const;
         bool usingPrerecordedPlan() const { return usingPrerecordedPlan_; }
+        PlanPtr getBestGuess() const;
+        //for testing
+        const FieldRangeSet &fbs() const { return *fbs_; }
     private:
         void addOtherPlans( bool checkFirst );
-        typedef boost::shared_ptr< QueryPlan > PlanPtr;
-        typedef vector< PlanPtr > PlanSet;
         void addPlan( PlanPtr plan, bool checkFirst ) {
             if ( checkFirst && plan->indexKey().woCompare( plans_[ 0 ]->indexKey() ) == 0 )
                 return;
@@ -147,14 +206,17 @@ namespace mongo {
         struct Runner {
             Runner( QueryPlanSet &plans, QueryOp &op );
             shared_ptr< QueryOp > run();
+            void mayYield( const vector< shared_ptr< QueryOp > > &ops );
             QueryOp &op_;
             QueryPlanSet &plans_;
             static void initOp( QueryOp &op );
             static void nextOp( QueryOp &op );
+            static bool prepareToYield( QueryOp &op );
+            static void recoverFromYield( QueryOp &op );
         };
         const char *ns;
-        BSONObj query_;
-        FieldRangeSet fbs_;
+        BSONObj _originalQuery;
+        auto_ptr< FieldRangeSet > fbs_;
         PlanSet plans_;
         bool mayRecordPlan_;
         bool usingPrerecordedPlan_;
@@ -165,16 +227,196 @@ namespace mongo {
         BSONObj min_;
         BSONObj max_;
         string _special;
+        bool _bestGuessOnly;
+        bool _mayYield;
+        ElapsedTracker _yieldSometimesTracker;
     };
 
+    // Handles $or type queries by generating a QueryPlanSet for each $or clause
+    // NOTE on our $or implementation: In our current qo implementation we don't
+    // keep statistics on our data, but we can conceptualize the problem of
+    // selecting an index when statistics exist for all index ranges.  The
+    // d-hitting set problem on k sets and n elements can be reduced to the
+    // problem of index selection on k $or clauses and n index ranges (where
+    // d is the max number of indexes, and the number of ranges n is unbounded).
+    // In light of the fact that d-hitting set is np complete, and we don't even
+    // track statistics (so cost calculations are expensive) our first
+    // implementation uses the following greedy approach: We take one $or clause
+    // at a time and treat each as a separate query for index selection purposes.
+    // But if an index range is scanned for a particular $or clause, we eliminate
+    // that range from all subsequent clauses.  One could imagine an opposite
+    // implementation where we select indexes based on the union of index ranges
+    // for all $or clauses, but this can have much poorer worst case behavior.
+    // (An index range that suits one $or clause may not suit another, and this
+    // is worse than the typical case of index range choice staleness because
+    // with $or the clauses may likely be logically distinct.)  The greedy
+    // implementation won't do any worse than all the $or clauses individually,
+    // and it can often do better.  In the first cut we are intentionally using
+    // QueryPattern tracking to record successful plans on $or clauses for use by
+    // subsequent $or clauses, even though there may be a significant aggregate
+    // $nor component that would not be represented in QueryPattern.
+    class MultiPlanScanner {
+    public:
+        MultiPlanScanner( const char *ns,
+                         const BSONObj &query,
+                         const BSONObj &order,
+                         const BSONElement *hint = 0,
+                         bool honorRecordedPlan = true,
+                         const BSONObj &min = BSONObj(),
+                         const BSONObj &max = BSONObj(),
+                         bool bestGuessOnly = false,
+                         bool mayYield = false);
+        shared_ptr< QueryOp > runOp( QueryOp &op );
+        template< class T >
+        shared_ptr< T > runOp( T &op ) {
+            return dynamic_pointer_cast< T >( runOp( static_cast< QueryOp& >( op ) ) );
+        }       
+        shared_ptr< QueryOp > runOpOnce( QueryOp &op );
+        template< class T >
+        shared_ptr< T > runOpOnce( T &op ) {
+            return dynamic_pointer_cast< T >( runOpOnce( static_cast< QueryOp& >( op ) ) );
+        }       
+        bool mayRunMore() const { return _or ? ( !_tableScanned && !_fros.orFinished() ) : _i == 0; }
+        BSONObj oldExplain() const { assertNotOr(); return _currentQps->explain(); }
+        // just report this when only one query op
+        bool usingPrerecordedPlan() const {
+            return !_or && _currentQps->usingPrerecordedPlan();
+        }
+        void setBestGuessOnly() { _bestGuessOnly = true; }
+        void mayYield( bool val ) { _mayYield = val; }
+    private:
+        void assertNotOr() const {
+            massert( 13266, "not implemented for $or query", !_or );
+        }
+        bool uselessOr( const BSONElement &hint ) const;
+        const char * _ns;
+        bool _or;
+        BSONObj _query;
+        FieldRangeOrSet _fros;
+        auto_ptr< QueryPlanSet > _currentQps;
+        int _i;
+        bool _honorRecordedPlan;
+        bool _bestGuessOnly;
+        BSONObj _hint;
+        bool _mayYield;
+        bool _tableScanned;
+    };
+    
+    class MultiCursor : public Cursor {
+    public:
+        class CursorOp : public QueryOp {
+        public:
+            CursorOp() {}
+            CursorOp( const QueryOp &other ) : QueryOp( other ) {}
+            virtual shared_ptr< Cursor > newCursor() const = 0;  
+        };
+        // takes ownership of 'op'
+        MultiCursor( const char *ns, const BSONObj &pattern, const BSONObj &order, shared_ptr< CursorOp > op = shared_ptr< CursorOp >(), bool mayYield = false )
+        : _mps( new MultiPlanScanner( ns, pattern, order, 0, true, BSONObj(), BSONObj(), !op.get(), mayYield ) ) {
+            if ( op.get() ) {
+                _op = op;
+            } else {
+                _op.reset( new NoOp() );
+            }
+            if ( _mps->mayRunMore() ) {
+                nextClause();
+                if ( !ok() ) {
+                    advance();
+                }
+            } else {
+                _c.reset( new BasicCursor( DiskLoc() ) );
+            }
+        }
+        // used to handoff a query to a getMore()
+        MultiCursor( auto_ptr< MultiPlanScanner > mps, const shared_ptr< Cursor > &c, const shared_ptr< CoveredIndexMatcher > &matcher, const QueryOp &op )
+        : _op( new NoOp( op ) ), _c( c ), _mps( mps ), _matcher( matcher ) {
+            _mps->setBestGuessOnly();
+            _mps->mayYield( false ); // with a NoOp, there's no need to yield in QueryPlanSet
+            if ( !ok() ) {
+                // would have been advanced by UserQueryOp if possible
+                advance();
+            }
+        }
+        virtual bool ok() { return _c->ok(); }
+        virtual Record* _current() { return _c->_current(); }
+        virtual BSONObj current() { return _c->current(); }
+        virtual DiskLoc currLoc() { return _c->currLoc(); }
+        virtual bool advance() {
+            _c->advance();
+            while( !ok() && _mps->mayRunMore() ) {
+                nextClause();
+            }
+            return ok();
+        }
+        virtual BSONObj currKey() const { return _c->currKey(); }
+        virtual DiskLoc refLoc() { return _c->refLoc(); }
+        virtual void noteLocation() {
+            _c->noteLocation();
+        }
+        virtual void checkLocation() {
+            _c->checkLocation();
+        }        
+        virtual bool supportGetMore() { return true; }
+        virtual bool supportYields() { return _c->supportYields(); }
+        // with update we could potentially get the same document on multiple
+        // indexes, but update appears to already handle this with seenObjects
+        // so we don't have to do anything special here.
+        virtual bool getsetdup(DiskLoc loc) {
+            return _c->getsetdup( loc );   
+        }
+        virtual CoveredIndexMatcher *matcher() const { return _matcher.get(); }
+        // just for testing
+        shared_ptr< Cursor > sub_c() const { return _c; }
+    private:
+        class NoOp : public CursorOp {
+        public:
+            NoOp() {}
+            NoOp( const QueryOp &other ) : CursorOp( other ) {}
+            virtual void _init() { setComplete(); }
+            virtual void next() {}
+            virtual bool mayRecordPlan() const { return false; }
+            virtual QueryOp *_createChild() const { return new NoOp(); }
+            virtual shared_ptr< Cursor > newCursor() const { return qp().newCursor(); }
+        };
+        void nextClause() {
+            shared_ptr< CursorOp > best = _mps->runOpOnce( *_op );
+            if ( ! best->complete() )
+                throw MsgAssertionException( best->exception() );
+            _c = best->newCursor();
+            _matcher = best->matcher();
+            _op = best;
+        }
+        shared_ptr< CursorOp > _op;
+        shared_ptr< Cursor > _c;
+        auto_ptr< MultiPlanScanner > _mps;
+        shared_ptr< CoveredIndexMatcher > _matcher;
+    };
+    
     // NOTE min, max, and keyPattern will be updated to be consistent with the selected index.
     IndexDetails *indexDetailsForRange( const char *ns, string &errmsg, BSONObj &min, BSONObj &max, BSONObj &keyPattern );
 
     inline bool isSimpleIdQuery( const BSONObj& query ){
-        return 
-            strcmp( query.firstElement().fieldName() , "_id" ) == 0 && 
-            query.nFields() == 1 && 
-            query.firstElement().isSimpleType();
+        BSONObjIterator i(query);
+        if( !i.more() ) return false;
+        BSONElement e = i.next();
+        if( i.more() ) return false;
+        if( strcmp("_id", e.fieldName()) != 0 ) return false;
+        return e.isSimpleType(); // e.g. not something like { _id : { $gt : ...
+    }
+    
+    // matcher() will always work on the returned cursor
+    inline shared_ptr< Cursor > bestGuessCursor( const char *ns, const BSONObj &query, const BSONObj &sort ) {
+        if( !query.getField( "$or" ).eoo() ) {
+            return shared_ptr< Cursor >( new MultiCursor( ns, query, sort ) );
+        } else {
+            auto_ptr< FieldRangeSet > frs( new FieldRangeSet( ns, query ) );
+            shared_ptr< Cursor > ret = QueryPlanSet( ns, frs, query, sort ).getBestGuess()->newCursor();
+            if ( !query.isEmpty() ) {
+                shared_ptr< CoveredIndexMatcher > matcher( new CoveredIndexMatcher( query, ret->indexKeyPattern() ) );
+                ret->setMatcher( matcher );
+            }
+            return ret;
+        }
     }
         
 } // namespace mongo
diff --git a/db/queryutil.cpp b/db/queryutil.cpp
index c01b89e..791096f 100644
--- a/db/queryutil.cpp
+++ b/db/queryutil.cpp
@@ -15,15 +15,18 @@
  *    limitations under the License.
  */
 
-#include "stdafx.h"
+#include "pch.h"
 
 #include "btree.h"
 #include "matcher.h"
 #include "pdfile.h"
 #include "queryoptimizer.h"
 #include "../util/unittest.h"
+#include "dbmessage.h"
 
 namespace mongo {
+    extern BSONObj staticNull;
+    
     /** returns a string that when used as a matcher, would match a super set of regex()
         returns "" for complex regular expressions
         used to optimize queries in some simple regex cases that start with '^'
@@ -35,11 +38,25 @@ namespace mongo {
 
         if (purePrefix) *purePrefix = false;
 
+        bool multilineOK;
+        if ( regex[0] == '\\' && regex[1] == 'A'){
+            multilineOK = true;
+            regex += 2;
+        } else if (regex[0] == '^') {
+            multilineOK = false;
+            regex += 1;
+        } else {
+            return r;
+        }
+
         bool extended = false;
         while (*flags){
             switch (*(flags++)){
                 case 'm': // multiline
-                    continue;
+                    if (multilineOK)
+                        continue;
+                    else
+                        return r;
                 case 'x': // extended
                     extended = true;
                     break;
@@ -48,9 +65,6 @@ namespace mongo {
             }
         }
 
-        if ( *(regex++) != '^' )
-            return r;
-
         stringstream ss;
 
         while(*regex){
@@ -131,7 +145,7 @@ namespace mongo {
             }
 
             for( set< BSONElement, element_lt >::const_iterator i = vals.begin(); i != vals.end(); ++i )
-                intervals_.push_back( FieldInterval(*i) );
+                _intervals.push_back( FieldInterval(*i) );
 
             for( vector< FieldRange >::const_iterator i = regexes.begin(); i != regexes.end(); ++i )
                 *this |= *i;
@@ -141,25 +155,25 @@ namespace mongo {
         
         if ( e.type() == Array && e.getGtLtOp() == BSONObj::Equality ){
             
-            intervals_.push_back( FieldInterval(e) );
+            _intervals.push_back( FieldInterval(e) );
             
             const BSONElement& temp = e.embeddedObject().firstElement();
             if ( ! temp.eoo() ){
                 if ( temp < e )
-                    intervals_.insert( intervals_.begin() , temp );
+                    _intervals.insert( _intervals.begin() , temp );
                 else
-                    intervals_.push_back( FieldInterval(temp) );
+                    _intervals.push_back( FieldInterval(temp) );
             }
             
             return;
         }
 
-        intervals_.push_back( FieldInterval() );
-        FieldInterval &initial = intervals_[ 0 ];
-        BSONElement &lower = initial.lower_.bound_;
-        bool &lowerInclusive = initial.lower_.inclusive_;
-        BSONElement &upper = initial.upper_.bound_;
-        bool &upperInclusive = initial.upper_.inclusive_;
+        _intervals.push_back( FieldInterval() );
+        FieldInterval &initial = _intervals[ 0 ];
+        BSONElement &lower = initial._lower._bound;
+        bool &lowerInclusive = initial._lower._inclusive;
+        BSONElement &upper = initial._upper._bound;
+        bool &upperInclusive = initial._upper._inclusive;
         lower = minKey.firstElement();
         lowerInclusive = true;
         upper = maxKey.firstElement();
@@ -190,13 +204,13 @@ namespace mongo {
                 // regex matches self - regex type > string type
                 if (e.type() == RegEx){
                     BSONElement re = addObj( BSON( "" << e ) ).firstElement();
-                    intervals_.push_back( FieldInterval(re) );
+                    _intervals.push_back( FieldInterval(re) );
                 } else {
                     BSONObj orig = e.embeddedObject();
                     BSONObjBuilder b;
                     b.appendRegex("", orig["$regex"].valuestrsafe(), orig["$options"].valuestrsafe());
                     BSONElement re = addObj( b.obj() ).firstElement();
-                    intervals_.push_back( FieldInterval(re) );
+                    _intervals.push_back( FieldInterval(re) );
                 }
 
             }
@@ -334,63 +348,67 @@ namespace mongo {
 
     }
 
+    void FieldRange::finishOperation( const vector< FieldInterval > &newIntervals, const FieldRange &other ) {
+        _intervals = newIntervals;
+        for( vector< BSONObj >::const_iterator i = other._objData.begin(); i != other._objData.end(); ++i )
+            _objData.push_back( *i );
+        if ( _special.size() == 0 && other._special.size() )
+            _special = other._special;
+    }
+    
     // as called, these functions find the max/min of a bound in the
     // opposite direction, so inclusive bounds are considered less
     // superlative
     FieldBound maxFieldBound( const FieldBound &a, const FieldBound &b ) {
-        int cmp = a.bound_.woCompare( b.bound_, false );
-        if ( ( cmp == 0 && !b.inclusive_ ) || cmp < 0 )
+        int cmp = a._bound.woCompare( b._bound, false );
+        if ( ( cmp == 0 && !b._inclusive ) || cmp < 0 )
             return b;
         return a;
     }
 
     FieldBound minFieldBound( const FieldBound &a, const FieldBound &b ) {
-        int cmp = a.bound_.woCompare( b.bound_, false );
-        if ( ( cmp == 0 && !b.inclusive_ ) || cmp > 0 )
+        int cmp = a._bound.woCompare( b._bound, false );
+        if ( ( cmp == 0 && !b._inclusive ) || cmp > 0 )
             return b;
         return a;
     }
 
     bool fieldIntervalOverlap( const FieldInterval &one, const FieldInterval &two, FieldInterval &result ) {
-        result.lower_ = maxFieldBound( one.lower_, two.lower_ );
-        result.upper_ = minFieldBound( one.upper_, two.upper_ );
-        return result.valid();
+        result._lower = maxFieldBound( one._lower, two._lower );
+        result._upper = minFieldBound( one._upper, two._upper );
+        return result.strictValid();
     }
     
 	// NOTE Not yet tested for complex $or bounds, just for simple bounds generated by $in
     const FieldRange &FieldRange::operator&=( const FieldRange &other ) {
         vector< FieldInterval > newIntervals;
-        vector< FieldInterval >::const_iterator i = intervals_.begin();
-        vector< FieldInterval >::const_iterator j = other.intervals_.begin();
-        while( i != intervals_.end() && j != other.intervals_.end() ) {
+        vector< FieldInterval >::const_iterator i = _intervals.begin();
+        vector< FieldInterval >::const_iterator j = other._intervals.begin();
+        while( i != _intervals.end() && j != other._intervals.end() ) {
             FieldInterval overlap;
             if ( fieldIntervalOverlap( *i, *j, overlap ) )
                 newIntervals.push_back( overlap );
-            if ( i->upper_ == minFieldBound( i->upper_, j->upper_ ) )
+            if ( i->_upper == minFieldBound( i->_upper, j->_upper ) )
                 ++i;
             else
                 ++j;      
         }
-        intervals_ = newIntervals;
-        for( vector< BSONObj >::const_iterator i = other.objData_.begin(); i != other.objData_.end(); ++i )
-            objData_.push_back( *i );
-        if ( _special.size() == 0 && other._special.size() )
-            _special = other._special;
+        finishOperation( newIntervals, other );
         return *this;
     }
     
     void handleInterval( const FieldInterval &lower, FieldBound &low, FieldBound &high, vector< FieldInterval > &newIntervals ) {
-        if ( low.bound_.eoo() ) {
-            low = lower.lower_; high = lower.upper_;
+        if ( low._bound.eoo() ) {
+            low = lower._lower; high = lower._upper;
         } else {
-            if ( high.bound_.woCompare( lower.lower_.bound_, false ) < 0 ) { // when equal but neither inclusive, just assume they overlap, since current btree scanning code just as efficient either way
+            if ( high._bound.woCompare( lower._lower._bound, false ) < 0 ) { // when equal but neither inclusive, just assume they overlap, since current btree scanning code just as efficient either way
                 FieldInterval tmp;
-                tmp.lower_ = low;
-                tmp.upper_ = high;
+                tmp._lower = low;
+                tmp._upper = high;
                 newIntervals.push_back( tmp );
-                low = lower.lower_; high = lower.upper_;                    
+                low = lower._lower; high = lower._upper;                    
             } else {
-                high = lower.upper_;
+                high = lower._upper;
             }
         }        
     }
@@ -399,11 +417,11 @@ namespace mongo {
         vector< FieldInterval > newIntervals;
         FieldBound low;
         FieldBound high;
-        vector< FieldInterval >::const_iterator i = intervals_.begin();
-        vector< FieldInterval >::const_iterator j = other.intervals_.begin();
-        while( i != intervals_.end() && j != other.intervals_.end() ) {
-            int cmp = i->lower_.bound_.woCompare( j->lower_.bound_, false );
-            if ( ( cmp == 0 && i->lower_.inclusive_ ) || cmp < 0 ) {
+        vector< FieldInterval >::const_iterator i = _intervals.begin();
+        vector< FieldInterval >::const_iterator j = other._intervals.begin();
+        while( i != _intervals.end() && j != other._intervals.end() ) {
+            int cmp = i->_lower._bound.woCompare( j->_lower._bound, false );
+            if ( ( cmp == 0 && i->_lower._inclusive ) || cmp < 0 ) {
                 handleInterval( *i, low, high, newIntervals );
                 ++i;
             } else {
@@ -411,34 +429,85 @@ namespace mongo {
                 ++j;
             } 
         }
-        while( i != intervals_.end() ) {
+        while( i != _intervals.end() ) {
             handleInterval( *i, low, high, newIntervals );
             ++i;            
         }
-        while( j != other.intervals_.end() ) {
+        while( j != other._intervals.end() ) {
             handleInterval( *j, low, high, newIntervals );
             ++j;            
         }
         FieldInterval tmp;
-        tmp.lower_ = low;
-        tmp.upper_ = high;
+        tmp._lower = low;
+        tmp._upper = high;
         newIntervals.push_back( tmp );        
-        intervals_ = newIntervals;
-        for( vector< BSONObj >::const_iterator i = other.objData_.begin(); i != other.objData_.end(); ++i )
-            objData_.push_back( *i );
-        if ( _special.size() == 0 && other._special.size() )
-            _special = other._special;
+        finishOperation( newIntervals, other );
+        return *this;        
+    }
+    
+    const FieldRange &FieldRange::operator-=( const FieldRange &other ) {
+        vector< FieldInterval >::iterator i = _intervals.begin();
+        vector< FieldInterval >::const_iterator j = other._intervals.begin();
+        while( i != _intervals.end() && j != other._intervals.end() ) {
+            int cmp = i->_lower._bound.woCompare( j->_lower._bound, false );
+            if ( cmp < 0 ||
+                ( cmp == 0 && i->_lower._inclusive && !j->_lower._inclusive ) ) {
+                int cmp2 = i->_upper._bound.woCompare( j->_lower._bound, false );
+                if ( cmp2 < 0 ) {
+                    ++i;
+                } else if ( cmp2 == 0 ) {
+                    if ( i->_upper._inclusive && j->_lower._inclusive ) {
+                        i->_upper._inclusive = false;
+                    }
+                    ++i;
+                } else {
+                    int cmp3 = i->_upper._bound.woCompare( j->_upper._bound, false );
+                    if ( cmp3 < 0 ||
+                        ( cmp3 == 0 && ( !i->_upper._inclusive || j->_upper._inclusive ) ) ) {
+                        i->_upper = j->_lower;
+                        i->_upper.flipInclusive();
+                        ++i;
+                    } else {
+                        ++j;
+                    }
+                }
+            } else {
+                int cmp2 = i->_lower._bound.woCompare( j->_upper._bound, false );
+                if ( cmp2 > 0 ||
+                    ( cmp2 == 0 && ( !i->_lower._inclusive || !j->_lower._inclusive ) ) ) {
+                    ++j;
+                } else {
+                    int cmp3 = i->_upper._bound.woCompare( j->_upper._bound, false );
+                    if ( cmp3 < 0 ||
+                        ( cmp3 == 0 && ( !i->_upper._inclusive || j->_upper._inclusive ) ) ) {
+                        i = _intervals.erase( i );
+                    } else {
+                        i->_lower = j->_upper;
+                        i->_lower.flipInclusive();                        
+                        ++j;
+                    }
+                }                
+            }
+        }
+        finishOperation( _intervals, other );
         return *this;        
     }
     
+    // TODO write a proper implementation that doesn't do a full copy
+    bool FieldRange::operator<=( const FieldRange &other ) {
+        FieldRange temp = *this;
+        temp -= other;
+        return temp.empty();
+    }
+    
     BSONObj FieldRange::addObj( const BSONObj &o ) {
-        objData_.push_back( o );
+        _objData.push_back( o );
         return o;
     }
-    
+        
     string FieldRangeSet::getSpecial() const {
         string s = "";
-        for ( map<string,FieldRange>::iterator i=ranges_.begin(); i!=ranges_.end(); i++ ){
+        for ( map<string,FieldRange>::iterator i=_ranges.begin(); i!=_ranges.end(); i++ ){
             if ( i->second.getSpecial().size() == 0 )
                 continue;
             uassert( 13033 , "can't have 2 special fields" , s.size() == 0 );
@@ -472,64 +541,99 @@ namespace mongo {
                 
                 int op3 = getGtLtOp( h );
                 if ( op3 == BSONObj::Equality ){
-                    ranges_[ fullname ] &= FieldRange( h , isNot , optimize );
+                    _ranges[ fullname ] &= FieldRange( h , isNot , optimize );
                 }
                 else {
                     BSONObjIterator l( h.embeddedObject() );
                     while ( l.more() ){
-                        ranges_[ fullname ] &= FieldRange( l.next() , isNot , optimize );
+                        _ranges[ fullname ] &= FieldRange( l.next() , isNot , optimize );
                     }
                 }
             }                        
         } else {
-            ranges_[ fieldName ] &= FieldRange( f , isNot , optimize );
+            _ranges[ fieldName ] &= FieldRange( f , isNot , optimize );
         }        
     }
     
+    void FieldRangeSet::processQueryField( const BSONElement &e, bool optimize ) {
+        bool equality = ( getGtLtOp( e ) == BSONObj::Equality );
+        if ( equality && e.type() == Object ) {
+            equality = ( strcmp( e.embeddedObject().firstElement().fieldName(), "$not" ) != 0 );
+        }
+        
+        if ( equality || ( e.type() == Object && !e.embeddedObject()[ "$regex" ].eoo() ) ) {
+            _ranges[ e.fieldName() ] &= FieldRange( e , false , optimize );
+        }
+        if ( !equality ) {
+            BSONObjIterator j( e.embeddedObject() );
+            while( j.more() ) {
+                BSONElement f = j.next();
+                if ( strcmp( f.fieldName(), "$not" ) == 0 ) {
+                    switch( f.type() ) {
+                        case Object: {
+                            BSONObjIterator k( f.embeddedObject() );
+                            while( k.more() ) {
+                                BSONElement g = k.next();
+                                uassert( 13034, "invalid use of $not", g.getGtLtOp() != BSONObj::Equality );
+                                processOpElement( e.fieldName(), g, true, optimize );
+                            }
+                            break;
+                        }
+                        case RegEx:
+                            processOpElement( e.fieldName(), f, true, optimize );
+                            break;
+                        default:
+                            uassert( 13041, "invalid use of $not", false );
+                    }
+                } else {
+                    processOpElement( e.fieldName(), f, false, optimize );
+                }
+            }                
+        }   
+    }
+    
     FieldRangeSet::FieldRangeSet( const char *ns, const BSONObj &query , bool optimize )
-        : ns_( ns ), query_( query.getOwned() ) {
-        BSONObjIterator i( query_ );
+        : _ns( ns ), _queries( 1, query.getOwned() ) {
+            BSONObjIterator i( _queries[ 0 ] );
+            
+            while( i.more() ) {
+                BSONElement e = i.next();
+                // e could be x:1 or x:{$gt:1}
+                
+                if ( strcmp( e.fieldName(), "$where" ) == 0 ) {
+                    continue;
+                }
+                
+                if ( strcmp( e.fieldName(), "$or" ) == 0 ) {                                                                                                                                                        
+                    continue;
+                }
+                
+                if ( strcmp( e.fieldName(), "$nor" ) == 0 ) {
+                    continue;
+                }
+                
+                processQueryField( e, optimize );
+            }   
+        }
+
+    FieldRangeOrSet::FieldRangeOrSet( const char *ns, const BSONObj &query , bool optimize )
+        : _baseSet( ns, query, optimize ), _orFound() {
+
+        BSONObjIterator i( _baseSet._queries[ 0 ] );
         
         while( i.more() ) {
             BSONElement e = i.next();
-            // e could be x:1 or x:{$gt:1}
-
-            if ( strcmp( e.fieldName(), "$where" ) == 0 )
+            if ( strcmp( e.fieldName(), "$or" ) == 0 ) {                                                                                                                                                        
+                massert( 13262, "$or requires nonempty array", e.type() == Array && e.embeddedObject().nFields() > 0 );                                                                                         
+                BSONObjIterator j( e.embeddedObject() );                                                                                                                                                        
+                while( j.more() ) {                                                                                                                                                                             
+                    BSONElement f = j.next();                                                                                                                                                                   
+                    massert( 13263, "$or array must contain objects", f.type() == Object );                                                                                                                     
+                    _orSets.push_back( FieldRangeSet( ns, f.embeddedObject(), optimize ) );
+                    massert( 13291, "$or may not contain 'special' query", _orSets.back().getSpecial().empty() );
+                }
+                _orFound = true;
                 continue;
-
-            bool equality = ( getGtLtOp( e ) == BSONObj::Equality );
-            if ( equality && e.type() == Object ) {
-                equality = ( strcmp( e.embeddedObject().firstElement().fieldName(), "$not" ) != 0 );
-            }
-            
-            if ( equality || ( e.type() == Object && !e.embeddedObject()[ "$regex" ].eoo() ) ) {
-                ranges_[ e.fieldName() ] &= FieldRange( e , false , optimize );
-            }
-            if ( !equality ) {
-                BSONObjIterator j( e.embeddedObject() );
-                while( j.more() ) {
-                    BSONElement f = j.next();
-                    if ( strcmp( f.fieldName(), "$not" ) == 0 ) {
-                        switch( f.type() ) {
-                            case Object: {
-                                BSONObjIterator k( f.embeddedObject() );
-                                while( k.more() ) {
-                                    BSONElement g = k.next();
-                                    uassert( 13034, "invalid use of $not", g.getGtLtOp() != BSONObj::Equality );
-                                    processOpElement( e.fieldName(), g, true, optimize );
-                                }
-                                break;
-                            }
-                            case RegEx:
-                                processOpElement( e.fieldName(), f, true, optimize );
-                                break;
-                            default:
-                                uassert( 13041, "invalid use of $not", false );
-                        }
-                    } else {
-                        processOpElement( e.fieldName(), f, false, optimize );
-                    }
-                }                
             }
         }
     }
@@ -545,8 +649,8 @@ namespace mongo {
         BSONObj fields = _fields;
         if ( fields.isEmpty() ) {
             BSONObjBuilder b;
-            for( map< string, FieldRange >::const_iterator i = ranges_.begin(); i != ranges_.end(); ++i ) {
-                b.append( i->first.c_str(), 1 );
+            for( map< string, FieldRange >::const_iterator i = _ranges.begin(); i != _ranges.end(); ++i ) {
+                b.append( i->first, 1 );
             }
             fields = b.obj();
         }
@@ -555,17 +659,19 @@ namespace mongo {
         while( i.more() ) {
             BSONElement e = i.next();
             const char *name = e.fieldName();
-            const FieldRange &range = ranges_[ name ];
+            const FieldRange &range = _ranges[ name ];
             assert( !range.empty() );
             if ( range.equality() )
                 b.appendAs( range.min(), name );
             else if ( range.nontrivial() ) {
+                BSONObj o;
                 BSONObjBuilder c;
                 if ( range.min().type() != MinKey )
                     c.appendAs( range.min(), range.minInclusive() ? "$gte" : "$gt" );
                 if ( range.max().type() != MaxKey )
                     c.appendAs( range.max(), range.maxInclusive() ? "$lte" : "$lt" );
-                b.append( name, c.done() );                
+                o = c.obj();
+                b.append( name, o );
             }
         }
         return b.obj();
@@ -573,58 +679,73 @@ namespace mongo {
     
     QueryPattern FieldRangeSet::pattern( const BSONObj &sort ) const {
         QueryPattern qp;
-        for( map< string, FieldRange >::const_iterator i = ranges_.begin(); i != ranges_.end(); ++i ) {
+        for( map< string, FieldRange >::const_iterator i = _ranges.begin(); i != _ranges.end(); ++i ) {
             assert( !i->second.empty() );
             if ( i->second.equality() ) {
-                qp.fieldTypes_[ i->first ] = QueryPattern::Equality;
+                qp._fieldTypes[ i->first ] = QueryPattern::Equality;
             } else if ( i->second.nontrivial() ) {
                 bool upper = i->second.max().type() != MaxKey;
                 bool lower = i->second.min().type() != MinKey;
                 if ( upper && lower )
-                    qp.fieldTypes_[ i->first ] = QueryPattern::UpperAndLowerBound;
+                    qp._fieldTypes[ i->first ] = QueryPattern::UpperAndLowerBound;
                 else if ( upper )
-                    qp.fieldTypes_[ i->first ] = QueryPattern::UpperBound;
+                    qp._fieldTypes[ i->first ] = QueryPattern::UpperBound;
                 else if ( lower )
-                    qp.fieldTypes_[ i->first ] = QueryPattern::LowerBound;                    
+                    qp._fieldTypes[ i->first ] = QueryPattern::LowerBound;                    
             }
         }
         qp.setSort( sort );
         return qp;
     }
     
+    // TODO get rid of this
     BoundList FieldRangeSet::indexBounds( const BSONObj &keyPattern, int direction ) const {
-        BSONObjBuilder equalityBuilder;
         typedef vector< pair< shared_ptr< BSONObjBuilder >, shared_ptr< BSONObjBuilder > > > BoundBuilders;
         BoundBuilders builders;
+        builders.push_back( make_pair( shared_ptr< BSONObjBuilder >( new BSONObjBuilder() ), shared_ptr< BSONObjBuilder >( new BSONObjBuilder() ) ) );        
         BSONObjIterator i( keyPattern );
+        bool ineq = false; // until ineq is true, we are just dealing with equality and $in bounds
         while( i.more() ) {
             BSONElement e = i.next();
             const FieldRange &fr = range( e.fieldName() );
             int number = (int) e.number(); // returns 0.0 if not numeric
             bool forward = ( ( number >= 0 ? 1 : -1 ) * ( direction >= 0 ? 1 : -1 ) > 0 );
-            if ( builders.empty() ) {
+            if ( !ineq ) {
                 if ( fr.equality() ) {
-                    equalityBuilder.appendAs( fr.min(), "" );
+                    for( BoundBuilders::const_iterator j = builders.begin(); j != builders.end(); ++j ) {
+                        j->first->appendAs( fr.min(), "" );
+                        j->second->appendAs( fr.min(), "" );
+                    }
                 } else {
-                    BSONObj equalityObj = equalityBuilder.done();
+                    if ( !fr.inQuery() ) {
+                        ineq = true;
+                    }
+                    BoundBuilders newBuilders;
                     const vector< FieldInterval > &intervals = fr.intervals();
-                    if ( forward ) {
-                        for( vector< FieldInterval >::const_iterator j = intervals.begin(); j != intervals.end(); ++j ) {
-                            builders.push_back( make_pair( shared_ptr< BSONObjBuilder >( new BSONObjBuilder() ), shared_ptr< BSONObjBuilder >( new BSONObjBuilder() ) ) );
-                            builders.back().first->appendElements( equalityObj );
-                            builders.back().second->appendElements( equalityObj );
-                            builders.back().first->appendAs( j->lower_.bound_, "" );
-                            builders.back().second->appendAs( j->upper_.bound_, "" );
+                    for( BoundBuilders::const_iterator i = builders.begin(); i != builders.end(); ++i ) {
+                        BSONObj first = i->first->obj();
+                        BSONObj second = i->second->obj();
+                        if ( forward ) {
+                            for( vector< FieldInterval >::const_iterator j = intervals.begin(); j != intervals.end(); ++j ) {
+                                uassert( 13303, "combinatorial limit of $in partitioning of result set exceeded", newBuilders.size() < 1000000 );
+                                newBuilders.push_back( make_pair( shared_ptr< BSONObjBuilder >( new BSONObjBuilder() ), shared_ptr< BSONObjBuilder >( new BSONObjBuilder() ) ) );
+                                newBuilders.back().first->appendElements( first );
+                                newBuilders.back().second->appendElements( second );
+                                newBuilders.back().first->appendAs( j->_lower._bound, "" );
+                                newBuilders.back().second->appendAs( j->_upper._bound, "" );
+                            }
+                        } else {
+                            for( vector< FieldInterval >::const_reverse_iterator j = intervals.rbegin(); j != intervals.rend(); ++j ) {
+                                uassert( 13304, "combinatorial limit of $in partitioning of result set exceeded", newBuilders.size() < 1000000 );
+                                newBuilders.push_back( make_pair( shared_ptr< BSONObjBuilder >( new BSONObjBuilder() ), shared_ptr< BSONObjBuilder >( new BSONObjBuilder() ) ) );
+                                newBuilders.back().first->appendElements( first );
+                                newBuilders.back().second->appendElements( second );
+                                newBuilders.back().first->appendAs( j->_upper._bound, "" );
+                                newBuilders.back().second->appendAs( j->_lower._bound, "" );
+                            }
                         }
-                    } else {
-                        for( vector< FieldInterval >::const_reverse_iterator j = intervals.rbegin(); j != intervals.rend(); ++j ) {
-                            builders.push_back( make_pair( shared_ptr< BSONObjBuilder >( new BSONObjBuilder() ), shared_ptr< BSONObjBuilder >( new BSONObjBuilder() ) ) );
-                            builders.back().first->appendElements( equalityObj );
-                            builders.back().second->appendElements( equalityObj );
-                            builders.back().first->appendAs( j->upper_.bound_, "" );
-                            builders.back().second->appendAs( j->lower_.bound_, "" );
-                        }                       
                     }
+                    builders = newBuilders;
                 }
             } else {
                 for( BoundBuilders::const_iterator j = builders.begin(); j != builders.end(); ++j ) {
@@ -633,19 +754,12 @@ namespace mongo {
                 }
             }
         }
-        if ( builders.empty() ) {
-            BSONObj equalityObj = equalityBuilder.done();
-            assert( !equalityObj.isEmpty() );
-            builders.push_back( make_pair( shared_ptr< BSONObjBuilder >( new BSONObjBuilder() ), shared_ptr< BSONObjBuilder >( new BSONObjBuilder() ) ) );
-            builders.back().first->appendElements( equalityObj );
-            builders.back().second->appendElements( equalityObj );            
-        }
         BoundList ret;
         for( BoundBuilders::const_iterator i = builders.begin(); i != builders.end(); ++i )
             ret.push_back( make_pair( i->first->obj(), i->second->obj() ) );
         return ret;
-    }
-
+    }    
+    
     ///////////////////
     // FieldMatcher //
     ///////////////////
@@ -658,16 +772,51 @@ namespace mongo {
         int true_false = -1;
         while ( i.more() ){
             BSONElement e = i.next();
-            add (e.fieldName(), e.trueValue());
 
-            // validate input
-            if (true_false == -1){
-                true_false = e.trueValue();
-                _include = !e.trueValue();
-            }
-            else{
-                uassert( 10053 , "You cannot currently mix including and excluding fields. Contact us if this is an issue." , 
-                         (bool)true_false == e.trueValue() );
+            if (e.type() == Object){
+                BSONObj obj = e.embeddedObject();
+                BSONElement e2 = obj.firstElement();
+                if ( strcmp(e2.fieldName(), "$slice") == 0 ){
+                    if (e2.isNumber()){
+                        int i = e2.numberInt();
+                        if (i < 0)
+                            add(e.fieldName(), i, -i); // limit is now positive
+                        else
+                            add(e.fieldName(), 0, i);
+
+                    } else if (e2.type() == Array) {
+                        BSONObj arr = e2.embeddedObject();
+                        uassert(13099, "$slice array wrong size", arr.nFields() == 2 );
+
+                        BSONObjIterator it(arr);
+                        int skip = it.next().numberInt();
+                        int limit = it.next().numberInt();
+                        uassert(13100, "$slice limit must be positive", limit > 0 );
+                        add(e.fieldName(), skip, limit);
+
+                    } else {
+                        uassert(13098, "$slice only supports numbers and [skip, limit] arrays", false);
+                    }
+                } else {
+                    uassert(13097, string("Unsupported projection option: ") + obj.firstElement().fieldName(), false);
+                }
+
+            } else if (!strcmp(e.fieldName(), "_id") && !e.trueValue()){
+                _includeID = false;
+
+            } else {
+
+                add (e.fieldName(), e.trueValue());
+
+                // validate input
+                if (true_false == -1){
+                    true_false = e.trueValue();
+                    _include = !e.trueValue();
+                }
+                else{
+                    uassert( 10053 , "You cannot currently mix including and excluding fields. Contact us if this is an issue." , 
+                             (bool)true_false == e.trueValue() );
+                }
             }
         }
     }
@@ -676,34 +825,71 @@ namespace mongo {
         if (field.empty()){ // this is the field the user referred to
             _include = include;
         } else {
+            _include = !include;
+
             const size_t dot = field.find('.');
             const string subfield = field.substr(0,dot);
             const string rest = (dot == string::npos ? "" : field.substr(dot+1,string::npos)); 
 
             boost::shared_ptr<FieldMatcher>& fm = _fields[subfield];
             if (!fm)
-                fm.reset(new FieldMatcher(!include));
+                fm.reset(new FieldMatcher());
 
             fm->add(rest, include);
         }
     }
 
+    void FieldMatcher::add(const string& field, int skip, int limit){
+        _special = true; // can't include or exclude whole object
+
+        if (field.empty()){ // this is the field the user referred to
+            _skip = skip;
+            _limit = limit;
+        } else {
+            const size_t dot = field.find('.');
+            const string subfield = field.substr(0,dot);
+            const string rest = (dot == string::npos ? "" : field.substr(dot+1,string::npos));
+
+            boost::shared_ptr<FieldMatcher>& fm = _fields[subfield];
+            if (!fm)
+                fm.reset(new FieldMatcher());
+
+            fm->add(rest, skip, limit);
+        }
+    }
+
     BSONObj FieldMatcher::getSpec() const{
         return _source;
     }
 
     //b will be the value part of an array-typed BSONElement
-    void FieldMatcher::appendArray( BSONObjBuilder& b , const BSONObj& a ) const {
+    void FieldMatcher::appendArray( BSONObjBuilder& b , const BSONObj& a , bool nested) const {
+        int skip  = nested ?  0 : _skip;
+        int limit = nested ? -1 : _limit;
+
+        if (skip < 0){
+            skip = max(0, skip + a.nFields());
+        }
+
         int i=0;
         BSONObjIterator it(a);
         while (it.more()){
             BSONElement e = it.next();
 
+            if (skip){
+                skip--;
+                continue;
+            }
+
+            if (limit != -1 && (limit-- == 0)){
+                break;
+            }
+
             switch(e.type()){
                 case Array:{
                     BSONObjBuilder subb;
-                    appendArray(subb , e.embeddedObject());
-                    b.appendArray(b.numStr(i++).c_str(), subb.obj());
+                    appendArray(subb , e.embeddedObject(), true);
+                    b.appendArray(b.numStr(i++), subb.obj());
                     break;
                 }
                 case Object:{
@@ -717,10 +903,8 @@ namespace mongo {
                 }
                 default:
                     if (_include)
-                        b.appendAs(e, b.numStr(i++).c_str());
+                        b.appendAs(e, b.numStr(i++));
             }
-            
-
         }
     }
 
@@ -734,7 +918,7 @@ namespace mongo {
         else {
             FieldMatcher& subfm = *field->second;
             
-            if (subfm._fields.empty() || !(e.type()==Object || e.type()==Array) ){
+            if ((subfm._fields.empty() && !subfm._special) || !(e.type()==Object || e.type()==Array) ){
                 if (subfm._include)
                     b.append(e);
             }
@@ -755,6 +939,173 @@ namespace mongo {
         }
     }
     
+    bool FieldRangeVector::matchesElement( const BSONElement &e, int i, bool forward ) const {
+        int l = matchingLowElement( e, i, forward );
+        return ( l % 2 == 0 ); // if we're inside an interval        
+    }
+    
+    int FieldRangeVector::matchingLowElement( const BSONElement &e, int i, bool forward ) const {
+        int l = -1;
+        int h = _ranges[ i ].intervals().size() * 2;
+        while( l + 1 < h ) {
+            int m = ( l + h ) / 2;
+            BSONElement toCmp;
+            if ( m % 2 == 0 ) {
+                toCmp = _ranges[ i ].intervals()[ m / 2 ]._lower._bound;
+            } else {
+                toCmp = _ranges[ i ].intervals()[ m / 2 ]._upper._bound;
+            }
+            int cmp = toCmp.woCompare( e, false );
+            if ( !forward ) {
+                cmp = -cmp;
+            }
+            if ( cmp < 0 ) {
+                l = m;
+            } else if ( cmp > 0 ) {
+                h = m;
+            } else {
+                return ( m % 2 == 0 ) ? m : m - 1;
+            }
+        }
+        assert( l + 1 == h );
+        return l;
+    }
+    
+    bool FieldRangeVector::matches( const BSONObj &obj ) const {
+        BSONObjIterator k( _keyPattern );
+        for( int i = 0; i < (int)_ranges.size(); ++i ) {
+            if ( _ranges[ i ].empty() ) {
+                return false;
+            }
+            BSONElement kk = k.next();
+            int number = (int) kk.number();
+            bool forward = ( number >= 0 ? 1 : -1 ) * ( _direction >= 0 ? 1 : -1 ) > 0;
+            BSONElement e = obj.getField( kk.fieldName() );
+            if ( e.eoo() ) {
+                e = staticNull.firstElement();
+            }
+            if ( e.type() == Array ) {
+                BSONObjIterator j( e.embeddedObject() );
+                bool match = false;
+                while( j.more() ) {
+                    if ( matchesElement( j.next(), i, forward ) ) {
+                        match = true;
+                        break;
+                    }
+                }
+                if ( !match ) {
+                    return false;
+                }
+            } else if ( !matchesElement( e, i, forward ) ) {
+                return false;
+            }
+        }
+        return true;
+    }
+    
+    // TODO optimize more
+    int FieldRangeVector::Iterator::advance( const BSONObj &curr ) {
+        BSONObjIterator j( curr );
+        BSONObjIterator o( _v._keyPattern );
+        int latestNonEndpoint = -1;
+        for( int i = 0; i < (int)_i.size(); ++i ) {
+            if ( i > 0 && !_v._ranges[ i - 1 ].intervals()[ _i[ i - 1 ] ].equality() ) {
+                // TODO if possible avoid this certain cases when field in prev key is the same
+                setMinus( i );
+            }
+            bool eq = false;
+            BSONElement oo = o.next();
+            bool reverse = ( ( oo.number() < 0 ) ^ ( _v._direction < 0 ) );
+            BSONElement jj = j.next();
+            if ( _i[ i ] == -1 ) {
+                int l = _v.matchingLowElement( jj, i, !reverse );
+                if ( l % 2 == 0 ) {
+                    _i[ i ] = l / 2;
+                    int diff = (int)_v._ranges[ i ].intervals().size() - _i[ i ];
+                    if ( diff > 1 ) {
+                        latestNonEndpoint = i;
+                    } else if ( diff == 1 ) {
+                        int x = _v._ranges[ i ].intervals()[ _i[ i ] ]._upper._bound.woCompare( jj, false );
+                        if ( x != 0 ) {
+                            latestNonEndpoint = i;
+                        }
+                    }
+                    continue;
+                } else {
+                    if ( l == (int)_v._ranges[ i ].intervals().size() * 2 - 1 ) {
+                        if ( latestNonEndpoint == -1 ) {
+                            return -2;
+                        }
+                        setZero( latestNonEndpoint + 1 );
+                        // skip to curr / latestNonEndpoint + 1 / superlative
+                        for( int j = latestNonEndpoint + 1; j < (int)_i.size(); ++j ) {
+                            _cmp[ j ] = _superlative[ j ];
+                        }
+                        return latestNonEndpoint + 1;                        
+                    }
+                    _i[ i ] = ( l + 1 ) / 2;
+                    // skip to curr / i / nextbounds
+                    _cmp[ i ] = &_v._ranges[ i ].intervals()[ _i[ i ] ]._lower._bound;
+                    for( int j = i + 1; j < (int)_i.size(); ++j ) {
+                        _cmp[ j ] = &_v._ranges[ j ].intervals().front()._lower._bound;
+                    }
+                    return i;                    
+                }
+            }
+            bool first = true;
+            while( _i[ i ] < (int)_v._ranges[ i ].intervals().size() ) {
+                int x = _v._ranges[ i ].intervals()[ _i[ i ] ]._upper._bound.woCompare( jj, false );
+                if ( reverse ) {
+                    x = -x;
+                }
+                if ( x == 0 ) {
+                    eq = true;
+                    break;
+                }
+                if ( x > 0 ) {
+                    if ( i == 0 && first ) {
+                        break; // the value of 1st field won't go backward
+                    }
+                    if ( !_v._ranges[ i ].intervals()[ _i[ i ] ].equality() ) {
+                        x = _v._ranges[ i ].intervals()[ _i[ i ] ]._lower._bound.woCompare( jj, false );
+                        if ( reverse ) {
+                            x = -x;
+                        }
+                    }
+                    if ( x > 0 ) {
+                        setZero( i + 1 );
+                        // skip to curr / i / nextbounds
+                        _cmp[ i ] = &_v._ranges[ i ].intervals()[ _i[ i ] ]._lower._bound;
+                        for( int j = i + 1; j < (int)_i.size(); ++j ) {
+                            _cmp[ j ] = &_v._ranges[ j ].intervals().front()._lower._bound;
+                        }
+                        return i;
+                    } else {
+                        break;
+                    }
+                }
+                ++_i[ i ];
+                setZero( i + 1 );
+                first = false;
+            }
+            int diff = (int)_v._ranges[ i ].intervals().size() - _i[ i ];
+            if ( diff > 1 || ( !eq && diff == 1 ) ) {
+                latestNonEndpoint = i;
+            } else if ( diff == 0 ) {
+                if ( latestNonEndpoint == -1 ) {
+                    return -2;
+                }
+                setZero( latestNonEndpoint + 1 );
+                // skip to curr / latestNonEndpoint + 1 / superlative
+                for( int j = latestNonEndpoint + 1; j < (int)_i.size(); ++j ) {
+                    _cmp[ j ] = _superlative[ j ];
+                }
+                return latestNonEndpoint + 1;
+            }
+        }
+        return -1;        
+    }
+    
     struct SimpleRegexUnitTest : UnitTest {
         void run(){
             {
@@ -783,22 +1134,88 @@ namespace mongo {
             }
             {
                 BSONObjBuilder b;
+                b.appendRegex("r", "\\Af", "");
+                BSONObj o = b.done();
+                assert( simpleRegex(o.firstElement()) == "f" );
+            }
+            {
+                BSONObjBuilder b;
                 b.appendRegex("r", "^f", "m");
                 BSONObj o = b.done();
+                assert( simpleRegex(o.firstElement()) == "" );
+            }
+            {
+                BSONObjBuilder b;
+                b.appendRegex("r", "\\Af", "m");
+                BSONObj o = b.done();
                 assert( simpleRegex(o.firstElement()) == "f" );
             }
             {
                 BSONObjBuilder b;
-                b.appendRegex("r", "^f", "mi");
+                b.appendRegex("r", "\\Af", "mi");
                 BSONObj o = b.done();
                 assert( simpleRegex(o.firstElement()) == "" );
             }
             {
                 BSONObjBuilder b;
-                b.appendRegex("r", "^f \t\vo\n\ro  \\ \\# #comment", "mx");
+                b.appendRegex("r", "\\Af \t\vo\n\ro  \\ \\# #comment", "mx");
                 BSONObj o = b.done();
                 assert( simpleRegex(o.firstElement()) == "foo #" );
             }
         }
     } simple_regex_unittest;
+
+
+    long long applySkipLimit( long long num , const BSONObj& cmd ){
+        BSONElement s = cmd["skip"];
+        BSONElement l = cmd["limit"];
+        
+        if ( s.isNumber() ){
+            num = num - s.numberLong();
+            if ( num < 0 ) {
+                num = 0;
+            }
+        }
+        
+        if ( l.isNumber() ){
+            long long limit = l.numberLong();
+            if ( limit < num ){
+                num = limit;
+            }
+        }
+
+        return num;        
+    }
+
+    string debugString( Message& m ){
+        stringstream ss;
+        ss << "op: " << opToString( m.operation() ) << " len: " << m.size();
+        if ( m.operation() >= 2000 && m.operation() < 2100 ){
+            DbMessage d(m);
+            ss << " ns: " << d.getns();
+            switch ( m.operation() ){
+            case dbUpdate: {
+                int flags = d.pullInt();
+                BSONObj q = d.nextJsObj();
+                ss << " flags: " << flags << " query: " << q;
+                break;
+            }
+            case dbInsert:
+                ss << d.nextJsObj();
+                break;
+            case dbDelete: {
+                int flags = d.pullInt();
+                BSONObj q = d.nextJsObj();
+                ss << " flags: " << flags << " query: " << q;
+                break;
+            }
+            default:
+                ss << " CANNOT HANDLE YET";
+            }
+                    
+                
+        }
+        return ss.str();
+    }    
+
 } // namespace mongo
diff --git a/db/queryutil.h b/db/queryutil.h
index 7d8be78..37dfa2a 100644
--- a/db/queryutil.h
+++ b/db/queryutil.h
@@ -22,26 +22,34 @@
 namespace mongo {
 
     struct FieldBound {
-        BSONElement bound_;
-        bool inclusive_;
+        BSONElement _bound;
+        bool _inclusive;
         bool operator==( const FieldBound &other ) const {
-            return bound_.woCompare( other.bound_ ) == 0 &&
-            inclusive_ == other.inclusive_;
+            return _bound.woCompare( other._bound ) == 0 &&
+            _inclusive == other._inclusive;
         }
+        void flipInclusive() { _inclusive = !_inclusive; }
     };
 
     struct FieldInterval {
-        FieldInterval(){}
-        FieldInterval( const BSONElement& e ){
-            lower_.bound_ = upper_.bound_ = e;
-            lower_.inclusive_ = upper_.inclusive_ = true;
+        FieldInterval() : _cachedEquality( -1 ) {}
+        FieldInterval( const BSONElement& e ) : _cachedEquality( -1 ) {
+            _lower._bound = _upper._bound = e;
+            _lower._inclusive = _upper._inclusive = true;
         }
-        FieldBound lower_;
-        FieldBound upper_;
-        bool valid() const {
-            int cmp = lower_.bound_.woCompare( upper_.bound_, false );
-            return ( cmp < 0 || ( cmp == 0 && lower_.inclusive_ && upper_.inclusive_ ) );
+        FieldBound _lower;
+        FieldBound _upper;
+        bool strictValid() const {
+            int cmp = _lower._bound.woCompare( _upper._bound, false );
+            return ( cmp < 0 || ( cmp == 0 && _lower._inclusive && _upper._inclusive ) );
         }
+        bool equality() const {
+            if ( _cachedEquality == -1 ) {
+                _cachedEquality = ( _lower._inclusive && _upper._inclusive && _lower._bound.woCompare( _upper._bound, false ) == 0 );
+            }
+            return _cachedEquality;
+        }
+        mutable int _cachedEquality;
     };
 
     // range of a field's value that may be determined from query -- used to
@@ -50,11 +58,16 @@ namespace mongo {
     public:
         FieldRange( const BSONElement &e = BSONObj().firstElement() , bool isNot=false , bool optimize=true );
         const FieldRange &operator&=( const FieldRange &other );
-    const FieldRange &operator|=( const FieldRange &other );
-        BSONElement min() const { assert( !empty() ); return intervals_[ 0 ].lower_.bound_; }
-        BSONElement max() const { assert( !empty() ); return intervals_[ intervals_.size() - 1 ].upper_.bound_; }
-        bool minInclusive() const { assert( !empty() ); return intervals_[ 0 ].lower_.inclusive_; }
-        bool maxInclusive() const { assert( !empty() ); return intervals_[ intervals_.size() - 1 ].upper_.inclusive_; }
+        const FieldRange &operator|=( const FieldRange &other );
+        // does not remove fully contained ranges (eg [1,3] - [2,2] doesn't remove anything)
+        // in future we can change so that an or on $in:[3] combined with $in:{$gt:2} doesn't scan 3 a second time
+        const FieldRange &operator-=( const FieldRange &other );
+        // true iff other includes this
+        bool operator<=( const FieldRange &other );
+        BSONElement min() const { assert( !empty() ); return _intervals[ 0 ]._lower._bound; }
+        BSONElement max() const { assert( !empty() ); return _intervals[ _intervals.size() - 1 ]._upper._bound; }
+        bool minInclusive() const { assert( !empty() ); return _intervals[ 0 ]._lower._inclusive; }
+        bool maxInclusive() const { assert( !empty() ); return _intervals[ _intervals.size() - 1 ]._upper._inclusive; }
         bool equality() const {
             return
                 !empty() &&
@@ -62,20 +75,51 @@ namespace mongo {
                 maxInclusive() &&
                 minInclusive();
         }
+        bool inQuery() const {
+            if ( equality() ) {
+                return true;
+            }
+            for( vector< FieldInterval >::const_iterator i = _intervals.begin(); i != _intervals.end(); ++i ) {            
+                if ( !i->equality() ) {
+                    return false;
+                }
+            }
+            return true;
+        }
         bool nontrivial() const {
             return
                 ! empty() && 
                 ( minKey.firstElement().woCompare( min(), false ) != 0 ||
                   maxKey.firstElement().woCompare( max(), false ) != 0 );
         }
-        bool empty() const { return intervals_.empty(); }
-		const vector< FieldInterval > &intervals() const { return intervals_; }
+        bool empty() const { return _intervals.empty(); }
+        void makeEmpty() { _intervals.clear(); }
+		const vector< FieldInterval > &intervals() const { return _intervals; }
         string getSpecial() const { return _special; }
-
+        void setExclusiveBounds() {
+            for( vector< FieldInterval >::iterator i = _intervals.begin(); i != _intervals.end(); ++i ) {
+                i->_lower._inclusive = false;
+                i->_upper._inclusive = false;
+            }
+        }
+        // constructs a range which is the reverse of the current one
+        // note - the resulting intervals may not be strictValid()
+        void reverse( FieldRange &ret ) const {
+            assert( _special.empty() );
+            ret._intervals.clear();
+            ret._objData = _objData;
+            for( vector< FieldInterval >::const_reverse_iterator i = _intervals.rbegin(); i != _intervals.rend(); ++i ) {
+                FieldInterval fi;
+                fi._lower = i->_upper;
+                fi._upper = i->_lower;
+                ret._intervals.push_back( fi );
+            }
+        }
     private:
         BSONObj addObj( const BSONObj &o );
-        vector< FieldInterval > intervals_;
-        vector< BSONObj > objData_;
+        void finishOperation( const vector< FieldInterval > &newIntervals, const FieldRange &other );
+        vector< FieldInterval > _intervals;
+        vector< BSONObj > _objData;
         string _special;
     };
     
@@ -101,10 +145,10 @@ namespace mongo {
             return !operator==( other );
         }
         bool operator<( const QueryPattern &other ) const {
-            map< string, Type >::const_iterator i = fieldTypes_.begin();
-            map< string, Type >::const_iterator j = other.fieldTypes_.begin();
-            while( i != fieldTypes_.end() ) {
-                if ( j == other.fieldTypes_.end() )
+            map< string, Type >::const_iterator i = _fieldTypes.begin();
+            map< string, Type >::const_iterator j = other._fieldTypes.begin();
+            while( i != _fieldTypes.end() ) {
+                if ( j == other._fieldTypes.end() )
                     return false;
                 if ( i->first < j->first )
                     return true;
@@ -117,14 +161,14 @@ namespace mongo {
                 ++i;
                 ++j;
             }
-            if ( j != other.fieldTypes_.end() )
+            if ( j != other._fieldTypes.end() )
                 return true;
-            return sort_.woCompare( other.sort_ ) < 0;
+            return _sort.woCompare( other._sort ) < 0;
         }
     private:
         QueryPattern() {}
         void setSort( const BSONObj sort ) {
-            sort_ = normalizeSort( sort );
+            _sort = normalizeSort( sort );
         }
         BSONObj static normalizeSort( const BSONObj &spec ) {
             if ( spec.isEmpty() )
@@ -140,73 +184,376 @@ namespace mongo {
             }
             return b.obj();
         }
-        map< string, Type > fieldTypes_;
-        BSONObj sort_;
+        map< string, Type > _fieldTypes;
+        BSONObj _sort;
     };
+
+    // a BoundList contains intervals specified by inclusive start
+    // and end bounds.  The intervals should be nonoverlapping and occur in
+    // the specified direction of traversal.  For example, given a simple index {i:1}
+    // and direction +1, one valid BoundList is: (1, 2); (4, 6).  The same BoundList
+    // would be valid for index {i:-1} with direction -1.
+    typedef vector< pair< BSONObj, BSONObj > > BoundList;	
     
     // ranges of fields' value that may be determined from query -- used to
     // determine index limits
     class FieldRangeSet {
     public:
+        friend class FieldRangeOrSet;
+        friend class FieldRangeVector;
         FieldRangeSet( const char *ns, const BSONObj &query , bool optimize=true );
+        bool hasRange( const char *fieldName ) const {
+            map< string, FieldRange >::const_iterator f = _ranges.find( fieldName );
+            return f != _ranges.end();
+        }
         const FieldRange &range( const char *fieldName ) const {
-            map< string, FieldRange >::const_iterator f = ranges_.find( fieldName );
-            if ( f == ranges_.end() )
+            map< string, FieldRange >::const_iterator f = _ranges.find( fieldName );
+            if ( f == _ranges.end() )
+                return trivialRange();
+            return f->second;            
+        }
+        FieldRange &range( const char *fieldName ) {
+            map< string, FieldRange >::iterator f = _ranges.find( fieldName );
+            if ( f == _ranges.end() )
                 return trivialRange();
-            return f->second;
+            return f->second;            
         }
         int nNontrivialRanges() const {
             int count = 0;
-            for( map< string, FieldRange >::const_iterator i = ranges_.begin(); i != ranges_.end(); ++i )
+            for( map< string, FieldRange >::const_iterator i = _ranges.begin(); i != _ranges.end(); ++i )
                 if ( i->second.nontrivial() )
                     ++count;
             return count;
         }
-        const char *ns() const { return ns_; }
-        BSONObj query() const { return query_; }
+        const char *ns() const { return _ns; }
         // if fields is specified, order fields of returned object to match those of 'fields'
         BSONObj simplifiedQuery( const BSONObj &fields = BSONObj() ) const;
         bool matchPossible() const {
-            for( map< string, FieldRange >::const_iterator i = ranges_.begin(); i != ranges_.end(); ++i )
+            for( map< string, FieldRange >::const_iterator i = _ranges.begin(); i != _ranges.end(); ++i )
                 if ( i->second.empty() )
                     return false;
             return true;
         }
         QueryPattern pattern( const BSONObj &sort = BSONObj() ) const;
-        BoundList indexBounds( const BSONObj &keyPattern, int direction ) const;
         string getSpecial() const;
+        const FieldRangeSet &operator-=( const FieldRangeSet &other ) {
+            int nUnincluded = 0;
+            string unincludedKey;
+            map< string, FieldRange >::iterator i = _ranges.begin();
+            map< string, FieldRange >::const_iterator j = other._ranges.begin();
+            while( nUnincluded < 2 && i != _ranges.end() && j != other._ranges.end() ) {
+                int cmp = i->first.compare( j->first );
+                if ( cmp == 0 ) {
+                    if ( i->second <= j->second ) {
+                        // nothing
+                    } else {
+                        ++nUnincluded;
+                        unincludedKey = i->first;
+                    }
+                    ++i;
+                    ++j;
+                } else if ( cmp < 0 ) {
+                    ++i;
+                } else {
+                    // other has a bound we don't, nothing can be done
+                    return *this;
+                }
+            }
+            if ( j != other._ranges.end() ) {
+                // other has a bound we don't, nothing can be done
+                return *this;                
+            }
+            if ( nUnincluded > 1 ) {
+                return *this;
+            }
+            if ( nUnincluded == 0 ) {
+                makeEmpty();
+                return *this;
+            }
+            // nUnincluded == 1
+            _ranges[ unincludedKey ] -= other._ranges[ unincludedKey ];
+            appendQueries( other );
+            return *this;
+        }
+        const FieldRangeSet &operator&=( const FieldRangeSet &other ) {
+            map< string, FieldRange >::iterator i = _ranges.begin();
+            map< string, FieldRange >::const_iterator j = other._ranges.begin();
+            while( i != _ranges.end() && j != other._ranges.end() ) {
+                int cmp = i->first.compare( j->first );
+                if ( cmp == 0 ) {
+                    i->second &= j->second;
+                    ++i;
+                    ++j;
+                } else if ( cmp < 0 ) {
+                    ++i;
+                } else {
+                    _ranges[ j->first ] = j->second;
+                    ++j;
+                }
+            }
+            while( j != other._ranges.end() ) {
+                _ranges[ j->first ] = j->second;
+                ++j;                
+            }
+            appendQueries( other );
+            return *this;
+        }
+        // TODO get rid of this
+        BoundList indexBounds( const BSONObj &keyPattern, int direction ) const;
     private:
+        void appendQueries( const FieldRangeSet &other ) {
+            for( vector< BSONObj >::const_iterator i = other._queries.begin(); i != other._queries.end(); ++i ) {
+                _queries.push_back( *i );                
+            }                        
+        }
+        void makeEmpty() {
+            for( map< string, FieldRange >::iterator i = _ranges.begin(); i != _ranges.end(); ++i ) {
+                i->second.makeEmpty();
+            }
+        }
+        void processQueryField( const BSONElement &e, bool optimize );
         void processOpElement( const char *fieldName, const BSONElement &f, bool isNot, bool optimize );
         static FieldRange *trivialRange_;
         static FieldRange &trivialRange();
-        mutable map< string, FieldRange > ranges_;
-        const char *ns_;
-        BSONObj query_;
+        mutable map< string, FieldRange > _ranges;
+        const char *_ns;
+        // make sure memory for FieldRange BSONElements is owned
+        vector< BSONObj > _queries;
     };
 
+    class FieldRangeVector {
+    public:
+        FieldRangeVector( const FieldRangeSet &frs, const BSONObj &keyPattern, int direction )
+        :_keyPattern( keyPattern ), _direction( direction >= 0 ? 1 : -1 )
+        {
+            _queries = frs._queries;
+            BSONObjIterator i( _keyPattern );
+            while( i.more() ) {
+                BSONElement e = i.next();
+                int number = (int) e.number(); // returns 0.0 if not numeric
+                bool forward = ( ( number >= 0 ? 1 : -1 ) * ( direction >= 0 ? 1 : -1 ) > 0 );
+                if ( forward ) {
+                    _ranges.push_back( frs.range( e.fieldName() ) );
+                } else {
+                    _ranges.push_back( FieldRange() );
+                    frs.range( e.fieldName() ).reverse( _ranges.back() );
+                }
+                assert( !_ranges.back().empty() );
+            }
+            uassert( 13385, "combinatorial limit of $in partitioning of result set exceeded", size() < 1000000 );
+        }
+        long long size() {
+            long long ret = 1;
+            for( vector< FieldRange >::const_iterator i = _ranges.begin(); i != _ranges.end(); ++i ) {
+                ret *= i->intervals().size();
+            }
+            return ret;
+        }        
+        BSONObj startKey() const {
+            BSONObjBuilder b;
+            for( vector< FieldRange >::const_iterator i = _ranges.begin(); i != _ranges.end(); ++i ) {
+                const FieldInterval &fi = i->intervals().front();
+                b.appendAs( fi._lower._bound, "" );
+            }
+            return b.obj();            
+        }
+        BSONObj endKey() const {
+            BSONObjBuilder b;
+            for( vector< FieldRange >::const_iterator i = _ranges.begin(); i != _ranges.end(); ++i ) {
+                const FieldInterval &fi = i->intervals().back();
+                b.appendAs( fi._upper._bound, "" );
+            }
+            return b.obj();            
+        }
+        BSONObj obj() const {
+            BSONObjBuilder b;
+            BSONObjIterator k( _keyPattern );
+            for( int i = 0; i < (int)_ranges.size(); ++i ) {
+                BSONArrayBuilder a( b.subarrayStart( k.next().fieldName() ) );
+                for( vector< FieldInterval >::const_iterator j = _ranges[ i ].intervals().begin();
+                    j != _ranges[ i ].intervals().end(); ++j ) {
+                    a << BSONArray( BSON_ARRAY( j->_lower._bound << j->_upper._bound ).clientReadable() );
+                }
+                a.done();
+            }
+            return b.obj();
+        }
+        bool matches( const BSONObj &obj ) const;
+        class Iterator {
+        public:
+            Iterator( const FieldRangeVector &v ) : _v( v ), _i( _v._ranges.size(), -1 ), _cmp( _v._ranges.size(), 0 ), _superlative( _v._ranges.size(), 0 ) {
+                static BSONObj minObj = minObject();
+                static BSONElement minElt = minObj.firstElement();
+                static BSONObj maxObj = maxObject();
+                static BSONElement maxElt = maxObj.firstElement();
+                BSONObjIterator i( _v._keyPattern );
+                for( int j = 0; j < (int)_superlative.size(); ++j ) {
+                    int number = (int) i.next().number();
+                    bool forward = ( ( number >= 0 ? 1 : -1 ) * ( _v._direction >= 0 ? 1 : -1 ) > 0 );
+                    _superlative[ j ] = forward ? &maxElt : &minElt;
+                }
+            }
+            static BSONObj minObject() {
+                BSONObjBuilder b;
+                b.appendMinKey( "" );
+                return b.obj();
+            }
+            static BSONObj maxObject() {
+                BSONObjBuilder b;
+                b.appendMaxKey( "" );
+                return b.obj();
+            }
+            bool advance() {
+                int i = _i.size() - 1;
+                while( i >= 0 && _i[ i ] >= ( (int)_v._ranges[ i ].intervals().size() - 1 ) ) {
+                    --i;
+                }
+                if( i >= 0 ) {
+                    _i[ i ]++;
+                    for( unsigned j = i + 1; j < _i.size(); ++j ) {
+                        _i[ j ] = 0;
+                    }
+                } else {
+                    _i[ 0 ] = _v._ranges[ 0 ].intervals().size();
+                }
+                return ok();
+            }
+            // return value
+            // -2 end of iteration
+            // -1 no skipping
+            // >= 0 skip parameter
+            int advance( const BSONObj &curr );
+            const vector< const BSONElement * > &cmp() const { return _cmp; }
+            void setZero( int i ) {
+                for( int j = i; j < (int)_i.size(); ++j ) {
+                    _i[ j ] = 0;
+                }
+            }
+            void setMinus( int i ) {
+                for( int j = i; j < (int)_i.size(); ++j ) {
+                    _i[ j ] = -1;
+                }
+            }
+            bool ok() {
+                return _i[ 0 ] < (int)_v._ranges[ 0 ].intervals().size();
+            }
+            BSONObj startKey() {
+                BSONObjBuilder b;
+                for( int unsigned i = 0; i < _i.size(); ++i ) {
+                    const FieldInterval &fi = _v._ranges[ i ].intervals()[ _i[ i ] ];
+                    b.appendAs( fi._lower._bound, "" );
+                }
+                return b.obj();
+            }
+            // temp
+            BSONObj endKey() {
+                BSONObjBuilder b;
+                for( int unsigned i = 0; i < _i.size(); ++i ) {
+                    const FieldInterval &fi = _v._ranges[ i ].intervals()[ _i[ i ] ];
+                    b.appendAs( fi._upper._bound, "" );
+                }
+                return b.obj();            
+            }
+            // check
+        private:
+            const FieldRangeVector &_v;
+            vector< int > _i;
+            vector< const BSONElement* > _cmp;
+            vector< const BSONElement* > _superlative;
+        };
+    private:
+        int matchingLowElement( const BSONElement &e, int i, bool direction ) const;
+        bool matchesElement( const BSONElement &e, int i, bool direction ) const;
+        vector< FieldRange > _ranges;
+        BSONObj _keyPattern;
+        int _direction;
+        vector< BSONObj > _queries; // make sure mem owned
+    };
+        
+    // generages FieldRangeSet objects, accounting for or clauses
+    class FieldRangeOrSet {
+    public:
+        FieldRangeOrSet( const char *ns, const BSONObj &query , bool optimize=true );
+        // if there's a useless or clause, we won't use or ranges to help with scanning
+        bool orFinished() const { return _orFound && _orSets.empty(); }
+        // removes first or clause, and removes the field ranges it covers from all subsequent or clauses
+        // this could invalidate the result of the last topFrs()
+        void popOrClause() {
+            massert( 13274, "no or clause to pop", !orFinished() );
+            const FieldRangeSet &toPop = _orSets.front();
+            list< FieldRangeSet >::iterator i = _orSets.begin();
+            ++i;
+            while( i != _orSets.end() ) {
+                *i -= toPop;
+                if( !i->matchPossible() ) {
+                    i = _orSets.erase( i );
+                } else {    
+                    ++i;
+                }
+            }
+            _oldOrSets.push_front( toPop );
+            _orSets.pop_front();
+        }
+        FieldRangeSet *topFrs() const {
+            FieldRangeSet *ret = new FieldRangeSet( _baseSet );
+            if (_orSets.size()){
+                *ret &= _orSets.front();
+            }
+            return ret;
+        }
+        void allClausesSimplified( vector< BSONObj > &ret ) const {
+            for( list< FieldRangeSet >::const_iterator i = _orSets.begin(); i != _orSets.end(); ++i ) {
+                if ( i->matchPossible() ) {
+                    ret.push_back( i->simplifiedQuery() );
+                }
+            }
+        }
+        string getSpecial() const { return _baseSet.getSpecial(); }
+
+        bool moreOrClauses() const { return !_orSets.empty(); }
+    private:
+        FieldRangeSet _baseSet;
+        list< FieldRangeSet > _orSets;
+        list< FieldRangeSet > _oldOrSets; // make sure memory is owned
+        bool _orFound;
+    };
+    
     /**
        used for doing field limiting
      */
     class FieldMatcher {
     public:
-
-        FieldMatcher(bool include=false) : _include(include){}
+        FieldMatcher()
+            : _include(true)
+            , _special(false)
+            , _includeID(true)
+            , _skip(0)
+            , _limit(-1)
+        {}
         
         void add( const BSONObj& o );
 
         void append( BSONObjBuilder& b , const BSONElement& e ) const;
 
         BSONObj getSpec() const;
+        bool includeID() { return _includeID; }
     private:
 
         void add( const string& field, bool include );
-        void appendArray( BSONObjBuilder& b , const BSONObj& a ) const;
+        void add( const string& field, int skip, int limit );
+        void appendArray( BSONObjBuilder& b , const BSONObj& a , bool nested=false) const;
 
         bool _include; // true if default at this level is to include
+        bool _special; // true if this level can't be skipped or included without recursing
         //TODO: benchmark vector<pair> vs map
         typedef map<string, boost::shared_ptr<FieldMatcher> > FieldMap;
         FieldMap _fields;
         BSONObj _source;
+        bool _includeID;
+
+        // used for $slice operator
+        int _skip;
+        int _limit;
     };
 
     /** returns a string that when used as a matcher, would match a super set of regex()
@@ -220,4 +567,6 @@ namespace mongo {
     /** returns the upper bound of a query that matches prefix */
     string simpleRegexEnd( string prefix );
 
+    long long applySkipLimit( long long num , const BSONObj& cmd );
+
 } // namespace mongo
diff --git a/db/rec.h b/db/rec.h
index ee75669..7b79c73 100644
--- a/db/rec.h
+++ b/db/rec.h
@@ -28,7 +28,7 @@
 #pragma once
 
 #include "reci.h"
-#include "reccache.h"
+//#include "reccache.h"
 
 namespace mongo { 
 
@@ -41,28 +41,28 @@ NamespaceDetails* nsdetails_notinline(const char *ns);
 
 class MongoMemMapped_RecStore : public RecStoreInterface { 
 public:
-    virtual char* get(DiskLoc d, unsigned len) { return d.rec()->data; }
+    VIRT char* get(DiskLoc d, unsigned len) { return d.rec()->data; }
 
-    virtual DiskLoc insert(const char *ns, const void *obuf, int len, bool god) { 
+    VIRT DiskLoc insert(const char *ns, const void *obuf, int len, bool god) { 
         return theDataFileMgr.insert(ns, obuf, len, god);
     }
 
-    virtual void deleteRecord(const char *ns, DiskLoc d) { 
+    VIRT void deleteRecord(const char *ns, DiskLoc d) { 
         theDataFileMgr._deleteRecord(nsdetails_notinline(ns), ns, d.rec(), d);
     }
 
-    virtual void modified(DiskLoc d) { }
+    VIRT void modified(DiskLoc d) { }
 
-    virtual void drop(const char *ns) { 
+    VIRT void drop(const char *ns) { 
         dropNS(ns);
     }
 
-    virtual void rename(const char *fromNs, const char *toNs) {
+    VIRT void rename(const char *fromNs, const char *toNs) {
       renameNamespace( fromNs, toNs );
     }
 
     /* close datafiles associated with the db specified. */
-    virtual void closeFiles(string dbname, string path) {
+    VIRT void closeFiles(string dbname, string path) {
         /* as this is only used for indexes so far, and we are in the same 
            PDFiles as the nonindex data, we just rely on them having been closed 
            at the same time.  one day this may need to change.
@@ -116,7 +116,9 @@ public:
 
 /* Glue btree to RecStoreInterface: ---------------------------- */
 
-extern RecStoreInterface *btreeStore;
+typedef MongoMemMapped_RecStore StoreToUse;
+
+extern StoreToUse *btreeStore;
 
 const int BucketSize = 8192;
 
diff --git a/db/reccache.cpp b/db/reccache.cpp
index 6e1f3de..eb20728 100644
--- a/db/reccache.cpp
+++ b/db/reccache.cpp
@@ -16,15 +16,17 @@
 
 // storage.cpp
 
-#include "stdafx.h"
+#include "pch.h"
 #include "pdfile.h"
-#include "reccache.h"
+//#include "reccache.h"
 #include "rec.h"
 #include "db.h"
 
+#error deprecated - do not include in project
+
 namespace mongo {
 
-RecCache theRecCache(BucketSize);
+//RecCache theRecCache(BucketSize);
 
 // 100k * 8KB = 800MB
 unsigned RecCache::MAXNODES = 50000;
diff --git a/db/reccache.h b/db/reccache.h
index d354587..d0fd118 100644
--- a/db/reccache.h
+++ b/db/reccache.h
@@ -32,6 +32,8 @@
 
 #pragma once
 
+#error deprecated 
+
 #include "reci.h"
 #include "recstore.h"
 
@@ -212,29 +214,29 @@ extern RecCache theRecCache;
 
 class CachedBasicRecStore : public RecStoreInterface { 
 public:
-    virtual char* get(DiskLoc d, unsigned len) { 
+    VIRT char* get(DiskLoc d, unsigned len) { 
         return theRecCache.get(d, len);
     }
 
-    virtual DiskLoc insert(const char *ns, const void *obuf, int len, bool god) { 
+    VIRT DiskLoc insert(const char *ns, const void *obuf, int len, bool god) { 
         return theRecCache.insert(ns, obuf, len, god);
     }
 
-    virtual void modified(DiskLoc d) { 
+    VIRT void modified(DiskLoc d) { 
         theRecCache.dirty(d);
     }
 
     /* drop collection */
-    virtual void drop(const char *ns) { 
+    VIRT void drop(const char *ns) { 
         theRecCache.drop(ns);
     }
 
-    virtual void rename(const char *fromNs, const char *toNs) {
+    VIRT void rename(const char *fromNs, const char *toNs) {
       massert( 10378 ,  "rename not yet implemented for CachedBasicRecStore", false );
     }
 
     /* close datafiles associated with the db specified. */
-    virtual void closeFiles(string dbname, string path) {
+    VIRT void closeFiles(string dbname, string path) {
         theRecCache.closeFiles(dbname, dbpath);
     }
 };
@@ -244,17 +246,17 @@ public:
    call 
 */
 
-inline void dbunlocking_read() { 
+//inline void dbunlocking_read() { 
     /*
     Client *c = currentClient.get();
     if ( c )
         c->top.clientStop();
     */
-}
+//}
 
-inline void dbunlocking_write() { 
-    theRecCache.ejectOld();
-	dbunlocking_read();
-}
+//inline void dbunlocking_write() { 
+    //theRecCache.ejectOld();
+//	dbunlocking_read();
+//}
 
 } /*namespace*/
diff --git a/db/reci.h b/db/reci.h
index 08dcece..a22f1f1 100644
--- a/db/reci.h
+++ b/db/reci.h
@@ -22,35 +22,38 @@
 
 namespace mongo { 
 
+// #define VIRT virtual
+#define VIRT 
+
 /* Subclass this and implement your real storage interface.
 */
 class RecStoreInterface {
 public:
-    virtual ~RecStoreInterface() {}
+    //VIRT ~RecStoreInterface() {}
 
     /* Get a pointer to the data at diskloc d.  Pointer guaranteed to stay in
        scope through the current database operation's life.
     */
-    virtual char* get(DiskLoc d, unsigned len) = 0;
+    //VIRT char* get(DiskLoc d, unsigned len) = 0;
 
     /* indicate that the diskloc specified has been updated. note that as-is today, the modification may come AFTER this 
        call -- we handle that currently -- until the dblock finishes.
     */
-    virtual void modified(DiskLoc d) = 0;
+    //VIRT void modified(DiskLoc d) = 0;
 
     /* insert specified data as a record */
-    virtual DiskLoc insert(const char *ns, const void *obuf, int len, bool god) = 0;
+    //VIRT DiskLoc insert(const char *ns, const void *obuf, int len, bool god) = 0;
 
-    virtual void deleteRecord(const char *ns, DiskLoc d) { massert( 10379 , "not implemented RecStoreInterface::deleteRecord", false); }
+    //VIRT void deleteRecord(const char *ns, DiskLoc d) { massert( 10379 , "not implemented RecStoreInterface::deleteRecord", false); }
 
     /* drop the collection */
-    virtual void drop(const char *ns) = 0;
+    //VIRT void drop(const char *ns) = 0;
 
     /* rename collection */
-    virtual void rename(const char *fromNs, const char *toNs) = 0;
+    //VIRT void rename(const char *fromNs, const char *toNs) = 0;
 
     /* close datafiles associated with the db specified. */
-    virtual void closeFiles(string dbname, string path) = 0;
+    //VIRT void closeFiles(string dbname, string path) = 0;
 
     /* todo add: 
        closeFiles(dbname)
diff --git a/db/recstore.h b/db/recstore.h
index bdb3d77..913070f 100644
--- a/db/recstore.h
+++ b/db/recstore.h
@@ -18,6 +18,8 @@
 
 #pragma once
 
+#error deprecated
+
 #include "../util/file.h"
 
 namespace mongo { 
diff --git a/db/repl.cpp b/db/repl.cpp
index efb078b..8cdd545 100644
--- a/db/repl.cpp
+++ b/db/repl.cpp
@@ -31,11 +31,12 @@
    local.pair.sync       - { initialsynccomplete: 1 }
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "jsobj.h"
 #include "../util/goodies.h"
 #include "repl.h"
 #include "../util/message.h"
+#include "../util/background.h"
 #include "../client/dbclient.h"
 #include "../client/connpool.h"
 #include "pdfile.h"
@@ -44,16 +45,17 @@
 #include "commands.h"
 #include "security.h"
 #include "cmdline.h"
+#include "repl_block.h"
+#include "repl/rs.h"
 
 namespace mongo {
     
     // our config from command line etc.
     ReplSettings replSettings;
 
-    void ensureHaveIdIndex(const char *ns);
-
     /* if 1 sync() is running */
-    int syncing = 0;
+    volatile int syncing = 0;
+	static volatile int relinquishSyncingSome = 0;
 
     /* if true replace our peer in a replication pair -- don't worry about if his
        local.oplog.$main is empty.
@@ -69,11 +71,9 @@ namespace mongo {
     
     IdTracker &idTracker = *( new IdTracker() );
     
-    int __findingStartInitialTimeout = 5; // configurable for testing    
-
 } // namespace mongo
 
-#include "replset.h"
+#include "replpair.h"
 
 namespace mongo {
 
@@ -102,7 +102,7 @@ namespace mongo {
             return;
         info = _comment;
         if ( n != state && !cmdLine.quiet )
-            log() << "pair: setting master=" << n << " was " << state << endl;
+            tlog() << "pair: setting master=" << n << " was " << state << endl;
         state = n;
     }
 
@@ -119,7 +119,7 @@ namespace mongo {
         auto_ptr<DBClientConnection> conn( newClientConnection() );
         string errmsg;
         if ( !conn->connect(arbHost.c_str(), errmsg) ) {
-            log() << "repl:   cantconn arbiter " << errmsg << endl;
+            tlog() << "repl:   cantconn arbiter " << errmsg << endl;
             setMasterLocked(State_CantArb, "can't connect to arb");
             return;
         }
@@ -131,18 +131,19 @@ namespace mongo {
 
     class CmdReplacePeer : public Command {
     public:
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        virtual bool adminOnly() {
+        virtual bool adminOnly() const {
             return true;
         }
         virtual bool logTheOp() {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; }
-        CmdReplacePeer() : Command("replacepeer") { }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        virtual LockType locktype() const { return WRITE; }
+        void help(stringstream&h) const { h << "replace a node in a replica pair"; }
+        CmdReplacePeer() : Command("replacePeer", false, "replacepeer") { }
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             if ( replPair == 0 ) {
                 errmsg = "not paired";
                 return false;
@@ -157,11 +158,12 @@ namespace mongo {
             }
             Timer t;
             while ( 1 ) {
-                if ( syncing == 0 || t.millis() > 20000 )
+                if ( syncing == 0 || t.millis() > 30000 )
                     break;
                 {
                     dbtemprelease t;
-                    sleepmillis(10);
+					relinquishSyncingSome = 1;
+					sleepmillis(1);
                 }
             }
             if ( syncing ) {
@@ -191,18 +193,19 @@ namespace mongo {
 
     class CmdForceDead : public Command {
     public:
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        virtual bool adminOnly() {
+        virtual bool adminOnly() const {
             return true;
         }
         virtual bool logTheOp() {
             return false;   
         }
-        virtual LockType locktype(){ return WRITE; }
+        virtual void help(stringstream& h) const { h << "internal"; }
+        virtual LockType locktype() const { return WRITE; }
         CmdForceDead() : Command("forcedead") { }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             replAllDead = "replication forced to stop by 'forcedead' command";
             log() << "*********************************************************\n";
             log() << "received 'forcedead' command, replication forced to stop" << endl;
@@ -213,18 +216,19 @@ namespace mongo {
     /* operator requested resynchronization of replication (on the slave).  { resync : 1 } */
     class CmdResync : public Command {
     public:
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        virtual bool adminOnly() {
+        virtual bool adminOnly() const {
             return true;
         }
         virtual bool logTheOp() {
             return false;
         }
-        virtual LockType locktype(){ return WRITE; }
+        virtual LockType locktype() const { return WRITE; }
+        void help(stringstream&h) const { h << "resync (from scratch) an out of date replica slave.\nhttp://www.mongodb.org/display/DOCS/Master+Slave"; }
         CmdResync() : Command("resync") { }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             if ( cmdObj.getBoolField( "force" ) ) {
                 if ( !waitForSyncToFinish( errmsg ) )
                     return false;
@@ -246,11 +250,12 @@ namespace mongo {
             // reloaded with new saved state on next pass.
             Timer t;
             while ( 1 ) {
-                if ( syncing == 0 || t.millis() > 20000 )
+                if ( syncing == 0 || t.millis() > 30000 )
                     break;
                 {
                     dbtemprelease t;
-                    sleepmillis(10);
+					relinquishSyncingSome = 1;
+                    sleepmillis(1);
                 }
             }
             if ( syncing ) {
@@ -268,7 +273,7 @@ namespace mongo {
     void appendReplicationInfo( BSONObjBuilder& result , bool authed , int level ){
         
         if ( replAllDead ) {
-            result.append("ismaster", 0.0);
+            result.append("ismaster", 0);
             if( authed ) { 
                 if ( replPair )
                     result.append("remote", replPair->remote);
@@ -281,20 +286,22 @@ namespace mongo {
             if( authed ) {
                 result.append("remote", replPair->remote);
                 if ( !replPair->info.empty() )
-                    result.append("info", replPair->info);
+                    result.append("info", replPair->info.toString());
             }
         }
         else {
-            result.append("ismaster", replSettings.slave ? 0 : 1);
-            result.append("msg", "not paired");
+            result.appendBool("ismaster", _isMaster() );
         }
-        
-        if ( level ){
+
+        if ( level && replSet ){
+            result.append( "info" , "is replica set" );
+        }
+        else if ( level ){
             BSONObjBuilder sources( result.subarrayStart( "sources" ) );
             
             readlock lk( "local.sources" );
             Client::Context ctx( "local.sources" );
-            auto_ptr<Cursor> c = findTableScan("local.sources", BSONObj());
+            shared_ptr<Cursor> c = findTableScan("local.sources", BSONObj());
             int n = 0;
             while ( c->ok() ){
                 BSONObj s = c->current();
@@ -336,17 +343,33 @@ namespace mongo {
     class CmdIsMaster : public Command {
     public:
         virtual bool requiresAuth() { return false; }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        virtual LockType locktype(){ return NONE; }
-        CmdIsMaster() : Command("ismaster") { }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) {
+        virtual void help( stringstream &help ) const {
+            help << "Check if this server is primary for a replica pair/set; also if it is --master or --slave in simple master/slave setups.\n";
+            help << "{ isMaster : 1 }";
+        }
+        virtual LockType locktype() const { return NONE; }
+        CmdIsMaster() : Command("isMaster", true, "ismaster") { }
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) {
 			/* currently request to arbiter is (somewhat arbitrarily) an ismaster request that is not 
 			   authenticated.
 			   we allow unauthenticated ismaster but we aren't as verbose informationally if 
 			   one is not authenticated for admin db to be safe.
 			*/
+
+            if( replSet ) {
+                if( theReplSet == 0 ) { 
+                    result.append("ismaster", false);
+                    result.append("secondary", false);
+                    errmsg = "replSet still trying to initialize";
+                    result.append("info", ReplSet::startupStatusMsg);
+                    return true;
+                }
+                theReplSet->fillIsMaster(result);
+                return true;
+            }
             
 			bool authed = cc().getAuthenticationInfo()->isAuthorizedReads("admin");
             appendReplicationInfo( result , authed );
@@ -357,12 +380,12 @@ namespace mongo {
     class CmdIsInitialSyncComplete : public Command {
     public:
         virtual bool requiresAuth() { return false; }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        virtual LockType locktype(){ return WRITE; }
+        virtual LockType locktype() const { return WRITE; }
         CmdIsInitialSyncComplete() : Command( "isinitialsynccomplete" ) {}
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) {
+        virtual bool run(const string&, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool /*fromRepl*/) {
             result.appendBool( "initialsynccomplete", getInitialSyncCompleted() );
             return true;
         }
@@ -388,14 +411,14 @@ namespace mongo {
     class CmdNegotiateMaster : public Command {
     public:
         CmdNegotiateMaster() : Command("negotiatemaster") { }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        virtual bool adminOnly() {
+        virtual bool adminOnly() const {
             return true;
         }
-        virtual LockType locktype(){ return WRITE; }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
+        virtual LockType locktype() const { return WRITE; }
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool) {
             if ( replPair == 0 ) {
                 massert( 10383 ,  "Another mongod instance believes incorrectly that this node is its peer", !cmdObj.getBoolField( "fromArbiter" ) );
                 // assume that we are an arbiter and should forward the request
@@ -430,7 +453,7 @@ namespace mongo {
                     if ( e.fieldName() != string( "ok" ) )
                         result.append( e );
                 }
-                return ( ret.getIntField("ok") == 1 );
+                return ret["ok"].trueValue();
             }
 
             int was = cmdObj.getIntField("i_was");
@@ -476,7 +499,7 @@ namespace mongo {
         b.append("your_port", remotePort);
         BSONObj cmd = b.done();
         BSONObj res = conn->findOne("admin.$cmd", cmd);
-        if ( res.getIntField("ok") != 1 ) {
+        if ( ! res["ok"].trueValue() ){
             string message = method + " negotiate failed";
             problem() << message << ": " << res.toString() << '\n';
             setMasterLocked(State_Confused, message.c_str());
@@ -495,20 +518,6 @@ namespace mongo {
         return remote;
     }
 
-    struct TestOpTime {
-        TestOpTime() {
-            OpTime t;
-            for ( int i = 0; i < 10; i++ ) {
-                OpTime s = OpTime::now();
-                assert( s != t );
-                t = s;
-            }
-            OpTime q = t;
-            assert( q == t );
-            assert( !(q != t) );
-        }
-    } testoptime;
-
     /* --------------------------------------------------------------*/
 
     ReplSource::ReplSource() {
@@ -573,7 +582,7 @@ namespace mongo {
         int n = 0;
         for ( set<string>::iterator i = addDbNextPass.begin(); i != addDbNextPass.end(); i++ ) {
             n++;
-            dbsNextPassBuilder.appendBool(i->c_str(), 1);
+            dbsNextPassBuilder.appendBool(*i, 1);
         }
         if ( n )
             b.append("dbsNextPass", dbsNextPassBuilder.done());
@@ -582,7 +591,7 @@ namespace mongo {
         n = 0;
         for ( set<string>::iterator i = incompleteCloneDbs.begin(); i != incompleteCloneDbs.end(); i++ ) {
             n++;
-            incompleteCloneDbsBuilder.appendBool(i->c_str(), 1);
+            incompleteCloneDbsBuilder.appendBool(*i, 1);
         }
         if ( n )
             b.append("incompleteCloneDbs", incompleteCloneDbsBuilder.done());
@@ -650,7 +659,7 @@ namespace mongo {
             // --source <host> specified.
             // check that no items are in sources other than that
             // add if missing
-            auto_ptr<Cursor> c = findTableScan("local.sources", BSONObj());
+            shared_ptr<Cursor> c = findTableScan("local.sources", BSONObj());
             int n = 0;
             while ( c->ok() ) {
                 n++;
@@ -694,7 +703,7 @@ namespace mongo {
             }
             // check that no items are in sources other than that
             // add if missing
-            auto_ptr<Cursor> c = findTableScan("local.sources", BSONObj());
+            shared_ptr<Cursor> c = findTableScan("local.sources", BSONObj());
             int n = 0;
             while ( c->ok() ) {
                 n++;
@@ -716,7 +725,7 @@ namespace mongo {
             }
         }
 
-        auto_ptr<Cursor> c = findTableScan("local.sources", BSONObj());
+        shared_ptr<Cursor> c = findTableScan("local.sources", BSONObj());
         while ( c->ok() ) {
             ReplSource tmp(c->current());
             if ( replPair && tmp.hostName == replPair->remote && tmp.sourceName() == "main" ) {
@@ -779,8 +788,9 @@ namespace mongo {
         BSONObj info;
         {
             dbtemprelease t;
-            connect();
-            bool ok = conn->runCommand( "admin", BSON( "listDatabases" << 1 ), info );
+            oplogReader.connect(hostName);
+            /* todo use getDatabaseNames() method here */
+            bool ok = oplogReader.conn()->runCommand( "admin", BSON( "listDatabases" << 1 ), info );
             massert( 10385 ,  "Unable to get database list", ok );
         }
         BSONObjIterator i( info.getField( "databases" ).embeddedObject() );
@@ -804,11 +814,9 @@ namespace mongo {
 
     string ReplSource::resyncDrop( const char *db, const char *requester ) {
         log() << "resync: dropping database " << db << endl;
-        string dummyns = string( db ) + ".";
-        Client::Context ctx(dummyns);
-        assert( cc().database()->name == db );
-        dropDatabase(dummyns.c_str());
-        return dummyns;
+        Client::Context ctx(db);
+        dropDatabase(db);
+        return db;
     }
     
     /* grab initial copy of a database from the master */
@@ -832,61 +840,8 @@ namespace mongo {
     }
 
     void ReplSource::applyOperation(const BSONObj& op) {
-        log( 6 ) << "applying op: " << op << endl;
-        OpDebug debug;
-        BSONObj o = op.getObjectField("o");
-        const char *ns = op.getStringField("ns");
-        // operation type -- see logOp() comments for types
-        const char *opType = op.getStringField("op");
         try {
-            if ( *opType == 'i' ) {
-                const char *p = strchr(ns, '.');
-                if ( p && strcmp(p, ".system.indexes") == 0 ) {
-                    // updates aren't allowed for indexes -- so we will do a regular insert. if index already
-                    // exists, that is ok.
-                    theDataFileMgr.insert(ns, (void*) o.objdata(), o.objsize());
-                }
-                else {
-                    // do upserts for inserts as we might get replayed more than once
-					BSONElement _id;
-					if( !o.getObjectID(_id) ) {
-						/* No _id.  This will be very slow. */
-                        Timer t;
-                        updateObjects(ns, o, o, true, false, false , debug );
-                        if( t.millis() >= 2 ) {
-                            RARELY OCCASIONALLY log() << "warning, repl doing slow updates (no _id field) for " << ns << endl;
-                        }
-                    }
-                    else {
-                        BSONObjBuilder b;
-						b.append(_id);
-                        
-                        /* erh 10/16/2009 - this is probably not relevant any more since its auto-created, but not worth removing */
-                        RARELY ensureHaveIdIndex(ns); // otherwise updates will be slow 
-
-                        updateObjects(ns, o, b.done(), true, false, false , debug );
-                    }
-                }
-            }
-            else if ( *opType == 'u' ) {
-                RARELY ensureHaveIdIndex(ns); // otherwise updates will be super slow
-                updateObjects(ns, o, op.getObjectField("o2"), op.getBoolField("b"), false, false , debug );
-            }
-            else if ( *opType == 'd' ) {
-                if ( opType[1] == 0 )
-                    deleteObjects(ns, o, op.getBoolField("b"));
-                else
-                    assert( opType[1] == 'b' ); // "db" advertisement
-            }
-            else if ( *opType == 'n' ) {
-                // no op
-            }
-            else {
-                BufBuilder bb;
-                BSONObjBuilder ob;
-                assert( *opType == 'c' );
-                _runCommands(ns, o, bb, ob, true, 0);
-            }
+            applyOperation_inlock( op );
         }
         catch ( UserException& e ) {
             log() << "sync: caught user assertion " << e << " while applying op: " << op << endl;;
@@ -894,8 +849,9 @@ namespace mongo {
         catch ( DBException& e ) {
             log() << "sync: caught db exception " << e << " while applying op: " << op << endl;;            
         }
+
     }
-    
+
     /* local.$oplog.main is of the form:
          { ts: ..., op: <optype>, ns: ..., o: <obj> , o2: <extraobj>, b: <boolflag> }
          ...
@@ -924,6 +880,42 @@ namespace mongo {
         if ( !only.empty() && only != clientName )
             return;
 
+        if( cmdLine.pretouch ) {
+            if( cmdLine.pretouch > 1 ) {
+                /* note: this is bad - should be put in ReplSource.  but this is first test... */
+                static int countdown;
+                if( countdown > 0 ) {
+                    countdown--; // was pretouched on a prev pass
+                    assert( countdown >= 0 );
+                } else {
+                    const int m = 4;
+                    if( tp.get() == 0 ) {
+                        int nthr = min(8, cmdLine.pretouch);
+                        nthr = max(nthr, 1);
+                        tp.reset( new ThreadPool(nthr) );
+                    }
+                    vector<BSONObj> v;
+                    oplogReader.peek(v, cmdLine.pretouch);
+                    unsigned a = 0;
+                    while( 1 ) {
+                        if( a >= v.size() ) break;
+                        unsigned b = a + m - 1; // v[a..b]
+                        if( b >= v.size() ) b = v.size() - 1;
+                        tp->schedule(pretouchN, v, a, b);
+                        DEV cout << "pretouch task: " << a << ".." << b << endl;
+                        a += m;
+                    }
+                    // we do one too...
+                    pretouchOperation(op);
+                    tp->join();
+                    countdown = v.size();
+                }
+            }
+            else {
+                pretouchOperation(op);
+            }
+        }
+
         dblock lk;
 
         if ( localLogTail && replPair && replPair->state == ReplPair::State_Master ) {
@@ -943,7 +935,8 @@ namespace mongo {
         bool empty = ctx.db()->isEmpty();
         bool incompleteClone = incompleteCloneDbs.count( clientName ) != 0;
 
-        log( 6 ) << "ns: " << ns << ", justCreated: " << ctx.justCreated() << ", empty: " << empty << ", incompleteClone: " << incompleteClone << endl;
+        if( logLevel >= 6 )
+            log(6) << "ns: " << ns << ", justCreated: " << ctx.justCreated() << ", empty: " << empty << ", incompleteClone: " << incompleteClone << endl;
         
         // always apply admin command command
         // this is a bit hacky -- the semantics of replication/commands aren't well specified
@@ -1050,17 +1043,17 @@ namespace mongo {
         if ( !only.empty() ) {
             b.appendRegex("ns", string("^") + only);
         }        
-        BSONObj last = conn->findOne( _ns.c_str(), Query( b.done() ).sort( BSON( "$natural" << -1 ) ) );
+        BSONObj last = oplogReader.findOne( _ns.c_str(), Query( b.done() ).sort( BSON( "$natural" << -1 ) ) );
         if ( !last.isEmpty() ) {
             BSONElement ts = last.getField( "ts" );
-            massert( 10386 ,  (string)"non Date ts found:" + last.jsonString() , ts.type() == Date || ts.type() == Timestamp );
+            massert( 10386 ,  "non Date ts found: " + last.toString(), ts.type() == Date || ts.type() == Timestamp );
             syncedTo = OpTime( ts.date() );
         }        
     }
     
     OpTime ReplSource::nextLastSavedLocalTs() const {
         Client::Context ctx( "local.oplog.$main" );
-        auto_ptr< Cursor > c = findTableScan( "local.oplog.$main", BSON( "$natural" << -1 ) );
+        shared_ptr<Cursor> c = findTableScan( "local.oplog.$main", BSON( "$natural" << -1 ) );
         if ( c->ok() )
             return OpTime( c->current().getField( "ts" ).date() );        
         return OpTime();
@@ -1076,19 +1069,19 @@ namespace mongo {
         log() << "Sending forcedead command to slave to stop its replication\n";
         log() << "Host: " << hostName << " paired: " << paired << endl;
         massert( 10387 ,  "request to kill slave replication failed",
-                conn->simpleCommand( "admin", 0, "forcedead" ) );        
+                oplogReader.conn()->simpleCommand( "admin", 0, "forcedead" ) );
         syncToTailOfRemoteLog();
         {
             dblock lk;
             setLastSavedLocalTs( nextLastSavedLocalTs() );
             save();
-            cursor.reset();
+            oplogReader.resetCursor();
         }
     }
     
     bool ReplSource::updateSetsWithLocalOps( OpTime &localLogTail, bool mayUnlock ) {
         Client::Context ctx( "local.oplog.$main" );
-        auto_ptr< Cursor > localLog = findTableScan( "local.oplog.$main", BSON( "$natural" << -1 ) );
+        shared_ptr<Cursor> localLog = findTableScan( "local.oplog.$main", BSON( "$natural" << -1 ) );
         OpTime newTail;
         for( ; localLog->ok(); localLog->advance() ) {
             BSONObj op = localLog->current();
@@ -1119,17 +1112,17 @@ namespace mongo {
     
     /* slave: pull some data from the master's oplog
        note: not yet in db mutex at this point. 
+       @return -1 error
+               0 ok, don't sleep
+               1 ok, sleep
     */
-    bool ReplSource::sync_pullOpLog(int& nApplied) {
+    int ReplSource::sync_pullOpLog(int& nApplied) {
+        int okResultCode = 1;
         string ns = string("local.oplog.$") + sourceName();
         log(2) << "repl: sync_pullOpLog " << ns << " syncedTo:" << syncedTo.toStringLong() << '\n';
 
         bool tailing = true;
-        DBClientCursor *c = cursor.get();
-        if ( c && c->isDead() ) {
-            log() << "repl:   old cursor isDead, initiating a new one\n";
-            c = 0;
-        }
+        oplogReader.tailCheck();
 
         if ( replPair && replPair->state == ReplPair::State_Master ) {
             dblock lk;
@@ -1139,12 +1132,12 @@ namespace mongo {
 
         bool initial = syncedTo.isNull();
         
-        if ( c == 0 || initial ) {
+        if ( !oplogReader.haveCursor() || initial ) {
             if ( initial ) {
                 // Important to grab last oplog timestamp before listing databases.
                 syncToTailOfRemoteLog();
                 BSONObj info;
-                bool ok = conn->runCommand( "admin", BSON( "listDatabases" << 1 ), info );
+                bool ok = oplogReader.conn()->runCommand( "admin", BSON( "listDatabases" << 1 ), info );
                 massert( 10389 ,  "Unable to get database list", ok );
                 BSONObjIterator i( info.getField( "databases" ).embeddedObject() );
                 while( i.moreWithEOO() ) {
@@ -1171,27 +1164,22 @@ namespace mongo {
             query.append("ts", q.done());
             if ( !only.empty() ) {
                // note we may here skip a LOT of data table scanning, a lot of work for the master.
-                query.appendRegex("ns", string("^") + only);
+                query.appendRegex("ns", string("^") + only); // maybe append "\\." here?
             }
             BSONObj queryObj = query.done();
-            // queryObj = { ts: { $gte: syncedTo } }
-
-            log(2) << "repl: " << ns << ".find(" << queryObj.toString() << ')' << '\n';
-            cursor = conn->query( ns.c_str(), queryObj, 0, 0, 0, 
-                                  QueryOption_CursorTailable | QueryOption_SlaveOk | QueryOption_OplogReplay |
-                                  QueryOption_AwaitData
-                                  );
-            c = cursor.get();
+            // e.g. queryObj = { ts: { $gte: syncedTo } }
+
+            oplogReader.tailingQuery(ns.c_str(), queryObj);
             tailing = false;
         }
         else {
             log(2) << "repl: tailing=true\n";
         }
 
-        if ( c == 0 ) {
-            problem() << "repl:   dbclient::query returns null (conn closed?)" << endl;
-            resetConnection();
-            return false;
+        if( !oplogReader.haveCursor() ) {
+            problem() << "repl: dbclient::query returns null (conn closed?)" << endl;
+            oplogReader.resetConnection();
+            return -1;
         }
 
         // show any deferred database creates from a previous pass
@@ -1206,9 +1194,12 @@ namespace mongo {
             }
         }
 
-        if ( !c->more() ) {
+        if ( !oplogReader.more() ) {
             if ( tailing ) {
                 log(2) << "repl: tailing & no new activity\n";
+                if( oplogReader.awaitCapable() )
+                    okResultCode = 0; // don't sleep
+
             } else {
                 log() << "repl:   " << ns << " oplog is empty\n";
             }
@@ -1217,24 +1208,31 @@ namespace mongo {
                 OpTime nextLastSaved = nextLastSavedLocalTs();
                 {
                     dbtemprelease t;
-                    if ( !c->more() ) {
+                    if ( !oplogReader.more() ) {
                         setLastSavedLocalTs( nextLastSaved );
                     }
                 }
                 save();            
             }
-            return true;
+            return okResultCode;
         }
         
         OpTime nextOpTime;
         {
-            BSONObj op = c->next();
+            BSONObj op = oplogReader.next();
             BSONElement ts = op.getField("ts");
             if ( ts.type() != Date && ts.type() != Timestamp ) {
                 string err = op.getStringField("$err");
                 if ( !err.empty() ) {
-                    problem() << "repl: $err reading remote oplog: " + err << '\n';
-                    massert( 10390 ,  "got $err reading remote oplog", false );
+                    // 13051 is "tailable cursor requested on non capped collection"
+                    if (op.getIntField("code") == 13051) {
+                        problem() << "trying to slave off of a non-master" << '\n';
+                        massert( 13344 ,  "trying to slave off of a non-master", false );
+                    }
+                    else {
+                        problem() << "repl: $err reading remote oplog: " + err << '\n';
+                        massert( 10390 ,  "got $err reading remote oplog", false );
+                    }
                 }
                 else {
                     problem() << "repl: bad object read from remote oplog: " << op.toString() << '\n';
@@ -1248,7 +1246,7 @@ namespace mongo {
                 if ( !tailing && !initial && next != syncedTo ) {
                     log() << "remote slave log filled, forcing slave resync" << endl;
                     resetSlave();
-                    return true;
+                    return 1;
                 }            
             
                 dblock lk;
@@ -1262,7 +1260,7 @@ namespace mongo {
                     log(1) << "repl:   initial run\n";
                 else
                     assert( syncedTo < nextOpTime );
-                c->putBack( op ); // op will be processed in the loop below
+                oplogReader.putBack( op ); // op will be processed in the loop below
                 nextOpTime = OpTime(); // will reread the op below
             }
             else if ( nextOpTime != syncedTo ) { // didn't get what we queried for - error
@@ -1305,17 +1303,21 @@ namespace mongo {
                    1) find most recent op in local log
                    2) more()?
                 */
-                if ( !c->more() ) {
+                if ( !oplogReader.more() ) {
                     dblock lk;
                     OpTime nextLastSaved = nextLastSavedLocalTs();
                     {
                         dbtemprelease t;
-                        if ( c->more() ) {
-                            continue;
+                        if ( oplogReader.more() ) {
+                            if ( getInitialSyncCompleted() ) { // if initial sync hasn't completed, break out of loop so we can set to completed or clone more dbs
+                                continue;
+                            }
                         } else {
                             setLastSavedLocalTs( nextLastSaved );
                         }
                     }
+                    if( oplogReader.awaitCapable() && tailing )
+                        okResultCode = 0; // don't sleep
                     syncedTo = nextOpTime;
                     save(); // note how far we are synced up to now
                     log() << "repl:   applied " << n << " operations" << endl;
@@ -1336,7 +1338,7 @@ namespace mongo {
 					n = 0;
 				}
 
-                BSONObj op = c->next();
+                BSONObj op = oplogReader.next();
                 BSONElement ts = op.getField("ts");
                 if( !( ts.type() == Date || ts.type() == Timestamp ) ) { 
                     log() << "sync error: problem querying remote oplog record\n";
@@ -1356,7 +1358,7 @@ namespace mongo {
                     uassert( 10123 , "replication error last applied optime at slave >= nextOpTime from master", false);
                 }
                 if ( replSettings.slavedelay && ( unsigned( time( 0 ) ) < nextOpTime.getSecs() + replSettings.slavedelay ) ) {
-                    c->putBack( op );
+                    oplogReader.putBack( op );
                     _sleepAdviceTime = nextOpTime.getSecs() + replSettings.slavedelay + 1;
                     dblock lk;
                     if ( n > 0 ) {
@@ -1374,11 +1376,11 @@ namespace mongo {
             }
         }
 
-        return true;
+        return okResultCode;
     }
 
 	BSONObj userReplQuery = fromjson("{\"user\":\"repl\"}");
-
+    
 	bool replAuthenticate(DBClientConnection *conn) {
 		if( ! cc().isAdmin() ){
 			log() << "replauthenticate: requires admin permissions, failing\n";
@@ -1399,6 +1401,7 @@ namespace mongo {
 					return false;
 				}
 			}
+            
 		}
 
 		string u = user.getStringField("user");
@@ -1413,12 +1416,37 @@ namespace mongo {
 		return true;
 	}
 
-    bool ReplSource::connect() {
-        if ( conn.get() == 0 ) {
-            conn = auto_ptr<DBClientConnection>(new DBClientConnection());
+    bool replHandshake(DBClientConnection *conn) {
+        
+        BSONObj me;
+        {
+            dblock l;
+            if ( ! Helpers::getSingleton( "local.me" , me ) ){
+                BSONObjBuilder b;
+                b.appendOID( "_id" , 0 , true );
+                me = b.obj();
+                Helpers::putSingleton( "local.me" , me );
+            }
+        }
+        
+        BSONObjBuilder cmd;
+        cmd.appendAs( me["_id"] , "handshake" );
+
+        BSONObj res;
+        bool ok = conn->runCommand( "admin" , cmd.obj() , res );
+        // ignoring for now on purpose for older versions
+        log(ok) << "replHandshake res not: " << ok << " res: " << res << endl;
+        return true;
+    }
+
+    bool OplogReader::connect(string hostName) {
+        if( conn() == 0 ) {
+            _conn = auto_ptr<DBClientConnection>(new DBClientConnection( false, 0, replPair ? 20 : 0 /* tcp timeout */));
             string errmsg;
             ReplInfo r("trying to connect to sync source");
-            if ( !conn->connect(hostName.c_str(), errmsg) || !replAuthenticate(conn.get()) ) {
+            if ( !_conn->connect(hostName.c_str(), errmsg) || 
+                 !replAuthenticate(_conn.get()) ||
+                 !replHandshake(_conn.get()) ) {
                 resetConnection();
                 log() << "repl:  " << errmsg << endl;
                 return false;
@@ -1428,9 +1456,10 @@ namespace mongo {
     }
     
     /* note: not yet in mutex at this point.
-       returns true if everything happy.  return false if you want to reconnect.
+       returns >= 0 if ok.  return -1 if you want to reconnect.
+       return value of zero indicates no sleep necessary before next call
     */
-    bool ReplSource::sync(int& nApplied) {
+    int ReplSource::sync(int& nApplied) {
         _sleepAdviceTime = 0;
         ReplInfo r("sync");
         if ( !cmdLine.quiet ) {
@@ -1447,24 +1476,24 @@ namespace mongo {
         if ( (string("localhost") == hostName || string("127.0.0.1") == hostName) && cmdLine.port == CmdLine::DefaultDBPort ) {
             log() << "repl:   can't sync from self (localhost). sources configuration may be wrong." << endl;
             sleepsecs(5);
-            return false;
+            return -1;
         }
 
-        if ( !connect() ) {
+        if ( !oplogReader.connect(hostName) ) {
 			log(4) << "repl:  can't connect to sync source" << endl;
             if ( replPair && paired ) {
                 assert( startsWith(hostName.c_str(), replPair->remoteHost.c_str()) );
                 replPair->arbitrate();
             }
-            return false;            
+            return -1;
         }
         
         if ( paired ) {
-            int remote = replPair->negotiate(conn.get(), "direct");
+            int remote = replPair->negotiate(oplogReader.conn(), "direct");
             int nMasters = ( remote == ReplPair::State_Master ) + ( replPair->state == ReplPair::State_Master );
             if ( getInitialSyncCompleted() && nMasters != 1 ) {
                 log() << ( nMasters == 0 ? "no master" : "two masters" ) << ", deferring oplog pull" << endl;
-                return true;
+                return 1;
             }
         }
 
@@ -1484,112 +1513,6 @@ namespace mongo {
         return sync_pullOpLog(nApplied);
     }
 
-    /* -- Logging of operations -------------------------------------*/
-
-// cached copies of these...so don't rename them
-    NamespaceDetails *localOplogMainDetails = 0;
-    Database *localOplogDB = 0;
-
-    void replCheckCloseDatabase( Database * db ){
-        localOplogDB = 0;
-        localOplogMainDetails = 0;
-    }
-
-    /* we write to local.opload.$main:
-         { ts : ..., op: ..., ns: ..., o: ... }
-       ts: an OpTime timestamp
-       op:
-        "i" insert
-        "u" update
-        "d" delete
-        "c" db cmd
-        "db" declares presence of a database (ns is set to the db name + '.')
-        "n" no op
-       logNS - e.g. "local.oplog.$main"
-       bb:
-         if not null, specifies a boolean to pass along to the other side as b: param.
-         used for "justOne" or "upsert" flags on 'd', 'u'
-       first: true
-         when set, indicates this is the first thing we have logged for this database.
-         thus, the slave does not need to copy down all the data when it sees this.
-    */
-    static void _logOp(const char *opstr, const char *ns, const char *logNS, const BSONObj& obj, BSONObj *o2, bool *bb, const OpTime &ts ) {
-        if ( strncmp(ns, "local.", 6) == 0 )
-            return;
-
-        DEV assertInWriteLock();
-
-        Client::Context context;
-
-        /* we jump through a bunch of hoops here to avoid copying the obj buffer twice --
-           instead we do a single copy to the destination position in the memory mapped file.
-        */
-
-        BSONObjBuilder b;
-        b.appendTimestamp("ts", ts.asDate());
-        b.append("op", opstr);
-        b.append("ns", ns);
-        if ( bb )
-            b.appendBool("b", *bb);
-        if ( o2 )
-            b.append("o2", *o2);
-        BSONObj partial = b.done();
-        int posz = partial.objsize();
-        int len = posz + obj.objsize() + 1 + 2 /*o:*/;
-
-        Record *r;
-        if ( strncmp( logNS, "local.", 6 ) == 0 ) { // For now, assume this is olog main
-            if ( localOplogMainDetails == 0 ) {
-                Client::Context ctx("local.", dbpath, 0, false);
-                localOplogDB = ctx.db();
-                localOplogMainDetails = nsdetails(logNS);
-            }
-            Client::Context ctx( "" , localOplogDB, false );
-            r = theDataFileMgr.fast_oplog_insert(localOplogMainDetails, logNS, len);
-        } else {
-            Client::Context ctx( logNS, dbpath, 0, false );
-            assert( nsdetails( logNS ) );
-            r = theDataFileMgr.fast_oplog_insert( nsdetails( logNS ), logNS, len);
-        }
-
-        char *p = r->data;
-        memcpy(p, partial.objdata(), posz);
-        *((unsigned *)p) += obj.objsize() + 1 + 2;
-        p += posz - 1;
-        *p++ = (char) Object;
-        *p++ = 'o';
-        *p++ = 0;
-        memcpy(p, obj.objdata(), obj.objsize());
-        p += obj.objsize();
-        *p = EOO;
-        
-        if ( logLevel >= 6 ) {
-            BSONObj temp(r);
-            log( 6 ) << "logging op:" << temp << endl;
-        }
-    }
-
-    static void logKeepalive() { 
-        BSONObj obj;
-        _logOp("n", "", "local.oplog.$main", obj, 0, 0, OpTime::now());
-    }
-
-    void logOp(const char *opstr, const char *ns, const BSONObj& obj, BSONObj *patt, bool *b) {
-        if ( replSettings.master ) {
-            _logOp(opstr, ns, "local.oplog.$main", obj, patt, b, OpTime::now());
-            char cl[ 256 ];
-            nsToDatabase( ns, cl );
-        }
-        NamespaceDetailsTransient &t = NamespaceDetailsTransient::get_w( ns );
-        if ( t.cllEnabled() ) {
-            try {
-                _logOp(opstr, ns, t.cllNS().c_str(), obj, patt, b, OpTime::now());
-            } catch ( const DBException & ) {
-                t.cllInvalidate();
-            }
-        }
-    }    
-    
     /* --------------------------------------------------------------*/
 
     /*
@@ -1620,11 +1543,11 @@ namespace mongo {
         int sleepAdvice = 1;
         for ( ReplSource::SourceVector::iterator i = sources.begin(); i != sources.end(); i++ ) {
             ReplSource *s = i->get();
-            bool ok = false;
+            int res = -1;
             try {
-                ok = s->sync(nApplied);
+                res = s->sync(nApplied);
                 bool moreToSync = s->haveMoreDbsToSync();
-                if( !ok ) { 
+                if( res < 0 ) { 
                     sleepAdvice = 3;
                 }
                 else if( moreToSync ) {
@@ -1633,7 +1556,9 @@ namespace mongo {
                 else if ( s->sleepAdvice() ) {
                     sleepAdvice = s->sleepAdvice();
                 }
-                if ( ok && !moreToSync /*&& !s->syncedTo.isNull()*/ ) {
+                else 
+                    sleepAdvice = res;
+                if ( res >= 0 && !moreToSync /*&& !s->syncedTo.isNull()*/ ) {
                     pairSync->setInitialSyncCompletedLocking();
                 }
             }
@@ -1663,8 +1588,8 @@ namespace mongo {
                 log() << "unexpected exception during replication.  replication will halt" << endl;
                 replAllDead = "caught unexpected exception during replication";
             }
-            if ( !ok )
-                s->resetConnection();
+            if ( res < 0 )
+                s->oplogReader.resetConnection();
         }
         return sleepAdvice;
     }
@@ -1702,6 +1627,12 @@ namespace mongo {
                 assert( syncing == 1 );
                 syncing--;
             }
+
+			if( relinquishSyncingSome )  { 
+				relinquishSyncingSome = 0;
+				s = 1; // sleep before going back in to syncing=1
+			}
+
             if ( s ) {
                 stringstream ss;
                 ss << "repl: sleep " << s << "sec before next pass";
@@ -1719,16 +1650,21 @@ namespace mongo {
     static void replMasterThread() {
         sleepsecs(4);
         Client::initThread("replmaster");
+        int toSleep = 10;
         while( 1 ) {
-            sleepsecs(10);
+
+            sleepsecs( toSleep );
             /* write a keep-alive like entry to the log.  this will make things like 
                printReplicationStatus() and printSlaveReplicationStatus() stay up-to-date
                even when things are idle.
             */
             {
-                writelocktry lk("");
+                writelocktry lk("",1);
                 if ( lk.got() ){
+                    toSleep = 10;
+                    
                     cc().getAuthenticationInfo()->authorize("admin");   
+                        
                     try { 
                         logKeepalive();
                     }
@@ -1736,6 +1672,10 @@ namespace mongo {
                         log() << "caught exception in replMasterThread()" << endl;
                     }
                 }
+                else {
+                    log(5) << "couldn't logKeepalive" << endl;
+                    toSleep = 1;
+                }
             }
         }
     }
@@ -1778,64 +1718,29 @@ namespace mongo {
         }
     }
 
-    void createOplog() {
-        dblock lk;
-
-        const char * ns = "local.oplog.$main";
-        Client::Context ctx(ns);
-        
-        if ( nsdetails( ns ) ) {
-            DBDirectClient c;
-            BSONObj lastOp = c.findOne( ns, Query().sort( BSON( "$natural" << -1 ) ) );
-            if ( !lastOp.isEmpty() ) {
-                OpTime::setLast( lastOp[ "ts" ].date() );
+    void newRepl();
+    void oldRepl();
+    void startReplication() {
+        /* if we are going to be a replica set, we aren't doing other forms of replication. */
+        if( !cmdLine._replSet.empty() ) {
+            if( replSettings.slave || replSettings.master || replPair ) { 
+                log() << "***" << endl;
+                log() << "ERROR: can't use --slave or --master replication options with --replSet" << endl;
+                log() << "***" << endl;
             }
+            createOplog();
+            newRepl();
             return;
         }
-        
-        /* create an oplog collection, if it doesn't yet exist. */
-        BSONObjBuilder b;
-        double sz;
-        if ( cmdLine.oplogSize != 0 )
-            sz = (double)cmdLine.oplogSize;
-        else {
-			/* not specified. pick a default size */
-            sz = 50.0 * 1000 * 1000;
-            if ( sizeof(int *) >= 8 ) {
-#if defined(__APPLE__)
-				// typically these are desktops (dev machines), so keep it smallish
-				sz = (256-64) * 1000 * 1000;
-#else
-                sz = 990.0 * 1000 * 1000;
-                boost::intmax_t free = freeSpace(); //-1 if call not supported.
-                double fivePct = free * 0.05;
-                if ( fivePct > sz )
-                    sz = fivePct;
-#endif
-            }
-        }
-
-        log() << "******\n";
-        log() << "creating replication oplog of size: " << (int)( sz / ( 1024 * 1024 ) ) << "MB (use --oplogSize to change)\n";
-        log() << "******" << endl;
-
-        b.append("size", sz);
-        b.appendBool("capped", 1);
-        b.appendBool("autoIndexId", false);
-
-        string err;
-        BSONObj o = b.done();
-        userCreateNS(ns, o, err, false);
-        logOp( "n", "dummy", BSONObj() );
-    }
-    
-    void startReplication() {
+
+        oldRepl();
+
         /* this was just to see if anything locks for longer than it should -- we need to be careful
            not to be locked when trying to connect() or query() the other side.
            */
         //boost::thread tempt(tempThread);
 
-        if ( !replSettings.slave && !replSettings.master && !replPair )
+        if( !replSettings.slave && !replSettings.master && !replPair )
             return;
 
         {
@@ -1846,11 +1751,11 @@ namespace mongo {
 
         if ( replSettings.slave || replPair ) {
             if ( replSettings.slave ) {
-				assert( replSettings.slave == SimpleSlave );
+                assert( replSettings.slave == SimpleSlave );
                 log(1) << "slave=true" << endl;
-			}
-			else
-				replSettings.slave = ReplPairSlave;
+            }
+            else
+                replSettings.slave = ReplPairSlave;
             boost::thread repl_thread(replSlaveThread);
         }
 
@@ -1871,56 +1776,29 @@ namespace mongo {
         replPair = new ReplPair(remoteEnd, arb);
     }
 
-    class CmdLogCollection : public Command {
-    public:
-        virtual bool slaveOk() {
-            return false;
-        }
-        virtual LockType locktype(){ return WRITE; }
-        CmdLogCollection() : Command( "logCollection" ) {}
-        virtual void help( stringstream &help ) const {
-            help << "examples: { logCollection: <collection ns>, start: 1 }, "
-                 << "{ logCollection: <collection ns>, validateComplete: 1 }";
-        }
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            string logCollection = cmdObj.getStringField( "logCollection" );
-            if ( logCollection.empty() ) {
-                errmsg = "missing logCollection spec";
-                return false;
-            }
-            bool start = !cmdObj.getField( "start" ).eoo();
-            bool validateComplete = !cmdObj.getField( "validateComplete" ).eoo();
-            if ( start ? validateComplete : !validateComplete ) {
-                errmsg = "Must specify exactly one of start:1 or validateComplete:1";
-                return false;
-            }
-            int logSizeMb = cmdObj.getIntField( "logSizeMb" );
-            NamespaceDetailsTransient &t = NamespaceDetailsTransient::get_w( logCollection.c_str() );
-            if ( start ) {
-                if ( t.cllNS().empty() ) {
-                    if ( logSizeMb == INT_MIN ) {
-                        t.cllStart();
-                    } else {
-                        t.cllStart( logSizeMb );
-                    }
-                } else {
-                    errmsg = "Log already started for ns: " + logCollection;
-                    return false;
-                }
-            } else {
-                if ( t.cllNS().empty() ) {
-                    errmsg = "No log to validateComplete for ns: " + logCollection;
-                    return false;
-                } else {
-                    if ( !t.cllValidateComplete() ) {
-                        errmsg = "Oplog failure, insufficient space allocated";
-                        return false;
-                    }
-                }
-            }
-            log() << "started logCollection with cmd obj: " << cmdObj << endl;
-            return true;
-        }
-    } cmdlogcollection;
+    void testPretouch() {
+        int nthr = min(8, 8);
+        nthr = max(nthr, 1);
+        int m = 8 / nthr;
+        ThreadPool tp(nthr);
+        vector<BSONObj> v;
+
+        BSONObj x = BSON( "ns" << "test.foo" << "o" << BSON( "_id" << 1 ) << "op" << "i" );
+
+        v.push_back(x);
+        v.push_back(x);
+        v.push_back(x);
+
+        unsigned a = 0;
+        while( 1 ) {
+            if( a >= v.size() ) break;
+            unsigned b = a + m - 1; // v[a..b]
+            if( b >= v.size() ) b = v.size() - 1;
+            tp.schedule(pretouchN, v, a, b);
+            DEV cout << "pretouch task: " << a << ".." << b << endl;
+            a += m;
+        }
+        tp.join();
+    }
     
 } // namespace mongo
diff --git a/db/repl.h b/db/repl.h
index eb1cb26..f33acad 100644
--- a/db/repl.h
+++ b/db/repl.h
@@ -23,7 +23,6 @@
 
    at the master:
      local.oplog.$<source>
-     local.oplog.$main is the default
 */
 
 #pragma once
@@ -33,16 +32,14 @@
 #include "dbhelpers.h"
 #include "query.h"
 #include "queryoptimizer.h"
-
 #include "../client/dbclient.h"
-
 #include "../util/optime.h"
+#include "oplog.h"
+#include "../util/concurrency/thread_pool.h"
+#include "oplogreader.h"
 
 namespace mongo {
 
-    class DBClientConnection;
-    class DBClientCursor;
-
 	/* replication slave? (possibly with slave or repl pair nonmaster)
        --slave cmd line setting -> SimpleSlave
 	*/
@@ -79,10 +76,9 @@ namespace mongo {
     /* A replication exception */
     class SyncException : public DBException {
     public:
-        virtual const char* what() const throw() { return "sync exception"; }
-        virtual int getCode(){ return 10001; }
+        SyncException() : DBException( "sync exception" , 10001 ){}
     };
-    
+
     /* A Source is a source from which we can pull (replicate) data.
        stored in collection local.sources.
 
@@ -94,16 +90,15 @@ namespace mongo {
        not done (always use main for now).
     */
     class ReplSource {
+        auto_ptr<ThreadPool> tp;
+
         bool resync(string db);
 
         /* pull some operations from the master's oplog, and apply them. */
-        bool sync_pullOpLog(int& nApplied);
+        int sync_pullOpLog(int& nApplied);
 
         void sync_pullOpLog_applyOperation(BSONObj& op, OpTime *localLogTail);
         
-        auto_ptr<DBClientConnection> conn;
-        auto_ptr<DBClientCursor> cursor;
-
         /* we only clone one database per pass, even if a lot need done.  This helps us
            avoid overflowing the master's transaction log by doing too much work before going
            back to read more transactions. (Imagine a scenario of slave startup where we try to
@@ -117,8 +112,6 @@ namespace mongo {
         
         // returns the dummy ns used to do the drop
         string resyncDrop( const char *db, const char *requester );
-        // returns true if connected on return
-        bool connect();
         // returns possibly unowned id spec for the operation.
         static BSONObj idForOp( const BSONObj &op, bool &mod );
         static void updateSetsWithOp( const BSONObj &op, bool mayUpdateStorage );
@@ -136,6 +129,8 @@ namespace mongo {
         unsigned _sleepAdviceTime;
         
     public:
+        OplogReader oplogReader;
+
         static void applyOperation(const BSONObj& op);
         bool replacing; // in "replace mode" -- see CmdReplacePeer
         bool paired; // --pair in use
@@ -162,12 +157,11 @@ namespace mongo {
         typedef vector< shared_ptr< ReplSource > > SourceVector;
         static void loadAll(SourceVector&);
         explicit ReplSource(BSONObj);
-        bool sync(int& nApplied);
+
+        /* -1 = error */
+        int sync(int& nApplied);
+
         void save(); // write ourself to local.sources
-        void resetConnection() {
-            cursor = auto_ptr<DBClientCursor>(0);
-            conn = auto_ptr<DBClientConnection>(0);
-        }
 
         // make a jsobj from our member fields of the form
         //   { host: ..., source: ..., syncedTo: ... }
@@ -176,7 +170,7 @@ namespace mongo {
         bool operator==(const ReplSource&r) const {
             return hostName == r.hostName && sourceName() == r.sourceName();
         }
-        operator string() const { return sourceName() + "@" + hostName; }
+        string toString() const { return sourceName() + "@" + hostName; }
         
         bool haveMoreDbsToSync() const { return !addDbNextPass.empty(); }        
         int sleepAdvice() const {
@@ -191,15 +185,6 @@ namespace mongo {
         void forceResync( const char *requester );
     };
 
-    /* Write operation to the log (local.oplog.$main)
-       "i" insert
-       "u" update
-       "d" delete
-       "c" db cmd
-       "db" declares presence of a database (ns is set to the db name + '.')
-    */
-    void logOp(const char *opstr, const char *ns, const BSONObj& obj, BSONObj *patt = 0, bool *b = 0);
-
     // class for managing a set of ids in memory
     class MemIds {
     public:
@@ -342,151 +327,5 @@ namespace mongo {
     bool anyReplEnabled();
     void appendReplicationInfo( BSONObjBuilder& result , bool authed , int level = 0 );
     
-    void replCheckCloseDatabase( Database * db );
     
-    extern int __findingStartInitialTimeout; // configurable for testing    
-
-    class FindingStartCursor {
-    public:
-        FindingStartCursor( const QueryPlan & qp ) : 
-        _qp( qp ),
-        _findingStart( true ),
-        _findingStartMode(),
-        _findingStartTimer( 0 ),
-        _findingStartCursor( 0 )
-        { init(); }
-        bool done() const { return !_findingStart; }
-        auto_ptr< Cursor > cRelease() { return _c; }
-        void next() {
-            if ( !_findingStartCursor || !_findingStartCursor->c->ok() ) {
-                _findingStart = false;
-                _c = _qp.newCursor(); // on error, start from beginning
-                destroyClientCursor();
-                return;
-            }
-            switch( _findingStartMode ) {
-                case Initial: {
-                    if ( !_matcher->matches( _findingStartCursor->c->currKey(), _findingStartCursor->c->currLoc() ) ) {
-                        _findingStart = false; // found first record out of query range, so scan normally
-                        _c = _qp.newCursor( _findingStartCursor->c->currLoc() );
-                        destroyClientCursor();
-                        return;
-                    }
-                    _findingStartCursor->c->advance();
-                    RARELY {
-                        if ( _findingStartTimer.seconds() >= __findingStartInitialTimeout ) {
-                            createClientCursor( startLoc( _findingStartCursor->c->currLoc() ) );
-                            _findingStartMode = FindExtent;
-                            return;
-                        }
-                    }
-                    maybeRelease();
-                    return;
-                }
-                case FindExtent: {
-                    if ( !_matcher->matches( _findingStartCursor->c->currKey(), _findingStartCursor->c->currLoc() ) ) {
-                        _findingStartMode = InExtent;
-                        return;
-                    }
-                    DiskLoc prev = prevLoc( _findingStartCursor->c->currLoc() );
-                    if ( prev.isNull() ) { // hit beginning, so start scanning from here
-                        createClientCursor();
-                        _findingStartMode = InExtent;
-                        return;
-                    }
-                    // There might be a more efficient implementation than creating new cursor & client cursor each time,
-                    // not worrying about that for now
-                    createClientCursor( prev );
-                    maybeRelease();
-                    return;
-                }
-                case InExtent: {
-                    if ( _matcher->matches( _findingStartCursor->c->currKey(), _findingStartCursor->c->currLoc() ) ) {
-                        _findingStart = false; // found first record in query range, so scan normally
-                        _c = _qp.newCursor( _findingStartCursor->c->currLoc() );
-                        destroyClientCursor();
-                        return;
-                    }
-                    _findingStartCursor->c->advance();
-                    maybeRelease();
-                    return;
-                }
-                default: {
-                    massert( 12600, "invalid _findingStartMode", false );
-                }
-            }                
-        }            
-    private:
-        enum FindingStartMode { Initial, FindExtent, InExtent };
-        const QueryPlan &_qp;
-        bool _findingStart;
-        FindingStartMode _findingStartMode;
-        auto_ptr< CoveredIndexMatcher > _matcher;
-        Timer _findingStartTimer;
-        ClientCursor * _findingStartCursor;
-        auto_ptr< Cursor > _c;
-        DiskLoc startLoc( const DiskLoc &rec ) {
-            Extent *e = rec.rec()->myExtent( rec );
-            if ( !_qp.nsd()->capLooped() || ( e->myLoc != _qp.nsd()->capExtent ) )
-                return e->firstRecord;
-            // Likely we are on the fresh side of capExtent, so return first fresh record.
-            // If we are on the stale side of capExtent, then the collection is small and it
-            // doesn't matter if we start the extent scan with capFirstNewRecord.
-            return _qp.nsd()->capFirstNewRecord;
-        }
-        
-        // should never have an empty extent in the oplog, so don't worry about that case
-        DiskLoc prevLoc( const DiskLoc &rec ) {
-            Extent *e = rec.rec()->myExtent( rec );
-            if ( _qp.nsd()->capLooped() ) {
-                if ( e->xprev.isNull() )
-                    e = _qp.nsd()->lastExtent.ext();
-                else
-                    e = e->xprev.ext();
-                if ( e->myLoc != _qp.nsd()->capExtent )
-                    return e->firstRecord;
-            } else {
-                if ( !e->xprev.isNull() ) {
-                    e = e->xprev.ext();
-                    return e->firstRecord;
-                }
-            }
-            return DiskLoc(); // reached beginning of collection
-        }
-        void createClientCursor( const DiskLoc &startLoc = DiskLoc() ) {
-            auto_ptr<Cursor> c = _qp.newCursor( startLoc );
-            _findingStartCursor = new ClientCursor(c, _qp.ns(), false);            
-        }
-        void destroyClientCursor() {
-            if ( _findingStartCursor ) {
-                ClientCursor::erase( _findingStartCursor->cursorid );
-                _findingStartCursor = 0;
-            }
-        }
-        void maybeRelease() {
-            RARELY {
-                CursorId id = _findingStartCursor->cursorid;
-                _findingStartCursor->updateLocation();
-                {
-                    dbtemprelease t;
-                }   
-                _findingStartCursor = ClientCursor::find( id, false );
-            }                                            
-        }
-        void init() {
-            // Use a ClientCursor here so we can release db mutex while scanning
-            // oplog (can take quite a while with large oplogs).
-            auto_ptr<Cursor> c = _qp.newReverseCursor();
-            _findingStartCursor = new ClientCursor(c, _qp.ns(), false);
-            _findingStartTimer.reset();
-            _findingStartMode = Initial;
-            BSONElement tsElt = _qp.query()[ "ts" ];
-            massert( 13044, "no ts field in query", !tsElt.eoo() );
-            BSONObjBuilder b;
-            b.append( tsElt );
-            BSONObj tsQuery = b.obj();
-            _matcher.reset(new CoveredIndexMatcher(tsQuery, _qp.indexKey()));
-        }
-    };
-
 } // namespace mongo
diff --git a/db/repl/connections.h b/db/repl/connections.h
new file mode 100644
index 0000000..95defe4
--- /dev/null
+++ b/db/repl/connections.h
@@ -0,0 +1,91 @@
+// @file 
+
+/*
+ *    Copyright (C) 2010 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <map>
+#include "../../client/dbclient.h"
+
+namespace mongo { 
+
+    /** here we keep a single connection (with reconnect) for a set of hosts, 
+        one each, and allow one user at a time per host.  if in use already for that 
+        host, we block.  so this is an easy way to keep a 1-deep pool of connections
+        that many threads can share.
+
+        thread-safe.
+
+        Example:
+        {
+            ScopedConn c("foo.acme.com:9999");
+            c->runCommand(...);
+        }
+
+        throws exception on connect error (but fine to try again later with a new
+        scopedconn object for same host).
+    */
+    class ScopedConn { 
+    public:
+        /** throws assertions if connect failure etc. */
+        ScopedConn(string hostport);
+        ~ScopedConn();
+        DBClientConnection* operator->();
+    private:
+        auto_ptr<scoped_lock> connLock;
+        static mutex mapMutex;
+        struct X { 
+            mutex z;
+            DBClientConnection cc;
+            X() : z("X"), cc(/*reconnect*/true, 0, /*timeout*/10) { 
+                cc._logLevel = 2;
+            }
+        } *x;
+        typedef map<string,ScopedConn::X*> M;
+        static M& _map;
+    };
+
+    inline ScopedConn::ScopedConn(string hostport) {
+        bool first = false;
+        {
+            scoped_lock lk(mapMutex);
+            x = _map[hostport];
+            if( x == 0 ) {
+                x = _map[hostport] = new X();
+                first = true;
+                connLock.reset( new scoped_lock(x->z) );
+            }
+        }
+        if( !first ) { 
+            connLock.reset( new scoped_lock(x->z) );
+            return;
+        }
+
+        // we already locked above...
+        string err;
+        x->cc.connect(hostport, err);
+    }
+
+    inline ScopedConn::~ScopedConn() { 
+        // conLock releases...
+    }
+
+    inline DBClientConnection* ScopedConn::operator->() { 
+        return &x->cc; 
+    }
+
+}
diff --git a/db/repl/consensus.cpp b/db/repl/consensus.cpp
new file mode 100644
index 0000000..4eba17d
--- /dev/null
+++ b/db/repl/consensus.cpp
@@ -0,0 +1,342 @@
+/**
+*    Copyright (C) 2010 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "../commands.h"
+#include "rs.h"
+#include "multicmd.h"
+
+namespace mongo { 
+
+    class CmdReplSetFresh : public ReplSetCommand { 
+    public:
+        CmdReplSetFresh() : ReplSetCommand("replSetFresh") { }
+    private:
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            if( !check(errmsg, result) )
+                return false;
+
+            if( cmdObj["set"].String() != theReplSet->name() ) { 
+                errmsg = "wrong repl set name";
+                return false;
+            }
+            string who = cmdObj["who"].String();
+            int cfgver = cmdObj["cfgver"].Int();
+			OpTime opTime(cmdObj["opTime"].Date());
+
+            bool weAreFresher = false;
+            if( theReplSet->config().version > cfgver ) { 
+                log() << "replSet member " << who << " is not yet aware its cfg version " << cfgver << " is stale" << rsLog;
+				result.append("info", "config version stale");
+                weAreFresher = true;
+            }
+            else if( opTime < theReplSet->lastOpTimeWritten )  { 
+				weAreFresher = true;
+			}
+            result.appendDate("opTime", theReplSet->lastOpTimeWritten.asDate());
+            result.append("fresher", weAreFresher);
+            return true;
+        }
+    } cmdReplSetFresh;
+
+    class CmdReplSetElect : public ReplSetCommand {
+    public:
+        CmdReplSetElect() : ReplSetCommand("replSetElect") { }
+    private:
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            if( !check(errmsg, result) )
+                return false;
+            //task::lam f = boost::bind(&Consensus::electCmdReceived, &theReplSet->elect, cmdObj, &result);
+            //theReplSet->mgr->call(f);
+            theReplSet->elect.electCmdReceived(cmdObj, &result);
+            return true;
+        }
+    } cmdReplSetElect;
+
+    int Consensus::totalVotes() const { 
+        static int complain = 0;
+        int vTot = rs._self->config().votes;
+        for( Member *m = rs.head(); m; m=m->next() ) 
+            vTot += m->config().votes;
+        if( vTot % 2 == 0 && vTot && complain++ == 0 )
+            log() << "replSet warning total number of votes is even - considering giving one member an extra vote" << rsLog;
+        return vTot;
+    }
+
+    bool Consensus::aMajoritySeemsToBeUp() const {
+        int vUp = rs._self->config().votes;
+        for( Member *m = rs.head(); m; m=m->next() ) 
+            vUp += m->hbinfo().up() ? m->config().votes : 0;
+        return vUp * 2 > totalVotes();
+    }
+
+    static const int VETO = -10000;
+
+    const time_t LeaseTime = 30;
+
+    unsigned Consensus::yea(unsigned memberId) /* throws VoteException */ {
+        Atomic<LastYea>::tran t(ly);
+        LastYea &ly = t.ref();
+        time_t now = time(0);
+        if( ly.when + LeaseTime >= now && ly.who != memberId ) {
+            log(1) << "replSet not voting yea for " << memberId <<
+                " voted for " << ly.who << ' ' << now-ly.when << " secs ago" << rsLog;
+            throw VoteException();
+        }
+        ly.when = now;
+        ly.who = memberId;
+        return rs._self->config().votes;
+    }
+
+    /* we vote for ourself at start of election.  once it fails, we can cancel the lease we had in 
+       place instead of leaving it for a long time.
+       */
+    void Consensus::electionFailed(unsigned meid) {
+        Atomic<LastYea>::tran t(ly);
+        LastYea &L = t.ref();
+        DEV assert( L.who == meid ); // this may not always always hold, so be aware, but adding for now as a quick sanity test
+        if( L.who == meid )
+            L.when = 0;
+    }
+
+    /* todo: threading **************** !!!!!!!!!!!!!!!! */
+    void Consensus::electCmdReceived(BSONObj cmd, BSONObjBuilder* _b) { 
+        BSONObjBuilder& b = *_b;
+        DEV log() << "replSet received elect msg " << cmd.toString() << rsLog;
+        else log(2) << "replSet received elect msg " << cmd.toString() << rsLog;
+        string set = cmd["set"].String();
+        unsigned whoid = cmd["whoid"].Int();
+        int cfgver = cmd["cfgver"].Int();
+        OID round = cmd["round"].OID();
+        int myver = rs.config().version;
+
+        int vote = 0;
+        if( set != rs.name() ) { 
+            log() << "replSet error received an elect request for '" << set << "' but our set name is '" << rs.name() << "'" << rsLog;
+
+        }
+        else if( myver < cfgver ) { 
+            // we are stale.  don't vote
+        }
+        else if( myver > cfgver ) { 
+            // they are stale!
+            log() << "replSet info got stale version # during election" << rsLog;
+            vote = -10000;
+        }
+        else {
+            try {
+                vote = yea(whoid);
+                rs.relinquish();
+                log() << "replSet info voting yea for " << whoid << rsLog;
+            }
+            catch(VoteException&) { 
+                log() << "replSet voting no already voted for another" << rsLog;
+            }
+        }
+
+        b.append("vote", vote);
+        b.append("round", round);
+    }
+
+    void ReplSetImpl::_getTargets(list<Target>& L, int& configVersion) {
+        configVersion = config().version;
+        for( Member *m = head(); m; m=m->next() )
+            if( m->hbinfo().up() )
+                L.push_back( Target(m->fullName()) );
+    }
+
+    /* config version is returned as it is ok to use this unlocked.  BUT, if unlocked, you would need 
+       to check later that the config didn't change. */
+    void ReplSetImpl::getTargets(list<Target>& L, int& configVersion) {
+        if( lockedByMe() ) { 
+            _getTargets(L, configVersion);
+            return;
+        }
+        lock lk(this);
+        _getTargets(L, configVersion);
+    }
+
+    /* Do we have the newest data of them all?
+       @param allUp - set to true if all members are up.  Only set if true returned.
+       @return true if we are freshest.  Note we may tie.
+    */
+    bool Consensus::weAreFreshest(bool& allUp, int& nTies) {
+        const OpTime ord = theReplSet->lastOpTimeWritten;
+        nTies = 0;
+		assert( !ord.isNull() );
+        BSONObj cmd = BSON(
+               "replSetFresh" << 1 <<
+               "set" << rs.name() << 
+			   "opTime" << Date_t(ord.asDate()) <<
+               "who" << rs._self->fullName() << 
+               "cfgver" << rs._cfg->version );
+        list<Target> L;
+        int ver;
+        rs.getTargets(L, ver);
+        multiCommand(cmd, L);
+        int nok = 0;
+        allUp = true;
+        for( list<Target>::iterator i = L.begin(); i != L.end(); i++ ) {
+            if( i->ok ) {
+                nok++;
+                if( i->result["fresher"].trueValue() )
+                    return false;
+                OpTime remoteOrd( i->result["opTime"].Date() );
+                if( remoteOrd == ord )
+                    nTies++;
+                assert( remoteOrd <= ord );
+            }
+            else {
+                DEV log() << "replSet freshest returns " << i->result.toString() << rsLog;
+                allUp = false;
+            }
+        }
+        DEV log() << "replSet dev we are freshest of up nodes, nok:" << nok << " nTies:" << nTies << rsLog; 
+        assert( ord <= theReplSet->lastOpTimeWritten ); // <= as this may change while we are working...
+        return true;
+    }
+
+    extern time_t started;
+
+    void Consensus::multiCommand(BSONObj cmd, list<Target>& L) { 
+        assert( !rs.lockedByMe() );
+        mongo::multiCommand(cmd, L);
+    }
+
+    void Consensus::_electSelf() {
+        if( time(0) < steppedDown ) 
+            return;
+
+        {
+            const OpTime ord = theReplSet->lastOpTimeWritten;
+            if( ord == 0 ) { 
+                log() << "replSet info not trying to elect self, do not yet have a complete set of data from any point in time" << rsLog;
+                return;
+            }
+        }
+
+        bool allUp;
+        int nTies;
+        if( !weAreFreshest(allUp, nTies) ) { 
+            log() << "replSet info not electing self, we are not freshest" << rsLog;
+            return;
+        }
+
+        rs.sethbmsg("",9);
+
+        if( !allUp && time(0) - started < 60 * 5 ) { 
+            /* the idea here is that if a bunch of nodes bounce all at once, we don't want to drop data 
+               if we don't have to -- we'd rather be offline and wait a little longer instead 
+               todo: make this configurable.
+               */
+            rs.sethbmsg("not electing self, not all members up and we have been up less than 5 minutes");
+            return;
+        }
+
+        Member& me = *rs._self;
+
+        if( nTies ) {
+            /* tie?  we then randomly sleep to try to not collide on our voting. */
+            /* todo: smarter. */
+            if( me.id() == 0 || sleptLast ) {
+                // would be fine for one node not to sleep 
+                // todo: biggest / highest priority nodes should be the ones that get to not sleep
+            } else {
+                assert( !rs.lockedByMe() ); // bad to go to sleep locked
+                unsigned ms = ((unsigned) rand()) % 1000 + 50;
+                DEV log() << "replSet tie " << nTies << " sleeping a little " << ms << "ms" << rsLog;
+                sleptLast = true;
+                sleepmillis(ms);
+                throw RetryAfterSleepException();
+            }
+        }
+        sleptLast = false;
+
+        time_t start = time(0);
+        unsigned meid = me.id();
+        int tally = yea( meid );
+        bool success = false;
+        try {
+            log() << "replSet info electSelf " << meid << rsLog;
+
+            BSONObj electCmd = BSON(
+                   "replSetElect" << 1 <<
+                   "set" << rs.name() << 
+                   "who" << me.fullName() << 
+                   "whoid" << me.hbinfo().id() << 
+                   "cfgver" << rs._cfg->version << 
+                   "round" << OID::gen() /* this is just for diagnostics */
+                );
+
+            int configVersion;
+            list<Target> L;
+            rs.getTargets(L, configVersion);
+            multiCommand(electCmd, L);
+
+            {
+                RSBase::lock lk(&rs);
+                for( list<Target>::iterator i = L.begin(); i != L.end(); i++ ) {
+                    DEV log() << "replSet elect res: " << i->result.toString() << rsLog;
+                    if( i->ok ) {
+                        int v = i->result["vote"].Int();
+                        tally += v;
+                    }
+                }
+                if( tally*2 <= totalVotes() ) {
+                    log() << "replSet couldn't elect self, only received " << tally << " votes" << rsLog;
+                }
+                else if( time(0) - start > 30 ) {
+                    // defensive; should never happen as we have timeouts on connection and operation for our conn
+                    log() << "replSet too much time passed during our election, ignoring result" << rsLog;
+                }
+                else if( configVersion != rs.config().version ) { 
+                    log() << "replSet config version changed during our election, ignoring result" << rsLog;
+                }
+                else {
+                    /* succeeded. */
+                    log(1) << "replSet election succeeded, assuming primary role" << rsLog;
+                    success = true;
+                    rs.assumePrimary();
+                } 
+            }
+        } catch( std::exception& ) { 
+            if( !success ) electionFailed(meid);
+            throw;
+        }
+        if( !success ) electionFailed(meid);
+    }
+
+    void Consensus::electSelf() {
+        assert( !rs.lockedByMe() );
+        assert( !rs.myConfig().arbiterOnly );
+        try { 
+            _electSelf(); 
+        } 
+        catch(RetryAfterSleepException&) { 
+            throw;
+        }
+        catch(VoteException& ) { 
+            log() << "replSet not trying to elect self as responded yea to someone else recently" << rsLog;
+        }
+        catch(DBException& e) { 
+            log() << "replSet warning caught unexpected exception in electSelf() " << e.toString() << rsLog;
+        }
+        catch(...) { 
+            log() << "replSet warning caught unexpected exception in electSelf()" << rsLog;
+        }
+    }
+
+}
diff --git a/db/repl/health.cpp b/db/repl/health.cpp
new file mode 100644
index 0000000..b0be25f
--- /dev/null
+++ b/db/repl/health.cpp
@@ -0,0 +1,389 @@
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,b
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "rs.h"
+#include "health.h"
+#include "../../util/background.h"
+#include "../../client/dbclient.h"
+#include "../commands.h"
+#include "../../util/concurrency/value.h"
+#include "../../util/concurrency/task.h"
+#include "../../util/mongoutils/html.h"
+#include "../../util/goodies.h"
+#include "../../util/ramlog.h"
+#include "../helpers/dblogger.h"
+#include "connections.h"
+#include "../../util/unittest.h"
+#include "../dbhelpers.h"
+
+namespace mongo {
+    /* decls for connections.h */
+    ScopedConn::M& ScopedConn::_map = *(new ScopedConn::M());    
+    mutex ScopedConn::mapMutex("ScopedConn::mapMutex");
+}
+
+namespace mongo { 
+
+    using namespace mongoutils::html;
+    using namespace bson;
+
+    static RamLog _rsLog;
+    Tee *rsLog = &_rsLog;
+
+    string ago(time_t t) { 
+        if( t == 0 ) return "";
+
+        time_t x = time(0) - t;
+        stringstream s;
+        if( x < 180 ) {
+            s << x << " sec";
+            if( x != 1 ) s << 's';
+        }
+        else if( x < 3600 ) {
+            s.precision(2);
+            s << x / 60.0 << " mins";
+        }
+        else { 
+            s.precision(2);
+            s << x / 3600.0 << " hrs";
+        }
+        return s.str();
+    }
+
+    void Member::summarizeMember(stringstream& s) const { 
+        s << tr();
+        {
+            stringstream u;
+            u << "http://" << h().host() << ':' << (h().port() + 1000) << "/_replSet";
+            s << td( a(u.str(), "", fullName()) );
+        }
+        s << td( id() );
+        double h = hbinfo().health;
+        bool ok = h > 0;
+        s << td(red(str::stream() << h,h == 0));
+        s << td(ago(hbinfo().upSince));
+        bool never = false;
+        {
+            string h;
+            time_t hb = hbinfo().lastHeartbeat;
+            if( hb == 0 ) {
+                h = "never";
+                never = true;
+            }
+            else h = ago(hb) + " ago";
+            s << td(h);
+        }
+        s << td(config().votes);
+        { 
+            string stateText = ReplSet::stateAsStr(state());
+            if( ok || stateText.empty() ) 
+                s << td(stateText); // text blank if we've never connected
+            else
+                s << td( grey(str::stream() << "(was " << ReplSet::stateAsStr(state()) << ')', true) );
+        }
+        s << td( grey(hbinfo().lastHeartbeatMsg,!ok) );
+        stringstream q;
+        q << "/_replSetOplog?" << id();
+        s << td( a(q.str(), "", never ? "?" : hbinfo().opTime.toString()) );
+        if( hbinfo().skew > INT_MIN ) {
+            s << td( grey(str::stream() << hbinfo().skew,!ok) );
+        } else
+            s << td("");
+        s << _tr();
+    }
+    
+    string ReplSetImpl::stateAsHtml(MemberState s) { 
+        if( s.s == MemberState::RS_STARTUP ) return a("", "serving still starting up, or still trying to initiate the set", "STARTUP");
+        if( s.s == MemberState::RS_PRIMARY ) return a("", "this server thinks it is primary", "PRIMARY");
+        if( s.s == MemberState::RS_SECONDARY ) return a("", "this server thinks it is a secondary (slave mode)", "SECONDARY");
+        if( s.s == MemberState::RS_RECOVERING ) return a("", "recovering/resyncing; after recovery usually auto-transitions to secondary", "RECOVERING");
+        if( s.s == MemberState::RS_FATAL ) return a("", "something bad has occurred and server is not completely offline with regard to the replica set.  fatal error.", "RS_FATAL");
+        if( s.s == MemberState::RS_STARTUP2 ) return a("", "loaded config, still determining who is primary", "RS_STARTUP2");
+        if( s.s == MemberState::RS_ARBITER ) return a("", "this server is an arbiter only", "ARBITER");
+        if( s.s == MemberState::RS_DOWN ) return a("", "member is down, slow, or unreachable", "DOWN");
+        return "";
+    }
+
+    string ReplSetImpl::stateAsStr(MemberState s) { 
+        if( s.s == MemberState::RS_STARTUP ) return "STARTUP";
+        if( s.s == MemberState::RS_PRIMARY ) return "PRIMARY";
+        if( s.s == MemberState::RS_SECONDARY ) return "SECONDARY";
+        if( s.s == MemberState::RS_RECOVERING ) return "RECOVERING";
+        if( s.s == MemberState::RS_FATAL ) return "FATAL";
+        if( s.s == MemberState::RS_STARTUP2 ) return "STARTUP2";
+        if( s.s == MemberState::RS_ARBITER ) return "ARBITER";
+        if( s.s == MemberState::RS_DOWN ) return "DOWN";
+        return "";
+    }
+
+    extern time_t started;
+
+    // oplogdiags in web ui
+    static void say(stringstream&ss, const bo& op) {
+        ss << "<tr>";
+
+        set<string> skip;
+        be e = op["ts"];
+        if( e.type() == Date || e.type() == Timestamp ) { 
+            OpTime ot = e._opTime();
+	    ss << td( time_t_to_String_short( ot.getSecs() ) );
+            ss << td( ot.toString() );
+            skip.insert("ts");
+        }
+        else ss << td("?") << td("?");
+
+        e = op["h"];
+        if( e.type() == NumberLong ) {
+            ss << "<td>" << hex << e.Long() << "</td>\n";
+            skip.insert("h");
+        } else
+            ss << td("?");
+
+        ss << td(op["op"].valuestrsafe());
+        ss << td(op["ns"].valuestrsafe());
+        skip.insert("op");
+        skip.insert("ns");
+
+        ss << "<td>";
+        for( bo::iterator i(op); i.more(); ) { 
+            be e = i.next();
+            if( skip.count(e.fieldName()) ) continue;
+            ss << e.toString() << ' ';
+        }
+        ss << "</td>";
+
+        ss << "</tr>";
+        ss << '\n';
+    }
+
+    void ReplSetImpl::_getOplogDiagsAsHtml(unsigned server_id, stringstream& ss) const { 
+        const Member *m = findById(server_id);
+        if( m == 0 ) { 
+            ss << "Error : can't find a member with id: " << server_id << '\n';
+            return;
+        }
+
+        ss << p("Server : " + m->fullName() + "<br>ns : " + rsoplog );
+
+        //const bo fields = BSON( "o" << false << "o2" << false );
+        const bo fields;
+
+        ScopedConn conn(m->fullName());        
+
+        auto_ptr<DBClientCursor> c = conn->query(rsoplog, Query().sort("$natural",1), 20, 0, &fields);
+        if( c.get() == 0 ) { 
+            ss << "couldn't query " << rsoplog;
+            return;
+        }
+        static const char *h[] = {"ts","optime", "h","op","ns","rest",0};
+
+        ss << "<style type=\"text/css\" media=\"screen\">"
+            "table { font-size:75% }\n"
+//            "th { background-color:#bbb; color:#000 }\n"
+//            "td,th { padding:.25em }\n"
+            "</style>\n";
+        
+        ss << table(h, true);
+        //ss << "<pre>\n";
+        int n = 0;
+        OpTime otFirst;
+        OpTime otLast;
+        OpTime otEnd;
+        while( c->more() ) {
+            bo o = c->next();
+            otLast = o["ts"]._opTime();
+            if( otFirst.isNull() ) 
+                otFirst = otLast;
+            say(ss, o);
+            n++;            
+        }
+        if( n == 0 ) {
+            ss << rsoplog << " is empty\n";
+        }
+        else { 
+            auto_ptr<DBClientCursor> c = conn->query(rsoplog, Query().sort("$natural",-1), 20, 0, &fields);
+            if( c.get() == 0 ) { 
+                ss << "couldn't query [2] " << rsoplog;
+                return;
+            }
+            string x;
+            bo o = c->next();
+            otEnd = o["ts"]._opTime();
+            while( 1 ) {
+                stringstream z;
+                if( o["ts"]._opTime() == otLast ) 
+                    break;
+                say(z, o);
+                x = z.str() + x;
+                if( !c->more() )
+                    break;
+                o = c->next();
+            }
+            if( !x.empty() ) {
+                ss << "<tr><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td></tr>\n" << x;
+                //ss << "\n...\n\n" << x;
+            }
+        }
+        ss << _table();
+        ss << p(time_t_to_String_short(time(0)) + " current time");
+
+        //ss << "</pre>\n";
+
+        if( !otEnd.isNull() ) {
+            ss << "<p>Log length in time: ";
+            unsigned d = otEnd.getSecs() - otFirst.getSecs();
+            double h = d / 3600.0;
+            ss.precision(3);
+            if( h < 72 )
+                ss << h << " hours";
+            else 
+                ss << h / 24.0 << " days";
+            ss << "</p>\n";
+        }
+
+    }
+
+    void ReplSetImpl::_summarizeAsHtml(stringstream& s) const { 
+        s << table(0, false);
+        s << tr("Set name:", _name);
+        s << tr("Majority up:", elect.aMajoritySeemsToBeUp()?"yes":"no" );
+        s << _table();
+
+        const char *h[] = {"Member", 
+            "<a title=\"member id in the replset config\">id</a>", 
+            "Up", 
+            "<a title=\"length of time we have been continuously connected to the other member with no reconnects (for self, shows uptime)\">cctime</a>", 
+            "<a title=\"when this server last received a heartbeat response - includes error code responses\">Last heartbeat</a>", 
+            "Votes", "State", "Status", 
+            "<a title=\"how up to date this server is.  this value polled every few seconds so actually lag is typically much lower than value shown here.\">optime</a>", 
+            "<a title=\"Clock skew in seconds relative to this server. Informational; server clock variances will make the diagnostics hard to read, but otherwise are benign..\">skew</a>", 
+            0};
+        s << table(h);
+
+        /* this is to sort the member rows by their ordinal _id, so they show up in the same 
+           order on all the different web ui's; that is less confusing for the operator. */
+        map<int,string> mp;
+
+        string myMinValid;
+        try {
+            readlocktry lk("local.replset.minvalid", 300);
+            if( lk.got() ) {
+                BSONObj mv;
+                if( Helpers::getSingleton("local.replset.minvalid", mv) ) { 
+                    myMinValid = "minvalid:" + mv["ts"]._opTime().toString();
+                }
+            }
+            else myMinValid = ".";
+        }
+        catch(...) { 
+            myMinValid = "exception fetching minvalid";
+        }
+
+        {
+            stringstream s;
+            /* self row */
+            s << tr() << td(_self->fullName() + " (me)") <<
+                td(_self->id()) <<
+  	        td("1") <<  //up
+                td(ago(started)) << 
+	        td("") << // last heartbeat
+                td(ToString(_self->config().votes)) << 
+                td(stateAsHtml(box.getState()));
+            s << td( _hbmsg );
+            stringstream q;
+            q << "/_replSetOplog?" << _self->id();
+            s << td( a(q.str(), myMinValid, theReplSet->lastOpTimeWritten.toString()) );
+            s << td(""); // skew
+            s << _tr();
+			mp[_self->hbinfo().id()] = s.str();
+        }
+        Member *m = head();
+        while( m ) {
+			stringstream s;
+            m->summarizeMember(s);
+			mp[m->hbinfo().id()] = s.str();
+            m = m->next();
+        }
+
+        for( map<int,string>::const_iterator i = mp.begin(); i != mp.end(); i++ )
+            s << i->second;
+        s << _table();
+    }
+
+
+    void fillRsLog(stringstream& s) {
+        _rsLog.toHTML( s );
+    }
+
+    const Member* ReplSetImpl::findById(unsigned id) const { 
+        if( id == _self->id() ) return _self;
+        for( Member *m = head(); m; m = m->next() )
+            if( m->id() == id ) 
+                return m;
+        return 0;
+    }
+
+    void ReplSetImpl::_summarizeStatus(BSONObjBuilder& b) const { 
+        vector<BSONObj> v;
+
+        // add self
+        {
+            HostAndPort h(getHostName(), cmdLine.port);
+
+            BSONObjBuilder bb;
+            bb.append("_id", (int) _self->id());
+            bb.append("name", h.toString());
+            bb.append("health", 1.0);
+            bb.append("state", (int) box.getState().s);
+            string s = _self->lhb();
+            if( !s.empty() )
+                bb.append("errmsg", s);
+            bb.append("self", true);
+            v.push_back(bb.obj());
+        }
+
+        Member *m =_members.head();
+        while( m ) {
+            BSONObjBuilder bb;
+            bb.append("_id", (int) m->id());
+            bb.append("name", m->fullName());
+            bb.append("health", m->hbinfo().health);
+            bb.append("state", (int) m->state().s);
+            bb.append("uptime", (unsigned) (m->hbinfo().upSince ? (time(0)-m->hbinfo().upSince) : 0));
+            bb.appendTimeT("lastHeartbeat", m->hbinfo().lastHeartbeat);
+            string s = m->lhb();
+            if( !s.empty() )
+                bb.append("errmsg", s);
+            v.push_back(bb.obj());
+            m = m->next();
+        }
+        sort(v.begin(), v.end());
+        b.append("set", name());
+        b.appendTimeT("date", time(0));
+        b.append("myState", box.getState().s);
+        b.append("members", v);
+    }
+
+    static struct Test : public UnitTest { 
+        void run() { 
+            HealthOptions a,b;
+            assert( a == b );
+            assert( a.isDefault() );
+        }
+    } test;
+
+}
diff --git a/db/repl/health.h b/db/repl/health.h
new file mode 100644
index 0000000..8b1005e
--- /dev/null
+++ b/db/repl/health.h
@@ -0,0 +1,50 @@
+// replset.h
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+namespace mongo {
+
+    /* throws */
+    bool requestHeartbeat(string setname, string fromHost, string memberFullName, BSONObj& result, int myConfigVersion, int& theirConfigVersion, bool checkEmpty = false);
+
+    struct HealthOptions { 
+        HealthOptions() { 
+            heartbeatSleepMillis = 2000;
+            heartbeatTimeoutMillis = 10000;
+            heartbeatConnRetries  = 3;
+        }
+
+        bool isDefault() const { return *this == HealthOptions(); }
+
+        // see http://www.mongodb.org/display/DOCS/Replica+Set+Internals
+        unsigned heartbeatSleepMillis;
+        unsigned heartbeatTimeoutMillis;
+        unsigned heartbeatConnRetries ;
+
+        void check() {
+            uassert(13112, "bad replset heartbeat option", heartbeatSleepMillis >= 10);
+            uassert(13113, "bad replset heartbeat option", heartbeatTimeoutMillis >= 10);
+        }
+
+        bool operator==(const HealthOptions& r) const { 
+            return heartbeatSleepMillis==r.heartbeatSleepMillis && heartbeatTimeoutMillis==r.heartbeatTimeoutMillis && heartbeatConnRetries==heartbeatConnRetries; 
+        }
+    };
+
+}
diff --git a/db/repl/heartbeat.cpp b/db/repl/heartbeat.cpp
new file mode 100644
index 0000000..78ce5d1
--- /dev/null
+++ b/db/repl/heartbeat.cpp
@@ -0,0 +1,257 @@
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,b
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "rs.h"
+#include "health.h"
+#include "../../util/background.h"
+#include "../../client/dbclient.h"
+#include "../commands.h"
+#include "../../util/concurrency/value.h"
+#include "../../util/concurrency/task.h"
+#include "../../util/concurrency/msg.h"
+#include "../../util/mongoutils/html.h"
+#include "../../util/goodies.h"
+#include "../../util/ramlog.h"
+#include "../helpers/dblogger.h"
+#include "connections.h"
+#include "../../util/unittest.h"
+#include "../instance.h"
+
+namespace mongo { 
+
+    using namespace bson;
+
+    extern bool replSetBlind;
+
+    // hacky
+    string *discoveredSeed = 0;
+
+    /* { replSetHeartbeat : <setname> } */
+    class CmdReplSetHeartbeat : public ReplSetCommand {
+    public:
+        virtual bool adminOnly() const { return false; }
+        CmdReplSetHeartbeat() : ReplSetCommand("replSetHeartbeat") { }
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            if( replSetBlind ) 
+                return false;
+
+            /* we don't call ReplSetCommand::check() here because heartbeat 
+               checks many things that are pre-initialization. */
+            if( !replSet ) {
+                errmsg = "not running with --replSet";
+                return false;
+            }
+            if( cmdObj["pv"].Int() != 1 ) { 
+                errmsg = "incompatible replset protocol version";
+                return false;
+            }
+            {
+                string s = string(cmdObj.getStringField("replSetHeartbeat"));
+                if( cmdLine.ourSetName() != s ) {
+                    errmsg = "repl set names do not match";
+                    log() << "cmdline: " << cmdLine._replSet << endl;
+                    log() << "s: " << s << endl;
+                    result.append("mismatch", true);
+                    return false;
+                }
+            }
+
+            result.append("rs", true);
+            if( cmdObj["checkEmpty"].trueValue() ) { 
+                result.append("hasData", replHasDatabases());
+            }
+            if( theReplSet == 0 ) {
+                string from( cmdObj.getStringField("from") );
+                if( !from.empty() && discoveredSeed == 0 ) {
+                    discoveredSeed = new string(from);
+                }
+                errmsg = "still initializing";
+                return false;
+            }
+
+            if( theReplSet->name() != cmdObj.getStringField("replSetHeartbeat") ) { 
+                errmsg = "repl set names do not match (2)";
+                result.append("mismatch", true);
+                return false;
+            }
+            result.append("set", theReplSet->name());
+            result.append("state", theReplSet->state().s);
+            result.append("hbmsg", theReplSet->hbmsg());
+            result.append("time", (int) time(0));
+            result.appendDate("opTime", theReplSet->lastOpTimeWritten.asDate());
+            int v = theReplSet->config().version;
+            result.append("v", v);
+            if( v > cmdObj["v"].Int() )
+                result << "config" << theReplSet->config().asBson();
+
+            return true;
+        }
+    } cmdReplSetHeartbeat;
+
+    /* throws dbexception */
+    bool requestHeartbeat(string setName, string from, string memberFullName, BSONObj& result, int myCfgVersion, int& theirCfgVersion, bool checkEmpty) { 
+        if( replSetBlind ) { 
+            //sleepmillis( rand() );
+            return false;
+        }
+
+        BSONObj cmd = BSON( "replSetHeartbeat" << setName << "v" << myCfgVersion << "pv" << 1 << "checkEmpty" << checkEmpty << "from" << from );
+
+        // we might be talking to ourself - generally not a great idea to do outbound waiting calls in a write lock
+        assert( !dbMutex.isWriteLocked() );
+
+        // these are slow (multisecond to respond), so generally we don't want to be locked, at least not without
+        // thinking acarefully about it first.
+        assert( theReplSet == 0 || !theReplSet->lockedByMe() );
+
+        ScopedConn conn(memberFullName);
+        return conn->runCommand("admin", cmd, result);
+    }
+
+    /* poll every other set member to check its status */
+    class ReplSetHealthPollTask : public task::Task {
+        HostAndPort h;
+        HeartbeatInfo m;
+    public:
+        ReplSetHealthPollTask(const HostAndPort& hh, const HeartbeatInfo& mm) : h(hh), m(mm) { }
+
+        string name() { return "ReplSetHealthPollTask"; }
+        void doWork() { 
+            if ( !theReplSet ) {
+                log(2) << "theReplSet not initialized yet, skipping health poll this round" << rsLog;
+                return;
+            }
+
+            HeartbeatInfo mem = m;
+            HeartbeatInfo old = mem;
+            try { 
+                BSONObj info;
+                int theirConfigVersion = -10000;
+
+                time_t before = time(0);
+
+                bool ok = requestHeartbeat(theReplSet->name(), theReplSet->selfFullName(), h.toString(), info, theReplSet->config().version, theirConfigVersion);
+
+                time_t after = mem.lastHeartbeat = time(0); // we set this on any response - we don't get this far if couldn't connect because exception is thrown
+
+                try {
+                    mem.skew = 0;
+                    long long t = info["time"].Long();
+                    if( t > after ) 
+                        mem.skew = (int) (t - after);
+                    else if( t < before ) 
+                        mem.skew = (int) (t - before); // negative
+                }
+                catch(...) { 
+                    mem.skew = INT_MIN;
+                }
+
+                {
+                    be state = info["state"];
+                    if( state.ok() )
+                        mem.hbstate = MemberState(state.Int());
+                }
+                if( ok ) {
+                    if( mem.upSince == 0 ) {
+                        log() << "replSet info " << h.toString() << " is now up" << rsLog;
+                        mem.upSince = mem.lastHeartbeat;
+                    }
+                    mem.health = 1.0;
+                    mem.lastHeartbeatMsg = info["hbmsg"].String();
+                    if( info.hasElement("opTime") )
+                        mem.opTime = info["opTime"].Date();
+
+                    be cfg = info["config"];
+                    if( cfg.ok() ) {
+                        // received a new config
+                        boost::function<void()> f = 
+                            boost::bind(&Manager::msgReceivedNewConfig, theReplSet->mgr, cfg.Obj().copy());
+                        theReplSet->mgr->send(f);
+                    }
+                }
+                else { 
+                    down(mem, info.getStringField("errmsg"));
+                }
+            }
+            catch(...) { 
+                down(mem, "connect/transport error");             
+            }
+            m = mem;
+
+            theReplSet->mgr->send( boost::bind(&ReplSet::msgUpdateHBInfo, theReplSet, mem) );
+
+            static time_t last = 0;
+            time_t now = time(0);
+            if( mem.changed(old) || now-last>4 ) {
+                last = now;
+                theReplSet->mgr->send( boost::bind(&Manager::msgCheckNewState, theReplSet->mgr) );
+            }
+        }
+
+    private:
+        void down(HeartbeatInfo& mem, string msg) {
+            mem.health = 0.0;
+            if( mem.upSince ) {
+                mem.upSince = 0;
+                log() << "replSet info " << h.toString() << " is now down (or slow to respond)" << rsLog;
+            }
+            mem.lastHeartbeatMsg = msg;
+        }
+    };
+
+    void ReplSetImpl::endOldHealthTasks() { 
+        unsigned sz = healthTasks.size();
+        for( set<ReplSetHealthPollTask*>::iterator i = healthTasks.begin(); i != healthTasks.end(); i++ )
+            (*i)->halt();
+        healthTasks.clear();
+        if( sz ) 
+            DEV log() << "replSet debug: cleared old tasks " << sz << endl;
+    }
+
+    void ReplSetImpl::startHealthTaskFor(Member *m) {
+        ReplSetHealthPollTask *task = new ReplSetHealthPollTask(m->h(), m->hbinfo());
+        healthTasks.insert(task);
+        task::repeat(task, 2000);
+    }
+
+    void startSyncThread();
+
+    /** called during repl set startup.  caller expects it to return fairly quickly. 
+        note ReplSet object is only created once we get a config - so this won't run 
+        until the initiation.
+    */
+    void ReplSetImpl::startThreads() {
+        task::fork(mgr);
+
+        /*Member* m = _members.head();
+        while( m ) {
+            ReplSetHealthPollTask *task = new ReplSetHealthPollTask(m->h(), m->hbinfo());
+            healthTasks.insert(task);
+            task::repeat(shared_ptr<task::Task>(task), 2000);
+            m = m->next();
+        }*/
+
+        mgr->send( boost::bind(&Manager::msgCheckNewState, theReplSet->mgr) );
+
+        boost::thread t(startSyncThread);
+    }
+
+}
+
+/* todo:
+   stop bg job and delete on removefromset
+*/
diff --git a/db/repl/manager.cpp b/db/repl/manager.cpp
new file mode 100644
index 0000000..e870688
--- /dev/null
+++ b/db/repl/manager.cpp
@@ -0,0 +1,179 @@
+/* @file manager.cpp 
+*/
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,b
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "rs.h"
+#include "../client.h"
+
+namespace mongo {
+
+    enum { 
+        NOPRIMARY = -2,
+        SELFPRIMARY = -1
+    };
+
+    /* check members OTHER THAN US to see if they think they are primary */
+    const Member * Manager::findOtherPrimary() { 
+        Member *m = rs->head();
+        Member *p = 0;
+        while( m ) {
+            if( m->state().primary() && m->hbinfo().up() ) {
+                if( p ) throw "twomasters"; // our polling is asynchronous, so this is often ok.
+                p = m;
+            }
+            m = m->next();
+        }
+        if( p ) 
+            noteARemoteIsPrimary(p);
+        return p;
+    }
+
+    Manager::Manager(ReplSetImpl *_rs) : 
+    task::Server("rs Manager"), rs(_rs), busyWithElectSelf(false), _primary(NOPRIMARY)
+    { 
+    }
+    
+    Manager::~Manager() { 
+        log() << "ERROR: ~Manager should never be called" << rsLog;
+        rs->mgr = 0;
+        assert(false);
+    }
+
+    void Manager::starting() { 
+        Client::initThread("rs Manager");
+    }
+
+    void Manager::noteARemoteIsPrimary(const Member *m) { 
+        if( rs->box.getPrimary() == m )
+            return;
+        rs->_self->lhb() = "";
+        rs->box.set(rs->iAmArbiterOnly() ? MemberState::RS_ARBITER : MemberState::RS_RECOVERING, m);
+    }
+
+    /** called as the health threads get new results */
+    void Manager::msgCheckNewState() {
+        {
+            theReplSet->assertValid();
+            rs->assertValid();
+
+            RSBase::lock lk(rs);
+
+            if( busyWithElectSelf ) return;
+
+            const Member *p = rs->box.getPrimary();
+            if( p && p != rs->_self ) {
+                if( !p->hbinfo().up() || 
+                    !p->hbinfo().hbstate.primary() ) 
+                {
+                    p = 0;
+                    rs->box.setOtherPrimary(0);
+                }
+            }
+
+            const Member *p2;
+            try { p2 = findOtherPrimary(); }
+            catch(string s) { 
+                /* two other nodes think they are primary (asynchronously polled) -- wait for things to settle down. */
+                log() << "replSet warning DIAG 2 primary" << s << rsLog;
+                return;
+            }
+
+            if( p2 ) {
+                /* someone else thinks they are primary. */
+                if( p == p2 ) { 
+                    // we thought the same; all set.
+                    return;
+                }
+                if( p == 0 ) {
+                    noteARemoteIsPrimary(p2); 
+                    return;
+                }
+                // todo xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+                if( p != rs->_self ) {
+                    // switch primary from oldremotep->newremotep2
+                    noteARemoteIsPrimary(p2); 
+                    return;
+                }
+                /* we thought we were primary, yet now someone else thinks they are. */
+                if( !rs->elect.aMajoritySeemsToBeUp() ) {
+                    /* we can't see a majority.  so the other node is probably the right choice. */
+                    noteARemoteIsPrimary(p2); 
+                    return;
+                }
+                /* ignore for now, keep thinking we are master. 
+                   this could just be timing (we poll every couple seconds) or could indicate 
+                   a problem?  if it happens consistently for a duration of time we should 
+                   alert the sysadmin.
+                */
+                return;
+            }
+
+            /* didn't find anyone who wants to be primary */
+
+            if( p ) { 
+                /* we are already primary */
+
+                if( p != rs->_self ) { 
+                    rs->sethbmsg("error p != rs->self in checkNewState");
+                    log() << "replSet " << p->fullName() << rsLog;
+                    log() << "replSet " << rs->_self->fullName() << rsLog;
+                    return;
+                }
+
+                if( !rs->elect.aMajoritySeemsToBeUp() ) { 
+                    log() << "replSet can't see a majority of the set, relinquishing primary" << rsLog;
+                    rs->relinquish();
+                }
+
+                return;
+            }
+
+            if( !rs->iAmPotentiallyHot() ) // if not we never try to be primary
+                return;
+
+            /* TODO : CHECK PRIORITY HERE.  can't be elected if priority zero. */
+
+            /* no one seems to be primary.  shall we try to elect ourself? */
+            if( !rs->elect.aMajoritySeemsToBeUp() ) { 
+                static time_t last;
+                static int n;
+                int ll = 0;
+                if( ++n > 5 ) ll++;
+                if( last + 60 > time(0 ) ) ll++;
+                log(ll) << "replSet can't see a majority, will not try to elect self" << rsLog;
+                last = time(0);
+                return;
+            }
+
+            busyWithElectSelf = true; // don't try to do further elections & such while we are already working on one.
+        }
+        try { 
+            rs->elect.electSelf(); 
+        }
+        catch(RetryAfterSleepException&) {
+            /* we want to process new inbounds before trying this again.  so we just put a checkNewstate in the queue for eval later. */
+            requeue();
+        }
+        catch(...) { 
+            log() << "replSet error unexpected assertion in rs manager" << rsLog; 
+        }
+        busyWithElectSelf = false;
+    }
+
+}
diff --git a/db/repl/multicmd.h b/db/repl/multicmd.h
new file mode 100644
index 0000000..61c9b5f
--- /dev/null
+++ b/db/repl/multicmd.h
@@ -0,0 +1,70 @@
+// @file multicmd.h
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include "../../util/background.h"
+#include "connections.h"
+
+namespace mongo { 
+
+    struct Target {
+        Target(string hostport) : toHost(hostport), ok(false) { }
+        Target() : ok(false) { }
+        string toHost;
+        bool ok;
+        BSONObj result;
+    };
+
+    /* -- implementation ------------- */
+
+    class _MultiCommandJob : public BackgroundJob { 
+    public:
+        BSONObj& cmd;
+        Target& d;
+        _MultiCommandJob(BSONObj& _cmd, Target& _d) : cmd(_cmd), d(_d) { }
+    private:
+        string name() { return "MultiCommandJob"; }
+        void run() {
+            try { 
+                ScopedConn c(d.toHost);
+                d.ok = c->runCommand("admin", cmd, d.result);
+            }
+            catch(DBException&) { 
+                DEV log() << "dev caught dbexception on multiCommand " << d.toHost << rsLog;
+            }
+        }
+    };
+
+    inline void multiCommand(BSONObj cmd, list<Target>& L) { 
+        typedef shared_ptr<_MultiCommandJob> P;
+        list<P> jobs;
+        list<BackgroundJob *> _jobs;
+
+        for( list<Target>::iterator i = L.begin(); i != L.end(); i++ ) { 
+            Target& d = *i;
+            _MultiCommandJob *j = new _MultiCommandJob(cmd, d);
+            jobs.push_back(P(j));
+            _jobs.push_back(j);
+        }
+
+        BackgroundJob::go(_jobs);
+        BackgroundJob::wait(_jobs,5);
+    }
+
+}
diff --git a/db/repl/replset_commands.cpp b/db/repl/replset_commands.cpp
new file mode 100644
index 0000000..f8f46d5
--- /dev/null
+++ b/db/repl/replset_commands.cpp
@@ -0,0 +1,293 @@
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "../cmdline.h"
+#include "../commands.h"
+#include "health.h"
+#include "rs.h"
+#include "rs_config.h"
+#include "../dbwebserver.h"
+#include "../../util/mongoutils/html.h"
+#include "../../client/dbclient.h"
+
+namespace mongo { 
+
+    void checkMembersUpForConfigChange(const ReplSetConfig& cfg, bool initial);
+
+    /* commands in other files:
+         replSetHeartbeat - health.cpp
+         replSetInitiate  - rs_mod.cpp
+    */
+
+    bool replSetBlind = false;
+
+    class CmdReplSetTest : public ReplSetCommand {
+    public:
+        virtual void help( stringstream &help ) const {
+            help << "Just for testing : do not use.\n";
+        }
+        CmdReplSetTest() : ReplSetCommand("replSetTest") { }
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            if( !check(errmsg, result) ) 
+                return false;
+            if( cmdObj.hasElement("blind") ) {
+                replSetBlind = cmdObj.getBoolField("blind");
+                log() << "replSet info replSetTest command received, replSetBlind=" << replSetBlind << rsLog;
+                return true;
+            }
+            return false;
+        }
+    } cmdReplSetTest;
+
+    class CmdReplSetGetRBID : public ReplSetCommand {
+    public:
+        int rbid;
+        virtual void help( stringstream &help ) const {
+            help << "internal";
+        }
+        CmdReplSetGetRBID() : ReplSetCommand("replSetGetRBID") { 
+            rbid = (int) curTimeMillis();
+        }
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            if( !check(errmsg, result) ) 
+                return false;
+	    result.append("rbid",rbid);
+            return true;
+        }
+    } cmdReplSetRBID;
+
+    using namespace bson;
+    int getRBID(DBClientConnection *c) { 
+        bo info;
+        c->simpleCommand("admin", &info, "replSetGetRBID");
+        return info["rbid"].numberInt();
+    } 
+
+    class CmdReplSetGetStatus : public ReplSetCommand {
+    public:
+        virtual void help( stringstream &help ) const {
+            help << "Report status of a replica set from the POV of this server\n";
+            help << "{ replSetGetStatus : 1 }";
+            help << "\nhttp://www.mongodb.org/display/DOCS/Replica+Set+Commands";
+        }
+        CmdReplSetGetStatus() : ReplSetCommand("replSetGetStatus", true) { }
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            if( !check(errmsg, result) ) 
+                return false;
+            theReplSet->summarizeStatus(result);
+            return true;
+        }
+    } cmdReplSetGetStatus;
+
+    class CmdReplSetReconfig : public ReplSetCommand {
+        RWLock mutex; /* we don't need rw but we wanted try capability. :-( */
+    public:
+        virtual void help( stringstream &help ) const {
+            help << "Adjust configuration of a replica set\n";
+            help << "{ replSetReconfig : config_object }";
+            help << "\nhttp://www.mongodb.org/display/DOCS/Replica+Set+Commands";
+        }
+        CmdReplSetReconfig() : ReplSetCommand("replSetReconfig"), mutex("rsreconfig") { }
+        virtual bool run(const string& a, BSONObj& b, string& errmsg, BSONObjBuilder& c, bool d) {
+            try { 
+                rwlock_try_write lk(mutex);
+                return _run(a,b,errmsg,c,d);
+            }
+            catch(rwlock_try_write::exception&) { }
+            errmsg = "a replSetReconfig is already in progress";
+            return false;
+        }
+    private:
+        bool _run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            if( !check(errmsg, result) ) 
+                return false;
+            if( !theReplSet->box.getState().primary() ) { 
+                errmsg = "replSetReconfig command must be sent to the current replica set primary.";
+                return false;
+            }
+
+            {
+                // just make sure we can get a write lock before doing anything else.  we'll reacquire one 
+                // later.  of course it could be stuck then, but this check lowers the risk if weird things 
+                // are up - we probably don't want a change to apply 30 minutes after the initial attempt.
+                time_t t = time(0);
+                writelock lk("");
+                if( time(0)-t > 20 ) {
+                    errmsg = "took a long time to get write lock, so not initiating.  Initiate when server less busy?";
+                    return false;
+                }
+            }
+
+            if( cmdObj["replSetReconfig"].type() != Object ) {
+                errmsg = "no configuration specified";
+                return false;
+            }
+
+            /** TODO
+                Support changes when a majority, but not all, members of a set are up.
+                Determine what changes should not be allowed as they would cause erroneous states.
+                What should be possible when a majority is not up?
+                */
+            try {
+                ReplSetConfig newConfig(cmdObj["replSetReconfig"].Obj());
+
+                log() << "replSet replSetReconfig config object parses ok, " << newConfig.members.size() << " members specified" << rsLog;
+
+                if( !ReplSetConfig::legalChange(theReplSet->getConfig(), newConfig, errmsg) ) { 
+                    return false;
+                }
+
+                checkMembersUpForConfigChange(newConfig,false);
+
+                log() << "replSet replSetReconfig [2]" << rsLog;
+
+                theReplSet->haveNewConfig(newConfig, true);
+                ReplSet::startupStatusMsg = "replSetReconfig'd";
+            }
+            catch( DBException& e ) { 
+                log() << "replSet replSetReconfig exception: " << e.what() << rsLog;
+                throw;
+            }
+
+            return true;
+        }
+    } cmdReplSetReconfig;
+
+    class CmdReplSetFreeze : public ReplSetCommand {
+    public:
+        virtual void help( stringstream &help ) const {
+            help << "Enable / disable failover for the set - locks current primary as primary even if issues occur.\nFor use during system maintenance.\n";
+            help << "{ replSetFreeze : <bool> }";
+            help << "\nhttp://www.mongodb.org/display/DOCS/Replica+Set+Commands";
+        }
+
+        CmdReplSetFreeze() : ReplSetCommand("replSetFreeze") { }
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            if( !check(errmsg, result) )
+                return false;
+            errmsg = "not yet implemented"; /*TODO*/
+            return false;
+        }
+    } cmdReplSetFreeze;
+
+    class CmdReplSetStepDown: public ReplSetCommand {
+    public:
+        virtual void help( stringstream &help ) const {
+            help << "Step down as primary.  Will not try to reelect self or 1 minute.\n";
+            help << "(If another member with same priority takes over in the meantime, it will stay primary.)\n";
+            help << "http://www.mongodb.org/display/DOCS/Replica+Set+Commands";
+        }
+
+        CmdReplSetStepDown() : ReplSetCommand("replSetStepDown") { }
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            if( !check(errmsg, result) )
+                return false;
+            if( !theReplSet->box.getState().primary() ) {
+                errmsg = "not primary so can't step down";
+                return false;
+            }
+            return theReplSet->stepDown();
+        }
+    } cmdReplSetStepDown;
+
+    using namespace bson;
+    using namespace mongoutils::html;
+    extern void fillRsLog(stringstream&);
+
+    class ReplSetHandler : public DbWebHandler {
+    public:
+        ReplSetHandler() : DbWebHandler( "_replSet" , 1 , true ){}
+
+        virtual bool handles( const string& url ) const {
+            return startsWith( url , "/_replSet" );
+        }
+
+        virtual void handle( const char *rq, string url, 
+                             string& responseMsg, int& responseCode,
+                             vector<string>& headers,  const SockAddr &from ){
+            
+            string s = str::after(url, "/_replSetOplog?");
+            if( !s.empty() )
+                responseMsg = _replSetOplog(s);
+            else
+                responseMsg = _replSet();
+            responseCode = 200;
+        }
+
+
+        string _replSetOplog(string parms) { 
+            stringstream s;
+            string t = "Replication oplog";
+            s << start(t);
+            s << p(t);
+
+            if( theReplSet == 0 ) { 
+                if( cmdLine._replSet.empty() ) 
+                    s << p("Not using --replSet");
+                else  {
+                    s << p("Still starting up, or else set is not yet " + a("http://www.mongodb.org/display/DOCS/Replica+Set+Configuration#InitialSetup", "", "initiated") 
+                           + ".<br>" + ReplSet::startupStatusMsg);
+                }
+            }
+            else {
+                try {
+                    theReplSet->getOplogDiagsAsHtml(stringToNum(parms.c_str()), s);
+                }
+                catch(std::exception& e) { 
+                    s << "error querying oplog: " << e.what() << '\n'; 
+                }
+            }
+
+            s << _end();
+            return s.str();
+        }
+
+        /* /_replSet show replica set status in html format */
+        string _replSet() { 
+            stringstream s;
+            s << start("Replica Set Status " + prettyHostName());
+            s << p( a("/", "back", "Home") + " | " + 
+                    a("/local/system.replset/?html=1", "", "View Replset Config") + " | " +
+                    a("/replSetGetStatus?text", "", "replSetGetStatus") + " | " +
+                    a("http://www.mongodb.org/display/DOCS/Replica+Sets", "", "Docs")
+                  );
+
+            if( theReplSet == 0 ) { 
+                if( cmdLine._replSet.empty() ) 
+                    s << p("Not using --replSet");
+                else  {
+                    s << p("Still starting up, or else set is not yet " + a("http://www.mongodb.org/display/DOCS/Replica+Set+Configuration#InitialSetup", "", "initiated") 
+                           + ".<br>" + ReplSet::startupStatusMsg);
+                }
+            }
+            else {
+                try {
+                    theReplSet->summarizeAsHtml(s);
+                }
+                catch(...) { s << "error summarizing replset status\n"; }
+            }
+            s << p("Recent replset log activity:");
+            fillRsLog(s);
+            s << _end();
+            return s.str();
+        }
+
+
+
+    } replSetHandler;
+
+}
diff --git a/db/repl/rs.cpp b/db/repl/rs.cpp
new file mode 100644
index 0000000..3e12e42
--- /dev/null
+++ b/db/repl/rs.cpp
@@ -0,0 +1,500 @@
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "../cmdline.h"
+#include "../../util/sock.h"
+#include "../client.h"
+#include "../../client/dbclient.h"
+#include "../dbhelpers.h"
+#include "rs.h"
+
+namespace mongo { 
+
+    using namespace bson;
+
+    bool replSet = false;
+    ReplSet *theReplSet = 0;
+    extern string *discoveredSeed;
+
+    void ReplSetImpl::sethbmsg(string s, int logLevel) { 
+      static time_t lastLogged;
+      if( s == _hbmsg ) { 
+	// unchanged
+	if( time(0)-lastLogged < 60 )
+	  return;
+      }
+
+      unsigned sz = s.size();
+      if( sz >= 256 ) 
+	memcpy(_hbmsg, s.c_str(), 255);
+      else {
+	_hbmsg[sz] = 0;
+	memcpy(_hbmsg, s.c_str(), sz);
+      }
+      if( !s.empty() ) {
+	lastLogged = time(0);
+	log(logLevel) << "replSet " << s << rsLog;
+      }
+    }
+
+    void ReplSetImpl::assumePrimary() { 
+        assert( iAmPotentiallyHot() );
+        writelock lk("admin."); // so we are synchronized with _logOp()
+        box.setSelfPrimary(_self);
+        log() << "replSet PRIMARY" << rsLog; // self (" << _self->id() << ") is now primary" << rsLog;
+    }
+
+    void ReplSetImpl::changeState(MemberState s) { box.change(s, _self); }
+
+    void ReplSetImpl::relinquish() { 
+        if( box.getState().primary() ) {
+            changeState(MemberState::RS_RECOVERING);
+            log() << "replSet info relinquished primary state" << rsLog;
+        }
+        else if( box.getState().startup2() ) {
+            // ? add comment
+            changeState(MemberState::RS_RECOVERING);
+        }
+    }
+
+    /* look freshly for who is primary - includes relinquishing ourself. */
+    void ReplSetImpl::forgetPrimary() { 
+        if( box.getState().primary() ) 
+            relinquish();
+        else {
+            box.setOtherPrimary(0);
+        }
+    }
+
+    bool ReplSetImpl::_stepDown() { 
+        lock lk(this);
+        if( box.getState().primary() ) { 
+            changeState(MemberState::RS_RECOVERING);
+            elect.steppedDown = time(0) + 60;
+            log() << "replSet info stepped down as primary" << rsLog;
+            return true;
+        }
+        return false;
+    }
+
+    void ReplSetImpl::msgUpdateHBInfo(HeartbeatInfo h) { 
+        for( Member *m = _members.head(); m; m=m->next() ) {
+            if( m->id() == h.id() ) {
+                m->_hbinfo = h;
+                return;
+            }
+        }
+    }
+
+    list<HostAndPort> ReplSetImpl::memberHostnames() const { 
+        list<HostAndPort> L;
+        L.push_back(_self->h());
+        for( Member *m = _members.head(); m; m = m->next() )
+            L.push_back(m->h());
+        return L;
+    }
+
+    void ReplSetImpl::_fillIsMasterHost(const Member *m, vector<string>& hosts, vector<string>& passives, vector<string>& arbiters) {
+        if( m->potentiallyHot() ) {
+            hosts.push_back(m->h().toString());
+        }
+        else if( !m->config().arbiterOnly ) {
+            passives.push_back(m->h().toString());
+        }
+        else {
+            arbiters.push_back(m->h().toString());
+        }
+    }
+
+    void ReplSetImpl::_fillIsMaster(BSONObjBuilder& b) {
+        const StateBox::SP sp = box.get();
+        bool isp = sp.state.primary();
+        b.append("ismaster", isp);
+        b.append("secondary", sp.state.secondary());
+        {
+            vector<string> hosts, passives, arbiters;
+            _fillIsMasterHost(_self, hosts, passives, arbiters);
+
+            for( Member *m = _members.head(); m; m = m->next() ) {
+                _fillIsMasterHost(m, hosts, passives, arbiters);
+            }
+
+            if( hosts.size() > 0 ) {
+                b.append("hosts", hosts);
+            }
+            if( passives.size() > 0 ) {
+                b.append("passives", passives);
+            }
+            if( arbiters.size() > 0 ) {
+                b.append("arbiters", arbiters);
+            }
+        }
+
+        if( !isp ) { 
+            const Member *m = sp.primary;
+            if( m )
+                b.append("primary", m->h().toString());
+        }
+        if( myConfig().arbiterOnly )
+            b.append("arbiterOnly", true);
+    }
+
+    /** @param cfgString <setname>/<seedhost1>,<seedhost2> */
+
+    void parseReplsetCmdLine(string cfgString, string& setname, vector<HostAndPort>& seeds, set<HostAndPort>& seedSet ) { 
+        const char *p = cfgString.c_str(); 
+        const char *slash = strchr(p, '/');
+        if( slash )
+            setname = string(p, slash-p);
+        else
+            setname = p;
+        uassert(13093, "bad --replSet config string format is: <setname>[/<seedhost1>,<seedhost2>,...]", !setname.empty());
+
+        if( slash == 0 )
+            return;
+
+        p = slash + 1;
+        while( 1 ) {
+            const char *comma = strchr(p, ',');
+            if( comma == 0 ) comma = strchr(p,0);
+            if( p == comma )
+                break;
+            {
+                HostAndPort m;
+                try {
+                    m = HostAndPort( string(p, comma-p) );
+                }
+                catch(...) {
+                    uassert(13114, "bad --replSet seed hostname", false);
+                }
+                uassert(13096, "bad --replSet command line config string - dups?", seedSet.count(m) == 0 );
+                seedSet.insert(m);
+                //uassert(13101, "can't use localhost in replset host list", !m.isLocalHost());
+                if( m.isSelf() ) {
+                    log(1) << "replSet ignoring seed " << m.toString() << " (=self)" << rsLog;
+                } else
+                    seeds.push_back(m);
+                if( *comma == 0 )
+                    break;
+                p = comma + 1;
+            }
+        }
+    }
+
+    ReplSetImpl::ReplSetImpl(ReplSetCmdline& replSetCmdline) : elect(this), 
+        _self(0), 
+        mgr( new Manager(this) )
+    {
+        memset(_hbmsg, 0, sizeof(_hbmsg));
+        *_hbmsg = '.'; // temp...just to see
+        lastH = 0;
+        changeState(MemberState::RS_STARTUP);
+
+        _seeds = &replSetCmdline.seeds;
+        //for( vector<HostAndPort>::iterator i = seeds->begin(); i != seeds->end(); i++ )
+        //    addMemberIfMissing(*i);
+
+        log(1) << "replSet beginning startup..." << rsLog;
+
+        loadConfig();
+
+        unsigned sss = replSetCmdline.seedSet.size();
+        for( Member *m = head(); m; m = m->next() ) {
+            replSetCmdline.seedSet.erase(m->h());
+        }
+        for( set<HostAndPort>::iterator i = replSetCmdline.seedSet.begin(); i != replSetCmdline.seedSet.end(); i++ ) {
+            if( i->isSelf() ) {
+                if( sss == 1 ) 
+                    log(1) << "replSet warning self is listed in the seed list and there are no other seeds listed did you intend that?" << rsLog;
+            } else
+                log() << "replSet warning command line seed " << i->toString() << " is not present in the current repl set config" << rsLog;
+        }
+    }
+
+    void newReplUp();
+
+    void ReplSetImpl::loadLastOpTimeWritten() { 
+        //assert( lastOpTimeWritten.isNull() );
+        readlock lk(rsoplog);
+        BSONObj o;
+        if( Helpers::getLast(rsoplog, o) ) { 
+            lastH = o["h"].numberLong();
+            lastOpTimeWritten = o["ts"]._opTime();
+            uassert(13290, "bad replSet oplog entry?", !lastOpTimeWritten.isNull());
+        }
+    }
+
+    /* call after constructing to start - returns fairly quickly after launching its threads */
+    void ReplSetImpl::_go() { 
+        try { 
+            loadLastOpTimeWritten();
+        }
+        catch(std::exception& e) { 
+            log() << "replSet ERROR FATAL couldn't query the local " << rsoplog << " collection.  Terminating mongod after 30 seconds." << rsLog;
+            log() << e.what() << rsLog;
+            sleepsecs(30);
+            dbexit( EXIT_REPLICATION_ERROR );
+            return;
+        }
+
+        changeState(MemberState::RS_STARTUP2);
+        startThreads();
+        newReplUp(); // oplog.cpp
+    }
+
+    ReplSetImpl::StartupStatus ReplSetImpl::startupStatus = PRESTART;
+    string ReplSetImpl::startupStatusMsg;
+
+    // true if ok; throws if config really bad; false if config doesn't include self
+    bool ReplSetImpl::initFromConfig(ReplSetConfig& c) {
+        lock lk(this);
+
+        {
+            int me = 0;
+            for( vector<ReplSetConfig::MemberCfg>::iterator i = c.members.begin(); i != c.members.end(); i++ ) { 
+                const ReplSetConfig::MemberCfg& m = *i;
+                if( m.h.isSelf() ) {
+                    me++;
+                }
+            }
+            if( me == 0 ) {
+                // log() << "replSet config : " << _cfg->toString() << rsLog;
+                log() << "replSet warning can't find self in the repl set configuration:" << rsLog;
+                log() << c.toString() << rsLog;
+                return false;
+            }
+            uassert( 13302, "replSet error self appears twice in the repl set configuration", me<=1 );
+        }
+
+        _cfg = new ReplSetConfig(c);
+        assert( _cfg->ok() );
+        assert( _name.empty() || _name == _cfg->_id );
+        _name = _cfg->_id;
+        assert( !_name.empty() );
+
+        // start with no members.  if this is a reconfig, drop the old ones.
+        _members.orphanAll();
+
+        endOldHealthTasks();
+
+        int oldPrimaryId = -1;
+        {
+            const Member *p = box.getPrimary();
+            if( p ) 
+                oldPrimaryId = p->id();
+        }
+        forgetPrimary();
+        _self = 0;
+        for( vector<ReplSetConfig::MemberCfg>::iterator i = _cfg->members.begin(); i != _cfg->members.end(); i++ ) { 
+            const ReplSetConfig::MemberCfg& m = *i;
+            Member *mi;
+            if( m.h.isSelf() ) {
+                assert( _self == 0 );
+                mi = _self = new Member(m.h, m._id, &m, true);
+                if( (int)mi->id() == oldPrimaryId )
+                    box.setSelfPrimary(mi);
+            } else {
+                mi = new Member(m.h, m._id, &m, false);
+                _members.push(mi);
+                startHealthTaskFor(mi);
+                if( (int)mi->id() == oldPrimaryId )
+                    box.setOtherPrimary(mi);
+            }
+        }
+        return true;
+    }
+
+    // Our own config must be the first one.
+    bool ReplSetImpl::_loadConfigFinish(vector<ReplSetConfig>& cfgs) { 
+        int v = -1;
+        ReplSetConfig *highest = 0;
+        int myVersion = -2000;
+        int n = 0;
+        for( vector<ReplSetConfig>::iterator i = cfgs.begin(); i != cfgs.end(); i++ ) { 
+            ReplSetConfig& cfg = *i;
+            if( ++n == 1 ) myVersion = cfg.version;
+            if( cfg.ok() && cfg.version > v ) { 
+                highest = &cfg;
+                v = cfg.version;
+            }
+        }
+        assert( highest );
+
+        if( !initFromConfig(*highest) ) 
+            return false;
+
+        if( highest->version > myVersion && highest->version >= 0 ) { 
+            log() << "replSet got config version " << highest->version << " from a remote, saving locally" << rsLog;
+            writelock lk("admin.");
+            highest->saveConfigLocally(BSONObj());
+        }
+        return true;
+    }
+
+    void ReplSetImpl::loadConfig() {
+        while( 1 ) {
+            startupStatus = LOADINGCONFIG;
+            startupStatusMsg = "loading " + rsConfigNs + " config (LOADINGCONFIG)";
+            try {
+                vector<ReplSetConfig> configs;
+                try { 
+                    configs.push_back( ReplSetConfig(HostAndPort::me()) );
+                }
+                catch(DBException& e) {
+                    log() << "replSet exception loading our local replset configuration object : " << e.toString() << rsLog;
+                    throw;
+                }
+                for( vector<HostAndPort>::const_iterator i = _seeds->begin(); i != _seeds->end(); i++ ) {
+                    try { 
+                        configs.push_back( ReplSetConfig(*i) );
+                    }
+                    catch( DBException& e ) { 
+                        log() << "replSet exception trying to load config from " << *i << " : " << e.toString() << rsLog;
+                    }
+                }
+
+                if( discoveredSeed ) { 
+                    try {
+                        configs.push_back( ReplSetConfig(HostAndPort(*discoveredSeed)) );
+                    }
+                    catch( DBException& ) { 
+                        log(1) << "replSet exception trying to load config from discovered seed " << *discoveredSeed << rsLog;
+                    }
+                }
+
+                int nok = 0;
+                int nempty = 0;
+                for( vector<ReplSetConfig>::iterator i = configs.begin(); i != configs.end(); i++ ) { 
+                    if( i->ok() )
+                        nok++;
+                    if( i->empty() )
+                        nempty++;
+                }
+                if( nok == 0 ) {
+
+                    if( nempty == (int) configs.size() ) {
+                        startupStatus = EMPTYCONFIG;
+                        startupStatusMsg = "can't get " + rsConfigNs + " config from self or any seed (EMPTYCONFIG)";
+                        log() << "replSet can't get " << rsConfigNs << " config from self or any seed (EMPTYCONFIG)" << rsLog;
+                        log(1) << "replSet have you ran replSetInitiate yet?" << rsLog;
+                        if( _seeds->size() == 0 )
+                            log(1) << "replSet info no seed hosts were specified on the --replSet command line" << rsLog;
+                    }
+                    else {
+                        startupStatus = EMPTYUNREACHABLE;
+                        startupStatusMsg = "can't currently get " + rsConfigNs + " config from self or any seed (EMPTYUNREACHABLE)";
+                        log() << "replSet can't get " << rsConfigNs << " config from self or any seed (yet)" << rsLog;
+                    }
+
+                    sleepsecs(10);
+                    continue;
+                }
+
+                if( !_loadConfigFinish(configs) ) { 
+                    log() << "replSet info Couldn't load config yet. Sleeping 20sec and will try again." << rsLog;
+                    sleepsecs(20);
+                    continue;
+                }
+            }
+            catch(DBException& e) { 
+                startupStatus = BADCONFIG;
+                startupStatusMsg = "replSet error loading set config (BADCONFIG)";
+                log() << "replSet error loading configurations " << e.toString() << rsLog;
+                log() << "replSet error replication will not start" << rsLog;
+                _fatal();
+                throw;
+            }
+            break;
+        }
+        startupStatusMsg = "? started";
+        startupStatus = STARTED;
+    }
+
+    void ReplSetImpl::_fatal() 
+    { 
+        //lock l(this);
+        box.set(MemberState::RS_FATAL, 0);
+        sethbmsg("fatal error");
+        log() << "replSet error fatal error, stopping replication" << rsLog; 
+    }
+
+
+    void ReplSet::haveNewConfig(ReplSetConfig& newConfig, bool addComment) { 
+        lock l(this); // convention is to lock replset before taking the db rwlock
+        writelock lk("");
+        bo comment;
+        if( addComment )
+            comment = BSON( "msg" << "Reconfig set" << "version" << newConfig.version );
+        newConfig.saveConfigLocally(comment);
+        try { 
+            initFromConfig(newConfig);
+            log() << "replSet replSetReconfig new config saved locally" << rsLog;
+        }
+        catch(DBException& e) { 
+            log() << "replSet error unexpected exception in haveNewConfig() : " << e.toString() << rsLog;
+            _fatal();
+        }
+        catch(...) { 
+            log() << "replSet error unexpected exception in haveNewConfig()" << rsLog;
+            _fatal();
+        }
+    }
+
+    void Manager::msgReceivedNewConfig(BSONObj o) {
+        log() << "replset msgReceivedNewConfig version: " << o["version"].toString() << rsLog;
+        ReplSetConfig c(o);
+        if( c.version > rs->config().version )
+            theReplSet->haveNewConfig(c, false);
+        else { 
+            log() << "replSet info msgReceivedNewConfig but version isn't higher " << 
+                c.version << ' ' << rs->config().version << rsLog;
+        }
+    }
+
+    /* forked as a thread during startup 
+       it can run quite a while looking for config.  but once found, 
+       a separate thread takes over as ReplSetImpl::Manager, and this thread
+       terminates.
+    */
+    void startReplSets(ReplSetCmdline *replSetCmdline) {
+        Client::initThread("startReplSets");
+        try { 
+            assert( theReplSet == 0 );
+            if( replSetCmdline == 0 ) {
+                assert(!replSet);
+                return;
+            }
+            (theReplSet = new ReplSet(*replSetCmdline))->go();
+        }
+        catch(std::exception& e) { 
+            log() << "replSet caught exception in startReplSets thread: " << e.what() << rsLog;
+            if( theReplSet ) 
+                theReplSet->fatal();
+        }
+        cc().shutdown();
+    }
+
+}
+
+namespace boost { 
+
+    void assertion_failed(char const * expr, char const * function, char const * file, long line)
+    {
+        mongo::log() << "boost assertion failure " << expr << ' ' << function << ' ' << file << ' ' << line << endl;
+    }
+
+}
diff --git a/db/repl/rs.h b/db/repl/rs.h
new file mode 100644
index 0000000..17a070c
--- /dev/null
+++ b/db/repl/rs.h
@@ -0,0 +1,415 @@
+// /db/repl/rs.h
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include "../../util/concurrency/list.h"
+#include "../../util/concurrency/value.h"
+#include "../../util/concurrency/msg.h"
+#include "../../util/hostandport.h"
+#include "../commands.h"
+#include "rs_exception.h"
+#include "rs_optime.h"
+#include "rs_member.h"
+#include "rs_config.h"
+
+namespace mongo {
+
+    struct HowToFixUp;
+    struct Target;
+    class DBClientConnection;
+    class ReplSetImpl;
+    class OplogReader;
+    extern bool replSet; // true if using repl sets
+    extern class ReplSet *theReplSet; // null until initialized
+    extern Tee *rsLog;
+
+    /* member of a replica set */
+    class Member : public List1<Member>::Base {
+    public:
+        Member(HostAndPort h, unsigned ord, const ReplSetConfig::MemberCfg *c, bool self);
+        string fullName() const { return h().toString(); }
+        const ReplSetConfig::MemberCfg& config() const { return *_config; }
+        const HeartbeatInfo& hbinfo() const { return _hbinfo; }
+        string lhb() { return _hbinfo.lastHeartbeatMsg; }
+        MemberState state() const { return _hbinfo.hbstate; }
+        const HostAndPort& h() const { return _h; }
+        unsigned id() const { return _hbinfo.id(); }
+        bool potentiallyHot() const { return _config->potentiallyHot(); } // not arbiter, not priority 0
+
+        void summarizeMember(stringstream& s) const;
+        friend class ReplSetImpl;
+    private:
+        const ReplSetConfig::MemberCfg *_config; /* todo: when this changes??? */
+        HostAndPort _h;
+        HeartbeatInfo _hbinfo;
+    };
+
+    class Manager : public task::Server {
+        ReplSetImpl *rs;
+        bool busyWithElectSelf;
+        int _primary;
+        const Member* findOtherPrimary();
+        void noteARemoteIsPrimary(const Member *);
+        virtual void starting();
+    public:
+        Manager(ReplSetImpl *rs);
+        ~Manager();
+        void msgReceivedNewConfig(BSONObj);
+        void msgCheckNewState();
+    };
+
+    struct Target;
+
+    class Consensus {
+        ReplSetImpl &rs;
+        struct LastYea { 
+            LastYea() : when(0), who(0xffffffff) { }
+            time_t when;
+            unsigned who;
+        };
+        Atomic<LastYea> ly;
+        unsigned yea(unsigned memberId); // throws VoteException
+        void electionFailed(unsigned meid);
+        void _electSelf();
+        bool weAreFreshest(bool& allUp, int& nTies);
+        bool sleptLast; // slept last elect() pass
+    public:
+        Consensus(ReplSetImpl *t) : rs(*t) { 
+            sleptLast = false;
+            steppedDown = 0;
+        }
+
+        /* if we've stepped down, this is when we are allowed to try to elect ourself again. 
+           todo: handle possible weirdnesses at clock skews etc.
+        */
+        time_t steppedDown;
+
+        int totalVotes() const;
+        bool aMajoritySeemsToBeUp() const;
+        void electSelf();
+        void electCmdReceived(BSONObj, BSONObjBuilder*);
+        void multiCommand(BSONObj cmd, list<Target>& L);
+    };
+
+    /** most operations on a ReplSet object should be done while locked. that logic implemented here. */
+    class RSBase : boost::noncopyable { 
+    public:
+        const unsigned magic;
+        void assertValid() { assert( magic == 0x12345677 ); }
+    private:
+        mutex m;
+        int _locked;
+        ThreadLocalValue<bool> _lockedByMe;
+    protected:
+        RSBase() : magic(0x12345677), m("RSBase"), _locked(0) { }
+        ~RSBase() { 
+            log() << "~RSBase should never be called?" << rsLog;
+            assert(false);
+        }
+
+        class lock { 
+            RSBase& rsbase;
+            auto_ptr<scoped_lock> sl;
+        public:
+            lock(RSBase* b) : rsbase(*b) { 
+                if( rsbase._lockedByMe.get() )
+                    return; // recursive is ok...
+
+                sl.reset( new scoped_lock(rsbase.m) );
+                DEV assert(rsbase._locked == 0);
+                rsbase._locked++; 
+                rsbase._lockedByMe.set(true);
+            }
+            ~lock() { 
+                if( sl.get() ) {
+                    assert( rsbase._lockedByMe.get() );
+                    DEV assert(rsbase._locked == 1);
+                    rsbase._lockedByMe.set(false);
+                    rsbase._locked--; 
+                }
+            }
+        };
+
+    public:
+        /* for asserts */
+        bool locked() const { return _locked != 0; }
+
+        /* if true, is locked, and was locked by this thread. note if false, it could be in the lock or not for another 
+           just for asserts & such so we can make the contracts clear on who locks what when.
+           we don't use these locks that frequently, so the little bit of overhead is fine.
+        */
+        bool lockedByMe() { return _lockedByMe.get(); } 
+    };
+
+    class ReplSetHealthPollTask;
+
+    /* safe container for our state that keeps member pointer and state variables always aligned */
+    class StateBox : boost::noncopyable {
+    public:
+        struct SP { // SP is like pair<MemberState,const Member *> but nicer
+            SP() : state(MemberState::RS_STARTUP), primary(0) { }
+            MemberState state;
+            const Member *primary;
+        };
+        const SP get() { 
+            scoped_lock lk(m);
+            return sp;
+        }
+        MemberState getState() const { return sp.state; }
+        const Member* getPrimary() const { return sp.primary; }
+        void change(MemberState s, const Member *self) { 
+            scoped_lock lk(m);
+            sp.state = s;
+            if( s.primary() ) { 
+                sp.primary = self;
+            }
+            else {
+                if( self == sp.primary )
+                    sp.primary = 0;
+            }
+        }
+        void set(MemberState s, const Member *p) { 
+            scoped_lock lk(m);
+            sp.state = s; sp.primary = p;
+        }
+        void setSelfPrimary(const Member *self) { change(MemberState::RS_PRIMARY, self); }
+        void setOtherPrimary(const Member *mem) { 
+            scoped_lock lk(m);
+            assert( !sp.state.primary() );
+            sp.primary = mem;
+        }
+        StateBox() : m("StateBox") { }
+    private:
+        mutex m;
+        SP sp;
+    };
+    
+    void parseReplsetCmdLine(string cfgString, string& setname, vector<HostAndPort>& seeds, set<HostAndPort>& seedSet );
+
+    /** Parameter given to the --replSet command line option (parsed).
+        Syntax is "<setname>/<seedhost1>,<seedhost2>"
+        where setname is a name and seedhost is "<host>[:<port>]" */
+    class ReplSetCmdline {
+    public:
+        ReplSetCmdline(string cfgString) { parseReplsetCmdLine(cfgString, setname, seeds, seedSet); }
+        string setname;
+        vector<HostAndPort> seeds;
+        set<HostAndPort> seedSet;
+    };
+
+    /* information about the entire repl set, such as the various servers in the set, and their state */
+    /* note: We currently do not free mem when the set goes away - it is assumed the replset is a 
+             singleton and long lived.
+    */
+    class ReplSetImpl : protected RSBase {
+    public:
+        /** info on our state if the replset isn't yet "up".  for example, if we are pre-initiation. */
+        enum StartupStatus { 
+            PRESTART=0, LOADINGCONFIG=1, BADCONFIG=2, EMPTYCONFIG=3, 
+            EMPTYUNREACHABLE=4, STARTED=5, SOON=6 
+        };
+        static StartupStatus startupStatus;
+        static string startupStatusMsg;
+        static string stateAsStr(MemberState state);
+        static string stateAsHtml(MemberState state);
+
+        /* todo thread */
+        void msgUpdateHBInfo(HeartbeatInfo);
+
+        StateBox box;
+
+        OpTime lastOpTimeWritten;
+        long long lastH; // hash we use to make sure we are reading the right flow of ops and aren't on an out-of-date "fork"
+    private:
+        set<ReplSetHealthPollTask*> healthTasks;
+        void endOldHealthTasks();
+        void startHealthTaskFor(Member *m);
+
+    private:
+        Consensus elect;
+        bool ok() const { return !box.getState().fatal(); }
+
+        void relinquish();
+        void forgetPrimary();
+
+    protected:
+        bool _stepDown();
+    private:
+        void assumePrimary();
+        void loadLastOpTimeWritten();
+        void changeState(MemberState s);
+
+    protected:
+        // "heartbeat message"
+        // sent in requestHeartbeat respond in field "hbm" 
+        char _hbmsg[256]; // we change this unlocked, thus not an stl::string
+    public:
+        void sethbmsg(string s, int logLevel = 0); 
+    protected:
+        bool initFromConfig(ReplSetConfig& c); // true if ok; throws if config really bad; false if config doesn't include self
+        void _fillIsMaster(BSONObjBuilder&);
+        void _fillIsMasterHost(const Member*, vector<string>&, vector<string>&, vector<string>&);
+        const ReplSetConfig& config() { return *_cfg; }
+        string name() const { return _name; } /* @return replica set's logical name */
+        MemberState state() const { return box.getState(); }
+        void _fatal();
+        void _getOplogDiagsAsHtml(unsigned server_id, stringstream& ss) const;
+        void _summarizeAsHtml(stringstream&) const;        
+        void _summarizeStatus(BSONObjBuilder&) const; // for replSetGetStatus command
+
+        /* throws exception if a problem initializing. */
+        ReplSetImpl(ReplSetCmdline&);
+
+        /* call afer constructing to start - returns fairly quickly after launching its threads */
+        void _go();
+
+    private:
+        string _name;
+        const vector<HostAndPort> *_seeds;
+        ReplSetConfig *_cfg;
+
+        /** load our configuration from admin.replset.  try seed machines too. 
+            @return true if ok; throws if config really bad; false if config doesn't include self
+        */
+        bool _loadConfigFinish(vector<ReplSetConfig>& v);
+        void loadConfig();
+
+        list<HostAndPort> memberHostnames() const;
+        const ReplSetConfig::MemberCfg& myConfig() const { return _self->config(); }
+        bool iAmArbiterOnly() const { return myConfig().arbiterOnly; }
+        bool iAmPotentiallyHot() const { return myConfig().potentiallyHot(); }
+    protected:
+        Member *_self;        
+    private:
+        List1<Member> _members; /* all members of the set EXCEPT self. */
+
+    public:
+        unsigned selfId() const { return _self->id(); }
+        Manager *mgr;
+
+    private:
+        Member* head() const { return _members.head(); }
+    public:
+        const Member* findById(unsigned id) const;
+    private:
+        void _getTargets(list<Target>&, int &configVersion);
+        void getTargets(list<Target>&, int &configVersion);
+        void startThreads();
+        friend class FeedbackThread;
+        friend class CmdReplSetElect;
+        friend class Member;
+        friend class Manager;
+        friend class Consensus;
+
+    private:
+        /* pulling data from primary related - see rs_sync.cpp */
+        bool initialSyncOplogApplication(string hn, const Member *primary, OpTime applyGTE, OpTime minValid);
+        void _syncDoInitialSync();
+        void syncDoInitialSync();
+        void _syncThread();
+        void syncTail();
+        void syncApply(const BSONObj &o);
+        void syncRollback(OplogReader& r);
+        void syncFixUp(HowToFixUp& h, OplogReader& r);
+    public:
+        void syncThread();
+    };
+
+    class ReplSet : public ReplSetImpl { 
+    public:
+        ReplSet(ReplSetCmdline& replSetCmdline) : ReplSetImpl(replSetCmdline) {  }
+
+        bool stepDown() { return _stepDown(); }
+
+        string selfFullName() { 
+            lock lk(this);
+            return _self->fullName();
+        }
+
+        /* call after constructing to start - returns fairly quickly after la[unching its threads */
+        void go() { _go(); }
+        void fatal() { _fatal(); }
+        bool isPrimary();
+        bool isSecondary();
+        MemberState state() const { return ReplSetImpl::state(); }
+        string name() const { return ReplSetImpl::name(); }
+        const ReplSetConfig& config() { return ReplSetImpl::config(); }
+        void getOplogDiagsAsHtml(unsigned server_id, stringstream& ss) const { _getOplogDiagsAsHtml(server_id,ss); }
+        void summarizeAsHtml(stringstream& ss) const { _summarizeAsHtml(ss); }
+        void summarizeStatus(BSONObjBuilder& b) const  { _summarizeStatus(b); }
+        void fillIsMaster(BSONObjBuilder& b) { _fillIsMaster(b); }
+
+        /* we have a new config (reconfig) - apply it. 
+           @param comment write a no-op comment to the oplog about it.  only makes sense if one is primary and initiating the reconf.
+        */
+        void haveNewConfig(ReplSetConfig& c, bool comment);
+
+        /* if we delete old configs, this needs to assure locking. currently we don't so it is ok. */
+        const ReplSetConfig& getConfig() { return config(); }
+
+        bool lockedByMe() { return RSBase::lockedByMe(); }
+
+        // heartbeat msg to send to others; descriptive diagnostic info
+        string hbmsg() const { return _hbmsg; }
+    };
+
+    /** base class for repl set commands.  checks basic things such as in rs mode before the command 
+        does its real work
+        */
+    class ReplSetCommand : public Command { 
+    protected:
+        ReplSetCommand(const char * s, bool show=false) : Command(s, show) { }
+        virtual bool slaveOk() const { return true; }
+        virtual bool adminOnly() const { return true; }
+        virtual bool logTheOp() { return false; }
+        virtual LockType locktype() const { return NONE; }
+        virtual void help( stringstream &help ) const { help << "internal"; }
+        bool check(string& errmsg, BSONObjBuilder& result) {
+            if( !replSet ) { 
+                errmsg = "not running with --replSet";
+                return false;
+            }
+            if( theReplSet == 0 ) {
+                result.append("startupStatus", ReplSet::startupStatus);
+                errmsg = ReplSet::startupStatusMsg.empty() ? "replset unknown error 2" : ReplSet::startupStatusMsg;
+                return false;
+            }
+            return true;
+        }
+    };
+
+    /** inlines ----------------- */
+
+    inline Member::Member(HostAndPort h, unsigned ord, const ReplSetConfig::MemberCfg *c, bool self) : 
+        _config(c), _h(h), _hbinfo(ord) { 
+            if( self ) { 
+                _hbinfo.health = 1.0;
+            }
+    }
+
+    inline bool ReplSet::isPrimary() {         
+        /* todo replset */
+        return box.getState().primary();
+    }
+
+    inline bool ReplSet::isSecondary() { 
+      return box.getState().secondary();
+    }
+
+}
diff --git a/db/repl/rs_config.cpp b/db/repl/rs_config.cpp
new file mode 100644
index 0000000..76b20a4
--- /dev/null
+++ b/db/repl/rs_config.cpp
@@ -0,0 +1,315 @@
+// rs_config.cpp
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "rs.h"
+#include "../../client/dbclient.h"
+#include "../../client/syncclusterconnection.h"
+#include "../../util/hostandport.h"
+#include "../dbhelpers.h"
+#include "connections.h"
+#include "../oplog.h"
+
+using namespace bson;
+
+namespace mongo { 
+
+    void logOpInitiate(const bo&);
+
+    list<HostAndPort> ReplSetConfig::otherMemberHostnames() const { 
+        list<HostAndPort> L;
+        for( vector<MemberCfg>::const_iterator i = members.begin(); i != members.end(); i++ ) {
+            if( !i->h.isSelf() )
+                L.push_back(i->h);
+        }
+        return L;
+    }
+    
+    /* comment MUST only be set when initiating the set by the initiator */
+    void ReplSetConfig::saveConfigLocally(bo comment) { 
+        check();
+        log() << "replSet info saving a newer config version to local.system.replset" << rsLog;
+        { 
+            writelock lk("");
+            Client::Context cx( rsConfigNs );
+            cx.db()->flushFiles(true);
+
+            //theReplSet->lastOpTimeWritten = ??;
+            //rather than above, do a logOp()? probably
+            BSONObj o = asBson();
+            Helpers::putSingletonGod(rsConfigNs.c_str(), o, false/*logOp=false; local db so would work regardless...*/);
+            if( !comment.isEmpty() )
+                logOpInitiate(comment);
+
+            cx.db()->flushFiles(true);
+        }
+        DEV log() << "replSet saveConfigLocally done" << rsLog;
+    }
+    
+    /*static*/ 
+    /*void ReplSetConfig::receivedNewConfig(BSONObj cfg) { 
+        if( theReplSet )
+            return; // this is for initial setup only, so far. todo
+
+        ReplSetConfig c(cfg);
+
+        writelock lk("admin.");
+        if( theReplSet ) 
+            return;
+        c.saveConfigLocally(bo());
+    }*/
+
+    bo ReplSetConfig::MemberCfg::asBson() const { 
+        bob b;
+        b << "_id" << _id;
+        b.append("host", h.toString());
+        if( votes != 1 ) b << "votes" << votes;
+        if( priority != 1.0 ) b << "priority" << priority;
+        if( arbiterOnly ) b << "arbiterOnly" << true;
+        return b.obj();
+    }
+
+    bo ReplSetConfig::asBson() const { 
+        bob b;
+        b.append("_id", _id).append("version", version);
+        if( !ho.isDefault() || !getLastErrorDefaults.isEmpty() ) {
+            bob settings;
+            if( !ho.isDefault() )
+                settings << "heartbeatConnRetries " << ho.heartbeatConnRetries  << 
+                             "heartbeatSleep" << ho.heartbeatSleepMillis / 1000 << 
+                             "heartbeatTimeout" << ho.heartbeatTimeoutMillis / 1000;
+            if( !getLastErrorDefaults.isEmpty() )
+                settings << "getLastErrorDefaults" << getLastErrorDefaults;
+            b << "settings" << settings.obj();
+        }
+
+        BSONArrayBuilder a;
+        for( unsigned i = 0; i < members.size(); i++ )
+            a.append( members[i].asBson() );
+        b.append("members", a.arr());
+
+        return b.obj();
+    }
+
+    static inline void mchk(bool expr) {
+        uassert(13126, "bad Member config", expr);
+    }
+
+    void ReplSetConfig::MemberCfg::check() const{ 
+        mchk(_id >= 0 && _id <= 255);
+        mchk(priority >= 0 && priority <= 1000);
+        mchk(votes >= 0 && votes <= 100);
+        uassert(13419, "this version of mongod only supports priorities 0 and 1", priority == 0 || priority == 1);
+    }
+
+    /*static*/ bool ReplSetConfig::legalChange(const ReplSetConfig& o, const ReplSetConfig& n, string& errmsg) { 
+        if( o._id != n._id ) { 
+            errmsg = "set name may not change"; 
+            return false;
+        }
+        /* TODO : wonder if we need to allow o.version < n.version only, which is more lenient.
+                  if someone had some intermediate config this node doesnt have, that could be 
+                  necessary.  but then how did we become primary?  so perhaps we are fine as-is.
+                  */
+        if( o.version + 1 != n.version ) { 
+            errmsg = "version number wrong";
+            return false;
+        }
+
+        /* TODO : MORE CHECKS HERE */
+
+        log() << "replSet TODO : don't allow removal of a node until we handle it at the removed node end?" << endl;
+        // we could change its votes to zero perhaps instead as a short term...
+
+        return true;
+    }
+
+    void ReplSetConfig::clear() { 
+        version = -5;
+        _ok = false;
+    }
+
+    void ReplSetConfig::check() const { 
+        uassert(13132,
+            "nonmatching repl set name in _id field; check --replSet command line",
+            _id == cmdLine.ourSetName());
+        uassert(13308, "replSet bad config version #", version > 0);
+        uassert(13133, "replSet bad config no members", members.size() >= 1);
+        uassert(13309, "replSet bad config maximum number of members is 7 (for now)", members.size() <= 7);
+    }
+
+    void ReplSetConfig::from(BSONObj o) {
+        md5 = o.md5();
+        _id = o["_id"].String();
+        if( o["version"].ok() ) {
+            version = o["version"].numberInt();
+            uassert(13115, "bad " + rsConfigNs + " config: version", version > 0);
+        }
+
+        if( o["settings"].ok() ) {
+            BSONObj settings = o["settings"].Obj();
+            if( settings["heartbeatConnRetries "].ok() )
+                ho.heartbeatConnRetries  = settings["heartbeatConnRetries "].numberInt();
+            if( settings["heartbeatSleep"].ok() )
+                ho.heartbeatSleepMillis = (unsigned) (settings["heartbeatSleep"].Number() * 1000);
+            if( settings["heartbeatTimeout"].ok() )
+                ho.heartbeatTimeoutMillis = (unsigned) (settings["heartbeatTimeout"].Number() * 1000);
+            ho.check();
+            try { getLastErrorDefaults = settings["getLastErrorDefaults"].Obj(); } catch(...) { }
+        }
+
+        set<string> hosts;
+        set<int> ords;
+        vector<BSONElement> members;
+        try {
+            members = o["members"].Array();
+        }
+        catch(...) {
+            uasserted(13131, "replSet error parsing (or missing) 'members' field in config object");
+        }
+
+        unsigned localhosts = 0;
+        for( unsigned i = 0; i < members.size(); i++ ) {
+            BSONObj mobj = members[i].Obj();
+            MemberCfg m;
+            try {
+                try { 
+                    m._id = (int) mobj["_id"].Number();
+                } catch(...) { 
+                    /* TODO: use of string exceptions may be problematic for reconfig case! */
+                    throw "_id must be numeric"; 
+                }
+                string s;
+                try {
+                    s = mobj["host"].String();
+                    m.h = HostAndPort(s);
+                }
+                catch(...) { 
+                    throw string("bad or missing host field? ") + mobj.toString();
+                }
+                if( m.h.isLocalHost() ) 
+                    localhosts++;
+                m.arbiterOnly = mobj.getBoolField("arbiterOnly");
+                if( mobj.hasElement("priority") )
+                    m.priority = mobj["priority"].Number();
+                if( mobj.hasElement("votes") )
+                    m.votes = (unsigned) mobj["votes"].Number();
+                m.check();
+            }
+            catch( const char * p ) { 
+                log() << "replSet cfg parsing exception for members[" << i << "] " << p << rsLog;
+                stringstream ss;
+                ss << "replSet members[" << i << "] " << p;
+                uassert(13107, ss.str(), false);
+            }
+            catch(DBException& e) { 
+                log() << "replSet cfg parsing exception for members[" << i << "] " << e.what() << rsLog;
+                stringstream ss;
+                ss << "bad config for member[" << i << "] " << e.what();
+                uassert(13135, ss.str(), false);
+            }
+            if( !(ords.count(m._id) == 0 && hosts.count(m.h.toString()) == 0) ) {
+                log() << "replSet " << o.toString() << rsLog;
+                uassert(13108, "bad replset config -- duplicate hosts in the config object?", false);
+            }
+            hosts.insert(m.h.toString());
+            ords.insert(m._id);
+            this->members.push_back(m);
+        }
+        uassert(13393, "can't use localhost in repl set member names except when using it for all members", localhosts == 0 || localhosts == members.size());
+        uassert(13117, "bad " + rsConfigNs + " config", !_id.empty());
+    }
+
+    static inline void configAssert(bool expr) {
+        uassert(13122, "bad repl set config?", expr);
+    }
+
+    ReplSetConfig::ReplSetConfig(BSONObj cfg) { 
+        clear();
+        from(cfg);
+        configAssert( version < 0 /*unspecified*/ || (version >= 1 && version <= 5000) );
+        if( version < 1 )
+            version = 1;
+        _ok = true;
+    }
+
+    ReplSetConfig::ReplSetConfig(const HostAndPort& h) {
+        clear();
+        int level = 2;
+        DEV level = 0;
+        //log(0) << "replSet load config from: " << h.toString() << rsLog;
+
+        auto_ptr<DBClientCursor> c;
+        int v = -5;
+        try {
+            if( h.isSelf() ) {
+                ;
+            }
+            else {
+                /* first, make sure other node is configured to be a replset. just to be safe. */
+                string setname = cmdLine.ourSetName();
+                BSONObj cmd = BSON( "replSetHeartbeat" << setname );
+                int theirVersion;
+                BSONObj info;
+                bool ok = requestHeartbeat(setname, "", h.toString(), info, -2, theirVersion);
+                if( info["rs"].trueValue() ) { 
+                    // yes, it is a replicate set, although perhaps not yet initialized
+                }
+                else {
+                    if( !ok ) {
+                        log() << "replSet TEMP !ok heartbeating " << h.toString() << " on cfg load" << rsLog;
+                        if( !info.isEmpty() ) 
+                            log() << "replSet info " << h.toString() << " : " << info.toString() << rsLog;
+                        return;
+                    }
+                    { 
+                        stringstream ss;
+                        ss << "replSet error: member " << h.toString() << " is not in --replSet mode";
+                        msgassertedNoTrace(13260, ss.str().c_str()); // not caught as not a user exception - we want it not caught
+                        //for python err# checker: uassert(13260, "", false);
+                    }
+                }
+            }
+
+            v = -4;
+            ScopedConn conn(h.toString());
+            v = -3;
+            c = conn->query(rsConfigNs);
+            if( c.get() == 0 ) {
+                version = v; return;
+            }
+            if( !c->more() ) {
+                version = EMPTYCONFIG;
+                return;
+            }
+            version = -1;
+        }
+        catch( DBException& e) { 
+            version = v;
+            log(level) << "replSet load config couldn't get from " << h.toString() << ' ' << e.what() << rsLog;
+            return;
+        }
+
+        BSONObj o = c->nextSafe();
+        uassert(13109, "multiple rows in " + rsConfigNs + " not supported", !c->more());
+        from(o);
+        _ok = true;
+        log(level) << "replSet load config ok from " << (h.isSelf() ? "self" : h.toString()) << rsLog;
+    }
+
+}
diff --git a/db/repl/rs_config.h b/db/repl/rs_config.h
new file mode 100644
index 0000000..38df772
--- /dev/null
+++ b/db/repl/rs_config.h
@@ -0,0 +1,88 @@
+// rs_config.h
+// repl set configuration
+//
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include "../../util/hostandport.h"
+#include "health.h"
+
+namespace mongo { 
+
+    /* singleton config object is stored here */
+    const string rsConfigNs = "local.system.replset";
+
+    class ReplSetConfig {
+        enum { EMPTYCONFIG = -2 };
+    public:
+        /* if something is misconfigured, throws an exception. 
+        if couldn't be queried or is just blank, ok() will be false.
+        */
+        ReplSetConfig(const HostAndPort& h);
+
+        ReplSetConfig(BSONObj cfg);
+
+        bool ok() const { return _ok; }
+
+        struct MemberCfg {
+            MemberCfg() : _id(-1), votes(1), priority(1.0), arbiterOnly(false) { }
+            int _id;              /* ordinal */
+            unsigned votes;       /* how many votes this node gets. default 1. */
+            HostAndPort h;
+            double priority;      /* 0 means can never be primary */
+            bool arbiterOnly;
+            void check() const;   /* check validity, assert if not. */
+            BSONObj asBson() const;
+            bool potentiallyHot() const { 
+                return !arbiterOnly && priority > 0;
+            }
+        };
+        vector<MemberCfg> members;
+        string _id;
+        int version;
+        HealthOptions ho;
+        string md5;
+        BSONObj getLastErrorDefaults;
+
+        list<HostAndPort> otherMemberHostnames() const; // except self
+
+        /** @return true if could connect, and there is no cfg object there at all */
+        bool empty() const { return version == EMPTYCONFIG; }
+
+        string toString() const { return asBson().toString(); }
+
+        /** validate the settings. does not call check() on each member, you have to do that separately. */
+        void check() const;
+
+        /** check if modification makes sense */
+        static bool legalChange(const ReplSetConfig& old, const ReplSetConfig& n, string& errmsg);
+
+        //static void receivedNewConfig(BSONObj);
+        void saveConfigLocally(BSONObj comment); // to local db
+        string saveConfigEverywhere(); // returns textual info on what happened
+
+        BSONObj asBson() const;
+
+    private:
+        bool _ok;
+        void from(BSONObj);
+        void clear();
+    };
+
+}
diff --git a/db/repl/rs_exception.h b/db/repl/rs_exception.h
new file mode 100755
index 0000000..e71cad2
--- /dev/null
+++ b/db/repl/rs_exception.h
@@ -0,0 +1,17 @@
+// @file rs_exception.h

+

+#pragma once

+

+namespace mongo { 

+

+    class VoteException : public std::exception { 
+    public:
+        const char * what() const throw () { return "VoteException"; }

+    };
+
+    class RetryAfterSleepException : public std::exception { 
+    public:
+        const char * what() const throw () { return "RetryAfterSleepException"; }
+    };
+
+}
diff --git a/db/repl/rs_initialsync.cpp b/db/repl/rs_initialsync.cpp
new file mode 100644
index 0000000..4c6bd4d
--- /dev/null
+++ b/db/repl/rs_initialsync.cpp
@@ -0,0 +1,214 @@
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "../client.h"
+#include "../../client/dbclient.h"
+#include "rs.h"
+#include "../oplogreader.h"
+#include "../../util/mongoutils/str.h"
+#include "../dbhelpers.h"
+#include "rs_optime.h"
+#include "../oplog.h"
+
+namespace mongo {
+
+    using namespace mongoutils;
+    using namespace bson;
+
+    void dropAllDatabasesExceptLocal();
+
+    // add try/catch with sleep
+
+    void isyncassert(const char *msg, bool expr) { 
+        if( !expr ) { 
+            string m = str::stream() << "initial sync " << msg;
+            theReplSet->sethbmsg(m, 0);
+            uasserted(13404, m);
+        }
+    }
+
+    void ReplSetImpl::syncDoInitialSync() { 
+        while( 1 ) {
+            try {
+                _syncDoInitialSync();
+                break;
+            }
+            catch(DBException& e) {
+                sethbmsg("initial sync exception " + e.toString(), 0);
+                sleepsecs(30);
+            }
+        }
+    }
+
+    bool cloneFrom(const char *masterHost, string& errmsg, const string& fromdb, bool logForReplication, 
+				   bool slaveOk, bool useReplAuth, bool snapshot);
+
+    /* todo : progress metering to sethbmsg. */
+    static bool clone(const char *master, string db) {
+        string err;
+        return cloneFrom(master, err, db, false, 
+            /*slaveok later can be true*/ false, true, false);
+    }
+
+    void _logOpObjRS(const BSONObj& op);
+
+    bool copyCollectionFromRemote(const string& host, const string& ns, const BSONObj& query, string errmsg);
+
+    static void emptyOplog() {
+        writelock lk(rsoplog);
+        Client::Context ctx(rsoplog);
+		NamespaceDetails *d = nsdetails(rsoplog);
+
+		// temp
+		if( d && d->nrecords == 0 )
+		  return; // already empty, ok.
+
+        log(1) << "replSet empty oplog" << rsLog;
+        d->emptyCappedCollection(rsoplog);
+
+        /*
+        string errmsg;
+        bob res;
+        dropCollection(rsoplog, errmsg, res);
+		log() << "replSet recreated oplog so it is empty.  todo optimize this..." << rsLog;
+		createOplog();*/
+
+      	// TEMP: restart to recreate empty oplog
+        //log() << "replSet FATAL error during initial sync.  mongod restart required." << rsLog;
+        //dbexit( EXIT_CLEAN );
+
+        /*
+        writelock lk(rsoplog);
+        Client::Context c(rsoplog, dbpath, 0, doauth/false);
+        NamespaceDetails *oplogDetails = nsdetails(rsoplog);
+        uassert(13412, str::stream() << "replSet error " << rsoplog << " is missing", oplogDetails != 0);
+        oplogDetails->cappedTruncateAfter(rsoplog, h.commonPointOurDiskloc, false);
+        */
+    }
+
+    void ReplSetImpl::_syncDoInitialSync() { 
+        sethbmsg("initial sync pending",0);
+
+        StateBox::SP sp = box.get();
+        assert( !sp.state.primary() ); // wouldn't make sense if we were.
+
+        const Member *cp = sp.primary;
+        if( cp == 0 ) {
+            sethbmsg("initial sync need a member to be primary",0);
+            sleepsecs(15);
+            return;
+        }
+
+        string masterHostname = cp->h().toString();
+        OplogReader r;
+        if( !r.connect(masterHostname) ) {
+            sethbmsg( str::stream() << "initial sync couldn't connect to " << cp->h().toString() , 0);
+            sleepsecs(15);
+            return;
+        }
+
+        BSONObj lastOp = r.getLastOp(rsoplog);
+        if( lastOp.isEmpty() ) { 
+            sethbmsg("initial sync couldn't read remote oplog", 0);
+            sleepsecs(15);
+            return;
+        }
+        OpTime startingTS = lastOp["ts"]._opTime();
+        
+        {
+            /* make sure things aren't too flappy */
+            sleepsecs(5);
+            isyncassert( "flapping?", box.getPrimary() == cp );
+            BSONObj o = r.getLastOp(rsoplog);
+            isyncassert( "flapping [2]?", !o.isEmpty() );
+        }
+
+        sethbmsg("initial sync drop all databases", 0);
+        dropAllDatabasesExceptLocal();
+
+//        sethbmsg("initial sync drop oplog", 0);
+//        emptyOplog();
+
+        list<string> dbs = r.conn()->getDatabaseNames();
+        for( list<string>::iterator i = dbs.begin(); i != dbs.end(); i++ ) {
+            string db = *i;
+            if( db != "local" ) {
+                sethbmsg( str::stream() << "initial sync cloning db: " << db , 0);
+                bool ok;
+                {
+                    writelock lk(db);
+                    Client::Context ctx(db);
+                    ok = clone(masterHostname.c_str(), db);
+                }
+                if( !ok ) { 
+                    sethbmsg( str::stream() << "initial sync error clone of " << db << " failed sleeping 5 minutes" ,0);
+                    sleepsecs(300);
+                    return;
+                }
+            }
+        }
+
+        sethbmsg("initial sync query minValid",0);
+
+        /* our cloned copy will be strange until we apply oplog events that occurred 
+           through the process.  we note that time point here. */
+        BSONObj minValid = r.getLastOp(rsoplog);
+        assert( !minValid.isEmpty() );
+        OpTime mvoptime = minValid["ts"]._opTime();
+        assert( !mvoptime.isNull() );
+
+        /* copy the oplog 
+        */
+        {
+            sethbmsg("initial sync copy+apply oplog");
+            if( ! initialSyncOplogApplication(masterHostname, cp, startingTS, mvoptime) ) { // note we assume here that this call does not throw
+                log() << "replSet initial sync failed during applyoplog" << rsLog;
+                emptyOplog(); // otherwise we'll be up!
+				lastOpTimeWritten = OpTime();
+				lastH = 0;
+                log() << "replSet cleaning up [1]" << rsLog;
+                {
+                    writelock lk("local.");
+                    Client::Context cx( "local." );
+                    cx.db()->flushFiles(true);            
+                }
+                log() << "replSet cleaning up [2]" << rsLog;
+                sleepsecs(2);
+                return;
+            }
+        }
+
+        sethbmsg("initial sync finishing up",0);
+        
+        assert( !box.getState().primary() ); // wouldn't make sense if we were.
+
+        {
+            writelock lk("local.");
+            Client::Context cx( "local." );
+            cx.db()->flushFiles(true);            
+            try {
+                log() << "replSet set minValid=" << minValid["ts"]._opTime().toString() << rsLog;
+            }
+            catch(...) { }
+            Helpers::putSingleton("local.replset.minvalid", minValid);
+            cx.db()->flushFiles(true);
+        }
+
+        sethbmsg("initial sync done",0);
+    }
+
+}
diff --git a/db/repl/rs_initiate.cpp b/db/repl/rs_initiate.cpp
new file mode 100644
index 0000000..9c74be0
--- /dev/null
+++ b/db/repl/rs_initiate.cpp
@@ -0,0 +1,238 @@
+/* @file rs_initiate.cpp
+   */
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "../cmdline.h"
+#include "../commands.h"
+#include "../../util/mmap.h"
+#include "../../util/mongoutils/str.h"
+#include "health.h"
+#include "rs.h"
+#include "rs_config.h"
+#include "../dbhelpers.h"
+
+using namespace bson;
+using namespace mongoutils;
+
+namespace mongo { 
+
+    /* called on a reconfig AND on initiate
+       throws 
+       @param initial true when initiating
+    */
+    void checkMembersUpForConfigChange(const ReplSetConfig& cfg, bool initial) {
+        int failures = 0;
+        int me = 0;
+        for( vector<ReplSetConfig::MemberCfg>::const_iterator i = cfg.members.begin(); i != cfg.members.end(); i++ ) {
+            if( i->h.isSelf() ) {
+                me++;
+                if( !i->potentiallyHot() ) { 
+                    uasserted(13420, "initiation and reconfiguration of a replica set must be sent to a node that can become primary");
+                }
+            }
+        }
+        uassert(13278, "bad config - dups?", me <= 1); // dups?
+        uassert(13279, "can't find self in the replset config", me == 1);
+
+        for( vector<ReplSetConfig::MemberCfg>::const_iterator i = cfg.members.begin(); i != cfg.members.end(); i++ ) {
+            BSONObj res;
+            {
+                bool ok = false;
+                try {
+                    int theirVersion = -1000;
+                    ok = requestHeartbeat(cfg._id, "", i->h.toString(), res, -1, theirVersion, initial/*check if empty*/); 
+                    if( theirVersion >= cfg.version ) { 
+                        stringstream ss;
+                        ss << "replSet member " << i->h.toString() << " has too new a config version (" << theirVersion << ") to reconfigure";
+                        uasserted(13259, ss.str());
+                    }
+                }
+                catch(DBException& e) { 
+                    log() << "replSet cmufcc requestHeartbeat " << i->h.toString() << " : " << e.toString() << rsLog;
+                }
+                catch(...) { 
+                    log() << "replSet cmufcc error exception in requestHeartbeat?" << rsLog;
+                }
+                if( res.getBoolField("mismatch") )
+                    uasserted(13145, "set name does not match the set name host " + i->h.toString() + " expects");
+                if( *res.getStringField("set") ) {
+                    if( cfg.version <= 1 ) {
+                        // this was to be initiation, no one shoudl be initiated already.
+                        uasserted(13256, "member " + i->h.toString() + " is already initiated");
+                    }
+                    else {
+                        // Assure no one has a newer config.
+                        if( res["v"].Int() >= cfg.version ) {
+                            uasserted(13341, "member " + i->h.toString() + " has a config version >= to the new cfg version; cannot change config");
+                        }
+                    }
+                }
+                if( !ok && !res["rs"].trueValue() ) {
+                    if( !res.isEmpty() ) {
+                        /* strange.  got a response, but not "ok". log it. */
+                        log() << "replSet warning " << i->h.toString() << " replied: " << res.toString() << rsLog;
+                    }
+
+                    bool allowFailure = false;
+                    failures++;
+                    if( res.isEmpty() && !initial && failures == 1 ) {
+                        /* for now we are only allowing 1 node to be down on a reconfig.  this can be made to be a minority
+                           trying to keep change small as release is near.
+                           */
+                        const Member* m = theReplSet->findById( i->_id );
+                        if( m ) { 
+                            // ok, so this was an existing member (wouldn't make sense to add to config a new member that is down)
+                            assert( m->h().toString() == i->h.toString() );
+                            allowFailure = true;
+                        }
+                    }
+
+                    if( !allowFailure ) {
+                        string msg = string("need members up to initiate, not ok : ") + i->h.toString();
+                        if( !initial )
+                            msg = string("need most members up to reconfigure, not ok : ") + i->h.toString();
+                        uasserted(13144, msg);
+                    }
+                }
+            }
+            if( initial ) {
+                bool hasData = res["hasData"].Bool();
+                uassert(13311, "member " + i->h.toString() + " has data already, cannot initiate set.  All members except initiator must be empty.", 
+                    !hasData || i->h.isSelf());
+            }
+        }
+    }
+
+    class CmdReplSetInitiate : public ReplSetCommand { 
+    public:
+        virtual LockType locktype() const { return NONE; }
+        CmdReplSetInitiate() : ReplSetCommand("replSetInitiate") { }
+        virtual void help(stringstream& h) const { 
+            h << "Initiate/christen a replica set."; 
+            h << "\nhttp://www.mongodb.org/display/DOCS/Replica+Set+Commands";
+        }
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+            log() << "replSet replSetInitiate admin command received from client" << rsLog;
+
+            if( !replSet ) { 
+                errmsg = "server is not running with --replSet";
+                return false;
+            }
+            if( theReplSet ) {
+                errmsg = "already initialized";
+                result.append("info", "try querying " + rsConfigNs + " to see current configuration");
+                return false;
+            }
+
+            {
+                // just make sure we can get a write lock before doing anything else.  we'll reacquire one 
+                // later.  of course it could be stuck then, but this check lowers the risk if weird things 
+                // are up.
+                time_t t = time(0);
+                writelock lk("");
+                if( time(0)-t > 10 ) { 
+                    errmsg = "took a long time to get write lock, so not initiating.  Initiate when server less busy?";
+                    return false;
+                }
+
+                /* check that we don't already have an oplog.  that could cause issues.
+                   it is ok if the initiating member has *other* data than that.
+                   */
+                BSONObj o;
+                if( Helpers::getFirst(rsoplog, o) ) { 
+                    errmsg = rsoplog + string(" is not empty on the initiating member.  cannot initiate.");
+                    return false;
+                }
+            }
+
+            if( ReplSet::startupStatus == ReplSet::BADCONFIG ) {
+                errmsg = "server already in BADCONFIG state (check logs); not initiating";
+                result.append("info", ReplSet::startupStatusMsg);
+                return false;
+            }
+            if( ReplSet::startupStatus != ReplSet::EMPTYCONFIG ) {
+                result.append("startupStatus", ReplSet::startupStatus);
+                errmsg = "all members and seeds must be reachable to initiate set";
+                result.append("info", cmdLine._replSet);
+                return false;
+            }
+
+            BSONObj configObj;
+
+            if( cmdObj["replSetInitiate"].type() != Object ) {
+                result.append("info2", "no configuration explicitly specified -- making one");
+                log() << "replSet info initiate : no configuration specified.  Using a default configuration for the set" << rsLog;
+
+                string name;
+                vector<HostAndPort> seeds;
+                set<HostAndPort> seedSet;
+                parseReplsetCmdLine(cmdLine._replSet, name, seeds, seedSet); // may throw...
+
+                bob b;
+                b.append("_id", name);
+                bob members;
+                members.append("0", BSON( "_id" << 0 << "host" << HostAndPort::Me().toString() ));
+                for( unsigned i = 0; i < seeds.size(); i++ )
+                    members.append(bob::numStr(i+1), BSON( "_id" << i+1 << "host" << seeds[i].toString()));
+                b.appendArray("members", members.obj());
+                configObj = b.obj();
+                log() << "replSet created this configuration for initiation : " << configObj.toString() << rsLog;
+            }
+            else { 
+                configObj = cmdObj["replSetInitiate"].Obj();
+            }
+
+            bool parsed = false;
+            try {
+                ReplSetConfig newConfig(configObj);
+                parsed = true;
+
+                if( newConfig.version > 1 ) { 
+                    errmsg = "can't initiate with a version number greater than 1";
+                    return false;
+                }
+
+                log() << "replSet replSetInitiate config object parses ok, " << newConfig.members.size() << " members specified" << rsLog;
+
+                checkMembersUpForConfigChange(newConfig, true);
+
+                log() << "replSet replSetInitiate all members seem up" << rsLog;
+
+                writelock lk("");
+                bo comment = BSON( "msg" << "initiating set");
+                newConfig.saveConfigLocally(comment);
+                log() << "replSet replSetInitiate config now saved locally.  Should come online in about a minute." << rsLog;
+                result.append("info", "Config now saved locally.  Should come online in about a minute.");
+                ReplSet::startupStatus = ReplSet::SOON;
+                ReplSet::startupStatusMsg = "Received replSetInitiate - should come online shortly.";
+            }
+            catch( DBException& e ) { 
+                log() << "replSet replSetInitiate exception: " << e.what() << rsLog;
+                if( !parsed ) 
+                    errmsg = string("couldn't parse cfg object ") + e.what();
+                else
+                    errmsg = string("couldn't initiate : ") + e.what();
+                return false;
+            }
+
+            return true;
+        }
+    } cmdReplSetInitiate;
+
+}
diff --git a/db/repl/rs_member.h b/db/repl/rs_member.h
new file mode 100644
index 0000000..4f6846a
--- /dev/null
+++ b/db/repl/rs_member.h
@@ -0,0 +1,91 @@
+// @file rsmember.h
+/*
+ *    Copyright (C) 2010 10gen Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/** replica set member */
+
+#pragma once
+
+namespace mongo {
+
+
+    /*
+        RS_STARTUP    serving still starting up, or still trying to initiate the set
+        RS_PRIMARY    this server thinks it is primary
+        RS_SECONDARY  this server thinks it is a secondary (slave mode)
+        RS_RECOVERING recovering/resyncing; after recovery usually auto-transitions to secondary
+        RS_FATAL      something bad has occurred and server is not completely offline with regard to the replica set.  fatal error.
+        RS_STARTUP2   loaded config, still determining who is primary
+    */
+    struct MemberState { 
+        enum MS { 
+            RS_STARTUP,
+            RS_PRIMARY,
+            RS_SECONDARY,
+            RS_RECOVERING,
+            RS_FATAL,
+            RS_STARTUP2,
+            RS_UNKNOWN, /* remote node not yet reached */
+            RS_ARBITER,
+            RS_DOWN /* node not reachable for a report */
+        } s;
+
+        MemberState(MS ms = RS_UNKNOWN) : s(ms) { }
+        explicit MemberState(int ms) : s((MS) ms) { }
+
+        bool primary() const { return s == RS_PRIMARY; }
+        bool secondary() const { return s == RS_SECONDARY; }
+        bool recovering() const { return s == RS_RECOVERING; }
+        bool startup2() const { return s == RS_STARTUP2; }
+        bool fatal() const { return s == RS_FATAL; }
+
+        bool operator==(const MemberState& r) const { return s == r.s; }
+        bool operator!=(const MemberState& r) const { return s != r.s; }
+    };
+
+    /* this is supposed to be just basic information on a member, 
+       and copy constructable. */
+    class HeartbeatInfo { 
+        unsigned _id;
+    public:
+        HeartbeatInfo() : _id(0xffffffff),skew(INT_MIN) { }
+        HeartbeatInfo(unsigned id);
+        bool up() const { return health > 0; }
+        unsigned id() const { return _id; }
+        MemberState hbstate;
+        double health;
+        time_t upSince;
+        time_t lastHeartbeat;
+        string lastHeartbeatMsg;
+        OpTime opTime;
+        int skew;
+
+        /* true if changed in a way of interest to the repl set manager. */
+        bool changed(const HeartbeatInfo& old) const;
+    };
+
+    inline HeartbeatInfo::HeartbeatInfo(unsigned id) : _id(id) { 
+          health = -1.0;
+          lastHeartbeat = upSince = 0; 
+          skew = INT_MIN;
+    }
+
+    inline bool HeartbeatInfo::changed(const HeartbeatInfo& old) const { 
+        return health != old.health ||
+               hbstate != old.hbstate;
+    }
+
+}
diff --git a/db/repl/rs_optime.h b/db/repl/rs_optime.h
new file mode 100644
index 0000000..b3607fa
--- /dev/null
+++ b/db/repl/rs_optime.h
@@ -0,0 +1,58 @@
+// @file rs_optime.h

+

+/*

+ *    Copyright (C) 2010 10gen Inc.

+ *

+ *    This program is free software: you can redistribute it and/or  modify

+ *    it under the terms of the GNU Affero General Public License, version 3,

+ *    as published by the Free Software Foundation.

+ *

+ *    This program is distributed in the hope that it will be useful,

+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of

+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ *    GNU Affero General Public License for more details.

+ *

+ *    You should have received a copy of the GNU Affero General Public License

+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.

+ */

+

+#pragma once

+

+#include "../../util/optime.h"

+

+namespace mongo {

+

+    const char rsoplog[] = "local.oplog.rs";
+

+    /*

+    class RSOpTime : public OpTime { 

+    public:

+        bool initiated() const { return getSecs() != 0; }

+    };*/

+

+    /*struct RSOpTime { 

+        unsigned long long ord;

+

+        RSOpTime() : ord(0) { }

+

+        bool initiated() const { return ord > 0; }

+

+        void initiate() { 

+            assert( !initiated() );

+            ord = 1000000;

+        }

+

+        ReplTime inc() {

+            DEV assertInWriteLock();

+            return ++ord;

+        }

+

+        string toString() const { return str::stream() << ord; }

+

+        // query the oplog and set the highest value herein.  acquires a db read lock. throws.

+        void load();

+    };

+

+    extern RSOpTime rsOpTime;*/

+

+}

diff --git a/db/repl/rs_rollback.cpp b/db/repl/rs_rollback.cpp
new file mode 100644
index 0000000..1bb7217
--- /dev/null
+++ b/db/repl/rs_rollback.cpp
@@ -0,0 +1,481 @@
+/* @file rs_rollback.cpp
+* 
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "../client.h"
+#include "../../client/dbclient.h"
+#include "rs.h"
+#include "../repl.h"
+#include "../query.h"
+
+/* Scenarios
+
+   We went offline with ops not replicated out.
+ 
+       F = node that failed and coming back.
+       P = node that took over, new primary
+
+   #1:
+       F : a b c d e f g
+       P : a b c d q
+
+   The design is "keep P".  One could argue here that "keep F" has some merits, however, in most cases P 
+   will have significantly more data.  Also note that P may have a proper subset of F's stream if there were 
+   no subsequent writes.
+
+   For now the model is simply : get F back in sync with P.  If P was really behind or something, we should have 
+   just chosen not to fail over anyway.
+
+   #2:
+       F : a b c d e f g                -> a b c d
+       P : a b c d
+
+   #3:
+       F : a b c d e f g                -> a b c d q r s t u v w x z
+       P : a b c d.q r s t u v w x z
+
+   Steps
+    find an event in common. 'd'.
+    undo our events beyond that by: 
+      (1) taking copy from other server of those objects
+      (2) do not consider copy valid until we pass reach an optime after when we fetched the new version of object 
+          -- i.e., reset minvalid.
+      (3) we could skip operations on objects that are previous in time to our capture of the object as an optimization.
+
+*/
+
+namespace mongo {
+
+    using namespace bson;
+
+    struct DocID {
+        const char *ns;
+        be _id;
+        bool operator<(const DocID& d) const { 
+            int c = strcmp(ns, d.ns);
+            if( c < 0 ) return true;
+            if( c > 0 ) return false;
+            return _id < d._id;
+        }
+    };
+
+    struct HowToFixUp {
+        /* note this is a set -- if there are many $inc's on a single document we need to rollback, we only 
+           need to refetch it once. */
+        set<DocID> toRefetch;
+
+        /* collections to drop */
+        set<string> toDrop;
+
+        OpTime commonPoint;
+        DiskLoc commonPointOurDiskloc;
+
+        int rbid; // remote server's current rollback sequence #
+    };
+
+    static void refetch(HowToFixUp& h, const BSONObj& ourObj) { 
+        const char *op = ourObj.getStringField("op");
+        if( *op == 'n' ) 
+            return;
+
+        unsigned long long totSize = 0;
+        totSize += ourObj.objsize();
+        if( totSize > 512 * 1024 * 1024 )
+            throw "rollback too large";
+
+        DocID d;
+        d.ns = ourObj.getStringField("ns");
+        if( *d.ns == 0 ) { 
+            log() << "replSet WARNING ignoring op on rollback no ns TODO : " << ourObj.toString() << rsLog;
+            return;
+        }
+
+        bo o = ourObj.getObjectField(*op=='u' ? "o2" : "o");
+        if( o.isEmpty() ) { 
+            log() << "replSet warning ignoring op on rollback : " << ourObj.toString() << rsLog;
+            return;
+        }
+
+        if( *op == 'c' ) { 
+            be first = o.firstElement();
+            NamespaceString s(d.ns); // foo.$cmd
+
+            if( string("create") == first.fieldName() ) {
+                /* Create collection operation 
+                   { ts: ..., h: ..., op: "c", ns: "foo.$cmd", o: { create: "abc", ... } }

+                */
+                string ns = s.db + '.' + o["create"].String(); // -> foo.abc
+                h.toDrop.insert(ns);
+                return;
+            }
+            else { 
+                log() << "replSet WARNING can't roll back this command yet: " << o.toString() << rsLog;
+            }
+        }
+
+        d._id = o["_id"];
+        if( d._id.eoo() ) {
+            log() << "replSet WARNING ignoring op on rollback no _id TODO : " << d.ns << ' '<< ourObj.toString() << rsLog;
+            return;
+        }
+
+        h.toRefetch.insert(d);
+    }
+
+    int getRBID(DBClientConnection*);
+
+    static void syncRollbackFindCommonPoint(DBClientConnection *them, HowToFixUp& h) { 
+        static time_t last;
+        if( time(0)-last < 60 ) { 
+            // this could put a lot of load on someone else, don't repeat too often
+            sleepsecs(10);
+            throw "findcommonpoint waiting a while before trying again";
+        }
+        last = time(0);
+
+        assert( dbMutex.atLeastReadLocked() );
+        Client::Context c(rsoplog, dbpath, 0, false);
+        NamespaceDetails *nsd = nsdetails(rsoplog);
+        assert(nsd);
+        ReverseCappedCursor u(nsd);
+        if( !u.ok() )
+            throw "our oplog empty or unreadable";
+
+        const Query q = Query().sort(reverseNaturalObj);
+        const bo fields = BSON( "ts" << 1 << "h" << 1 );
+
+        //auto_ptr<DBClientCursor> u = us->query(rsoplog, q, 0, 0, &fields, 0, 0);
+
+        h.rbid = getRBID(them);
+        auto_ptr<DBClientCursor> t = them->query(rsoplog, q, 0, 0, &fields, 0, 0);
+
+        if( t.get() == 0 || !t->more() ) throw "remote oplog empty or unreadable";
+
+        BSONObj ourObj = u.current();
+        OpTime ourTime = ourObj["ts"]._opTime();
+        BSONObj theirObj = t->nextSafe();
+        OpTime theirTime = theirObj["ts"]._opTime();
+
+        if( 1 ) {
+            long long diff = (long long) ourTime.getSecs() - ((long long) theirTime.getSecs());
+            /* diff could be positive, negative, or zero */
+            log() << "replSet info syncRollback diff in end of log times : " << diff << " seconds" << rsLog;
+            if( diff > 3600 ) { 
+                log() << "replSet syncRollback too long a time period for a rollback." << rsLog;
+                throw "error not willing to roll back more than one hour of data";
+            }
+        }
+
+        unsigned long long scanned = 0;
+        while( 1 ) {
+            scanned++;
+            /* todo add code to assure no excessive scanning for too long */
+            if( ourTime == theirTime ) { 
+                if( ourObj["h"].Long() == theirObj["h"].Long() ) { 
+                    // found the point back in time where we match.
+                    // todo : check a few more just to be careful about hash collisions.
+                    log() << "replSet rollback found matching events at " << ourTime.toStringPretty() << rsLog;
+                    log() << "replSet rollback findcommonpoint scanned : " << scanned << rsLog;
+                    h.commonPoint = ourTime;
+                    h.commonPointOurDiskloc = u.currLoc();
+                    return;
+                }
+
+                refetch(h, ourObj);
+
+                theirObj = t->nextSafe();
+                theirTime = theirObj["ts"]._opTime();
+
+                u.advance();
+                if( !u.ok() ) throw "reached beginning of local oplog";
+                ourObj = u.current();
+                ourTime = ourObj["ts"]._opTime();
+            }
+            else if( theirTime > ourTime ) { 
+                /* todo: we could hit beginning of log here.  exception thrown is ok but not descriptive, so fix up */
+                theirObj = t->nextSafe();
+                theirTime = theirObj["ts"]._opTime();
+            }
+            else { 
+                // theirTime < ourTime
+                refetch(h, ourObj);
+                u.advance();
+                if( !u.ok() ) throw "reached beginning of local oplog";
+                ourObj = u.current();
+                ourTime = ourObj["ts"]._opTime();
+            }
+        }
+    }
+
+    struct X { 
+        const bson::bo *op;
+        bson::bo goodVersionOfObject;
+    };
+
+   void ReplSetImpl::syncFixUp(HowToFixUp& h, OplogReader& r) {
+       DBClientConnection *them = r.conn();
+
+       // fetch all first so we needn't handle interruption in a fancy way
+
+       unsigned long long totSize = 0;
+
+       list< pair<DocID,bo> > goodVersions;
+
+       bo newMinValid;
+
+       DocID d;
+       unsigned long long n = 0;
+       try {
+           for( set<DocID>::iterator i = h.toRefetch.begin(); i != h.toRefetch.end(); i++ ) { 
+               d = *i;
+
+               assert( !d._id.eoo() );
+
+               {
+                   /* TODO : slow.  lots of round trips. */
+                   n++;
+                   bo good= them->findOne(d.ns, d._id.wrap()).getOwned();
+                   totSize += good.objsize();
+                   uassert( 13410, "replSet too much data to roll back", totSize < 300 * 1024 * 1024 );
+
+                   // note good might be eoo, indicating we should delete it
+                   goodVersions.push_back(pair<DocID,bo>(d,good));
+               }
+           }
+           newMinValid = r.getLastOp(rsoplog);
+           if( newMinValid.isEmpty() ) { 
+               sethbmsg("syncRollback error newMinValid empty?");
+               return;
+           }
+       }
+       catch(DBException& e) {
+           sethbmsg(str::stream() << "syncRollback re-get objects: " << e.toString(),0);
+           log() << "syncRollback couldn't re-get ns:" << d.ns << " _id:" << d._id << ' ' << n << '/' << h.toRefetch.size() << rsLog;
+           throw e;
+       }
+
+       sethbmsg("syncRollback 3.5");
+       if( h.rbid != getRBID(r.conn()) ) { 
+           // our source rolled back itself.  so the data we received isn't necessarily consistent.
+           sethbmsg("syncRollback rbid on source changed during rollback, cancelling this attempt");
+           return;
+       }
+
+       // update them
+       sethbmsg(str::stream() << "syncRollback 4 n:" << goodVersions.size());
+
+       bool warn = false;
+
+       assert( !h.commonPointOurDiskloc.isNull() );
+
+       MemoryMappedFile::flushAll(true);
+
+       dbMutex.assertWriteLocked();
+
+       /* we have items we are writing that aren't from a point-in-time.  thus best not to come online 
+	      until we get to that point in freshness. */
+       try {
+           log() << "replSet set minvalid=" << newMinValid["ts"]._opTime().toString() << rsLog;
+       }
+       catch(...){}
+       Helpers::putSingleton("local.replset.minvalid", newMinValid);
+
+       /** first drop collections to drop - that might make things faster below actually if there were subsequent inserts */
+       for( set<string>::iterator i = h.toDrop.begin(); i != h.toDrop.end(); i++ ) { 
+           Client::Context c(*i, dbpath, 0, /*doauth*/false);
+           try {
+               bob res;
+               string errmsg;
+               log(1) << "replSet rollback drop: " << *i << rsLog;
+               dropCollection(*i, errmsg, res);
+           }
+           catch(...) { 
+               log() << "replset rollback error dropping collection " << *i << rsLog;
+           }
+       }
+
+       Client::Context c(rsoplog, dbpath, 0, /*doauth*/false);
+       NamespaceDetails *oplogDetails = nsdetails(rsoplog);
+       uassert(13423, str::stream() << "replSet error in rollback can't find " << rsoplog, oplogDetails);
+
+       map<string,shared_ptr<RemoveSaver> > removeSavers;
+
+       unsigned deletes = 0, updates = 0;
+       for( list<pair<DocID,bo> >::iterator i = goodVersions.begin(); i != goodVersions.end(); i++ ) {
+           const DocID& d = i->first;
+           bo pattern = d._id.wrap(); // { _id : ... }
+           try { 
+               assert( d.ns && *d.ns );
+               
+               shared_ptr<RemoveSaver>& rs = removeSavers[d.ns];
+               if ( ! rs )
+                   rs.reset( new RemoveSaver( "rollback" , "" , d.ns ) );
+
+               // todo: lots of overhead in context, this can be faster
+               Client::Context c(d.ns, dbpath, 0, /*doauth*/false);
+               if( i->second.isEmpty() ) {
+                   // wasn't on the primary; delete.
+                   /* TODO1.6 : can't delete from a capped collection.  need to handle that here. */
+                   deletes++;
+
+                   NamespaceDetails *nsd = nsdetails(d.ns);
+                   if( nsd ) {
+                       if( nsd->capped ) { 
+                           /* can't delete from a capped collection - so we truncate instead. if this item must go, 
+                           so must all successors!!! */
+                           try { 
+                               /** todo: IIRC cappedTrunateAfter does not handle completely empty.  todo. */
+                               // this will crazy slow if no _id index.
+                               long long start = Listener::getElapsedTimeMillis();
+                               DiskLoc loc = Helpers::findOne(d.ns, pattern, false);
+                               if( Listener::getElapsedTimeMillis() - start > 200 ) 
+                                   log() << "replSet warning roll back slow no _id index for " << d.ns << rsLog; 
+                               //would be faster but requires index: DiskLoc loc = Helpers::findById(nsd, pattern);
+                               if( !loc.isNull() ) {
+                                   try {
+                                       nsd->cappedTruncateAfter(d.ns, loc, true);
+                                   }
+                                   catch(DBException& e) { 
+                                       if( e.getCode() == 13415 ) {
+                                           // hack: need to just make cappedTruncate do this...
+                                           nsd->emptyCappedCollection(d.ns);
+                                       } else {
+                                           throw;
+                                       }
+                                   }
+                               }
+                           }
+                           catch(DBException& e) { 
+                               log() << "replSet error rolling back capped collection rec " << d.ns << ' ' << e.toString() << rsLog;
+                           }
+                       }
+                       else {
+                           try { 
+                               deletes++;
+                               deleteObjects(d.ns, pattern, /*justone*/true, /*logop*/false, /*god*/true, rs.get() );
+                           }
+                           catch(...) { 
+                               log() << "replSet error rollback delete failed ns:" << d.ns << rsLog;
+                           }
+                       }
+                       // did we just empty the collection?  if so let's check if it even exists on the source.
+                       if( nsd->nrecords == 0 ) {
+                           try { 
+                               string sys = cc().database()->name + ".system.namespaces";
+                               bo o = them->findOne(sys, QUERY("name"<<d.ns));
+                               if( o.isEmpty() ) { 
+                                   // we should drop
+                                   try {
+                                       bob res;
+                                       string errmsg;
+                                       dropCollection(d.ns, errmsg, res);
+                                   }
+                                   catch(...) { 
+                                       log() << "replset error rolling back collection " << d.ns << rsLog;
+                                   }
+                               }
+                           }
+                           catch(DBException& ) { 
+                               /* this isn't *that* big a deal, but is bad. */
+                               log() << "replSet warning rollback error querying for existence of " << d.ns << " at the primary, ignoring" << rsLog;
+                           }
+                       }
+                   }
+               }
+               else {
+                   // todo faster...
+                   OpDebug debug;
+                   updates++;
+                   _updateObjects(/*god*/true, d.ns, i->second, pattern, /*upsert=*/true, /*multi=*/false , /*logtheop=*/false , debug, rs.get() );
+               }
+           }
+           catch(DBException& e) { 
+               log() << "replSet exception in rollback ns:" << d.ns << ' ' << pattern.toString() << ' ' << e.toString() << " ndeletes:" << deletes << rsLog;
+               warn = true;
+           }
+       }
+
+       removeSavers.clear(); // this effectively closes all of them
+
+       sethbmsg(str::stream() << "syncRollback 5 d:" << deletes << " u:" << updates);
+       MemoryMappedFile::flushAll(true);
+       sethbmsg("syncRollback 6");
+
+       // clean up oplog
+       log(2) << "replSet rollback truncate oplog after " << h.commonPoint.toStringPretty() << rsLog;
+       // todo: fatal error if this throws?
+       oplogDetails->cappedTruncateAfter(rsoplog, h.commonPointOurDiskloc, false);
+
+       /* reset cached lastoptimewritten and h value */
+       loadLastOpTimeWritten();
+
+       sethbmsg("syncRollback 7");
+       MemoryMappedFile::flushAll(true);
+
+       // done
+       if( warn ) 
+           sethbmsg("issues during syncRollback, see log");
+       else
+           sethbmsg("syncRollback done");
+   }
+
+    void ReplSetImpl::syncRollback(OplogReader&r) { 
+        assert( !lockedByMe() );
+        assert( !dbMutex.atLeastReadLocked() );
+
+        sethbmsg("syncRollback 0");
+
+        writelocktry lk(rsoplog, 20000);
+        if( !lk.got() ) {
+            sethbmsg("syncRollback couldn't get write lock in a reasonable time");
+            sleepsecs(2);
+            return;
+        }
+
+        HowToFixUp how;
+        sethbmsg("syncRollback 1");
+        {
+            r.resetCursor();
+            /*DBClientConnection us(false, 0, 0);
+            string errmsg;
+            if( !us.connect(HostAndPort::me().toString(),errmsg) ) { 
+                sethbmsg("syncRollback connect to self failure" + errmsg);
+                return;
+            }*/
+
+            sethbmsg("syncRollback 2 FindCommonPoint");
+            try {
+                syncRollbackFindCommonPoint(r.conn(), how);
+            }
+            catch( const char *p ) { 
+                sethbmsg(string("syncRollback 2 error ") + p);
+                sleepsecs(10);
+                return;
+            }
+            catch( DBException& e ) { 
+                sethbmsg(string("syncRollback 2 exception ") + e.toString() + "; sleeping 1 min");
+                sleepsecs(60);
+                throw;
+            }
+        }
+
+        sethbmsg("replSet syncRollback 3 fixup");
+
+        syncFixUp(how, r);
+    }
+
+}
diff --git a/db/repl/rs_sync.cpp b/db/repl/rs_sync.cpp
new file mode 100644
index 0000000..bece96c
--- /dev/null
+++ b/db/repl/rs_sync.cpp
@@ -0,0 +1,328 @@
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "../client.h"
+#include "../../client/dbclient.h"
+#include "rs.h"
+#include "../repl.h"
+
+namespace mongo {
+
+    using namespace bson;
+
+    void startSyncThread() { 
+        Client::initThread("rs_sync");
+        theReplSet->syncThread();
+        cc().shutdown();
+    }
+
+    void ReplSetImpl::syncApply(const BSONObj &o) {
+        //const char *op = o.getStringField("op");
+        
+        char db[MaxDatabaseLen];
+        const char *ns = o.getStringField("ns");
+        nsToDatabase(ns, db);
+
+        if ( *ns == '.' || *ns == 0 ) {
+		    if( *o.getStringField("op") == 'n' )
+			    return;
+            log() << "replSet skipping bad op in oplog: " << o.toString() << endl;
+            return;
+        }
+
+        Client::Context ctx(ns);
+        ctx.getClient()->curop()->reset();
+
+        /* todo : if this asserts, do we want to ignore or not? */
+        applyOperation_inlock(o);
+    }
+
+    bool ReplSetImpl::initialSyncOplogApplication(
+        string hn, 
+        const Member *primary,
+        OpTime applyGTE,
+        OpTime minValid)
+    { 
+        if( primary == 0 ) return false;
+
+        OpTime ts;
+        try {
+            OplogReader r;
+            if( !r.connect(hn) ) { 
+                log(2) << "replSet can't connect to " << hn << " to read operations" << rsLog;
+                return false;
+            }
+
+            r.query(rsoplog, bo());
+            assert( r.haveCursor() );
+
+            /* we lock outside the loop to avoid the overhead of locking on every operation.  server isn't usable yet anyway! */
+            writelock lk("");
+
+            {
+                if( !r.more() ) { 
+                    sethbmsg("replSet initial sync error reading remote oplog");
+                    return false;
+                }
+                bo op = r.next();
+                OpTime t = op["ts"]._opTime();
+                r.putBack(op);
+                assert( !t.isNull() );
+                if( t > applyGTE ) {
+                    sethbmsg(str::stream() << "error " << hn << " oplog wrapped during initial sync");
+                    return false;
+                }
+            }
+
+            // todo : use exhaust
+            unsigned long long n = 0;
+            while( 1 ) { 
+                if( !r.more() )
+                    break;
+                BSONObj o = r.nextSafe(); /* note we might get "not master" at some point */
+                {
+                    //writelock lk("");
+
+                    ts = o["ts"]._opTime();
+
+                    /* if we have become primary, we dont' want to apply things from elsewhere
+                        anymore. assumePrimary is in the db lock so we are safe as long as 
+                        we check after we locked above. */
+					const Member *p1 = box.getPrimary();
+                    if( p1 != primary ) {
+					  log() << "replSet primary was:" << primary->fullName() << " now:" << 
+						(p1 != 0 ? p1->fullName() : "none") << rsLog;
+                        throw DBException("primary changed",0);
+                    }
+
+                    if( ts >= applyGTE ) {
+                        // optimes before we started copying need not be applied.
+                        syncApply(o);
+                    }
+                    _logOpObjRS(o);   /* with repl sets we write the ops to our oplog too */
+                }
+                if( ++n % 100000 == 0 ) { 
+                    // simple progress metering
+                    log() << "replSet initialSyncOplogApplication " << n << rsLog;
+                }
+            }
+        }
+        catch(DBException& e) { 
+            if( ts <= minValid ) {
+                // didn't make it far enough
+                log() << "replSet initial sync failing, error applying oplog " << e.toString() << rsLog;
+                return false;
+            }
+        }
+        return true;
+    }
+
+    void ReplSetImpl::syncTail() { 
+        // todo : locking vis a vis the mgr...
+
+        const Member *primary = box.getPrimary();
+        if( primary == 0 ) return;
+        string hn = primary->h().toString();
+        OplogReader r;
+        if( !r.connect(primary->h().toString()) ) { 
+            log(2) << "replSet can't connect to " << hn << " to read operations" << rsLog;
+            return;
+        }
+
+        /* first make sure we are not hopelessly out of sync by being very stale. */
+        {
+            BSONObj remoteOldestOp = r.findOne(rsoplog, Query());
+            OpTime ts = remoteOldestOp["ts"]._opTime();
+            DEV log() << "remoteOldestOp: " << ts.toStringPretty() << endl;
+            else log(3) << "remoteOldestOp: " << ts.toStringPretty() << endl;
+            if( lastOpTimeWritten < ts ) { 
+                log() << "replSet error too stale to catch up, at least from primary " << hn << rsLog;
+                log() << "replSet our last optime : " << lastOpTimeWritten.toStringPretty() << rsLog;
+                log() << "replSet oldest at " << hn << " : " << ts.toStringPretty() << rsLog;
+                log() << "replSet See http://www.mongodb.org/display/DOCS/Resyncing+a+Very+Stale+Replica+Set+Member" << rsLog;
+                sethbmsg("error too stale to catch up");
+                sleepsecs(120);
+                return;
+            }
+        }
+
+        r.tailingQueryGTE(rsoplog, lastOpTimeWritten);
+        assert( r.haveCursor() );
+        assert( r.awaitCapable() );
+
+        {
+            if( !r.more() ) {
+                /* maybe we are ahead and need to roll back? */
+                try {
+                    bo theirLastOp = r.getLastOp(rsoplog);
+                    if( theirLastOp.isEmpty() ) {
+                        log() << "replSet error empty query result from " << hn << " oplog" << rsLog;
+                        sleepsecs(2);
+                        return;
+                    }
+                    OpTime theirTS = theirLastOp["ts"]._opTime();
+                    if( theirTS < lastOpTimeWritten ) { 
+                        log() << "replSet we are ahead of the primary, will try to roll back" << rsLog;
+                        syncRollback(r);
+                        return;
+                    }
+                    /* we're not ahead?  maybe our new query got fresher data.  best to come back and try again */
+                    log() << "replSet syncTail condition 1" << rsLog;
+                    sleepsecs(1);
+                }
+                catch(DBException& e) { 
+                    log() << "replSet error querying " << hn << ' ' << e.toString() << rsLog;
+                    sleepsecs(2);
+                }
+                return;
+                /*
+                log() << "replSet syncTail error querying oplog >= " << lastOpTimeWritten.toString() << " from " << hn << rsLog;
+                try {
+                    log() << "replSet " << hn << " last op: " << r.getLastOp(rsoplog).toString() << rsLog;
+                }
+                catch(...) { }
+                sleepsecs(1);
+                return;*/
+            }
+
+            BSONObj o = r.nextSafe();
+            OpTime ts = o["ts"]._opTime();
+            long long h = o["h"].numberLong();
+            if( ts != lastOpTimeWritten || h != lastH ) { 
+                log(1) << "TEMP our last op time written: " << lastOpTimeWritten.toStringPretty() << endl;
+                log(1) << "TEMP primary's GTE: " << ts.toStringPretty() << endl;
+                /*
+                }*/
+
+                syncRollback(r);
+                return;
+            }
+        }
+
+        while( 1 ) { 
+            while( 1 ) {
+                if( !r.moreInCurrentBatch() ) { 
+                    /* we need to occasionally check some things. between 
+                       batches is probably a good time. */
+
+                    /* perhaps we should check this earlier? but not before the rollback checks. */
+                    if( state().recovering() ) { 
+                        /* can we go to RS_SECONDARY state?  we can if not too old and if minvalid achieved */
+                        bool golive = false;			
+                        OpTime minvalid;
+                        {
+                            readlock lk("local.replset.minvalid");
+                            BSONObj mv;
+                            if( Helpers::getSingleton("local.replset.minvalid", mv) ) { 
+                                minvalid = mv["ts"]._opTime();
+                                if( minvalid <= lastOpTimeWritten ) { 
+                                    golive=true;
+                                }
+                            }
+                            else 
+                                golive = true; /* must have been the original member */
+                        }
+                        if( golive ) {
+                            sethbmsg("");
+                            log() << "replSet SECONDARY" << rsLog;
+                            changeState(MemberState::RS_SECONDARY);
+                        }
+                        else { 
+                            sethbmsg(str::stream() << "still syncing, not yet to minValid optime " << minvalid.toString());
+                        }
+
+                        /* todo: too stale capability */
+                    }
+
+                    if( box.getPrimary() != primary ) 
+                        return;
+                }
+                if( !r.more() )
+                    break;
+                { 
+                    BSONObj o = r.nextSafe(); /* note we might get "not master" at some point */
+                    {
+                        writelock lk("");
+
+                        /* if we have become primary, we dont' want to apply things from elsewhere
+                           anymore. assumePrimary is in the db lock so we are safe as long as 
+                           we check after we locked above. */
+                        if( box.getPrimary() != primary ) {
+                            if( box.getState().primary() )
+                                log(0) << "replSet stopping syncTail we are now primary" << rsLog;
+                            return;
+                        }
+
+                        syncApply(o);
+                        _logOpObjRS(o);   /* with repl sets we write the ops to our oplog too: */                   
+                    }
+                }
+            }
+            r.tailCheck();
+            if( !r.haveCursor() ) {
+                log() << "replSet TEMP end syncTail pass with " << hn << rsLog;
+                // TODO : reuse our cnonection to the primary.
+                return;
+            }
+            if( box.getPrimary() != primary )
+                return;
+            // looping back is ok because this is a tailable cursor
+        }
+    }
+
+    void ReplSetImpl::_syncThread() {
+        StateBox::SP sp = box.get();
+        if( sp.state.primary() ) {
+            sleepsecs(1);
+            return;
+        }
+
+        /* later, we can sync from up secondaries if we want. tbd. */
+        if( sp.primary == 0 )
+            return;
+
+        /* do we have anything at all? */
+        if( lastOpTimeWritten.isNull() ) {
+            syncDoInitialSync();
+            return; // _syncThread will be recalled, starts from top again in case sync failed.
+        }
+
+        /* we have some data.  continue tailing. */
+        syncTail();
+    }
+
+    void ReplSetImpl::syncThread() {
+        if( myConfig().arbiterOnly )
+            return;
+        while( 1 ) { 
+            try {
+                _syncThread();
+            }
+            catch(DBException& e) { 
+                sethbmsg("syncThread: " + e.toString());
+                sleepsecs(10);
+            }
+            catch(...) { 
+                sethbmsg("unexpected exception in syncThread()");
+                // TODO : SET NOT SECONDARY here.
+                sleepsecs(60);
+            }
+            sleepsecs(1);
+        }
+    }
+
+}
diff --git a/db/repl/test.html b/db/repl/test.html
new file mode 100644
index 0000000..295ad2e
--- /dev/null
+++ b/db/repl/test.html
@@ -0,0 +1,11 @@
+<HTML>

+<BODY>

+<!-- see also jstests/rs/ -->

+<iframe src="http://127.0.0.1:28000/_replSet" width="100%" height="50%" frameborder=1>

+</iframe>

+

+<iframe src="http://127.0.0.1:28001/_replSet" width="100%" height="50%" frameborder=1>

+</iframe>

+

+</BODY>

+</HTML>

diff --git a/db/repl/testing.js b/db/repl/testing.js
new file mode 100644
index 0000000..d741cf3
--- /dev/null
+++ b/db/repl/testing.js
@@ -0,0 +1,42 @@
+// helpers for testing repl sets

+// run

+//   mongo --shell <host:port> testing.js

+

+cfg = {

+    _id: 'asdf',

+    members: [

+        { _id : 0, host : "dm_hp" },

+        { _id : 2, host : "dm_hp:27002" }

+        ]

+};

+c2 = {

+    _id: 'asdf',

+    members: [

+        { _id: 0, host: "dmthink" },

+        { _id: 2, host: "dmthink:27002" }

+        ]

+};

+

+db = db.getSisterDB("admin");

+local = db.getSisterDB("local");

+

+print("\n\ndb = admin db on localhost:27017");

+print("b = admin on localhost:27002");

+print("rc(x) = db.runCommand(x)");

+print("cfg = samp replset config");

+print("i() = replSetInitiate(cfg)");

+print("ism() = rc('ismaster')");

+print("\n\n");

+

+function rc(c) { return db.runCommand(c); }

+function i() { return rc({ replSetInitiate: cfg }); }

+function ism() { return rc("isMaster"); }

+

+b = 0;

+try {

+    b = new Mongo("localhost:27002").getDB("admin");

+}

+catch (e) {

+    print("\nCouldn't connect to b mongod instance\n");

+}

+

diff --git a/db/repl_block.cpp b/db/repl_block.cpp
new file mode 100644
index 0000000..9cff24f
--- /dev/null
+++ b/db/repl_block.cpp
@@ -0,0 +1,207 @@
+// repl_block.cpp
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "repl.h"
+#include "repl_block.h"
+#include "instance.h"
+#include "dbhelpers.h"
+#include "../util/background.h"
+#include "../util/mongoutils/str.h"
+#include "../client/dbclient.h"
+#include "replpair.h"
+
+//#define REPLDEBUG(x) log() << "replBlock: "  << x << endl;
+#define REPLDEBUG(x)
+
+namespace mongo {
+
+    using namespace mongoutils;
+
+    class SlaveTracking : public BackgroundJob {
+    public:
+        string name() { return "SlaveTracking"; }
+
+        static const char * NS;
+
+        struct Ident {
+            
+            Ident(BSONObj r,string h,string n){
+                BSONObjBuilder b;
+                b.appendElements( r );
+                b.append( "host" , h );
+                b.append( "ns" , n );
+                obj = b.obj();
+            }
+
+            bool operator<( const Ident& other ) const {
+                return obj.woCompare( other.obj ) < 0;
+            }
+            
+            BSONObj obj;
+        };
+
+        struct Info {
+            Info() : loc(0){}
+            ~Info(){
+                if ( loc && owned ){
+                    delete loc;
+                }
+            }
+            bool owned;
+            OpTime * loc;
+        };
+
+        SlaveTracking() : _mutex("SlaveTracking") {
+            _dirty = false;
+            _started = false;
+        }
+
+        void run(){
+            Client::initThread( "slaveTracking" );
+            DBDirectClient db;
+            while ( ! inShutdown() ){
+                sleepsecs( 1 );
+
+                if ( ! _dirty )
+                    continue;
+                
+                writelock lk(NS);
+
+                list< pair<BSONObj,BSONObj> > todo;
+                
+                {
+                    scoped_lock mylk(_mutex);
+                    
+                    for ( map<Ident,Info>::iterator i=_slaves.begin(); i!=_slaves.end(); i++ ){
+                        BSONObjBuilder temp;
+                        temp.appendTimestamp( "syncedTo" , i->second.loc[0].asDate() );
+                        todo.push_back( pair<BSONObj,BSONObj>( i->first.obj.getOwned() , 
+                                                               BSON( "$set" << temp.obj() ).getOwned() ) );
+                    }
+                    
+                    _slaves.clear();
+                }
+
+                for ( list< pair<BSONObj,BSONObj> >::iterator i=todo.begin(); i!=todo.end(); i++ ){
+                    db.update( NS , i->first , i->second , true );
+                }
+
+                _dirty = false;
+            }
+        }
+
+        void reset(){
+            scoped_lock mylk(_mutex);
+            _slaves.clear();
+        }
+
+        void update( const BSONObj& rid , const string& host , const string& ns , OpTime last ){
+            REPLDEBUG( host << " " << rid << " " << ns << " " << last );
+
+            scoped_lock mylk(_mutex);
+            
+#ifdef _DEBUG
+            MongoFileAllowWrites allowWrites;
+#endif
+
+            Ident ident(rid,host,ns);
+            Info& i = _slaves[ ident ];
+            if ( i.loc ){
+                i.loc[0] = last;
+                return;
+            }
+            
+            dbMutex.assertAtLeastReadLocked();
+
+            BSONObj res;
+            if ( Helpers::findOne( NS , ident.obj , res ) ){
+                assert( res["syncedTo"].type() );
+                i.owned = false;
+                i.loc = (OpTime*)res["syncedTo"].value();
+                i.loc[0] = last;
+                return;
+            }
+            
+            i.owned = true;
+            i.loc = new OpTime[1];
+            i.loc[0] = last;
+            _dirty = true;
+
+            if ( ! _started ){
+                // start background thread here since we definitely need it
+                _started = true;
+                go();
+            }
+
+        }
+        
+        bool opReplicatedEnough( OpTime op , int w ){
+            RARELY {
+                REPLDEBUG( "looking for : " << op << " w=" << w );
+            }
+
+            if ( w <= 1 || ! _isMaster() )
+                return true;
+
+            w--; // now this is the # of slaves i need
+            scoped_lock mylk(_mutex);
+            for ( map<Ident,Info>::iterator i=_slaves.begin(); i!=_slaves.end(); i++){
+                OpTime s = *(i->second.loc);
+                if ( s < op ){
+                    continue;
+                }
+                if ( --w == 0 )
+                    return true;
+            }
+            return w <= 0;
+        }
+        
+        // need to be careful not to deadlock with this
+        mongo::mutex _mutex;
+        map<Ident,Info> _slaves;
+        bool _dirty;
+        bool _started;
+
+    } slaveTracking;
+
+    const char * SlaveTracking::NS = "local.slaves";
+
+    void updateSlaveLocation( CurOp& curop, const char * ns , OpTime lastOp ){
+        if ( lastOp.isNull() )
+            return;
+        
+        assert( str::startsWith(ns, "local.oplog.") );
+        
+        Client * c = curop.getClient();
+        assert(c);
+        BSONObj rid = c->getRemoteID();
+        if ( rid.isEmpty() )
+            return;
+
+        slaveTracking.update( rid , curop.getRemoteString( false ) , ns , lastOp );
+    }
+
+    bool opReplicatedEnough( OpTime op , int w ){
+        return slaveTracking.opReplicatedEnough( op , w );
+    }
+
+    void resetSlaveCache(){
+        slaveTracking.reset();
+    }
+}
diff --git a/db/repl_block.h b/db/repl_block.h
new file mode 100644
index 0000000..e9a990a
--- /dev/null
+++ b/db/repl_block.h
@@ -0,0 +1,34 @@
+// repl_block.h - blocking on writes for replication
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include "../pch.h"
+#include "client.h"
+#include "curop.h"
+
+/**
+   local.slaves  - current location for all slaves
+   
+ */
+namespace mongo {
+    
+    void updateSlaveLocation( CurOp& curop, const char * ns , OpTime lastOp );
+    bool opReplicatedEnough( OpTime op , int w );
+    void resetSlaveCache();
+}
diff --git a/db/replset.h b/db/replpair.h
index 66a8604..19b79bd 100644
--- a/db/replset.h
+++ b/db/replpair.h
@@ -22,6 +22,7 @@
 #include "../client/dbclient.h"
 #include "repl.h"
 #include "cmdline.h"
+#include "repl/rs.h"
 
 namespace mongo {
 
@@ -54,7 +55,7 @@ namespace mongo {
         int remotePort;
         string remoteHost;
         string remote; // host:port if port specified.
-//    int date; // -1 not yet set; 0=slave; 1=master
+	//    int date; // -1 not yet set; 0=slave; 1=master
         
         string getInfo() {
             stringstream ss;
@@ -110,23 +111,23 @@ namespace mongo {
 
        If 'client' is not specified, the current client is used.
     */
-    inline bool isMaster( const char *client = 0 ) {
-		if( ! replSettings.slave ) 
-			return true;
-
-        if ( !client ) {
-            Database *database = cc().database();
-            assert( database );
-            client = database->name.c_str();
+    inline bool _isMaster() {
+        if( replSet ) {
+            if( theReplSet ) 
+                return theReplSet->isPrimary();
+            return false;
         }
 
+        if( ! replSettings.slave ) 
+            return true;
+
         if ( replAllDead )
-            return strcmp( client, "local" ) == 0;
+            return false;
 
         if ( replPair ) {
-			if( replPair->state == ReplPair::State_Master )
-				return true;
-		}
+            if( replPair->state == ReplPair::State_Master )
+                return true;
+        }
         else { 
             if( replSettings.master ) {
                 // if running with --master --slave, allow.  note that master is also true 
@@ -138,8 +139,37 @@ namespace mongo {
         if ( cc().isGod() )
             return true;
         
+        return false;
+    }
+    inline bool isMaster(const char *client = 0) {
+        if( _isMaster() )
+            return true;
+        if ( !client ) {
+            Database *database = cc().database();
+            assert( database );
+            client = database->name.c_str();
+        }
         return strcmp( client, "local" ) == 0;
     }
+
+    inline void notMasterUnless(bool expr) { 
+        uassert( 10107 , "not master" , expr );
+    }
+
+    /* we allow queries to SimpleSlave's -- but not to the slave (nonmaster) member of a replica pair 
+       so that queries to a pair are realtime consistent as much as possible.  use setSlaveOk() to 
+       query the nonmaster member of a replica pair.
+    */
+    inline void replVerifyReadsOk(ParsedQuery& pq) {
+        if( replSet ) {
+            /* todo: speed up the secondary case.  as written here there are 2 mutex entries, it can be 1. */
+            if( isMaster() ) return;
+            notMasterUnless( pq.hasOption(QueryOption_SlaveOk) && theReplSet->isSecondary() );
+        } else {
+            notMasterUnless(isMaster() || pq.hasOption(QueryOption_SlaveOk) || replSettings.slave == SimpleSlave );
+        }
+    }
+
     inline bool isMasterNs( const char *ns ) {
         char cl[ 256 ];
         nsToDatabase( ns, cl );
@@ -191,15 +221,14 @@ namespace mongo {
             BSONObj o = fromjson("{\"initialsynccomplete\":1}");
             Helpers::putSingleton("local.pair.sync", o);
             initialsynccomplete = 1;
+            tlog() << "pair: initial sync complete" << endl;
         }
 
         void setInitialSyncCompletedLocking() {
             if ( initialsynccomplete == 1 )
                 return;
             dblock lk;
-            BSONObj o = fromjson("{\"initialsynccomplete\":1}");
-            Helpers::putSingleton("local.pair.sync", o);
-            initialsynccomplete = 1;
+            setInitialSyncCompleted();
         }
     };
 
diff --git a/db/resource.h b/db/resource.h
index 59b9f5c..bee8d30 100644..100755
--- a/db/resource.h
+++ b/db/resource.h
@@ -1,34 +1,16 @@
-//{{NO_DEPENDENCIES}}
-// Microsoft Visual C++ generated include file.
-// Used by db.rc
-
-/**
-*    Copyright (C) 2008 10gen Inc.
-*
-*    This program is free software: you can redistribute it and/or  modify
-*    it under the terms of the GNU Affero General Public License, version 3,
-*    as published by the Free Software Foundation.
-*
-*    This program is distributed in the hope that it will be useful,
-*    but WITHOUT ANY WARRANTY; without even the implied warranty of
-*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-*    GNU Affero General Public License for more details.
-*
-*    You should have received a copy of the GNU Affero General Public License
-*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-namespace mongo {
-
-// Next default values for new objects
-//
-#ifdef APSTUDIO_INVOKED
-#ifndef APSTUDIO_READONLY_SYMBOLS
-#define _APS_NEXT_RESOURCE_VALUE        101
-#define _APS_NEXT_COMMAND_VALUE         40001
-#define _APS_NEXT_CONTROL_VALUE         1001
-#define _APS_NEXT_SYMED_VALUE           101
-#endif
-#endif
-
-} // namespace mongo
+//{{NO_DEPENDENCIES}}

+// Microsoft Visual C++ generated include file.

+// Used by db.rc

+//

+#define IDI_ICON2                       102

+

+// Next default values for new objects

+// 

+#ifdef APSTUDIO_INVOKED

+#ifndef APSTUDIO_READONLY_SYMBOLS

+#define _APS_NEXT_RESOURCE_VALUE        104

+#define _APS_NEXT_COMMAND_VALUE         40001

+#define _APS_NEXT_CONTROL_VALUE         1001

+#define _APS_NEXT_SYMED_VALUE           101

+#endif

+#endif

diff --git a/db/restapi.cpp b/db/restapi.cpp
new file mode 100644
index 0000000..3fd39c2
--- /dev/null
+++ b/db/restapi.cpp
@@ -0,0 +1,310 @@
+/** @file resetapi.cpp
+    web rest api
+*/
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "pch.h"
+#include "../util/miniwebserver.h"
+#include "../util/mongoutils/html.h"
+#include "../util/md5.hpp"
+#include "instance.h"
+#include "dbwebserver.h"
+#include "dbhelpers.h"
+#include "repl.h"
+#include "replpair.h"
+#include "clientcursor.h"
+#include "background.h"
+
+namespace mongo {
+
+    extern const char *replInfo;
+    bool getInitialSyncCompleted();
+
+    using namespace bson;
+    using namespace mongoutils::html;
+
+    class RESTHandler : public DbWebHandler {
+    public:
+        RESTHandler() : DbWebHandler( "DUMMY REST" , 1000 , true ){}
+
+        virtual bool handles( const string& url ) const { 
+            return 
+                url[0] == '/' &&
+                url.find_last_of( '/' ) > 0;
+        }
+
+        virtual void handle( const char *rq, string url, 
+                             string& responseMsg, int& responseCode,
+                             vector<string>& headers,  const SockAddr &from ){
+
+            string::size_type first = url.find( "/" , 1 );
+            if ( first == string::npos ) {
+                responseCode = 400;
+                return;
+            }
+
+            string method = MiniWebServer::parseMethod( rq );
+            string dbname = url.substr( 1 , first - 1 );
+            string coll = url.substr( first + 1 );
+            string action = "";
+
+            BSONObj params;
+            if ( coll.find( "?" ) != string::npos ) {
+                MiniWebServer::parseParams( params , coll.substr( coll.find( "?" ) + 1 ) );
+                coll = coll.substr( 0 , coll.find( "?" ) );
+            }
+
+            string::size_type last = coll.find_last_of( "/" );
+            if ( last == string::npos ) {
+                action = coll;
+                coll = "_defaultCollection";
+            }
+            else {
+                action = coll.substr( last + 1 );
+                coll = coll.substr( 0 , last );
+            }
+
+            for ( string::size_type i=0; i<coll.size(); i++ )
+                if ( coll[i] == '/' )
+                    coll[i] = '.';
+
+            string fullns = MiniWebServer::urlDecode(dbname + "." + coll);
+
+            headers.push_back( (string)"x-action: " + action );
+            headers.push_back( (string)"x-ns: " + fullns );
+
+            bool html = false;
+
+
+            stringstream ss;
+
+            if ( method == "GET" ) {
+                responseCode = 200;
+                html = handleRESTQuery( fullns , action , params , responseCode , ss  );
+            }
+            else if ( method == "POST" ) {
+                responseCode = 201;
+                handlePost( fullns , MiniWebServer::body( rq ) , params , responseCode , ss  );
+            }
+            else {
+                responseCode = 400;
+                headers.push_back( "X_err: bad request" );
+                ss << "don't know how to handle a [" << method << "]";
+                out() << "don't know how to handle a [" << method << "]" << endl;
+            }
+
+            if( html ) 
+                headers.push_back("Content-Type: text/html;charset=utf-8");
+            else
+                headers.push_back("Content-Type: text/plain;charset=utf-8");
+
+            responseMsg = ss.str();
+        }
+
+        bool handleRESTQuery( string ns , string action , BSONObj & params , int & responseCode , stringstream & out ) {
+            Timer t;
+
+            int html = _getOption( params["html"] , 0 ); 
+            int skip = _getOption( params["skip"] , 0 );
+            int num  = _getOption( params["limit"] , _getOption( params["count" ] , 1000 ) ); // count is old, limit is new
+
+            int one = 0;
+            if ( params["one"].type() == String && tolower( params["one"].valuestr()[0] ) == 't' ) {
+                num = 1;
+                one = 1;
+            }
+
+            BSONObjBuilder queryBuilder;
+
+            BSONObjIterator i(params);
+            while ( i.more() ){
+                BSONElement e = i.next();
+                string name = e.fieldName();
+                if ( ! name.find( "filter_" ) == 0 )
+                    continue;
+
+                string field = name.substr(7);
+                const char * val = e.valuestr();
+
+                char * temp;
+
+                // TODO: this is how i guess if something is a number.  pretty lame right now
+                double number = strtod( val , &temp );
+                if ( temp != val )
+                    queryBuilder.append( field , number );
+                else
+                    queryBuilder.append( field , val );
+            }
+
+            BSONObj query = queryBuilder.obj();
+            auto_ptr<DBClientCursor> cursor = db.query( ns.c_str() , query, num , skip );
+            uassert( 13085 , "query failed for dbwebserver" , cursor.get() );
+
+            if ( one ) {
+                if ( cursor->more() ) {
+                    BSONObj obj = cursor->next();
+                    out << obj.jsonString(Strict,html?1:0) << '\n';
+                }
+                else {
+                    responseCode = 404;
+                }
+                return html != 0;
+            }
+
+            if( html )  {
+                string title = string("query ") + ns;
+                out << start(title) 
+                    << p(title)
+                    << "<pre>";
+            } else {
+                out << "{\n";
+                out << "  \"offset\" : " << skip << ",\n";
+                out << "  \"rows\": [\n";
+            }
+
+            int howMany = 0;
+            while ( cursor->more() ) {
+                if ( howMany++ && html == 0 )
+                    out << " ,\n";
+                BSONObj obj = cursor->next();
+                if( html ) {
+                    if( out.tellp() > 4 * 1024 * 1024 ) {
+                        out << "Stopping output: more than 4MB returned and in html mode\n";
+                        break;
+                    }
+                    out << obj.jsonString(Strict, html?1:0) << "\n\n";
+                }
+                else {
+                    if( out.tellp() > 50 * 1024 * 1024 ) // 50MB limit - we are using ram
+                        break;
+                    out << "    " << obj.jsonString();
+                }
+            }
+
+            if( html ) { 
+                out << "</pre>\n";
+                if( howMany == 0 ) out << p("Collection is empty");
+                out << _end();
+            }
+            else {
+                out << "\n  ],\n\n";
+                out << "  \"total_rows\" : " << howMany << " ,\n";
+                out << "  \"query\" : " << query.jsonString() << " ,\n";
+                out << "  \"millis\" : " << t.millis() << '\n';
+                out << "}\n";
+            }
+
+            return html != 0;
+        }
+
+        // TODO Generate id and revision per couch POST spec
+        void handlePost( string ns, const char *body, BSONObj& params, int & responseCode, stringstream & out ) {
+            try {
+                BSONObj obj = fromjson( body );
+                db.insert( ns.c_str(), obj );
+            } catch ( ... ) {
+                responseCode = 400; // Bad Request.  Seems reasonable for now.
+                out << "{ \"ok\" : false }";
+                return;
+            }
+
+            responseCode = 201;
+            out << "{ \"ok\" : true }";
+        }
+
+        int _getOption( BSONElement e , int def ) {
+            if ( e.isNumber() )
+                return e.numberInt();
+            if ( e.type() == String )
+                return atoi( e.valuestr() );
+            return def;
+        }
+        
+        DBDirectClient db;
+
+    } restHandler;
+
+    bool webHaveAdminUsers(){
+        readlocktryassert rl("admin.system.users", 10000);
+        Client::Context cx( "admin.system.users" );
+        return ! Helpers::isEmpty("admin.system.users");
+    }
+
+    BSONObj webGetAdminUser( const string& username ){
+        Client::GodScope gs;
+        readlocktryassert rl("admin.system.users", 10000);
+        Client::Context cx( "admin.system.users" );
+        BSONObj user;
+        if ( Helpers::findOne( "admin.system.users" , BSON( "user" << username ) , user ) )
+            return user.copy();
+        return BSONObj();
+    }
+
+    class LowLevelMongodStatus : public WebStatusPlugin {
+    public:
+        LowLevelMongodStatus() : WebStatusPlugin( "low level" , 5 , "requires read lock" ){}
+
+        virtual void init(){}
+
+        void _gotLock( int millis , stringstream& ss ){
+            ss << "<pre>\n";
+            ss << "time to get readlock: " << millis << "ms\n";
+            
+            ss << "# databases: " << dbHolder.size() << '\n';
+            
+            if( ClientCursor::byLocSize()>500 )
+                ss << "Cursors byLoc.size(): " << ClientCursor::byLocSize() << '\n';
+            
+            ss << "\nreplication: ";
+            if( *replInfo )
+                ss << "\nreplInfo:  " << replInfo << "\n\n";
+            if( replSet ) {
+                ss << a("", "see replSetGetStatus link top of page") << "--replSet </a>" << cmdLine._replSet << '\n';
+            }
+            if ( replAllDead )
+                ss << "<b>replication replAllDead=" << replAllDead << "</b>\n";
+
+            else {
+                ss << "\nmaster: " << replSettings.master << '\n';
+                ss << "slave:  " << replSettings.slave << '\n';
+                if ( replPair ) {
+                    ss << "replpair:\n";
+                    ss << replPair->getInfo();
+                }
+                bool seemCaughtUp = getInitialSyncCompleted();
+                if ( !seemCaughtUp ) ss << "<b>";
+                ss <<   "initialSyncCompleted: " << seemCaughtUp;
+                if ( !seemCaughtUp ) ss << "</b>";
+                ss << '\n';
+            }
+
+            BackgroundOperation::dump(ss);
+            ss << "</pre>\n";
+        }
+
+        virtual void run( stringstream& ss ){
+            Timer t;
+            readlocktry lk( "" , 300 );
+            if ( lk.got() ){
+                _gotLock( t.millis() , ss );
+            }
+            else {
+                ss << "\n<b>timed out getting lock</b>\n";
+            }
+        }
+    } lowLevelMongodStatus;
+}
diff --git a/db/scanandorder.h b/db/scanandorder.h
index f038069..48f5aa6 100644
--- a/db/scanandorder.h
+++ b/db/scanandorder.h
@@ -50,25 +50,31 @@ namespace mongo {
        _ response size limit from runquery; push it up a bit.
     */
 
-    inline void fillQueryResultFromObj(BufBuilder& bb, FieldMatcher *filter, BSONObj& js) {
+    inline void fillQueryResultFromObj(BufBuilder& bb, FieldMatcher *filter, BSONObj& js, DiskLoc* loc=NULL) {
         if ( filter ) {
             BSONObjBuilder b( bb );
             BSONObjIterator i( js );
-            bool gotId = false;
             while ( i.more() ){
                 BSONElement e = i.next();
                 const char * fname = e.fieldName();
                 
                 if ( strcmp( fname , "_id" ) == 0 ){
-                    b.append( e );
-                    gotId = true;
+                    if (filter->includeID())
+                        b.append( e );
                 } else {
                     filter->append( b , e );
                 }
             }
+            if (loc)
+                b.append("$diskLoc", loc->toBSONObj());
+            b.done();
+        } else if (loc) {
+            BSONObjBuilder b( bb );
+            b.appendElements(js);
+            b.append("$diskLoc", loc->toBSONObj());
             b.done();
         } else {
-            bb.append((void*) js.objdata(), js.objsize());
+            bb.appendBuf((void*) js.objdata(), js.objsize());
         }
     }
     
@@ -80,17 +86,24 @@ namespace mongo {
         KeyType order;
         unsigned approxSize;
 
-        void _add(BSONObj& k, BSONObj o) {
-            best.insert(make_pair(k,o));
+        void _add(BSONObj& k, BSONObj o, DiskLoc* loc) {
+            if (!loc){
+                best.insert(make_pair(k,o));
+            } else {
+                BSONObjBuilder b;
+                b.appendElements(o);
+                b.append("$diskLoc", loc->toBSONObj());
+                best.insert(make_pair(k, b.obj()));
+            }
         }
 
-        void _addIfBetter(BSONObj& k, BSONObj o, BestMap::iterator i) {
+        void _addIfBetter(BSONObj& k, BSONObj o, BestMap::iterator i, DiskLoc* loc) {
             const BSONObj& worstBestKey = i->first;
             int c = worstBestKey.woCompare(k, order.pattern);
             if ( c > 0 ) {
                 // k is better, 'upgrade'
                 best.erase(i);
-                _add(k, o);
+                _add(k, o, loc);
             }
         }
 
@@ -106,19 +119,20 @@ namespace mongo {
             return best.size();
         }
 
-        void add(BSONObj o) {
+        void add(BSONObj o, DiskLoc* loc) {
+            assert( o.isValid() );
             BSONObj k = order.getKeyFromObject(o);
             if ( (int) best.size() < limit ) {
                 approxSize += k.objsize();
                 uassert( 10128 ,  "too much key data for sort() with no index.  add an index or specify a smaller limit", approxSize < 1 * 1024 * 1024 );
-                _add(k, o);
+                _add(k, o, loc);
                 return;
             }
             BestMap::iterator i;
             assert( best.end() != best.begin() );
             i = best.end();
             i--;
-            _addIfBetter(k, o, i);
+            _addIfBetter(k, o, i, loc);
         }
 
         void _fill(BufBuilder& b, FieldMatcher *filter, int& nout, BestMap::iterator begin, BestMap::iterator end) {
diff --git a/db/security.cpp b/db/security.cpp
index 6a01627..c552b53 100644
--- a/db/security.cpp
+++ b/db/security.cpp
@@ -16,7 +16,7 @@
  *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "security.h"
 #include "instance.h"
 #include "client.h"
@@ -31,9 +31,9 @@ namespace mongo {
 	int AuthenticationInfo::warned = 0;
 
     void AuthenticationInfo::print(){
-        cout << "AuthenticationInfo: " << this << "\n";
+        cout << "AuthenticationInfo: " << this << '\n';
         for ( map<string,Auth>::iterator i=m.begin(); i!=m.end(); i++ ){
-            cout << "\t" << i->first << "\t" << i->second.level << "\n";
+            cout << "\t" << i->first << "\t" << i->second.level << '\n';
         }
         cout << "END" << endl;
     }
diff --git a/db/security.h b/db/security.h
index 261b123..a6a9103 100644
--- a/db/security.h
+++ b/db/security.h
@@ -18,10 +18,6 @@
 
 #pragma once
 
-#include <boost/thread/tss.hpp>
-#undef assert
-#define assert xassert
-
 #include "nonce.h"
 #include "concurrency.h"
 
@@ -42,7 +38,7 @@ namespace mongo {
 		static int warned;
     public:
 		bool isLocalHost;
-        AuthenticationInfo() { isLocalHost = false; }
+        AuthenticationInfo() : _lock("AuthenticationInfo") { isLocalHost = false; }
         ~AuthenticationInfo() {
         }
         void logout(const string& dbname ) { 
diff --git a/db/security_commands.cpp b/db/security_commands.cpp
index 326d6e4..af03294 100644
--- a/db/security_commands.cpp
+++ b/db/security_commands.cpp
@@ -19,7 +19,7 @@
 
 // security.cpp
 
-#include "stdafx.h"
+#include "pch.h"
 #include "security.h"
 #include "../util/md5.hpp"
 #include "json.h" 
@@ -52,12 +52,13 @@ namespace mongo {
         virtual bool logTheOp() {
             return false;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        virtual LockType locktype(){ return NONE; }
+        void help(stringstream& h) const { h << "internal"; }
+        virtual LockType locktype() const { return NONE; }
         CmdGetNonce() : Command("getnonce") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        bool run(const string&, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             nonce *n = new nonce(security.getNonce());
             stringstream ss;
             ss << hex << *n;
@@ -72,15 +73,15 @@ namespace mongo {
         virtual bool logTheOp() {
             return false;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        virtual LockType locktype(){ return NONE; }
+        void help(stringstream& h) const { h << "de-authenticate"; }
+        virtual LockType locktype() const { return NONE; }
         CmdLogout() : Command("logout") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
-            // database->name is the one we are logging out...
+        bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             AuthenticationInfo *ai = cc().getAuthenticationInfo();
-            ai->logout(nsToDatabase(ns));
+            ai->logout(dbname);
             return true;
         }
     } cmdLogout;
@@ -91,12 +92,13 @@ namespace mongo {
         virtual bool logTheOp() {
             return false;
         }
-        virtual bool slaveOk() {
+        virtual bool slaveOk() const {
             return true;
         }
-        virtual LockType locktype(){ return WRITE; } // TODO: make this READ
+        virtual LockType locktype() const { return WRITE; } // TODO: make this READ
+        virtual void help(stringstream& ss) const { ss << "internal"; }
         CmdAuthenticate() : Command("authenticate") {}
-        bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
+        bool run(const string& dbname , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl) {
             log(1) << " authenticate: " << cmdObj << endl;
 
             string user = cmdObj.getStringField("user");
@@ -105,7 +107,7 @@ namespace mongo {
             
             if( user.empty() || key.empty() || received_nonce.empty() ) { 
                 log() << "field missing/wrong type in received authenticate command " 
-                    << cc().database()->name
+                    << dbname
                     << endl;               
                 errmsg = "auth fails";
                 sleepmillis(10);
@@ -119,9 +121,11 @@ namespace mongo {
                 nonce *ln = lastNonce.release();
                 if ( ln == 0 ) {
                     reject = true;
+                    log(1) << "auth: no lastNonce" << endl;
                 } else {
                     digestBuilder << hex << *ln;
                     reject = digestBuilder.str() != received_nonce;
+                    if ( reject ) log(1) << "auth: different lastNonce" << endl;
                 }
                     
                 if ( reject ) {
@@ -133,7 +137,7 @@ namespace mongo {
             }
 
             static BSONObj userPattern = fromjson("{\"user\":1}");
-            string systemUsers = cc().database()->name + ".system.users";
+            string systemUsers = dbname + ".system.users";
             OCCASIONALLY Helpers::ensureIndex(systemUsers.c_str(), userPattern, false, "user_1");
 
             BSONObj userObj;
@@ -164,7 +168,7 @@ namespace mongo {
             string computed = digestToString( d );
             
             if ( key != computed ){
-                log() << "auth: key mismatch " << user << ", ns:" << ns << endl;
+                log() << "auth: key mismatch " << user << ", ns:" << dbname << endl;
                 errmsg = "auth fails";
                 return false;
             }
@@ -172,13 +176,7 @@ namespace mongo {
             AuthenticationInfo *ai = cc().getAuthenticationInfo();
             
             if ( userObj[ "readOnly" ].isBoolean() && userObj[ "readOnly" ].boolean() ) {
-                if ( readLockSupported() ){
-                    ai->authorizeReadOnly( cc().database()->name.c_str() );
-                }
-                else {
-                    log() << "warning: old version of boost, read-only users not supported" << endl;
-                    ai->authorize( cc().database()->name.c_str() );
-                }
+                ai->authorizeReadOnly( cc().database()->name.c_str() );
             } else {
                 ai->authorize( cc().database()->name.c_str() );
             }
diff --git a/db/stats/counters.cpp b/db/stats/counters.cpp
index 8e90902..a2d4cfb 100644
--- a/db/stats/counters.cpp
+++ b/db/stats/counters.cpp
@@ -16,7 +16,7 @@
  */
 
 
-#include "stdafx.h"
+#include "pch.h"
 #include "../jsobj.h"
 #include "counters.h"
 
@@ -34,17 +34,17 @@ namespace mongo {
         b.append( "command" , zero );
         _obj = b.obj();
 
-        _insert = (int*)_obj["insert"].value();
-        _query = (int*)_obj["query"].value();
-        _update = (int*)_obj["update"].value();
-        _delete = (int*)_obj["delete"].value();
-        _getmore = (int*)_obj["getmore"].value();
-        _command = (int*)_obj["command"].value();
+        _insert = (AtomicUInt*)_obj["insert"].value();
+        _query = (AtomicUInt*)_obj["query"].value();
+        _update = (AtomicUInt*)_obj["update"].value();
+        _delete = (AtomicUInt*)_obj["delete"].value();
+        _getmore = (AtomicUInt*)_obj["getmore"].value();
+        _command = (AtomicUInt*)_obj["command"].value();
     }
 
     void OpCounters::gotOp( int op , bool isCommand ){
         switch ( op ){
-        case dbInsert: gotInsert(); break;
+        case dbInsert: /*gotInsert();*/ break; // need to handle multi-insert
         case dbQuery: 
             if ( isCommand )
                 gotCommand();
@@ -123,6 +123,24 @@ namespace mongo {
         b.appendNumber( "last_ms" , _last_time );
         b.append("last_finished", _last);
     }
+
+
+    void GenericCounter::hit( const string& name , int count ){
+        scoped_lock lk( _mutex );
+        _counts[name]++;
+    }
+    
+    BSONObj GenericCounter::getObj() {
+        BSONObjBuilder b(128);
+        {
+            mongo::mutex::scoped_lock lk( _mutex );
+            for ( map<string,long long>::iterator i=_counts.begin(); i!=_counts.end(); i++ ){
+                b.appendNumber( i->first , i->second );
+            }
+        }
+        return b.obj();
+    }
+
     
 
     OpCounters globalOpCounters;
diff --git a/db/stats/counters.h b/db/stats/counters.h
index 41c2cd2..2704464 100644
--- a/db/stats/counters.h
+++ b/db/stats/counters.h
@@ -15,8 +15,9 @@
  *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#pragma once
 
-#include "../../stdafx.h"
+#include "../../pch.h"
 #include "../jsobj.h"
 #include "../../util/message.h"
 #include "../../util/processinfo.h"
@@ -32,12 +33,12 @@ namespace mongo {
         
         OpCounters();
 
-        int * getInsert(){ return _insert; }
-        int * getQuery(){ return _query; }
-        int * getUpdate(){ return _update; }
-        int * getDelete(){ return _delete; }
-        int * getGetMore(){ return _getmore; }
-        int * getCommand(){ return _command; }
+        AtomicUInt * getInsert(){ return _insert; }
+        AtomicUInt * getQuery(){ return _query; }
+        AtomicUInt * getUpdate(){ return _update; }
+        AtomicUInt * getDelete(){ return _delete; }
+        AtomicUInt * getGetMore(){ return _getmore; }
+        AtomicUInt * getCommand(){ return _command; }
         
         void gotInsert(){ _insert[0]++; }
         void gotQuery(){ _query[0]++; }
@@ -51,12 +52,12 @@ namespace mongo {
         BSONObj& getObj(){ return _obj; }
     private:
         BSONObj _obj;
-        int * _insert;
-        int * _query;
-        int * _update;
-        int * _delete;
-        int * _getmore;
-        int * _command;
+        AtomicUInt * _insert;
+        AtomicUInt * _query;
+        AtomicUInt * _update;
+        AtomicUInt * _delete;
+        AtomicUInt * _getmore;
+        AtomicUInt * _command;
     };
     
     extern OpCounters globalOpCounters;
@@ -118,4 +119,15 @@ namespace mongo {
     };
 
     extern FlushCounters globalFlushCounters;
+
+
+    class GenericCounter {
+    public:
+        GenericCounter() : _mutex("GenericCounter") { }
+        void hit( const string& name , int count=0 );
+        BSONObj getObj();
+    private:
+        map<string,long long> _counts; // TODO: replace with thread safe map
+        mongo::mutex _mutex;
+    };
 }
diff --git a/db/stats/fine_clock.h b/db/stats/fine_clock.h
new file mode 100644
index 0000000..1f23175
--- /dev/null
+++ b/db/stats/fine_clock.h
@@ -0,0 +1,66 @@
+// fine_clock.h
+
+/**
+*    Copyright (C) 2008 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef DB_STATS_FINE_CLOCK_HEADER
+#define DB_STATS_FINE_CLOCK_HEADER
+
+#include <time.h>  // struct timespec
+
+namespace mongo {
+
+    /**
+     * This is a nano-second precision clock. We're skipping the
+     * harware TSC in favor of clock_gettime() which in some systems
+     * does not involve a trip to the OS (VDSO).
+     *
+     * We're exporting a type WallTime that is and should remain
+     * opaque. The business of getting accurate time is still ongoing
+     * and we may change the internal representation of this class.
+     * (http://lwn.net/Articles/388188/)
+     *
+     * Really, you shouldn't be using this class in hot code paths for
+     * platforms you're not sure whether the overhead is low.
+     */
+    class FineClock{
+    public:
+
+        typedef timespec WallTime;
+
+        static WallTime now(){
+            struct timespec ts;
+            clock_gettime(CLOCK_MONOTONIC, &ts);
+            return ts;
+        }
+
+        static uint64_t diffInNanos( WallTime end, WallTime start ){
+            uint64_t diff;
+            if ( end.tv_nsec < start.tv_nsec ){
+                diff = 1000000000 * ( end.tv_sec - start.tv_sec - 1);
+                diff += 1000000000 + end.tv_nsec - start.tv_nsec;
+            } else {
+                diff = 1000000000 * ( end.tv_sec - start.tv_sec );
+                diff += end.tv_nsec - start.tv_nsec;
+            }
+            return diff;
+        }
+  
+    };
+}
+
+#endif  // DB_STATS_FINE_CLOCK_HEADER
+
diff --git a/db/stats/service_stats.cpp b/db/stats/service_stats.cpp
new file mode 100644
index 0000000..5574ecb
--- /dev/null
+++ b/db/stats/service_stats.cpp
@@ -0,0 +1,68 @@
+// service_stats.cpp
+
+/**
+*    Copyright (C) 2010 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <sstream>
+
+#include "../../util/histogram.h"
+#include "service_stats.h"
+
+namespace mongo {
+
+    using std::ostringstream;
+
+    ServiceStats::ServiceStats(){
+        // Time histogram covers up to 128msec in exponential intervals
+        // starting at 125usec.
+        Histogram::Options timeOpts;
+        timeOpts.numBuckets = 12;
+        timeOpts.bucketSize = 125;
+        timeOpts.exponential = true;
+        _timeHistogram = new Histogram( timeOpts );
+
+        // Space histogram covers up to 1MB in exponentialintervals starting
+        // at 1K.
+        Histogram::Options spaceOpts;
+        spaceOpts.numBuckets = 12;
+        spaceOpts.bucketSize = 1024;
+        spaceOpts.exponential = true;
+        _spaceHistogram = new Histogram( spaceOpts );
+    }
+
+    ServiceStats::~ServiceStats(){
+        delete _timeHistogram;
+        delete _spaceHistogram;
+    }
+
+    void ServiceStats::logResponse( uint64_t duration, uint64_t bytes ){
+        _spinLock.lock();
+        _timeHistogram->insert( duration / 1000 /* in usecs */ );
+        _spaceHistogram->insert( bytes );
+        _spinLock.unlock();
+    }
+
+    string ServiceStats::toHTML() const {
+        ostringstream res ;
+        res << "Cumulative wire stats\n"
+            << "Response times\n" << _timeHistogram->toHTML()
+            << "Response sizes\n" << _spaceHistogram->toHTML()
+            << '\n';
+
+        return res.str();
+    }
+
+}  // mongo
diff --git a/db/stats/service_stats.h b/db/stats/service_stats.h
new file mode 100644
index 0000000..5b0e75f
--- /dev/null
+++ b/db/stats/service_stats.h
@@ -0,0 +1,66 @@
+// service_stats.h
+
+/**
+*    Copyright (C) 2010 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef DB_STATS_SERVICE_STATS_HEADER
+#define DB_STATS_SERVICE_STATS_HEADER
+
+#include <string>
+
+#include "../../util/concurrency/spin_lock.h"
+
+namespace mongo {
+
+    using std::string;
+
+    class Histogram;
+
+    /**
+     * ServiceStats keeps track of the time a request/response message
+     * took inside a service as well as the size of the response
+     * generated.
+     */
+    class ServiceStats {
+    public:
+        ServiceStats();
+        ~ServiceStats();
+
+        /**
+         * Record the 'duration' in microseconds a request/response
+         * message took and the size in bytes of the generated
+         * response.
+         */
+        void logResponse( uint64_t duration, uint64_t bytes );
+
+        /**
+         * Render the histogram as string that can be used inside an
+         * HTML doc.
+         */
+        string toHTML() const;
+
+    private:
+        SpinLock   _spinLock;         // protects state below
+        Histogram* _timeHistogram;
+        Histogram* _spaceHistogram;
+
+        ServiceStats( const ServiceStats& );
+        ServiceStats operator=( const ServiceStats& );
+    };
+
+}  // namespace mongo
+
+#endif  //  DB_STATS_SERVICE_STATS_HEADER
diff --git a/db/stats/snapshots.cpp b/db/stats/snapshots.cpp
index 71ddd72..b439b03 100644
--- a/db/stats/snapshots.cpp
+++ b/db/stats/snapshots.cpp
@@ -16,10 +16,12 @@
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "snapshots.h"
 #include "../client.h"
 #include "../clientcursor.h"
+#include "../dbwebserver.h"
+#include "../../util/mongoutils/html.h"
 
 /**
    handles snapshotting performance metrics and other such things
@@ -55,7 +57,7 @@ namespace mongo {
     }
 
     Snapshots::Snapshots(int n)
-        : _n(n)
+        : _lock("Snapshots"), _n(n)
         , _snapshots(new SnapshotData[n])
         , _loc(0)
         , _stored(0)
@@ -87,19 +89,16 @@ namespace mongo {
 
     void Snapshots::outputLockInfoHTML( stringstream& ss ){
         scoped_lock lk(_lock);
-        ss << "\n<table>";
-        ss << "<tr><th>elapsed(ms)</th><th>% write locked</th></tr>\n";
-        
+        ss << "\n<div>";
         for ( int i=0; i<numDeltas(); i++ ){
             SnapshotDelta d( getPrev(i+1) , getPrev(i) );
-            ss << "<tr>"
-               << "<td>" << ( d.elapsed() / 1000 ) << "</td>"
-               << "<td>" << (unsigned)(100*d.percentWriteLocked()) << "%</td>"
-               << "</tr>"
-                ;
+            unsigned e = (unsigned) d.elapsed() / 1000;
+            ss << (unsigned)(100*d.percentWriteLocked());
+            if( e < 3900 || e > 4100 ) 
+                ss << '(' << e / 1000.0 << "s)";
+            ss << ' ';
         }
-        
-        ss << "</table>\n";
+        ss << "</div>\n";
     }
 
     void SnapshotThread::run(){
@@ -122,8 +121,6 @@ namespace mongo {
                         log() << "cpu: elapsed:" << (elapsed/1000) <<"  writelock: " << (int)(100*d.percentWriteLocked()) << "%" << endl;
                     }
 
-                    // TODO: this should really be somewhere else, like in a special ClientCursor thread
-                    ClientCursor::idleTimeReport( (unsigned)(elapsed/1000) );
                 }
 
                 prev = s;
@@ -139,6 +136,89 @@ namespace mongo {
         client.shutdown();
     }
 
+    using namespace mongoutils::html;
+
+    class WriteLockStatus : public WebStatusPlugin {
+    public:
+        WriteLockStatus() : WebStatusPlugin( "write lock" , 51 , "% time in write lock, by 4 sec periods" ){}
+        virtual void init(){}
+
+        virtual void run( stringstream& ss ){
+            statsSnapshots.outputLockInfoHTML( ss );
+
+            ss << "<a "
+                  "href=\"http://www.mongodb.org/pages/viewpage.action?pageId=7209296\" " 
+                  "title=\"snapshot: was the db in the write lock when this page was generated?\">";
+            ss << "write locked now:</a> " << (dbMutex.info().isLocked() ? "true" : "false") << "\n";
+        }
+
+    } writeLockStatus;
+
+    class DBTopStatus : public WebStatusPlugin {
+    public:
+        DBTopStatus() : WebStatusPlugin( "dbtop" , 50 , "(occurences|percent of elapsed)" ){}
+
+        void display( stringstream& ss , double elapsed , const Top::UsageData& usage ){
+            ss << "<td>";
+            ss << usage.count;
+            ss << "</td><td>";
+            double per = 100 * ((double)usage.time)/elapsed;
+            ss << setprecision(4) << fixed << per << "%";
+            ss << "</td>";
+        }
+
+        void display( stringstream& ss , double elapsed , const string& ns , const Top::CollectionData& data ){
+            if ( ns != "GLOBAL" && data.total.count == 0 )
+                return;
+            ss << "<tr><th>" << ns << "</th>";
+            
+            display( ss , elapsed , data.total );
+
+            display( ss , elapsed , data.readLock );
+            display( ss , elapsed , data.writeLock );
+
+            display( ss , elapsed , data.queries );
+            display( ss , elapsed , data.getmore );
+            display( ss , elapsed , data.insert );
+            display( ss , elapsed , data.update );
+            display( ss , elapsed , data.remove );
+            
+            ss << "</tr>\n";
+        }
+
+        void run( stringstream& ss ){
+            auto_ptr<SnapshotDelta> delta = statsSnapshots.computeDelta();
+            if ( ! delta.get() )
+                return;
+            
+            ss << "<table border=1 cellpadding=2 cellspacing=0>";
+            ss << "<tr align='left'><th>";
+            ss << a("http://www.mongodb.org/display/DOCS/Developer+FAQ#DeveloperFAQ-What%27sa%22namespace%22%3F", "namespace") << 
+                "NS</a></th>"
+                "<th colspan=2>total</th>"
+                "<th colspan=2>Reads</th>"
+                "<th colspan=2>Writes</th>"
+                "<th colspan=2>Queries</th>"
+                "<th colspan=2>GetMores</th>"
+                "<th colspan=2>Inserts</th>"
+                "<th colspan=2>Updates</th>"
+                "<th colspan=2>Removes</th>";
+            ss << "</tr>\n";
+            
+            display( ss , (double) delta->elapsed() , "GLOBAL" , delta->globalUsageDiff() );
+            
+            Top::UsageMap usage = delta->collectionUsageDiff();
+            for ( Top::UsageMap::iterator i=usage.begin(); i != usage.end(); i++ ){
+                display( ss , (double) delta->elapsed() , i->first , i->second );
+            }
+            
+            ss << "</table>";
+        
+        }
+
+        virtual void init(){}
+    } dbtopStatus;
+
     Snapshots statsSnapshots;
     SnapshotThread snapshotThread;    
 }
diff --git a/db/stats/snapshots.h b/db/stats/snapshots.h
index 542318a..6d8e23d 100644
--- a/db/stats/snapshots.h
+++ b/db/stats/snapshots.h
@@ -17,7 +17,7 @@
 */
 
 #pragma once
-#include "../../stdafx.h"
+#include "../../pch.h"
 #include "../jsobj.h"
 #include "top.h"
 #include "../../util/background.h"
@@ -103,6 +103,7 @@ namespace mongo {
 
     class SnapshotThread : public BackgroundJob {
     public:
+        string name() { return "snapshot"; }
         void run();
     };
     
diff --git a/db/stats/top.cpp b/db/stats/top.cpp
index 462bfe6..3e65261 100644
--- a/db/stats/top.cpp
+++ b/db/stats/top.cpp
@@ -16,7 +16,7 @@
  */
 
 
-#include "stdafx.h"
+#include "pch.h"
 #include "top.h"
 #include "../../util/message.h"
 #include "../commands.h"
@@ -96,10 +96,11 @@ namespace mongo {
         case dbDelete:
             c.remove.inc( micros );
             break;
+        case dbKillCursors:
+            break;
         case opReply: 
         case dbMsg:
-        case dbKillCursors:
-            //log() << "unexpected op in Top::record: " << op << endl;
+            log() << "unexpected op in Top::record: " << op << endl;
             break;
         default:
             log() << "unknown op in Top::record: " << op << endl;
@@ -148,14 +149,14 @@ namespace mongo {
 
     class TopCmd : public Command {
     public:
-        TopCmd() : Command( "top" ){}
+        TopCmd() : Command( "top", true ){}
 
-        virtual bool slaveOk(){ return true; }
-        virtual bool adminOnly(){ return true; }
-        virtual LockType locktype(){ return READ; } 
+        virtual bool slaveOk() const { return true; }
+        virtual bool adminOnly() const { return true; }
+        virtual LockType locktype() const { return READ; } 
         virtual void help( stringstream& help ) const { help << "usage by collection"; }
 
-        virtual bool run(const char *ns, BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl){
+        virtual bool run(const string& , BSONObj& cmdObj, string& errmsg, BSONObjBuilder& result, bool fromRepl){
             {
                 BSONObjBuilder b( result.subobjStart( "totals" ) );
                 Top::global.append( b );
@@ -175,7 +176,7 @@ namespace mongo {
     TopOld::UsageMap TopOld::_snapshotB;
     TopOld::UsageMap &TopOld::_snapshot = TopOld::_snapshotA;
     TopOld::UsageMap &TopOld::_nextSnapshot = TopOld::_snapshotB;
-    mongo::mutex TopOld::topMutex;
+    mongo::mutex TopOld::topMutex("topMutex");
 
 
 }
diff --git a/db/stats/top.h b/db/stats/top.h
index 8dab3b0..135e8f8 100644
--- a/db/stats/top.h
+++ b/db/stats/top.h
@@ -19,7 +19,7 @@
 
 #include <boost/date_time/posix_time/posix_time.hpp>
 #undef assert
-#define assert xassert
+#define assert MONGO_assert
 
 namespace mongo {
 
@@ -29,6 +29,8 @@ namespace mongo {
     class Top {
 
     public:
+        Top() : _lock("Top") { }
+
         class UsageData {
         public:
             UsageData() : time(0) , count(0){}
diff --git a/db/storage.cpp b/db/storage.cpp
index 7ddfc65..63e7639 100644
--- a/db/storage.cpp
+++ b/db/storage.cpp
@@ -16,28 +16,30 @@
  */
 
 
-#include "stdafx.h"
+#include "pch.h"
 #include "pdfile.h"
-#include "reccache.h"
+//#include "reccache.h"
 #include "rec.h"
 #include "db.h"
 
 namespace mongo {
 
-void writerThread();
-
-#if defined(_RECSTORE)
-    static int inited;
-#endif
-
 // pick your store for indexes by setting this typedef
 // this doesn't need to be an ifdef, we can make it dynamic
 #if defined(_RECSTORE)
 RecStoreInterface *btreeStore = new CachedBasicRecStore();
 #else
-RecStoreInterface *btreeStore = new MongoMemMapped_RecStore();
+MongoMemMapped_RecStore *btreeStore = new MongoMemMapped_RecStore();
 #endif
 
+#if 0
+
+#if defined(_RECSTORE)
+    static int inited;
+#endif
+
+void writerThread();
+
 void BasicRecStore::init(const char *fn, unsigned recsize)
 { 
     massert( 10394 ,  "compile packing problem recstore?", sizeof(RecStoreHeader) == 8192);
@@ -74,4 +76,6 @@ void BasicRecStore::init(const char *fn, unsigned recsize)
 #endif
 }
 
+#endif
+
 }
diff --git a/db/tests.cpp b/db/tests.cpp
index 81cc363..1218f1b 100644
--- a/db/tests.cpp
+++ b/db/tests.cpp
@@ -19,7 +19,7 @@
    unit test & such
 */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "../util/mmap.h"
 
 namespace mongo {
diff --git a/db/update.cpp b/db/update.cpp
index 6dcfc78..cbf93ba 100644
--- a/db/update.cpp
+++ b/db/update.cpp
@@ -16,13 +16,14 @@
  *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "stdafx.h"
+#include "pch.h"
 #include "query.h"
 #include "pdfile.h"
 #include "jsobjmanipulator.h"
 #include "queryoptimizer.h"
 #include "repl.h"
 #include "update.h"
+#include "btree.h"
 
 //#define DEBUGUPDATE(x) cout << x << endl;
 #define DEBUGUPDATE(x)
@@ -236,7 +237,7 @@ namespace mongo {
                 }
 
                 while( i.more() ) {
-                    bb.appendAs( i.next() , bb.numStr( n - 1 ).c_str() );
+                    bb.appendAs( i.next() , bb.numStr( n - 1 ) );
                     n++;
                 }
             }
@@ -306,8 +307,10 @@ namespace mongo {
     }
 
     auto_ptr<ModSetState> ModSet::prepare(const BSONObj &obj) const {
+        DEBUGUPDATE( "\t start prepare" );
         ModSetState * mss = new ModSetState( obj );
-
+        
+        
         // Perform this check first, so that we don't leave a partially modified object on uassert.
         for ( ModHolder::const_iterator i = _mods.begin(); i != _mods.end(); ++i ) {
             DEBUGUPDATE( "\t\t prepare : " << i->first );
@@ -407,9 +410,41 @@ namespace mongo {
                 mss->amIInPlacePossible( false );
             }
         }
+
+        DEBUGUPDATE( "\t mss\n" << mss->toString() << "\t--" );
         
         return auto_ptr<ModSetState>( mss );
     }
+
+    void ModState::appendForOpLog( BSONObjBuilder& b ) const {
+        if ( incType ){
+            DEBUGUPDATE( "\t\t\t\t\t appendForOpLog inc fieldname: " << m->fieldName << " short:" << m->shortFieldName );
+            BSONObjBuilder bb( b.subobjStart( "$set" ) );
+            appendIncValue( bb , true );
+            bb.done();
+            return;
+        }
+        
+        const char * name = fixedOpName ? fixedOpName : Mod::modNames[op()];
+
+        DEBUGUPDATE( "\t\t\t\t\t appendForOpLog name:" << name << " fixed: " << fixed << " fn: " << m->fieldName );
+
+        BSONObjBuilder bb( b.subobjStart( name ) );
+        if ( fixed )
+            bb.appendAs( *fixed , m->fieldName );
+        else
+            bb.appendAs( m->elt , m->fieldName );
+        bb.done();
+    }
+
+    string ModState::toString() const {
+        stringstream ss;
+        if ( fixedOpName )
+            ss << " fixedOpName: " << fixedOpName;
+        if ( fixed )
+            ss << " fixed: " << fixed;
+        return ss.str();
+    }
     
     void ModSetState::applyModsInPlace() {
         for ( ModStateHolder::iterator i = _mods.begin(); i != _mods.end(); ++i ) {
@@ -492,7 +527,7 @@ namespace mongo {
             string field = root + e.fieldName();
             FieldCompareResult cmp = compareDottedFieldNames( m->second.m->fieldName , field );
 
-            DEBUGUPDATE( "\t\t\t" << field << "\t" << m->second.m->fieldName << "\t" << cmp );
+            DEBUGUPDATE( "\t\t\t field:" << field << "\t mod:" << m->second.m->fieldName << "\t cmp:" << cmp );
             
             switch ( cmp ){
                 
@@ -518,15 +553,18 @@ namespace mongo {
                 continue;
             }
             case LEFT_BEFORE: // Mod on a field that doesn't exist
+                DEBUGUPDATE( "\t\t\t\t creating new field for: " << m->second.m->fieldName );
                 _appendNewFromMods( root , m->second , b , onedownseen );
                 m++;
                 continue;
             case SAME:
+                DEBUGUPDATE( "\t\t\t\t applying mod on: " << m->second.m->fieldName );
                 m->second.apply( b , e );
                 e = es.next();
                 m++;
                 continue;
             case RIGHT_BEFORE: // field that doesn't have a MOD
+                DEBUGUPDATE( "\t\t\t\t just copying" );
                 b.append( e ); // if array, ignore field name
                 e = es.next();
                 continue;
@@ -540,12 +578,14 @@ namespace mongo {
         
         // finished looping the mods, just adding the rest of the elements
         while ( e.type() ){
+            DEBUGUPDATE( "\t\t\t copying: " << e.fieldName() );
             b.append( e );  // if array, ignore field name
             e = es.next();
         }
         
         // do mods that don't have fields already
         for ( ; m != mend; m++ ){
+            DEBUGUPDATE( "\t\t\t\t appending from mod at end: " << m->second.m->fieldName );
             _appendNewFromMods( root , m->second , b , onedownseen );
         }
     }
@@ -556,6 +596,14 @@ namespace mongo {
         return b.obj();
     }
 
+    string ModSetState::toString() const {
+        stringstream ss;
+        for ( ModStateHolder::const_iterator i=_mods.begin(); i!=_mods.end(); ++i ){
+            ss << "\t\t" << i->first << "\t" << i->second.toString() << "\n";
+        }
+        return ss.str();
+    }
+
     BSONObj ModSet::createNewFromQuery( const BSONObj& query ){
         BSONObj newObj;
 
@@ -565,6 +613,8 @@ namespace mongo {
             BSONObjIteratorSorted i( query );
             while ( i.more() ){
                 BSONElement e = i.next();
+                if ( e.fieldName()[0] == '$' ) // for $atomic and anything else we add
+                    continue;
 
                 if ( e.type() == Object && e.embeddedObject().firstElement().fieldName()[0] == '$' ){
                     // this means this is a $gt type filter, so don't make part of the new object
@@ -610,6 +660,7 @@ namespace mongo {
             
             uassert( 10147 ,  "Invalid modifier specified" + string( fn ), e.type() == Object );
             BSONObj j = e.embeddedObject();
+            DEBUGUPDATE( "\t" << j );
             
             BSONObjIterator jt(j);
             Mod::Op op = opFromStr( fn );
@@ -622,7 +673,7 @@ namespace mongo {
                 uassert( 10148 ,  "Mod on _id not allowed", strcmp( fieldName, "_id" ) != 0 );
                 uassert( 10149 ,  "Invalid mod field name, may not end in a period", fieldName[ strlen( fieldName ) - 1 ] != '.' );
                 uassert( 10150 ,  "Field name duplication not allowed with modifiers", ! haveModForField( fieldName ) );
-                uassert( 10151 ,  "have conflict mod" , ! haveConflictingMod( fieldName ) );
+                uassert( 10151 ,  "have conflicting mods in update" , ! haveConflictingMod( fieldName ) );
                 uassert( 10152 ,  "Modifier $inc allowed for numbers only", f.isNumber() || op != Mod::INC );
                 uassert( 10153 ,  "Modifier $pushAll/pullAll allowed for arrays only", f.type() == Array || ( op != Mod::PUSH_ALL && op != Mod::PULL_ALL ) );
                 
@@ -639,7 +690,7 @@ namespace mongo {
 
                 _mods[m.fieldName] = m;
 
-                DEBUGUPDATE( "\t\t " << fieldName << "\t" << _hasDynamicArray );
+                DEBUGUPDATE( "\t\t " << fieldName << "\t" << m.fieldName << "\t" << _hasDynamicArray );
             }
         }
 
@@ -677,54 +728,152 @@ namespace mongo {
         }
     }
     
-    class UpdateOp : public QueryOp {
+    class UpdateOp : public MultiCursor::CursorOp {
     public:
-        UpdateOp() : _nscanned() {}
-        virtual void init() {
-            BSONObj pattern = qp().query();
-            _c.reset( qp().newCursor().release() );
-            if ( ! _c->ok() )
+        UpdateOp( bool hasPositionalField ) : _nscanned(), _hasPositionalField( hasPositionalField ){}
+        virtual void _init() {
+            _c = qp().newCursor();
+            if ( ! _c->ok() ) {
                 setComplete();
-            else
-                _matcher.reset( new CoveredIndexMatcher( pattern, qp().indexKey() ) );
+            }
         }
+        virtual bool prepareToYield() {
+            if ( ! _cc ) {
+                _cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , _c , qp().ns() ) );
+            }
+            return _cc->prepareToYield( _yieldData );
+        }        
+        virtual void recoverFromYield() {
+            if ( !ClientCursor::recoverFromYield( _yieldData ) ) {
+                _c.reset();
+                _cc.reset();
+                massert( 13339, "cursor dropped during update", false );
+            }
+        }        
         virtual void next() {
             if ( ! _c->ok() ) {
                 setComplete();
                 return;
             }
             _nscanned++;
-            if ( _matcher->matches(_c->currKey(), _c->currLoc(), &_details ) ) {
+            if ( matcher()->matches(_c->currKey(), _c->currLoc(), &_details ) ) {
                 setComplete();
                 return;
             }
             _c->advance();
         }
-        bool curMatches(){
-            return _matcher->matches(_c->currKey(), _c->currLoc() , &_details );
-        }
+
         virtual bool mayRecordPlan() const { return false; }
-        virtual QueryOp *clone() const {
-            return new UpdateOp();
+        virtual QueryOp *_createChild() const {
+            return new UpdateOp( _hasPositionalField );
         }
-        shared_ptr< Cursor > c() { return _c; }
-        long long nscanned() const { return _nscanned; }
-        MatchDetails& getMatchDetails(){ return _details; }
+        // already scanned to the first match, so return _c
+        virtual shared_ptr< Cursor > newCursor() const { return _c; }
+        virtual bool alwaysUseRecord() const { return _hasPositionalField; }
     private:
         shared_ptr< Cursor > _c;
         long long _nscanned;
-        auto_ptr< CoveredIndexMatcher > _matcher;
+        bool _hasPositionalField;
         MatchDetails _details;
+        ClientCursor::CleanupPointer _cc;
+        ClientCursor::YieldData _yieldData;
     };
 
-    
-    UpdateResult updateObjects(const char *ns, const BSONObj& updateobj, BSONObj patternOrig, bool upsert, bool multi, bool logop , OpDebug& debug ) {
+    static void checkTooLarge(const BSONObj& newObj) {
+        uassert( 12522 , "$ operator made object too large" , newObj.objsize() <= ( 4 * 1024 * 1024 ) );
+    }
+
+    /* note: this is only (as-is) called for 
+
+             - not multi
+             - not mods is indexed
+             - not upsert
+    */
+    static UpdateResult _updateById(bool isOperatorUpdate, int idIdxNo, ModSet *mods, int profile, NamespaceDetails *d, 
+                                    NamespaceDetailsTransient *nsdt,
+                                    bool god, const char *ns, 
+                                    const BSONObj& updateobj, BSONObj patternOrig, bool logop, OpDebug& debug) 
+    {
+        DiskLoc loc;
+        {
+            IndexDetails& i = d->idx(idIdxNo);
+            BSONObj key = i.getKeyFromQuery( patternOrig );
+            loc = i.head.btree()->findSingle(i, i.head, key);
+            if( loc.isNull() ) { 
+                // no upsert support in _updateById yet, so we are done.
+                return UpdateResult(0, 0, 0);
+            }
+        }
+
+        Record *r = loc.rec();
+                
+        /* look for $inc etc.  note as listed here, all fields to inc must be this type, you can't set some
+           regular ones at the moment. */
+        if ( isOperatorUpdate ) {                   
+            const BSONObj& onDisk = loc.obj();                    
+            auto_ptr<ModSetState> mss = mods->prepare( onDisk );
+                    
+            if( mss->canApplyInPlace() ) {
+                mss->applyModsInPlace();                    
+                DEBUGUPDATE( "\t\t\t updateById doing in place update" );
+                /*if ( profile )
+                    ss << " fastmod "; */
+            } 
+            else {
+                BSONObj newObj = mss->createNewFromMods();
+                checkTooLarge(newObj);
+                bool changedId;
+                assert(nsdt);
+                DiskLoc newLoc = theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , newObj.objdata(), newObj.objsize(), debug, changedId);                        
+            }
+                    
+            if ( logop ) {
+                DEV assert( mods->size() );
+                 
+                BSONObj pattern = patternOrig;
+                if ( mss->haveArrayDepMod() ) {
+                    BSONObjBuilder patternBuilder;
+                    patternBuilder.appendElements( pattern );
+                    mss->appendSizeSpecForArrayDepMods( patternBuilder );
+                    pattern = patternBuilder.obj();                        
+                }
+                        
+                if( mss->needOpLogRewrite() ) {
+                    DEBUGUPDATE( "\t rewrite update: " << mss->getOpLogRewrite() );
+                    logOp("u", ns, mss->getOpLogRewrite() , &pattern );
+                }
+                else {
+                    logOp("u", ns, updateobj, &pattern );
+                }
+            }
+            return UpdateResult( 1 , 1 , 1);
+        } // end $operator update
+                
+        // regular update
+        BSONElementManipulator::lookForTimestamps( updateobj );
+        checkNoMods( updateobj );
+        bool changedId = false;
+        assert(nsdt);
+        theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , updateobj.objdata(), updateobj.objsize(), debug, changedId);
+        if ( logop ) {
+            if ( !changedId ) {
+                logOp("u", ns, updateobj, &patternOrig );
+            } else {
+                logOp("d", ns, patternOrig );
+                logOp("i", ns, updateobj );                    
+            }
+        }
+        return UpdateResult( 1 , 0 , 1 );
+    }
+ 
+    UpdateResult _updateObjects(bool god, const char *ns, const BSONObj& updateobj, BSONObj patternOrig, bool upsert, bool multi, bool logop , OpDebug& debug, RemoveSaver* rs ) {
         DEBUGUPDATE( "update: " << ns << " update: " << updateobj << " query: " << patternOrig << " upsert: " << upsert << " multi: " << multi );
-        int profile = cc().database()->profile;
+        Client& client = cc();
+        int profile = client.database()->profile;
         StringBuilder& ss = debug.str;
 
         if ( logLevel > 2 )
-            ss << " update: " << updateobj;
+            ss << " update: " << updateobj.toString();
         
         /* idea with these here it to make them loop invariant for multi updates, and thus be a bit faster for that case */
         /* NOTE: when yield() is added herein, these must be refreshed after each call to yield! */
@@ -732,12 +881,6 @@ namespace mongo {
         NamespaceDetailsTransient *nsdt = &NamespaceDetailsTransient::get_w(ns);
         /* end note */
         
-        uassert( 10155 , "cannot update reserved $ collection", strchr(ns, '$') == 0 );
-        if ( strstr(ns, ".system.") ) {
-            /* dm: it's very important that system.indexes is never updated as IndexDetails has pointers into it */
-            uassert( 10156 , "cannot update system collection", legalClientSystemNS( ns , true ) );
-        }
-
         auto_ptr<ModSet> mods;
         bool isOperatorUpdate = updateobj.firstElement().fieldName()[0] == '$';
         int modsIsIndexed = false; // really the # of indexes
@@ -753,31 +896,63 @@ namespace mongo {
             modsIsIndexed = mods->isIndexed();
         }
 
+        if( !upsert && !multi && isSimpleIdQuery(patternOrig) && d && !modsIsIndexed ) {
+            int idxNo = d->findIdIndex();
+            if( idxNo >= 0 ) {
+                ss << " byid ";
+                return _updateById(isOperatorUpdate, idxNo, mods.get(), profile, d, nsdt, god, ns, updateobj, patternOrig, logop, debug);
+            }
+        }
+
         set<DiskLoc> seenObjects;
         
-        QueryPlanSet qps( ns, patternOrig, BSONObj() );
-        UpdateOp original;
-        shared_ptr< UpdateOp > u = qps.runOp( original );
-        massert( 10401 ,  u->exceptionMessage(), u->complete() );
-        shared_ptr< Cursor > c = u->c();
         int numModded = 0;
+        long long nscanned = 0;
+        MatchDetails details;
+        shared_ptr< MultiCursor::CursorOp > opPtr( new UpdateOp( mods.get() && mods->hasDynamicArray() ) );
+        shared_ptr< MultiCursor > c( new MultiCursor( ns, patternOrig, BSONObj(), opPtr, true ) );
+        
+        auto_ptr<ClientCursor> cc;
+            
         while ( c->ok() ) {
-            if ( numModded > 0 && ! u->curMatches() ){
+            nscanned++;
+
+            bool atomic = c->matcher()->docMatcher().atomic();
+                
+            // May have already matched in UpdateOp, but do again to get details set correctly
+            if ( ! c->matcher()->matches( c->currKey(), c->currLoc(), &details ) ){
                 c->advance();
+                    
+                if ( nscanned % 256 == 0 && ! atomic ){
+                    if ( cc.get() == 0 ) {
+                        shared_ptr< Cursor > cPtr = c;
+                        cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , cPtr , ns ) );
+                    }
+                    if ( ! cc->yield() ){
+                        cc.release();
+                        // TODO should we assert or something?
+                        break;
+                    }
+                    if ( !c->ok() ) {
+                        break;
+                    }
+                }
                 continue;
             }
+            
             Record *r = c->_current();
             DiskLoc loc = c->currLoc();
-
+                
+            // TODO Maybe this is unnecessary since we have seenObjects
             if ( c->getsetdup( loc ) ){
                 c->advance();
                 continue;
             }
-
+                
             BSONObj js(r);
-            
+                
             BSONObj pattern = patternOrig;
-
+                
             if ( logop ) {
                 BSONObjBuilder idPattern;
                 BSONElement id;
@@ -793,43 +968,47 @@ namespace mongo {
                     uassert( 10157 ,  "multi-update requires all modified objects to have an _id" , ! multi );
                 }
             }
-            
+                
             if ( profile )
-                ss << " nscanned:" << u->nscanned();
-            
+                ss << " nscanned:" << nscanned;
+                
             /* look for $inc etc.  note as listed here, all fields to inc must be this type, you can't set some
-               regular ones at the moment. */
+                regular ones at the moment. */
             if ( isOperatorUpdate ) {
-                
+                    
                 if ( multi ){
                     c->advance(); // go to next record in case this one moves
                     if ( seenObjects.count( loc ) )
                         continue;
                 }
-                
+                    
                 const BSONObj& onDisk = loc.obj();
-                
+                    
                 ModSet * useMods = mods.get();
                 bool forceRewrite = false;
-
+                    
                 auto_ptr<ModSet> mymodset;
-                if ( u->getMatchDetails().elemMatchKey && mods->hasDynamicArray() ){
-                    useMods = mods->fixDynamicArray( u->getMatchDetails().elemMatchKey );
+                if ( details.elemMatchKey && mods->hasDynamicArray() ){
+                    useMods = mods->fixDynamicArray( details.elemMatchKey );
                     mymodset.reset( useMods );
                     forceRewrite = true;
                 }
-
-                     
+                    
                 auto_ptr<ModSetState> mss = useMods->prepare( onDisk );
-                
+                    
                 bool indexHack = multi && ( modsIsIndexed || ! mss->canApplyInPlace() );
-                
-                if ( indexHack )
-                    c->noteLocation();
-                
+                    
+                if ( indexHack ){
+                    if ( cc.get() )
+                        cc->updateLocation();
+                    else
+                        c->noteLocation();
+                }
+                    
                 if ( modsIsIndexed <= 0 && mss->canApplyInPlace() ){
                     mss->applyModsInPlace();// const_cast<BSONObj&>(onDisk) );
-
+                    
+                    DEBUGUPDATE( "\t\t\t doing in place update" );
                     if ( profile )
                         ss << " fastmod ";
                     
@@ -838,26 +1017,30 @@ namespace mongo {
                     }
                 } 
                 else {
+                    if ( rs )
+                        rs->goingToDelete( onDisk );
+
                     BSONObj newObj = mss->createNewFromMods();
-                    uassert( 12522 , "$ operator made object too large" , newObj.objsize() <= ( 4 * 1024 * 1024 ) );
-                    DiskLoc newLoc = theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , newObj.objdata(), newObj.objsize(), debug);
+                    checkTooLarge(newObj);
+                    bool changedId;
+                    DiskLoc newLoc = theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , newObj.objdata(), newObj.objsize(), debug, changedId);
                     if ( newLoc != loc || modsIsIndexed ) {
                         // object moved, need to make sure we don' get again
                         seenObjects.insert( newLoc );
                     }
                         
                 }
-                
+                    
                 if ( logop ) {
                     DEV assert( mods->size() );
-
+                        
                     if ( mss->haveArrayDepMod() ) {
                         BSONObjBuilder patternBuilder;
                         patternBuilder.appendElements( pattern );
                         mss->appendSizeSpecForArrayDepMods( patternBuilder );
                         pattern = patternBuilder.obj();                        
                     }
-                    
+                        
                     if ( forceRewrite || mss->needOpLogRewrite() ){
                         DEBUGUPDATE( "\t rewrite update: " << mss->getOpLogRewrite() );
                         logOp("u", ns, mss->getOpLogRewrite() , &pattern );
@@ -868,19 +1051,42 @@ namespace mongo {
                 }
                 numModded++;
                 if ( ! multi )
-                    break;
+                    return UpdateResult( 1 , 1 , numModded );
                 if ( indexHack )
                     c->checkLocation();
+                    
+                if ( nscanned % 64 == 0 && ! atomic ){
+                    if ( cc.get() == 0 ) {
+                        shared_ptr< Cursor > cPtr = c;
+                        cc.reset( new ClientCursor( QueryOption_NoCursorTimeout , cPtr , ns ) );
+                    }
+                    if ( ! cc->yield() ){
+                        cc.release();
+                        break;
+                    }
+                    if ( !c->ok() ) {
+                        break;
+                    }
+                }
+                
                 continue;
             } 
-            
+                
             uassert( 10158 ,  "multi update only works with $ operators" , ! multi );
-
+                
             BSONElementManipulator::lookForTimestamps( updateobj );
             checkNoMods( updateobj );
-            theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , updateobj.objdata(), updateobj.objsize(), debug);
-            if ( logop )
-                logOp("u", ns, updateobj, &pattern );
+            bool changedId = false;
+            theDataFileMgr.updateRecord(ns, d, nsdt, r, loc , updateobj.objdata(), updateobj.objsize(), debug, changedId, god);
+            if ( logop ) {
+                DEV if( god ) log() << "REALLY??" << endl; // god doesn't get logged, this would be bad.
+                if ( !changedId ) {
+                    logOp("u", ns, updateobj, &pattern );
+                } else {
+                    logOp("d", ns, pattern );
+                    logOp("i", ns, updateobj );                    
+                }
+            }
             return UpdateResult( 1 , 0 , 1 );
         }
         
@@ -889,7 +1095,7 @@ namespace mongo {
 
         
         if ( profile )
-            ss << " nscanned:" << u->nscanned();
+            ss << " nscanned:" << nscanned;
         
         if ( upsert ) {
             if ( updateobj.firstElement().fieldName()[0] == '$' ) {
@@ -897,24 +1103,32 @@ namespace mongo {
                 BSONObj newObj = mods->createNewFromQuery( patternOrig );
                 if ( profile )
                     ss << " fastmodinsert ";
-                theDataFileMgr.insert(ns, newObj);
-                if ( profile )
-                    ss << " fastmodinsert ";
+                theDataFileMgr.insertWithObjMod(ns, newObj, god);
                 if ( logop )
                     logOp( "i", ns, newObj );
-                return UpdateResult( 0 , 1 , 1 );
+                
+                return UpdateResult( 0 , 1 , 1 , newObj );
             }
             uassert( 10159 ,  "multi update only works with $ operators" , ! multi );
             checkNoMods( updateobj );
             if ( profile )
                 ss << " upsert ";
             BSONObj no = updateobj;
-            theDataFileMgr.insert(ns, no);
+            theDataFileMgr.insertWithObjMod(ns, no, god);
             if ( logop )
                 logOp( "i", ns, no );
-            return UpdateResult( 0 , 0 , 1 );
+            return UpdateResult( 0 , 0 , 1 , no );
         }
         return UpdateResult( 0 , 0 , 0 );
     }
-    
+ 
+    UpdateResult updateObjects(const char *ns, const BSONObj& updateobj, BSONObj patternOrig, bool upsert, bool multi, bool logop , OpDebug& debug ) {
+        uassert( 10155 , "cannot update reserved $ collection", strchr(ns, '$') == 0 );
+        if ( strstr(ns, ".system.") ) {
+            /* dm: it's very important that system.indexes is never updated as IndexDetails has pointers into it */
+            uassert( 10156 , "cannot update system collection", legalClientSystemNS( ns , true ) );
+        }
+        return _updateObjects(false, ns, updateobj, patternOrig, upsert, multi, logop, debug);
+    }
+   
 }
diff --git a/db/update.h b/db/update.h
index bfec7cd..b7950de 100644
--- a/db/update.h
+++ b/db/update.h
@@ -16,7 +16,7 @@
  *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "../stdafx.h"
+#include "../pch.h"
 #include "jsobj.h"
 #include "../util/embedded_builder.h"
 #include "matcher.h"
@@ -129,6 +129,15 @@ namespace mongo {
                 StringBuilder buf( fullName.size() + 1 );
                 for ( size_t i=0; i<fullName.size(); i++ ){
                     char c = fullName[i];
+                    
+                    if ( c == '$' && 
+                         i > 0 && fullName[i-1] == '.' &&
+                         i+1<fullName.size() && 
+                         fullName[i+1] == '.' ){
+                        i++;
+                        continue;
+                    }
+
                     buf << c;
 
                     if ( c != '.' )
@@ -410,23 +419,7 @@ namespace mongo {
             }
         }
         
-        void appendForOpLog( BSONObjBuilder& b ) const {
-            if ( incType ){
-                BSONObjBuilder bb( b.subobjStart( "$set" ) );
-                appendIncValue( bb , true );
-                bb.done();
-                return;
-            }
-            
-            const char * name = fixedOpName ? fixedOpName : Mod::modNames[op()];
-
-            BSONObjBuilder bb( b.subobjStart( name ) );
-            if ( fixed )
-                bb.appendAs( *fixed , m->fieldName );
-            else
-                bb.appendAs( m->elt , m->fieldName );
-            bb.done();
-        }
+        void appendForOpLog( BSONObjBuilder& b ) const;
 
         template< class Builder >
         void apply( Builder& b , BSONElement in ){
@@ -436,7 +429,7 @@ namespace mongo {
         template< class Builder >
         void appendIncValue( Builder& b , bool useFullName ) const {
             const char * n = useFullName ? m->fieldName : m->shortFieldName;
-            
+
             switch ( incType ){
             case NumberDouble:
                 b.append( n , incdouble ); break;
@@ -448,6 +441,8 @@ namespace mongo {
                 assert(0);
             }
         }
+
+        string toString() const;
     };
     
     /**
@@ -578,6 +573,7 @@ namespace mongo {
             }
         }
 
+        string toString() const;
 
         friend class ModSet;
     };