summaryrefslogtreecommitdiff
path: root/util/file_allocator.h
diff options
context:
space:
mode:
Diffstat (limited to 'util/file_allocator.h')
-rw-r--r--util/file_allocator.h278
1 files changed, 53 insertions, 225 deletions
diff --git a/util/file_allocator.h b/util/file_allocator.h
index b0267d9..6cc7b2d 100644
--- a/util/file_allocator.h
+++ b/util/file_allocator.h
@@ -16,246 +16,74 @@
*/
#include "../pch.h"
-#include <fcntl.h>
-#include <errno.h>
-#if defined(__freebsd__) || defined(__openbsd__)
-#include <sys/stat.h>
-#endif
-
-#ifndef O_NOATIME
-#define O_NOATIME 0
-#endif
namespace mongo {
- /* Handles allocation of contiguous files on disk. Allocation may be
- requested asynchronously or synchronously.
- */
- class FileAllocator {
- /* The public functions may not be called concurrently. The allocation
- functions may be called multiple times per file, but only the first
- size specified per file will be used.
+ /*
+ * Handles allocation of contiguous files on disk. Allocation may be
+ * requested asynchronously or synchronously.
+ * singleton
+ */
+ class FileAllocator : boost::noncopyable {
+ /*
+ * The public functions may not be called concurrently. The allocation
+ * functions may be called multiple times per file, but only the first
+ * size specified per file will be used.
*/
public:
-#if !defined(_WIN32)
- FileAllocator() : pendingMutex_("FileAllocator"), failed_() {}
-#endif
- void start() {
-#if !defined(_WIN32)
- Runner r( *this );
- boost::thread t( r );
-#endif
- }
- // May be called if file exists. If file exists, or its allocation has
- // been requested, size is updated to match existing file size.
- void requestAllocation( const string &name, long &size ) {
- /* Some of the system calls in the file allocator don't work in win,
- so no win support - 32 or 64 bit. Plus we don't seem to need preallocation
- on windows anyway as we don't have to pre-zero the file there.
- */
-#if !defined(_WIN32)
- scoped_lock lk( pendingMutex_ );
- if ( failed_ )
- return;
- long oldSize = prevSize( name );
- if ( oldSize != -1 ) {
- size = oldSize;
- return;
- }
- pending_.push_back( name );
- pendingSize_[ name ] = size;
- pendingUpdated_.notify_all();
-#endif
- }
- // Returns when file has been allocated. If file exists, size is
- // updated to match existing file size.
- void allocateAsap( const string &name, long &size ) {
-#if !defined(_WIN32)
- scoped_lock lk( pendingMutex_ );
- long oldSize = prevSize( name );
- if ( oldSize != -1 ) {
- size = oldSize;
- if ( !inProgress( name ) )
- return;
- }
- checkFailure();
- pendingSize_[ name ] = size;
- if ( pending_.size() == 0 )
- pending_.push_back( name );
- else if ( pending_.front() != name ) {
- pending_.remove( name );
- list< string >::iterator i = pending_.begin();
- ++i;
- pending_.insert( i, name );
- }
- pendingUpdated_.notify_all();
- while( inProgress( name ) ) {
- checkFailure();
- pendingUpdated_.wait( lk.boost() );
- }
-#endif
- }
+ void start();
- void waitUntilFinished() const {
-#if !defined(_WIN32)
- if ( failed_ )
- return;
- scoped_lock lk( pendingMutex_ );
- while( pending_.size() != 0 )
- pendingUpdated_.wait( lk.boost() );
-#endif
- }
-
- static void ensureLength( int fd , long size ){
+ /**
+ * May be called if file exists. If file exists, or its allocation has
+ * been requested, size is updated to match existing file size.
+ */
+ void requestAllocation( const string &name, long &size );
-#if defined(_WIN32)
- // we don't zero on windows
- // TODO : we should to avoid fragmentation
-#else
-#if defined(__linux__)
- int ret = posix_fallocate(fd,0,size);
- if ( ret == 0 )
- return;
-
- log() << "posix_fallocate failed: " << errnoWithDescription( ret ) << " falling back" << endl;
-#endif
-
- off_t filelen = lseek(fd, 0, SEEK_END);
- if ( filelen < size ) {
- if (filelen != 0) {
- stringstream ss;
- ss << "failure creating new datafile; lseek failed for fd " << fd << " with errno: " << errnoWithDescription();
- massert( 10440 , ss.str(), filelen == 0 );
- }
- // Check for end of disk.
- massert( 10441 , "Unable to allocate file of desired size",
- size - 1 == lseek(fd, size - 1, SEEK_SET) );
- massert( 10442 , "Unable to allocate file of desired size",
- 1 == write(fd, "", 1) );
- lseek(fd, 0, SEEK_SET);
-
- const long z = 256 * 1024;
- const boost::scoped_array<char> buf_holder (new char[z]);
- char* buf = buf_holder.get();
- memset(buf, 0, z);
- long left = size;
- while ( left > 0 ) {
- long towrite = left;
- if ( towrite > z )
- towrite = z;
-
- int written = write( fd , buf , towrite );
- massert( 10443 , errnoWithPrefix("write failed" ), written > 0 );
- left -= written;
- }
- }
-#endif
- }
-
+ /**
+ * Returns when file has been allocated. If file exists, size is
+ * updated to match existing file size.
+ */
+ void allocateAsap( const string &name, unsigned long long &size );
+
+ void waitUntilFinished() const;
+
+ static void ensureLength(int fd , long size);
+
+ /** @return the singletone */
+ static FileAllocator * get();
+
private:
+
+ FileAllocator();
+
#if !defined(_WIN32)
- void checkFailure() {
- massert( 12520, "file allocation failure", !failed_ );
- }
-
- // caller must hold pendingMutex_ lock. Returns size if allocated or
+ void checkFailure();
+
+ // caller must hold pendingMutex_ lock. Returns size if allocated or
// allocation requested, -1 otherwise.
- long prevSize( const string &name ) const {
- if ( pendingSize_.count( name ) > 0 )
- return pendingSize_[ name ];
- if ( boost::filesystem::exists( name ) )
- return boost::filesystem::file_size( name );
- return -1;
- }
-
+ long prevSize( const string &name ) const;
+
// caller must hold pendingMutex_ lock.
- bool inProgress( const string &name ) const {
- for( list< string >::const_iterator i = pending_.begin(); i != pending_.end(); ++i )
- if ( *i == name )
- return true;
- return false;
- }
+ bool inProgress( const string &name ) const;
- mutable mongo::mutex pendingMutex_;
- mutable boost::condition pendingUpdated_;
- list< string > pending_;
- mutable map< string, long > pendingSize_;
- bool failed_;
-
- struct Runner {
- Runner( FileAllocator &allocator ) : a_( allocator ) {}
- FileAllocator &a_;
- void operator()() {
- while( 1 ) {
- {
- scoped_lock lk( a_.pendingMutex_ );
- if ( a_.pending_.size() == 0 )
- a_.pendingUpdated_.wait( lk.boost() );
- }
- while( 1 ) {
- string name;
- long size;
- {
- scoped_lock lk( a_.pendingMutex_ );
- if ( a_.pending_.size() == 0 )
- break;
- name = a_.pending_.front();
- size = a_.pendingSize_[ name ];
- }
- try {
- log() << "allocating new datafile " << name << ", filling with zeroes..." << endl;
- long fd = open(name.c_str(), O_CREAT | O_RDWR | O_NOATIME, S_IRUSR | S_IWUSR);
- if ( fd <= 0 ) {
- stringstream ss;
- ss << "couldn't open " << name << ' ' << errnoWithDescription();
- massert( 10439 , ss.str(), fd <= 0 );
- }
+ /** called from the worked thread */
+ static void run( FileAllocator * fa );
-#if defined(POSIX_FADV_DONTNEED)
- if( posix_fadvise(fd, 0, size, POSIX_FADV_DONTNEED) ) {
- log() << "warning: posix_fadvise fails " << name << ' ' << errnoWithDescription() << endl;
- }
-#endif
-
- Timer t;
-
- /* make sure the file is the full desired length */
- ensureLength( fd , size );
+ mutable mongo::mutex _pendingMutex;
+ mutable boost::condition _pendingUpdated;
+
+ list< string > _pending;
+ mutable map< string, long > _pendingSize;
- log() << "done allocating datafile " << name << ", "
- << "size: " << size/1024/1024 << "MB, "
- << " took " << ((double)t.millis())/1000.0 << " secs"
- << endl;
+ bool _failed;
+#endif
+
+ static FileAllocator* _instance;
- close( fd );
-
- } catch ( ... ) {
- problem() << "Failed to allocate new file: " << name
- << ", size: " << size << ", aborting." << endl;
- try {
- BOOST_CHECK_EXCEPTION( boost::filesystem::remove( name ) );
- } catch ( ... ) {
- }
- scoped_lock lk( a_.pendingMutex_ );
- a_.failed_ = true;
- // not erasing from pending
- a_.pendingUpdated_.notify_all();
- return; // no more allocation
- }
-
- {
- scoped_lock lk( a_.pendingMutex_ );
- a_.pendingSize_.erase( name );
- a_.pending_.pop_front();
- a_.pendingUpdated_.notify_all();
- }
- }
- }
- }
- };
-#endif
};
-
- FileAllocator &theFileAllocator();
+
+ /** like "mkdir -p" but on parent dir of p rather than p itself */
+ void ensureParentDirCreated(const boost::filesystem::path& p);
+
} // namespace mongo
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-03 02:17:43 +0000