From a5cc350dcf6cf83a84b5845e0f61230f1f42e7c7 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sun, 4 Jul 2010 15:03:35 +0000 Subject: 2010-07-04 Tatsuhiro Tsujikawa When allocating disk space, for Linux system with fallocate() system call, first check file system supports fallocate. This just run fallocate with small chunk and see it succeeds or fails. If it succeeds, use fallocate() to allocate entire file otherwise fall back to traditional slower method: writing zeros. This behavior is enabled in --file-allocation=prealloc, so this is enabled by default for most modern Linux. * configure.ac * src/AbstractDiskWriter.cc * src/AbstractDiskWriter.h * src/AbstractSingleDiskAdaptor.cc * src/AdaptiveFileAllocationIterator.cc * src/AdaptiveFileAllocationIterator.h * src/DefaultPieceStorage.cc * src/DiskAdaptor.cc * src/DiskAdaptor.h * src/FallocFileAllocationIterator.cc * src/Makefile.am * src/MultiFileAllocationIterator.cc * src/OptionHandlerFactory.cc * test/FallocFileAllocationIteratorTest.cc * test/Makefile.am --- src/AbstractDiskWriter.cc | 16 +++++- src/AbstractDiskWriter.h | 2 - src/AbstractSingleDiskAdaptor.cc | 17 +++--- src/AdaptiveFileAllocationIterator.cc | 102 ++++++++++++++++++++++++++++++++++ src/AdaptiveFileAllocationIterator.h | 80 ++++++++++++++++++++++++++ src/DefaultPieceStorage.cc | 2 - src/DiskAdaptor.cc | 2 - src/DiskAdaptor.h | 6 +- src/FallocFileAllocationIterator.cc | 11 ++-- src/Makefile.am | 7 ++- src/Makefile.in | 26 +++++---- src/MultiFileAllocationIterator.cc | 29 +++++----- src/OptionHandlerFactory.cc | 4 +- 13 files changed, 248 insertions(+), 56 deletions(-) create mode 100644 src/AdaptiveFileAllocationIterator.cc create mode 100644 src/AdaptiveFileAllocationIterator.h (limited to 'src') diff --git a/src/AbstractDiskWriter.cc b/src/AbstractDiskWriter.cc index e06e4732..1c4fdc1d 100644 --- a/src/AbstractDiskWriter.cc +++ b/src/AbstractDiskWriter.cc @@ -193,19 +193,31 @@ void AbstractDiskWriter::truncate(uint64_t length) #endif } -#ifdef HAVE_POSIX_FALLOCATE void AbstractDiskWriter::allocate(off_t offset, uint64_t length) { +#ifdef HAVE_SOME_FALLOCATE if(fd_ == -1) { throw DL_ABORT_EX("File not yet opened."); } +# ifdef HAVE_FALLOCATE + // For linux, we use fallocate to detect file system supports + // fallocate or not. + int r = fallocate(fd_, 0, offset, length); + if(r == -1) { + throw DL_ABORT_EX(StringFormat("fallocate failed. cause: %s", + strerror(errno)).str()); + } +# elif HAVE_POSIX_FALLOCATE int r = posix_fallocate(fd_, offset, length); if(r != 0) { throw DL_ABORT_EX(StringFormat("posix_fallocate failed. cause: %s", strerror(r)).str()); } +# else +# error "no *_fallocate function available." +# endif +#endif // HAVE_SOME_FALLOCATE } -#endif // HAVE_POSIX_FALLOCATE uint64_t AbstractDiskWriter::size() { diff --git a/src/AbstractDiskWriter.h b/src/AbstractDiskWriter.h index 89c5a64b..5fab1f6b 100644 --- a/src/AbstractDiskWriter.h +++ b/src/AbstractDiskWriter.h @@ -75,10 +75,8 @@ public: virtual void truncate(uint64_t length); -#ifdef HAVE_POSIX_FALLOCATE // File must be opened before calling this function. virtual void allocate(off_t offset, uint64_t length); -#endif // HAVE_POSIX_FALLOCATE virtual uint64_t size(); diff --git a/src/AbstractSingleDiskAdaptor.cc b/src/AbstractSingleDiskAdaptor.cc index 8a5be61b..0dfae344 100644 --- a/src/AbstractSingleDiskAdaptor.cc +++ b/src/AbstractSingleDiskAdaptor.cc @@ -34,10 +34,10 @@ /* copyright --> */ #include "AbstractSingleDiskAdaptor.h" #include "File.h" -#include "SingleFileAllocationIterator.h" -#ifdef HAVE_POSIX_FALLOCATE +#include "AdaptiveFileAllocationIterator.h" +#ifdef HAVE_SOME_FALLOCATE # include "FallocFileAllocationIterator.h" -#endif // HAVE_POSIX_FALLOCATE +#endif // HAVE_SOME_FALLOCATE #include "DiskWriter.h" #include "FileEntry.h" @@ -95,22 +95,21 @@ void AbstractSingleDiskAdaptor::truncate(uint64_t length) diskWriter_->truncate(length); } -FileAllocationIteratorHandle +SharedHandle AbstractSingleDiskAdaptor::fileAllocationIterator() { -#ifdef HAVE_POSIX_FALLOCATE +#ifdef HAVE_SOME_FALLOCATE if(doesFallocate()) { SharedHandle h (new FallocFileAllocationIterator (diskWriter_.get(), size() ,totalLength_)); return h; } else -#endif // HAVE_POSIX_FALLOCATE +#endif // HAVE_SOME_FALLOCATE { - SingleFileAllocationIteratorHandle h - (new SingleFileAllocationIterator + SharedHandle h + (new AdaptiveFileAllocationIterator (diskWriter_.get(), size(), totalLength_)); - h->init(); return h; } } diff --git a/src/AdaptiveFileAllocationIterator.cc b/src/AdaptiveFileAllocationIterator.cc new file mode 100644 index 00000000..3ec80fb6 --- /dev/null +++ b/src/AdaptiveFileAllocationIterator.cc @@ -0,0 +1,102 @@ +/* */ +#include "AdaptiveFileAllocationIterator.h" +#include "BinaryStream.h" +#ifdef HAVE_FALLOCATE +# include "FallocFileAllocationIterator.h" +#endif // HAVE_FALLOCATE +#include "SingleFileAllocationIterator.h" +#include "RecoverableException.h" +#include "LogFactory.h" +#include "Logger.h" + +namespace aria2 { + +AdaptiveFileAllocationIterator::AdaptiveFileAllocationIterator +(BinaryStream* stream, off_t offset, uint64_t totalLength): + stream_(stream), offset_(offset), totalLength_(totalLength), + logger_(LogFactory::getInstance()) {} + +AdaptiveFileAllocationIterator::~AdaptiveFileAllocationIterator() {} + +void AdaptiveFileAllocationIterator::allocateChunk() +{ + if(allocator_.isNull()) { +#ifdef HAVE_FALLOCATE + try { + if(logger_->debug()) { + logger_->debug("Testing file system supports fallocate."); + } + if(static_cast(offset_) < totalLength_) { + off_t len = std::min(totalLength_-offset_, static_cast(4096)); + stream_->allocate(offset_, len); + offset_ += len; + } + if(logger_->debug()) { + logger_->debug("File system supports fallocate."); + } + allocator_.reset + (new FallocFileAllocationIterator(stream_, offset_, totalLength_)); + } catch(RecoverableException& e) { + if(logger_->debug()) { + logger_->debug("File system does not support fallocate."); + } + SharedHandle salloc + (new SingleFileAllocationIterator(stream_, offset_, totalLength_)); + salloc->init(); + allocator_ = salloc; + } +#else // !HAVE_FALLOCATE + SharedHandle salloc + (new SingleFileAllocationIterator(stream_, offset_, totalLength_)); + salloc->init(); + allocator_ = salloc; +#endif // !HAVE_FALLOCATE + allocator_->allocateChunk(); + } else { + allocator_->allocateChunk(); + } +} + +bool AdaptiveFileAllocationIterator::finished() +{ + if(allocator_.isNull()) { + return false; + } else { + return allocator_->finished(); + } +} + +} // namespace aria2 diff --git a/src/AdaptiveFileAllocationIterator.h b/src/AdaptiveFileAllocationIterator.h new file mode 100644 index 00000000..7a5cd51d --- /dev/null +++ b/src/AdaptiveFileAllocationIterator.h @@ -0,0 +1,80 @@ +/* */ +#ifndef D_ADAPTIVE_FILE_ALLOCATION_ITERATOR_H +#define D_ADAPTIVE_FILE_ALLOCATION_ITERATOR_H + +#include "FileAllocationIterator.h" + +namespace aria2 { + +class BinaryStream; +class Logger; + +class AdaptiveFileAllocationIterator:public FileAllocationIterator +{ +private: + SharedHandle allocator_; + + BinaryStream* stream_; + + off_t offset_; + + uint64_t totalLength_; + + Logger* logger_; +public: + AdaptiveFileAllocationIterator + (BinaryStream* stream, off_t offset, uint64_t totalLength); + + virtual ~AdaptiveFileAllocationIterator(); + + virtual void allocateChunk(); + + virtual bool finished(); + + virtual off_t getCurrentLength() + { + return offset_; + } + + virtual uint64_t getTotalLength() + { + return totalLength_; + } +}; + +} // namespace aria2 + +#endif // D_ADAPTIVE_FILE_ALLOCATION_ITERATOR_H diff --git a/src/DefaultPieceStorage.cc b/src/DefaultPieceStorage.cc index 40e90ca4..7ae7f799 100644 --- a/src/DefaultPieceStorage.cc +++ b/src/DefaultPieceStorage.cc @@ -509,11 +509,9 @@ void DefaultPieceStorage::initStorage() (option_->getAsInt(PREF_BT_MAX_OPEN_FILES)); diskAdaptor_ = multiDiskAdaptor; } -#ifdef HAVE_POSIX_FALLOCATE if(option_->get(PREF_FILE_ALLOCATION) == V_FALLOC) { diskAdaptor_->enableFallocate(); } -#endif // HAVE_POSIX_FALLOCATE } void DefaultPieceStorage::setBitfield(const unsigned char* bitfield, diff --git a/src/DiskAdaptor.cc b/src/DiskAdaptor.cc index 8c910c5f..d9722f4b 100644 --- a/src/DiskAdaptor.cc +++ b/src/DiskAdaptor.cc @@ -40,9 +40,7 @@ namespace aria2 { DiskAdaptor::DiskAdaptor(): -#ifdef HAVE_POSIX_FALLOCATE fallocate_(false), -#endif // HAVE_POSIX_FALLOCATE logger_(LogFactory::getInstance()) {} } // namespace aria2 diff --git a/src/DiskAdaptor.h b/src/DiskAdaptor.h index 8dde24ea..48477f75 100644 --- a/src/DiskAdaptor.h +++ b/src/DiskAdaptor.h @@ -51,9 +51,9 @@ class FileAllocationIterator; class DiskAdaptor:public BinaryStream { private: std::vector > fileEntries_; -#ifdef HAVE_POSIX_FALLOCATE + bool fallocate_; -#endif // HAVE_POSIX_FALLOCATE + Logger* logger_; protected: Logger* getLogger() const @@ -110,7 +110,6 @@ public: // successfully changed. virtual size_t utime(const Time& actime, const Time& modtime) = 0; -#ifdef HAVE_POSIX_FALLOCATE void enableFallocate() { fallocate_ = true; @@ -125,7 +124,6 @@ public: { return fallocate_; } -#endif // HAVE_POSIX_FALLOCATE }; typedef SharedHandle DiskAdaptorHandle; diff --git a/src/FallocFileAllocationIterator.cc b/src/FallocFileAllocationIterator.cc index 1fd53d68..3e4e24d1 100644 --- a/src/FallocFileAllocationIterator.cc +++ b/src/FallocFileAllocationIterator.cc @@ -43,12 +43,13 @@ FallocFileAllocationIterator::FallocFileAllocationIterator void FallocFileAllocationIterator::allocateChunk() { - if(static_cast(offset_) > totalLength_) { - throw DL_ABORT_EX("FallocFileAllocationIterator: offset is larger than" - " totalLength"); + if(static_cast(offset_) < totalLength_) { + stream_->allocate(offset_, totalLength_-offset_); + offset_ = totalLength_; + } else { + stream_->truncate(totalLength_); + offset_ = totalLength_; } - stream_->allocate(offset_, totalLength_-offset_); - offset_ = totalLength_; } bool FallocFileAllocationIterator::finished() diff --git a/src/Makefile.am b/src/Makefile.am index 430af281..8274e869 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -205,7 +205,8 @@ SRCS = Socket.h\ timespec.h\ ValueBase.cc ValueBase.h\ ContextAttribute.h\ - TorrentAttribute.h + TorrentAttribute.h\ + AdaptiveFileAllocationIterator.cc AdaptiveFileAllocationIterator.h if ENABLE_XML_RPC SRCS += XmlRpcRequestParserController.cc XmlRpcRequestParserController.h\ @@ -235,9 +236,9 @@ endif # HAVE_LIBEXPAT endif # ENABLE_XML_RPC -if HAVE_POSIX_FALLOCATE +if HAVE_SOME_FALLOCATE SRCS += FallocFileAllocationIterator.cc FallocFileAllocationIterator.h -endif # HAVE_POSIX_FALLOCATE +endif # HAVE_SOME_FALLOCATE if HAVE_EPOLL SRCS += EpollEventPoll.cc EpollEventPoll.h diff --git a/src/Makefile.in b/src/Makefile.in index f77d3cd9..53a5834d 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -52,11 +52,12 @@ bin_PROGRAMS = aria2c$(EXEEXT) @ENABLE_XML_RPC_TRUE@ HttpListenCommand.cc HttpListenCommand.h\ @ENABLE_XML_RPC_TRUE@ HttpServerCommand.cc HttpServerCommand.h\ @ENABLE_XML_RPC_TRUE@ HttpServerResponseCommand.cc HttpServerResponseCommand.h\ -@ENABLE_XML_RPC_TRUE@ HttpServer.cc HttpServer.h +@ENABLE_XML_RPC_TRUE@ HttpServer.cc HttpServer.h\ +@ENABLE_XML_RPC_TRUE@ AdaptiveFileAllocationIterator.cc AdaptiveFileAllocationIterator.h @ENABLE_XML_RPC_TRUE@@HAVE_LIBXML2_TRUE@am__append_2 = Xml2XmlRpcRequestProcessor.cc Xml2XmlRpcRequestProcessor.h @ENABLE_XML_RPC_TRUE@@HAVE_LIBEXPAT_TRUE@am__append_3 = ExpatXmlRpcRequestProcessor.cc ExpatXmlRpcRequestProcessor.h -@HAVE_POSIX_FALLOCATE_TRUE@am__append_4 = FallocFileAllocationIterator.cc FallocFileAllocationIterator.h +@HAVE_SOME_FALLOCATE_TRUE@am__append_4 = FallocFileAllocationIterator.cc FallocFileAllocationIterator.h @HAVE_EPOLL_TRUE@am__append_5 = EpollEventPoll.cc EpollEventPoll.h @ENABLE_SSL_TRUE@am__append_6 = TLSContext.h @HAVE_LIBGNUTLS_TRUE@am__append_7 = LibgnutlsTLSContext.cc LibgnutlsTLSContext.h @@ -452,13 +453,14 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ HttpListenCommand.cc HttpListenCommand.h HttpServerCommand.cc \ HttpServerCommand.h HttpServerResponseCommand.cc \ HttpServerResponseCommand.h HttpServer.cc HttpServer.h \ - Xml2XmlRpcRequestProcessor.cc Xml2XmlRpcRequestProcessor.h \ - ExpatXmlRpcRequestProcessor.cc ExpatXmlRpcRequestProcessor.h \ - FallocFileAllocationIterator.cc FallocFileAllocationIterator.h \ - EpollEventPoll.cc EpollEventPoll.h TLSContext.h \ - LibgnutlsTLSContext.cc LibgnutlsTLSContext.h \ - LibsslTLSContext.cc LibsslTLSContext.h GZipDecoder.cc \ - GZipDecoder.h GZipEncoder.cc GZipEncoder.h \ + AdaptiveFileAllocationIterator.cc \ + AdaptiveFileAllocationIterator.h Xml2XmlRpcRequestProcessor.cc \ + Xml2XmlRpcRequestProcessor.h ExpatXmlRpcRequestProcessor.cc \ + ExpatXmlRpcRequestProcessor.h FallocFileAllocationIterator.cc \ + FallocFileAllocationIterator.h EpollEventPoll.cc \ + EpollEventPoll.h TLSContext.h LibgnutlsTLSContext.cc \ + LibgnutlsTLSContext.h LibsslTLSContext.cc LibsslTLSContext.h \ + GZipDecoder.cc GZipDecoder.h GZipEncoder.cc GZipEncoder.h \ Sqlite3MozCookieParser.cc Sqlite3MozCookieParser.h \ AsyncNameResolver.cc AsyncNameResolver.h \ IteratableChunkChecksumValidator.cc \ @@ -626,10 +628,11 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ @ENABLE_XML_RPC_TRUE@ HttpListenCommand.$(OBJEXT) \ @ENABLE_XML_RPC_TRUE@ HttpServerCommand.$(OBJEXT) \ @ENABLE_XML_RPC_TRUE@ HttpServerResponseCommand.$(OBJEXT) \ -@ENABLE_XML_RPC_TRUE@ HttpServer.$(OBJEXT) +@ENABLE_XML_RPC_TRUE@ HttpServer.$(OBJEXT) \ +@ENABLE_XML_RPC_TRUE@ AdaptiveFileAllocationIterator.$(OBJEXT) @ENABLE_XML_RPC_TRUE@@HAVE_LIBXML2_TRUE@am__objects_2 = Xml2XmlRpcRequestProcessor.$(OBJEXT) @ENABLE_XML_RPC_TRUE@@HAVE_LIBEXPAT_TRUE@am__objects_3 = ExpatXmlRpcRequestProcessor.$(OBJEXT) -@HAVE_POSIX_FALLOCATE_TRUE@am__objects_4 = FallocFileAllocationIterator.$(OBJEXT) +@HAVE_SOME_FALLOCATE_TRUE@am__objects_4 = FallocFileAllocationIterator.$(OBJEXT) @HAVE_EPOLL_TRUE@am__objects_5 = EpollEventPoll.$(OBJEXT) am__objects_6 = @HAVE_LIBGNUTLS_TRUE@am__objects_7 = LibgnutlsTLSContext.$(OBJEXT) @@ -1343,6 +1346,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AbstractProxyResponseCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AbstractSingleDiskAdaptor.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ActivePeerConnectionCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AdaptiveFileAllocationIterator.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AdaptiveURISelector.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AnnounceList.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/AsyncNameResolver.Po@am__quote@ diff --git a/src/MultiFileAllocationIterator.cc b/src/MultiFileAllocationIterator.cc index 8fa15a97..9a0dc3f5 100644 --- a/src/MultiFileAllocationIterator.cc +++ b/src/MultiFileAllocationIterator.cc @@ -35,15 +35,16 @@ #include "MultiFileAllocationIterator.h" #include "MultiDiskAdaptor.h" #include "FileEntry.h" -#include "SingleFileAllocationIterator.h" -#ifdef HAVE_POSIX_FALLOCATE +#include "AdaptiveFileAllocationIterator.h" +#ifdef HAVE_SOME_FALLOCATE # include "FallocFileAllocationIterator.h" -#endif // HAVE_POSIX_FALLOCATE +#endif // HAVE_SOME_FALLOCATE #include "DiskWriter.h" namespace aria2 { -MultiFileAllocationIterator::MultiFileAllocationIterator(MultiDiskAdaptor* diskAdaptor): +MultiFileAllocationIterator::MultiFileAllocationIterator +(MultiDiskAdaptor* diskAdaptor): diskAdaptor_(diskAdaptor), entries_(diskAdaptor_->diskWriterEntries_.begin(), diskAdaptor_->diskWriterEntries_.end()), @@ -54,7 +55,8 @@ MultiFileAllocationIterator::~MultiFileAllocationIterator() {} void MultiFileAllocationIterator::allocateChunk() { - while(fileAllocationIterator_.isNull() || fileAllocationIterator_->finished()) { + while(fileAllocationIterator_.isNull() || + fileAllocationIterator_->finished()) { if(entries_.empty()) { break; } @@ -65,21 +67,19 @@ void MultiFileAllocationIterator::allocateChunk() diskAdaptor_->openIfNot(entry, &DiskWriterEntry::openFile); if(entry->needsFileAllocation() && entry->size() < fileEntry->getLength()) { // Calling private function of MultiDiskAdaptor. -#ifdef HAVE_POSIX_FALLOCATE +#ifdef HAVE_SOME_FALLOCATE if(diskAdaptor_->doesFallocate()) { fileAllocationIterator_.reset (new FallocFileAllocationIterator(entry->getDiskWriter().get(), entry->size(), fileEntry->getLength())); } else -#endif // HAVE_POSIX_FALLOCATE +#endif // HAVE_SOME_FALLOCATE { - SharedHandle fa - (new SingleFileAllocationIterator(entry->getDiskWriter().get(), - entry->size(), - fileEntry->getLength())); - fa->init(); - fileAllocationIterator_ = fa; + fileAllocationIterator_.reset + (new AdaptiveFileAllocationIterator(entry->getDiskWriter().get(), + entry->size(), + fileEntry->getLength())); } } } @@ -91,7 +91,8 @@ void MultiFileAllocationIterator::allocateChunk() bool MultiFileAllocationIterator::finished() { - return entries_.empty() && (fileAllocationIterator_.isNull() || fileAllocationIterator_->finished()); + return entries_.empty() && + (fileAllocationIterator_.isNull() || fileAllocationIterator_->finished()); } off_t MultiFileAllocationIterator::getCurrentLength() diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index afd88610..8a42f905 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -244,9 +244,9 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers() TEXT_FILE_ALLOCATION, V_PREALLOC, V_NONE, V_PREALLOC, -#ifdef HAVE_POSIX_FALLOCATE +#ifdef HAVE_SOME_FALLOCATE V_FALLOC, -#endif // HAVE_POSIX_FALLOCATE +#endif // HAVE_SOME_FALLOCATE 'a')); op->addTag(TAG_BASIC); op->addTag(TAG_FILE); -- cgit