diff options
| author | Tatsuhiro Tsujikawa <tatsuhiro.t@gmail.com> | 2009-03-25 05:43:07 +0000 |
|---|---|---|
| committer | Tatsuhiro Tsujikawa <tatsuhiro.t@gmail.com> | 2009-03-25 05:43:07 +0000 |
| commit | 29c5ef9215be004b31cd53ef8841680c3f015d57 (patch) | |
| tree | acae3135c9240d627b30629c5a7594951a4cb688 /src | |
| parent | fe614395d60da75836dbfb3f0a593d91b4024a89 (diff) | |
2009-03-25 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Added 'falloc' parameter for --file-allocation option. 'falloc'
allocation mode uses posix_fallocate() system call to allocate
file on disk. If you are using newer file systems such as ext4
(with extents support), btrfs or xfs, 'falloc' is your best
choice. It allocates large(few GiB) files almost instantly.
Don't use 'falloc' with legacy file systems such as ext3 because
it takes almost same time as 'prealloc' and it blocks aria2
entirely until allocation finishes. 'falloc' may not be
available if your system doesn't have posix_fallocate() system
call.
* configure.ac
* src/AbstractDiskWriter.cc
* src/AbstractDiskWriter.h
* src/AbstractSingleDiskAdaptor.cc
* src/BinaryStream.h
* src/BtCheckIntegrityEntry.cc
* src/ByteArrayDiskWriter.h
* src/CheckIntegrityEntry.cc
* src/CheckIntegrityEntry.h
* src/DefaultPieceStorage.cc
* src/DiskAdaptor.cc
* src/DiskAdaptor.h
* src/DiskWriter.h
* src/FallocFileAllocationIterator.cc
* src/FallocFileAllocationIterator.h
* src/FileAllocationEntry.cc
* src/FileAllocationEntry.h
* src/Makefile.am
* src/MultiFileAllocationIterator.cc
* src/MultiFileAllocationIterator.h
* src/OptionHandlerFactory.cc
* src/RequestGroup.cc
* src/StreamCheckIntegrityEntry.cc
* src/prefs.cc
* src/prefs.h
* src/usage_text.h
* test/FallocFileAllocationIteratorTest.cc
* test/Makefile.am
Diffstat (limited to 'src')
| -rw-r--r-- | src/AbstractDiskWriter.cc | 14 | ||||
| -rw-r--r-- | src/AbstractDiskWriter.h | 5 | ||||
| -rw-r--r-- | src/AbstractSingleDiskAdaptor.cc | 16 | ||||
| -rw-r--r-- | src/BinaryStream.h | 12 | ||||
| -rw-r--r-- | src/BtCheckIntegrityEntry.cc | 20 | ||||
| -rw-r--r-- | src/ByteArrayDiskWriter.h | 3 | ||||
| -rw-r--r-- | src/CheckIntegrityEntry.cc | 21 | ||||
| -rw-r--r-- | src/CheckIntegrityEntry.h | 5 | ||||
| -rw-r--r-- | src/DefaultPieceStorage.cc | 3 | ||||
| -rw-r--r-- | src/DiskAdaptor.cc | 3 | ||||
| -rw-r--r-- | src/DiskAdaptor.h | 19 | ||||
| -rw-r--r-- | src/DiskWriter.h | 2 | ||||
| -rw-r--r-- | src/FallocFileAllocationIterator.cc | 69 | ||||
| -rw-r--r-- | src/FallocFileAllocationIterator.h | 65 | ||||
| -rw-r--r-- | src/FileAllocationEntry.cc | 5 | ||||
| -rw-r--r-- | src/FileAllocationEntry.h | 2 | ||||
| -rw-r--r-- | src/Makefile.am | 4 | ||||
| -rw-r--r-- | src/Makefile.in | 98 | ||||
| -rw-r--r-- | src/MultiFileAllocationIterator.cc | 19 | ||||
| -rw-r--r-- | src/MultiFileAllocationIterator.h | 3 | ||||
| -rw-r--r-- | src/OptionHandlerFactory.cc | 3 | ||||
| -rw-r--r-- | src/RequestGroup.cc | 6 | ||||
| -rw-r--r-- | src/StreamCheckIntegrityEntry.cc | 11 | ||||
| -rw-r--r-- | src/prefs.cc | 5 | ||||
| -rw-r--r-- | src/prefs.h | 3 | ||||
| -rw-r--r-- | src/usage_text.h | 11 |
26 files changed, 327 insertions, 100 deletions
diff --git a/src/AbstractDiskWriter.cc b/src/AbstractDiskWriter.cc index cec3bdaf..1c8d070f 100644 --- a/src/AbstractDiskWriter.cc +++ b/src/AbstractDiskWriter.cc @@ -192,6 +192,20 @@ void AbstractDiskWriter::truncate(uint64_t length) #endif } +#ifdef HAVE_POSIX_FALLOCATE +void AbstractDiskWriter::allocate(off_t offset, uint64_t length) +{ + if(fd == -1) { + throw DlAbortEx("File not yet opened."); + } + int r = posix_fallocate(fd, offset, length); + if(r != 0) { + throw DlAbortEx(StringFormat("posix_fallocate failed. cause: %s", + strerror(r)).str()); + } +} +#endif // HAVE_POSIX_FALLOCATE + // TODO the file descriptor fd must be opened before calling this function. uint64_t AbstractDiskWriter::size() { diff --git a/src/AbstractDiskWriter.h b/src/AbstractDiskWriter.h index ef967687..ff9f3ca1 100644 --- a/src/AbstractDiskWriter.h +++ b/src/AbstractDiskWriter.h @@ -74,6 +74,11 @@ public: virtual void truncate(uint64_t length); +#ifdef HAVE_POSIX_FALLOCATE + // File must be opened before calling this function. + virtual void allocate(off_t offset, uint64_t length); +#endif // HAVE_POSIX_FALLOCATE + virtual uint64_t size(); virtual void enableDirectIO(); diff --git a/src/AbstractSingleDiskAdaptor.cc b/src/AbstractSingleDiskAdaptor.cc index d1c4d8a5..647cd33a 100644 --- a/src/AbstractSingleDiskAdaptor.cc +++ b/src/AbstractSingleDiskAdaptor.cc @@ -35,6 +35,7 @@ #include "AbstractSingleDiskAdaptor.h" #include "File.h" #include "SingleFileAllocationIterator.h" +#include "FallocFileAllocationIterator.h" #include "DiskWriter.h" namespace aria2 { @@ -90,10 +91,17 @@ void AbstractSingleDiskAdaptor::truncate(uint64_t length) FileAllocationIteratorHandle AbstractSingleDiskAdaptor::fileAllocationIterator() { - SingleFileAllocationIteratorHandle h - (new SingleFileAllocationIterator(this, size(), totalLength)); - h->init(); - return h; + + if(_fallocate) { + SharedHandle<FallocFileAllocationIterator> h + (new FallocFileAllocationIterator(this, size(), totalLength)); + return h; + } else { + SingleFileAllocationIteratorHandle h + (new SingleFileAllocationIterator(this, size(), totalLength)); + h->init(); + return h; + } } void AbstractSingleDiskAdaptor::enableDirectIO() diff --git a/src/BinaryStream.h b/src/BinaryStream.h index 9af8b3bf..38688e44 100644 --- a/src/BinaryStream.h +++ b/src/BinaryStream.h @@ -36,9 +36,11 @@ #define _D_BINARY_STREAM_H_ #include "common.h" -#include "SharedHandle.h" + #include <unistd.h> +#include "SharedHandle.h" + namespace aria2 { class BinaryStream { @@ -49,7 +51,13 @@ public: virtual ssize_t readData(unsigned char* data, size_t len, off_t offset) = 0; - virtual void truncate(uint64_t length) = 0; + // Truncates a file to given length. The default implementation does + // nothing. + virtual void truncate(uint64_t length) {} + + // Allocates given length bytes of disk space from given offset. The + // default implementation does nothing. + virtual void allocate(off_t offset, uint64_t length) {} virtual void enableDirectIO() = 0; diff --git a/src/BtCheckIntegrityEntry.cc b/src/BtCheckIntegrityEntry.cc index bed582c5..1be5e251 100644 --- a/src/BtCheckIntegrityEntry.cc +++ b/src/BtCheckIntegrityEntry.cc @@ -37,7 +37,6 @@ #include "RequestGroup.h" #include "PieceStorage.h" #include "DownloadEngine.h" -#include "FileAllocationMan.h" #include "DiskAdaptor.h" #include "prefs.h" #include "Option.h" @@ -49,17 +48,6 @@ BtCheckIntegrityEntry::BtCheckIntegrityEntry(RequestGroup* requestGroup): BtCheckIntegrityEntry::~BtCheckIntegrityEntry() {} -static void proceedFileAllocation -(std::deque<Command*>& commands, DownloadEngine* e, RequestGroup* requestGroup) -{ - FileAllocationEntryHandle entry(new BtFileAllocationEntry(requestGroup)); - if(requestGroup->needsFileAllocation()) { - e->_fileAllocationMan->pushEntry(entry); - } else { - entry->prepareForNextAction(commands, e); - } -} - void BtCheckIntegrityEntry::onDownloadIncomplete(std::deque<Command*>& commands, DownloadEngine* e) { @@ -68,7 +56,9 @@ void BtCheckIntegrityEntry::onDownloadIncomplete(std::deque<Command*>& commands, _requestGroup->getPieceStorage()->getDiskAdaptor()->disableReadOnly(); _requestGroup->getPieceStorage()->getDiskAdaptor()->openFile(); - proceedFileAllocation(commands, e, _requestGroup); + SharedHandle<BtFileAllocationEntry> entry + (new BtFileAllocationEntry(_requestGroup)); + proceedFileAllocation(commands, entry, e); } void BtCheckIntegrityEntry::onDownloadFinished(std::deque<Command*>& commands, @@ -80,7 +70,9 @@ void BtCheckIntegrityEntry::onDownloadFinished(std::deque<Command*>& commands, // to exit rather than doing seeding. So, it would be good to toggle this // behavior. if(e->option->getAsBool(PREF_BT_HASH_CHECK_SEED)) { - proceedFileAllocation(commands, e, _requestGroup); + SharedHandle<BtFileAllocationEntry> entry + (new BtFileAllocationEntry(_requestGroup)); + proceedFileAllocation(commands, entry, e); } } diff --git a/src/ByteArrayDiskWriter.h b/src/ByteArrayDiskWriter.h index 2f8007c8..2ac323d8 100644 --- a/src/ByteArrayDiskWriter.h +++ b/src/ByteArrayDiskWriter.h @@ -60,9 +60,6 @@ public: virtual void writeData(const unsigned char* data, size_t len, off_t position); virtual ssize_t readData(unsigned char* data, size_t len, off_t position); - // Not implemented yet - virtual void truncate(uint64_t length) {} - virtual uint64_t size(); virtual void enableDirectIO() {} diff --git a/src/CheckIntegrityEntry.cc b/src/CheckIntegrityEntry.cc index 5bfec8c5..7d71b7f8 100644 --- a/src/CheckIntegrityEntry.cc +++ b/src/CheckIntegrityEntry.cc @@ -37,6 +37,10 @@ #include "RequestGroup.h" #include "PieceStorage.h" #include "DiskAdaptor.h" +#include "prefs.h" +#include "FileAllocationEntry.h" +#include "DownloadEngine.h" +#include "Option.h" namespace aria2 { @@ -80,4 +84,21 @@ void CheckIntegrityEntry::cutTrailingGarbage() _requestGroup->getPieceStorage()->getDiskAdaptor()->cutTrailingGarbage(); } +void CheckIntegrityEntry::proceedFileAllocation +(std::deque<Command*>& commands, + const SharedHandle<FileAllocationEntry>& entry, + DownloadEngine* e) +{ + if(_requestGroup->needsFileAllocation()) { + e->_fileAllocationMan->pushEntry(entry); + } else { + entry->prepareForNextAction(commands, e); + } + // Disable directIO when fallocation() is going to be used. + if(e->option->get(PREF_FILE_ALLOCATION) == V_FALLOC) { + entry->disableDirectIO(); + } +} + + } // namespace aria2 diff --git a/src/CheckIntegrityEntry.h b/src/CheckIntegrityEntry.h index 300c7536..a2991eb6 100644 --- a/src/CheckIntegrityEntry.h +++ b/src/CheckIntegrityEntry.h @@ -43,11 +43,16 @@ namespace aria2 { class IteratableValidator; class DownloadEngine; +class FileAllocationEntry; class CheckIntegrityEntry : public RequestGroupEntry, public ProgressAwareEntry { protected: SharedHandle<IteratableValidator> _validator; + + void proceedFileAllocation(std::deque<Command*>& commands, + const SharedHandle<FileAllocationEntry>& entry, + DownloadEngine* e); public: CheckIntegrityEntry(RequestGroup* requestGroup, Command* nextCommand = 0); diff --git a/src/DefaultPieceStorage.cc b/src/DefaultPieceStorage.cc index c8f31568..ce4b37b5 100644 --- a/src/DefaultPieceStorage.cc +++ b/src/DefaultPieceStorage.cc @@ -498,6 +498,9 @@ void DefaultPieceStorage::initStorage() } diskAdaptor->setStoreDir(downloadContext->getDir()); diskAdaptor->setFileEntries(downloadContext->getFileEntries()); + if(option->get(PREF_FILE_ALLOCATION) == V_FALLOC) { + diskAdaptor->enableFallocate(); + } } void DefaultPieceStorage::setBitfield(const unsigned char* bitfield, diff --git a/src/DiskAdaptor.cc b/src/DiskAdaptor.cc index fef04d50..d90b1b9e 100644 --- a/src/DiskAdaptor.cc +++ b/src/DiskAdaptor.cc @@ -42,7 +42,8 @@ namespace aria2 { -DiskAdaptor::DiskAdaptor():logger(LogFactory::getInstance()) {} +DiskAdaptor::DiskAdaptor(): + _fallocate(false), logger(LogFactory::getInstance()) {} DiskAdaptor::~DiskAdaptor() {} diff --git a/src/DiskAdaptor.h b/src/DiskAdaptor.h index 307ce155..dd102274 100644 --- a/src/DiskAdaptor.h +++ b/src/DiskAdaptor.h @@ -50,6 +50,7 @@ class DiskAdaptor:public BinaryStream { protected: std::string storeDir; std::deque<SharedHandle<FileEntry> > fileEntries; + bool _fallocate; Logger* logger; public: DiskAdaptor(); @@ -69,9 +70,6 @@ public: virtual uint64_t size() = 0; - // optional behavior - virtual void truncate(uint64_t length) {} - void setFileEntries(const std::deque<SharedHandle<FileEntry> >& fileEntries); SharedHandle<FileEntry> getFileEntryFromPath(const std::string& fileEntryPath) const; @@ -110,6 +108,21 @@ public: // Returns the number of files, the actime and modtime of which are // successfully changed. virtual size_t utime(const Time& actime, const Time& modtime) = 0; + + void enableFallocate() + { + _fallocate = true; + } + + void disableFallocate() + { + _fallocate = false; + } + + bool doesFallocate() const + { + return _fallocate; + } }; typedef SharedHandle<DiskAdaptor> DiskAdaptorHandle; diff --git a/src/DiskWriter.h b/src/DiskWriter.h index dfd9fd97..ede90fcf 100644 --- a/src/DiskWriter.h +++ b/src/DiskWriter.h @@ -74,8 +74,6 @@ public: */ virtual void openExistingFile(const std::string& filename, uint64_t totalLength = 0) = 0; - virtual void truncate(uint64_t length) = 0; - // Returns file length virtual uint64_t size() = 0; diff --git a/src/FallocFileAllocationIterator.cc b/src/FallocFileAllocationIterator.cc new file mode 100644 index 00000000..ca0a0161 --- /dev/null +++ b/src/FallocFileAllocationIterator.cc @@ -0,0 +1,69 @@ +/* <!-- copyright */ +/* + * aria2 - The high speed download utility + * + * Copyright (C) 2009 Tatsuhiro Tsujikawa + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * In addition, as a special exception, the copyright holders give + * permission to link the code of portions of this program with the + * OpenSSL library under certain conditions as described in each + * individual source file, and distribute linked combinations + * including the two. + * You must obey the GNU General Public License in all respects + * for all of the code used other than OpenSSL. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you + * do not wish to do so, delete this exception statement from your + * version. If you delete this exception statement from all source + * files in the program, then also delete it here. + */ +/* copyright --> */ +#include "FallocFileAllocationIterator.h" +#include "DlAbortEx.h" + +namespace aria2 { + +FallocFileAllocationIterator::FallocFileAllocationIterator +(BinaryStream* stream, off_t offset, uint64_t totalLength): + _stream(stream), _offset(offset), _totalLength(totalLength) {} + +void FallocFileAllocationIterator::allocateChunk() +{ + if(static_cast<uint64_t>(_offset) > _totalLength) { + throw DlAbortEx("FallocFileAllocationIterator: offset is larger than" + " totalLength"); + } + _stream->allocate(_offset, _totalLength-_offset); + _offset = _totalLength; +} + +bool FallocFileAllocationIterator::finished() +{ + return static_cast<uint64_t>(_offset) == _totalLength; +} + +off_t FallocFileAllocationIterator::getCurrentLength() +{ + return _offset; +} + +uint64_t FallocFileAllocationIterator::getTotalLength() +{ + return _totalLength; +} + +} // namespace aria2 diff --git a/src/FallocFileAllocationIterator.h b/src/FallocFileAllocationIterator.h new file mode 100644 index 00000000..6f6cfa65 --- /dev/null +++ b/src/FallocFileAllocationIterator.h @@ -0,0 +1,65 @@ +/* <!-- copyright */ +/* + * aria2 - The high speed download utility + * + * Copyright (C) 2009 Tatsuhiro Tsujikawa + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * In addition, as a special exception, the copyright holders give + * permission to link the code of portions of this program with the + * OpenSSL library under certain conditions as described in each + * individual source file, and distribute linked combinations + * including the two. + * You must obey the GNU General Public License in all respects + * for all of the code used other than OpenSSL. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you + * do not wish to do so, delete this exception statement from your + * version. If you delete this exception statement from all source + * files in the program, then also delete it here. + */ +/* copyright --> */ +#ifndef _D_FALLOC_FILE_ALLOCATION_ITERATOR_H_ +#define _D_FALLOC_FILE_ALLOCATION_ITERATOR_H_ + +#include "FileAllocationIterator.h" +#include "BinaryStream.h" + +namespace aria2 { + +// Allocate disk space using posix_fallocate() system call. +class FallocFileAllocationIterator:public FileAllocationIterator { +private: + BinaryStream* _stream; + off_t _offset; + uint64_t _totalLength; +public: + FallocFileAllocationIterator(BinaryStream* stream, off_t offset, + uint64_t totalLength); + + virtual void allocateChunk(); + + virtual bool finished(); + + virtual off_t getCurrentLength(); + + virtual uint64_t getTotalLength(); +}; + +} // namespace aria2 + +#endif // _D_FALLOC_FILE_ALLOCATION_ITERATOR_H_ + diff --git a/src/FileAllocationEntry.cc b/src/FileAllocationEntry.cc index a0e1df05..6a6cd732 100644 --- a/src/FileAllocationEntry.cc +++ b/src/FileAllocationEntry.cc @@ -73,4 +73,9 @@ void FileAllocationEntry::allocateChunk() _fileAllocationIterator->allocateChunk(); } +void FileAllocationEntry::disableDirectIO() +{ + _requestGroup->getPieceStorage()->getDiskAdaptor()->disableDirectIO(); +} + } // namespace aria2 diff --git a/src/FileAllocationEntry.h b/src/FileAllocationEntry.h index 97e1d594..56a42617 100644 --- a/src/FileAllocationEntry.h +++ b/src/FileAllocationEntry.h @@ -63,6 +63,8 @@ public: virtual void prepareForNextAction(std::deque<Command*>& commands, DownloadEngine* e) = 0; + + void disableDirectIO(); }; typedef SharedHandle<FileAllocationEntry> FileAllocationEntryHandle; diff --git a/src/Makefile.am b/src/Makefile.am index 0ae20f20..c8e94f3f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -207,6 +207,10 @@ SRCS = Socket.h\ PieceSelector.h\ LongestSequencePieceSelector.cc LongestSequencePieceSelector.h +if HAVE_POSIX_FALLOCATE +SRCS += FallocFileAllocationIterator.cc FallocFileAllocationIterator.h +endif # HAVE_POSIX_FALLOCATE + if HAVE_EPOLL SRCS += EpollEventPoll.cc EpollEventPoll.h endif # HAVE_EPOLL diff --git a/src/Makefile.in b/src/Makefile.in index f14f8051..33ecc216 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -35,14 +35,15 @@ build_triplet = @build@ host_triplet = @host@ target_triplet = @target@ bin_PROGRAMS = aria2c$(EXEEXT) -@HAVE_EPOLL_TRUE@am__append_1 = EpollEventPoll.cc EpollEventPoll.h -@ENABLE_SSL_TRUE@am__append_2 = TLSContext.h -@HAVE_LIBGNUTLS_TRUE@am__append_3 = LibgnutlsTLSContext.cc LibgnutlsTLSContext.h -@HAVE_LIBSSL_TRUE@am__append_4 = LibsslTLSContext.cc LibsslTLSContext.h -@HAVE_LIBZ_TRUE@am__append_5 = GZipDecoder.cc GZipDecoder.h -@HAVE_SQLITE3_TRUE@am__append_6 = Sqlite3MozCookieParser.cc Sqlite3MozCookieParser.h -@ENABLE_ASYNC_DNS_TRUE@am__append_7 = AsyncNameResolver.cc AsyncNameResolver.h -@ENABLE_MESSAGE_DIGEST_TRUE@am__append_8 = IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\ +@HAVE_POSIX_FALLOCATE_TRUE@am__append_1 = FallocFileAllocationIterator.cc FallocFileAllocationIterator.h +@HAVE_EPOLL_TRUE@am__append_2 = EpollEventPoll.cc EpollEventPoll.h +@ENABLE_SSL_TRUE@am__append_3 = TLSContext.h +@HAVE_LIBGNUTLS_TRUE@am__append_4 = LibgnutlsTLSContext.cc LibgnutlsTLSContext.h +@HAVE_LIBSSL_TRUE@am__append_5 = LibsslTLSContext.cc LibsslTLSContext.h +@HAVE_LIBZ_TRUE@am__append_6 = GZipDecoder.cc GZipDecoder.h +@HAVE_SQLITE3_TRUE@am__append_7 = Sqlite3MozCookieParser.cc Sqlite3MozCookieParser.h +@ENABLE_ASYNC_DNS_TRUE@am__append_8 = AsyncNameResolver.cc AsyncNameResolver.h +@ENABLE_MESSAGE_DIGEST_TRUE@am__append_9 = IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\ @ENABLE_MESSAGE_DIGEST_TRUE@ IteratableChecksumValidator.cc IteratableChecksumValidator.h\ @ENABLE_MESSAGE_DIGEST_TRUE@ CheckIntegrityDispatcherCommand.cc CheckIntegrityDispatcherCommand.h\ @ENABLE_MESSAGE_DIGEST_TRUE@ CheckIntegrityCommand.cc CheckIntegrityCommand.h\ @@ -52,7 +53,7 @@ bin_PROGRAMS = aria2c$(EXEEXT) @ENABLE_MESSAGE_DIGEST_TRUE@ Checksum.h\ @ENABLE_MESSAGE_DIGEST_TRUE@ ChunkChecksum.h -@ENABLE_BITTORRENT_TRUE@am__append_9 = PeerMessageUtil.cc PeerMessageUtil.h\ +@ENABLE_BITTORRENT_TRUE@am__append_10 = PeerMessageUtil.cc PeerMessageUtil.h\ @ENABLE_BITTORRENT_TRUE@ PeerAbstractCommand.cc PeerAbstractCommand.h\ @ENABLE_BITTORRENT_TRUE@ PeerInitiateConnectionCommand.cc PeerInitiateConnectionCommand.h\ @ENABLE_BITTORRENT_TRUE@ PeerInteractionCommand.cc PeerInteractionCommand.h\ @@ -219,7 +220,7 @@ bin_PROGRAMS = aria2c$(EXEEXT) @ENABLE_BITTORRENT_TRUE@ RangeBtMessageValidator.h\ @ENABLE_BITTORRENT_TRUE@ IndexBtMessageValidator.h -@ENABLE_METALINK_TRUE@am__append_10 = Metalinker.cc Metalinker.h\ +@ENABLE_METALINK_TRUE@am__append_11 = Metalinker.cc Metalinker.h\ @ENABLE_METALINK_TRUE@ MetalinkEntry.cc MetalinkEntry.h\ @ENABLE_METALINK_TRUE@ MetalinkResource.cc MetalinkResource.h\ @ENABLE_METALINK_TRUE@ MetalinkProcessor.h\ @@ -248,17 +249,17 @@ bin_PROGRAMS = aria2c$(EXEEXT) @ENABLE_METALINK_TRUE@ MetalinkPostDownloadHandler.cc MetalinkPostDownloadHandler.h\ @ENABLE_METALINK_TRUE@ MetalinkHelper.cc MetalinkHelper.h -@ENABLE_LIBXML2_TRUE@am__append_11 = XML2SAXMetalinkProcessor.cc XML2SAXMetalinkProcessor.h -@ENABLE_LIBEXPAT_TRUE@am__append_12 = ExpatMetalinkProcessor.cc ExpatMetalinkProcessor.h -@HAVE_ASCTIME_R_FALSE@am__append_13 = asctime_r.c asctime_r.h -@HAVE_BASENAME_FALSE@am__append_14 = libgen.c libgen.h -@HAVE_GETADDRINFO_FALSE@am__append_15 = getaddrinfo.c getaddrinfo.h -@HAVE_GAI_STRERROR_FALSE@am__append_16 = gai_strerror.c gai_strerror.h -@HAVE_GETTIMEOFDAY_FALSE@am__append_17 = gettimeofday.c gettimeofday.h -@HAVE_INET_ATON_FALSE@am__append_18 = inet_aton.c inet_aton.h -@HAVE_LOCALTIME_R_FALSE@am__append_19 = localtime_r.c localtime_r.h -@HAVE_STRPTIME_FALSE@am__append_20 = strptime.c strptime.h -@HAVE_TIMEGM_FALSE@am__append_21 = timegm.c timegm.h +@ENABLE_LIBXML2_TRUE@am__append_12 = XML2SAXMetalinkProcessor.cc XML2SAXMetalinkProcessor.h +@ENABLE_LIBEXPAT_TRUE@am__append_13 = ExpatMetalinkProcessor.cc ExpatMetalinkProcessor.h +@HAVE_ASCTIME_R_FALSE@am__append_14 = asctime_r.c asctime_r.h +@HAVE_BASENAME_FALSE@am__append_15 = libgen.c libgen.h +@HAVE_GETADDRINFO_FALSE@am__append_16 = getaddrinfo.c getaddrinfo.h +@HAVE_GAI_STRERROR_FALSE@am__append_17 = gai_strerror.c gai_strerror.h +@HAVE_GETTIMEOFDAY_FALSE@am__append_18 = gettimeofday.c gettimeofday.h +@HAVE_INET_ATON_FALSE@am__append_19 = inet_aton.c inet_aton.h +@HAVE_LOCALTIME_R_FALSE@am__append_20 = localtime_r.c localtime_r.h +@HAVE_STRPTIME_FALSE@am__append_21 = strptime.c strptime.h +@HAVE_TIMEGM_FALSE@am__append_22 = timegm.c timegm.h subdir = src DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in alloca.c ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 @@ -416,7 +417,8 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ HttpServerCommand.h HttpServerResponseCommand.cc \ HttpServerResponseCommand.h HttpServer.cc HttpServer.h \ PieceSelector.h LongestSequencePieceSelector.cc \ - LongestSequencePieceSelector.h EpollEventPoll.cc \ + LongestSequencePieceSelector.h FallocFileAllocationIterator.cc \ + FallocFileAllocationIterator.h EpollEventPoll.cc \ EpollEventPoll.h TLSContext.h LibgnutlsTLSContext.cc \ LibgnutlsTLSContext.h LibsslTLSContext.cc LibsslTLSContext.h \ GZipDecoder.cc GZipDecoder.h Sqlite3MozCookieParser.cc \ @@ -576,21 +578,22 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ gai_strerror.h gettimeofday.c gettimeofday.h inet_aton.c \ inet_aton.h localtime_r.c localtime_r.h strptime.c strptime.h \ timegm.c timegm.h -@HAVE_EPOLL_TRUE@am__objects_1 = EpollEventPoll.$(OBJEXT) -am__objects_2 = -@HAVE_LIBGNUTLS_TRUE@am__objects_3 = LibgnutlsTLSContext.$(OBJEXT) -@HAVE_LIBSSL_TRUE@am__objects_4 = LibsslTLSContext.$(OBJEXT) -@HAVE_LIBZ_TRUE@am__objects_5 = GZipDecoder.$(OBJEXT) -@HAVE_SQLITE3_TRUE@am__objects_6 = Sqlite3MozCookieParser.$(OBJEXT) -@ENABLE_ASYNC_DNS_TRUE@am__objects_7 = AsyncNameResolver.$(OBJEXT) -@ENABLE_MESSAGE_DIGEST_TRUE@am__objects_8 = IteratableChunkChecksumValidator.$(OBJEXT) \ +@HAVE_POSIX_FALLOCATE_TRUE@am__objects_1 = FallocFileAllocationIterator.$(OBJEXT) +@HAVE_EPOLL_TRUE@am__objects_2 = EpollEventPoll.$(OBJEXT) +am__objects_3 = +@HAVE_LIBGNUTLS_TRUE@am__objects_4 = LibgnutlsTLSContext.$(OBJEXT) +@HAVE_LIBSSL_TRUE@am__objects_5 = LibsslTLSContext.$(OBJEXT) +@HAVE_LIBZ_TRUE@am__objects_6 = GZipDecoder.$(OBJEXT) +@HAVE_SQLITE3_TRUE@am__objects_7 = Sqlite3MozCookieParser.$(OBJEXT) +@ENABLE_ASYNC_DNS_TRUE@am__objects_8 = AsyncNameResolver.$(OBJEXT) +@ENABLE_MESSAGE_DIGEST_TRUE@am__objects_9 = IteratableChunkChecksumValidator.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ IteratableChecksumValidator.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ CheckIntegrityDispatcherCommand.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ CheckIntegrityCommand.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ ChecksumCheckIntegrityEntry.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ messageDigest.$(OBJEXT) \ @ENABLE_MESSAGE_DIGEST_TRUE@ MessageDigestHelper.$(OBJEXT) -@ENABLE_BITTORRENT_TRUE@am__objects_9 = PeerMessageUtil.$(OBJEXT) \ +@ENABLE_BITTORRENT_TRUE@am__objects_10 = PeerMessageUtil.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ PeerAbstractCommand.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ PeerInitiateConnectionCommand.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ PeerInteractionCommand.$(OBJEXT) \ @@ -701,7 +704,7 @@ am__objects_2 = @ENABLE_BITTORRENT_TRUE@ RangeBtMessage.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ IndexBtMessage.$(OBJEXT) \ @ENABLE_BITTORRENT_TRUE@ ZeroBtMessage.$(OBJEXT) -@ENABLE_METALINK_TRUE@am__objects_10 = Metalinker.$(OBJEXT) \ +@ENABLE_METALINK_TRUE@am__objects_11 = Metalinker.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkEntry.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkResource.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkProcessorFactory.$(OBJEXT) \ @@ -727,20 +730,20 @@ am__objects_2 = @ENABLE_METALINK_TRUE@ Metalink2RequestGroup.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkPostDownloadHandler.$(OBJEXT) \ @ENABLE_METALINK_TRUE@ MetalinkHelper.$(OBJEXT) -@ENABLE_LIBXML2_TRUE@am__objects_11 = \ +@ENABLE_LIBXML2_TRUE@am__objects_12 = \ @ENABLE_LIBXML2_TRUE@ XML2SAXMetalinkProcessor.$(OBJEXT) -@ENABLE_LIBEXPAT_TRUE@am__objects_12 = \ +@ENABLE_LIBEXPAT_TRUE@am__objects_13 = \ @ENABLE_LIBEXPAT_TRUE@ ExpatMetalinkProcessor.$(OBJEXT) -@HAVE_ASCTIME_R_FALSE@am__objects_13 = asctime_r.$(OBJEXT) -@HAVE_BASENAME_FALSE@am__objects_14 = libgen.$(OBJEXT) -@HAVE_GETADDRINFO_FALSE@am__objects_15 = getaddrinfo.$(OBJEXT) -@HAVE_GAI_STRERROR_FALSE@am__objects_16 = gai_strerror.$(OBJEXT) -@HAVE_GETTIMEOFDAY_FALSE@am__objects_17 = gettimeofday.$(OBJEXT) -@HAVE_INET_ATON_FALSE@am__objects_18 = inet_aton.$(OBJEXT) -@HAVE_LOCALTIME_R_FALSE@am__objects_19 = localtime_r.$(OBJEXT) -@HAVE_STRPTIME_FALSE@am__objects_20 = strptime.$(OBJEXT) -@HAVE_TIMEGM_FALSE@am__objects_21 = timegm.$(OBJEXT) -am__objects_22 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ +@HAVE_ASCTIME_R_FALSE@am__objects_14 = asctime_r.$(OBJEXT) +@HAVE_BASENAME_FALSE@am__objects_15 = libgen.$(OBJEXT) +@HAVE_GETADDRINFO_FALSE@am__objects_16 = getaddrinfo.$(OBJEXT) +@HAVE_GAI_STRERROR_FALSE@am__objects_17 = gai_strerror.$(OBJEXT) +@HAVE_GETTIMEOFDAY_FALSE@am__objects_18 = gettimeofday.$(OBJEXT) +@HAVE_INET_ATON_FALSE@am__objects_19 = inet_aton.$(OBJEXT) +@HAVE_LOCALTIME_R_FALSE@am__objects_20 = localtime_r.$(OBJEXT) +@HAVE_STRPTIME_FALSE@am__objects_21 = strptime.$(OBJEXT) +@HAVE_TIMEGM_FALSE@am__objects_22 = timegm.$(OBJEXT) +am__objects_23 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ AbstractCommand.$(OBJEXT) \ InitiateConnectionCommandFactory.$(OBJEXT) \ DownloadCommand.$(OBJEXT) \ @@ -825,8 +828,8 @@ am__objects_22 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ $(am__objects_11) $(am__objects_12) $(am__objects_13) \ $(am__objects_14) $(am__objects_15) $(am__objects_16) \ $(am__objects_17) $(am__objects_18) $(am__objects_19) \ - $(am__objects_20) $(am__objects_21) -am_libaria2c_a_OBJECTS = $(am__objects_22) + $(am__objects_20) $(am__objects_21) $(am__objects_22) +am_libaria2c_a_OBJECTS = $(am__objects_23) libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS) am__installdirs = "$(DESTDIR)$(bindir)" binPROGRAMS_INSTALL = $(INSTALL_PROGRAM) @@ -1161,7 +1164,7 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \ $(am__append_12) $(am__append_13) $(am__append_14) \ $(am__append_15) $(am__append_16) $(am__append_17) \ $(am__append_18) $(am__append_19) $(am__append_20) \ - $(am__append_21) + $(am__append_21) $(am__append_22) noinst_LIBRARIES = libaria2c.a libaria2c_a_SOURCES = $(SRCS) aria2c_LDADD = libaria2c.a @LIBINTL@ @ALLOCA@ @LIBGNUTLS_LIBS@\ @@ -1390,6 +1393,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/EpollEventPoll.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Exception.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ExpatMetalinkProcessor.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FallocFileAllocationIterator.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeatureConfig.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeedbackURISelector.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/File.Po@am__quote@ diff --git a/src/MultiFileAllocationIterator.cc b/src/MultiFileAllocationIterator.cc index 63654a0e..647d4ebf 100644 --- a/src/MultiFileAllocationIterator.cc +++ b/src/MultiFileAllocationIterator.cc @@ -36,6 +36,7 @@ #include "MultiDiskAdaptor.h" #include "FileEntry.h" #include "SingleFileAllocationIterator.h" +#include "FallocFileAllocationIterator.h" #include "DiskWriter.h" namespace aria2 { @@ -61,11 +62,19 @@ void MultiFileAllocationIterator::allocateChunk() _diskAdaptor->openIfNot(entry, &DiskWriterEntry::openFile); if(entry->needsFileAllocation() && entry->size() < fileEntry->getLength()) { // Calling private function of MultiDiskAdaptor. - _fileAllocationIterator.reset - (new SingleFileAllocationIterator(entry->getDiskWriter().get(), - entry->size(), - fileEntry->getLength())); - _fileAllocationIterator->init(); + if(_diskAdaptor->doesFallocate()) { + _fileAllocationIterator.reset + (new FallocFileAllocationIterator(entry->getDiskWriter().get(), + entry->size(), + fileEntry->getLength())); + } else { + SharedHandle<SingleFileAllocationIterator> fa + (new SingleFileAllocationIterator(entry->getDiskWriter().get(), + entry->size(), + fileEntry->getLength())); + fa->init(); + _fileAllocationIterator = fa; + } } } if(finished()) { diff --git a/src/MultiFileAllocationIterator.h b/src/MultiFileAllocationIterator.h index 1f32ff04..682e6686 100644 --- a/src/MultiFileAllocationIterator.h +++ b/src/MultiFileAllocationIterator.h @@ -42,14 +42,13 @@ namespace aria2 { class MultiDiskAdaptor; class DiskWriterEntry; -class SingleFileAllocationIterator; class MultiFileAllocationIterator:public FileAllocationIterator { private: MultiDiskAdaptor* _diskAdaptor; std::deque<SharedHandle<DiskWriterEntry> > _entries; - SharedHandle<SingleFileAllocationIterator> _fileAllocationIterator; + SharedHandle<FileAllocationIterator> _fileAllocationIterator; off_t _offset; public: diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index 84625d61..b4a92f97 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -203,6 +203,9 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers() TEXT_FILE_ALLOCATION, V_PREALLOC, V_NONE, V_PREALLOC, +#ifdef HAVE_POSIX_FALLOCATE + V_FALLOC, +#endif // HAVE_POSIX_FALLOCATE 'a')); op->addTag(TAG_BASIC); handlers.push_back(op); diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 09299505..871c0d41 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -134,11 +134,7 @@ RequestGroup::RequestGroup(const Option* option, _option(option), _logger(LogFactory::getInstance()) { - if(_option->get(PREF_FILE_ALLOCATION) == V_PREALLOC) { - _fileAllocationEnabled = true; - } else { - _fileAllocationEnabled = false; - } + _fileAllocationEnabled = _option->get(PREF_FILE_ALLOCATION) != V_NONE; // Add types to be sent as a Accept header value here. // It would be good to put this value in Option so that user can tweak // and add this list. diff --git a/src/StreamCheckIntegrityEntry.cc b/src/StreamCheckIntegrityEntry.cc index e8685422..5a4014c3 100644 --- a/src/StreamCheckIntegrityEntry.cc +++ b/src/StreamCheckIntegrityEntry.cc @@ -36,7 +36,6 @@ #include "RequestGroup.h" #include "DownloadEngine.h" #include "StreamFileAllocationEntry.h" -#include "FileAllocationMan.h" #include "Request.h" namespace aria2 { @@ -50,17 +49,13 @@ StreamCheckIntegrityEntry::StreamCheckIntegrityEntry(const RequestHandle& curren StreamCheckIntegrityEntry::~StreamCheckIntegrityEntry() {} -void StreamCheckIntegrityEntry::onDownloadIncomplete(std::deque<Command*>& commands, - DownloadEngine* e) +void StreamCheckIntegrityEntry::onDownloadIncomplete +(std::deque<Command*>& commands, DownloadEngine* e) { FileAllocationEntryHandle entry (new StreamFileAllocationEntry(_currentRequest, _requestGroup, popNextCommand())); - if(_requestGroup->needsFileAllocation()) { - e->_fileAllocationMan->pushEntry(entry); - } else { - entry->prepareForNextAction(commands, e); - } + proceedFileAllocation(commands, entry, e); } void StreamCheckIntegrityEntry::onDownloadFinished(std::deque<Command*>& commands, diff --git a/src/prefs.cc b/src/prefs.cc index 9ef9ed07..0cca4040 100644 --- a/src/prefs.cc +++ b/src/prefs.cc @@ -81,10 +81,11 @@ const std::string PREF_MAX_OVERALL_DOWNLOAD_LIMIT("max-overall-download-limit"); const std::string PREF_MAX_DOWNLOAD_LIMIT("max-download-limit"); // value: 1*digit const std::string PREF_STARTUP_IDLE_TIME("startup-idle-time"); -// value: prealloc | none +// value: prealloc | fallc | none const std::string PREF_FILE_ALLOCATION("file-allocation"); const std::string V_PREALLOC("prealloc"); -#// value: 1*digit +const std::string V_FALLOC("falloc"); +// value: 1*digit const std::string PREF_NO_FILE_ALLOCATION_LIMIT("no-file-allocation-limit"); // value: true | false const std::string PREF_ALLOW_OVERWRITE("allow-overwrite"); diff --git a/src/prefs.h b/src/prefs.h index 3c45e39f..c7974c3e 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -83,9 +83,10 @@ extern const std::string PREF_SEGMENT_SIZE; extern const std::string PREF_MAX_DOWNLOAD_LIMIT; // value: 1*digit extern const std::string PREF_STARTUP_IDLE_TIME; -// value: prealloc | none +// value: prealloc | falloc | none extern const std::string PREF_FILE_ALLOCATION; extern const std::string V_PREALLOC; +extern const std::string V_FALLOC; // value: 1*digit extern const std::string PREF_NO_FILE_ALLOCATION_LIMIT; // value: true | false diff --git a/src/usage_text.h b/src/usage_text.h index 43a5d66e..e78441af 100644 --- a/src/usage_text.h +++ b/src/usage_text.h @@ -120,7 +120,16 @@ _(" --file-allocation=METHOD Specify file allocation method.\n"\ " 'none' doesn't pre-allocate file space. 'prealloc'\n"\ " pre-allocates file space before download begins.\n"\ " This may take some time depending on the size of\n"\ - " the file.") + " the file.\n"\ + " If you are using newer file systems such as ext4\n"\ + " (with extents support), btrfs or xfs, 'falloc' is\n"\ + " your best choice. It allocates large(few GiB)\n"\ + " files almost instantly. Don't use 'falloc' with\n"\ + " legacy file systems such as ext3 because it takes\n"\ + " almost same time as 'prealloc' and it blocks aria2\n"\ + " entirely until allocation finishes. 'falloc' may\n"\ + " not be available if your system doesn't have\n"\ + " posix_fallocate() system call.") #define TEXT_NO_FILE_ALLOCATION_LIMIT \ _(" --no-file-allocation-limit=SIZE No file allocation is made for files whose\n"\ " size is smaller than SIZE.\n"\ |
