diff options
| author | Tatsuhiro Tsujikawa <tatsuhiro.t@gmail.com> | 2009-06-23 15:35:45 +0000 |
|---|---|---|
| committer | Tatsuhiro Tsujikawa <tatsuhiro.t@gmail.com> | 2009-06-23 15:35:45 +0000 |
| commit | 90471d68050ebd05439435330e4d0dbb19c6fd1e (patch) | |
| tree | 7324c9a0e4d57328c15d32cb1efaa2a91ad774a9 | |
| parent | 6b2ab3ffc0d96d66723fa8b43552fd09d2108781 (diff) | |
2009-06-24 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Added experimental support of WEB-Seeding for multi-file torrent.
Due to fundamental changes in file handling in HTTP/FTP code, many
functions are not working: PeerStat, ServerHost, proxy..etc
* src/AbstractCommand.cc
* src/AbstractCommand.h
* src/BitfieldMan.cc
* src/BitfieldMan.h
* src/CreateRequestCommand.cc
* src/CreateRequestCommand.h
* src/DefaultPieceStorage.cc
* src/DefaultPieceStorage.h
* src/DownloadCommand.cc
* src/DownloadCommand.h
* src/DownloadContext.cc
* src/DownloadContext.h
* src/FileEntry.cc
* src/FileEntry.h
* src/FtpDownloadCommand.cc
* src/FtpDownloadCommand.h
* src/FtpFinishDownloadCommand.cc
* src/FtpFinishDownloadCommand.h
* src/FtpInitiateConnectionCommand.cc
* src/FtpInitiateConnectionCommand.h
* src/FtpNegotiationCommand.cc
* src/FtpNegotiationCommand.h
* src/FtpTunnelResponseCommand.cc
* src/HttpDownloadCommand.cc
* src/HttpDownloadCommand.h
* src/HttpInitiateConnectionCommand.cc
* src/HttpInitiateConnectionCommand.h
* src/HttpProxyResponseCommand.cc
* src/HttpRequest.cc
* src/HttpRequest.h
* src/HttpRequestCommand.cc
* src/HttpRequestCommand.h
* src/HttpResponseCommand.cc
* src/HttpResponseCommand.h
* src/HttpSkipResponseCommand.cc
* src/HttpSkipResponseCommand.h
* src/InitiateConnectionCommand.cc
* src/InitiateConnectionCommand.h
* src/InitiateConnectionCommandFactory.cc
* src/InitiateConnectionCommandFactory.h
* src/Makefile.am
* src/PieceStorage.h
* src/RequestGroup.cc
* src/RequestGroup.h
* src/RequestGroupMan.cc
* src/SegmentMan.cc
* src/SegmentMan.h
* src/SingleFileDownloadContext.h
* src/StreamFileAllocationEntry.cc
* src/TrackerWatcherCommand.cc
* src/UnknownLengthPieceStorage.cc
* src/UnknownLengthPieceStorage.h
* src/array_fun.h
* src/bitfield.h
* src/download_helper.cc
* test/DownloadContextTest.cc
* test/Makefile.am
* test/MockDownloadContext.h
61 files changed, 936 insertions, 184 deletions
@@ -1,5 +1,69 @@ 2009-06-24 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net> + Added experimental support of WEB-Seeding for multi-file torrent. + Due to fundamental changes in file handling in HTTP/FTP code, many + functions are not working: PeerStat, ServerHost, proxy..etc + * src/AbstractCommand.cc + * src/AbstractCommand.h + * src/BitfieldMan.cc + * src/BitfieldMan.h + * src/CreateRequestCommand.cc + * src/CreateRequestCommand.h + * src/DefaultPieceStorage.cc + * src/DefaultPieceStorage.h + * src/DownloadCommand.cc + * src/DownloadCommand.h + * src/DownloadContext.cc + * src/DownloadContext.h + * src/FileEntry.cc + * src/FileEntry.h + * src/FtpDownloadCommand.cc + * src/FtpDownloadCommand.h + * src/FtpFinishDownloadCommand.cc + * src/FtpFinishDownloadCommand.h + * src/FtpInitiateConnectionCommand.cc + * src/FtpInitiateConnectionCommand.h + * src/FtpNegotiationCommand.cc + * src/FtpNegotiationCommand.h + * src/FtpTunnelResponseCommand.cc + * src/HttpDownloadCommand.cc + * src/HttpDownloadCommand.h + * src/HttpInitiateConnectionCommand.cc + * src/HttpInitiateConnectionCommand.h + * src/HttpProxyResponseCommand.cc + * src/HttpRequest.cc + * src/HttpRequest.h + * src/HttpRequestCommand.cc + * src/HttpRequestCommand.h + * src/HttpResponseCommand.cc + * src/HttpResponseCommand.h + * src/HttpSkipResponseCommand.cc + * src/HttpSkipResponseCommand.h + * src/InitiateConnectionCommand.cc + * src/InitiateConnectionCommand.h + * src/InitiateConnectionCommandFactory.cc + * src/InitiateConnectionCommandFactory.h + * src/Makefile.am + * src/PieceStorage.h + * src/RequestGroup.cc + * src/RequestGroup.h + * src/RequestGroupMan.cc + * src/SegmentMan.cc + * src/SegmentMan.h + * src/SingleFileDownloadContext.h + * src/StreamFileAllocationEntry.cc + * src/TrackerWatcherCommand.cc + * src/UnknownLengthPieceStorage.cc + * src/UnknownLengthPieceStorage.h + * src/array_fun.h + * src/bitfield.h + * src/download_helper.cc + * test/DownloadContextTest.cc + * test/Makefile.am + * test/MockDownloadContext.h + +2009-06-24 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net> + Added tellWaiting XML-RPC method. * src/SingleFileDownloadContext.cc * src/XmlRpcMethodFactory.cc diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index 39aa7a9e..b1004a48 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -47,7 +47,7 @@ #include "DlAbortEx.h" #include "DlRetryEx.h" #include "DownloadFailureException.h" -#include "InitiateConnectionCommandFactory.h" +#include "CreateRequestCommand.h" #include "SleepCommand.h" #ifdef ENABLE_ASYNC_DNS #include "AsyncNameResolver.h" @@ -62,9 +62,12 @@ #include "RequestGroupMan.h" #include "A2STR.h" #include "Util.h" +#include "LogFactory.h" +#include "DownloadContext.h" namespace aria2 { +// TODO1.5 Remove this AbstractCommand::AbstractCommand(int32_t cuid, const SharedHandle<Request>& req, RequestGroup* requestGroup, @@ -83,6 +86,25 @@ AbstractCommand::AbstractCommand(int32_t cuid, _requestGroup->increaseNumCommand(); } +AbstractCommand::AbstractCommand(int32_t cuid, + const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, + RequestGroup* requestGroup, + DownloadEngine* e, + const SocketHandle& s): + Command(cuid), _requestGroup(requestGroup), + req(req), _fileEntry(fileEntry), e(e), socket(s), + checkSocketIsReadable(false), checkSocketIsWritable(false), + nameResolverCheck(false) +{ + if(!socket.isNull() && socket->isOpen()) { + setReadCheckSocket(socket); + } + timeout = _requestGroup->getTimeout(); + _requestGroup->increaseStreamConnection(); + _requestGroup->increaseNumCommand(); +} + AbstractCommand::~AbstractCommand() { disableReadCheckSocket(); disableWriteCheckSocket(); @@ -125,7 +147,8 @@ bool AbstractCommand::execute() { if(!_requestGroup->getPieceStorage().isNull()) { _segments.clear(); _requestGroup->getSegmentMan()->getInFlightSegment(_segments, cuid); - while(_segments.size() < req->getMaxPipelinedRequest()) { + size_t maxSegments = req.isNull()?1:req->getMaxPipelinedRequest(); + while(_segments.size() < maxSegments) { SegmentHandle segment = _requestGroup->getSegmentMan()->getSegment(cuid); if(segment.isNull()) { break; @@ -135,7 +158,7 @@ bool AbstractCommand::execute() { if(_segments.empty()) { // TODO socket could be pooled here if pipelining is enabled... logger->info(MSG_NO_SEGMENT_AVAILABLE, cuid); - return prepareForRetry(1); + return true; } } return executeInternal(); @@ -146,6 +169,7 @@ bool AbstractCommand::execute() { } else { if(checkPoint.elapsed(timeout)) { // timeout triggers ServerStat error state. + SharedHandle<ServerStat> ss = e->_requestGroupMan->getOrCreateServerStat(req->getHost(), req->getProtocol()); @@ -157,13 +181,18 @@ bool AbstractCommand::execute() { return false; } } catch(DlAbortEx& err) { - logger->error(MSG_DOWNLOAD_ABORTED, - DL_ABORT_EX2(StringFormat - ("URI=%s", req->getCurrentUrl().c_str()).str(),err), - cuid, req->getUrl().c_str()); - _requestGroup->addURIResult(req->getUrl(), err.getCode()); + if(req.isNull()) { + logger->debug(EX_EXCEPTION_CAUGHT, err); + } else { + logger->error(MSG_DOWNLOAD_ABORTED, + DL_ABORT_EX2(StringFormat + ("URI=%s", req->getCurrentUrl().c_str()).str(),err), + cuid, req->getUrl().c_str()); + _requestGroup->addURIResult(req->getUrl(), err.getCode()); + } onAbort(); - req->resetUrl(); + // TODO Do we need this? + //req->resetUrl(); tryReserved(); return true; } catch(DlRetryEx& err) { @@ -202,6 +231,16 @@ bool AbstractCommand::execute() { void AbstractCommand::tryReserved() { _requestGroup->removeServerHost(cuid); + if(_requestGroup->getDownloadContext()->getFileMode() == DownloadContext::SINGLE) { + const SharedHandle<FileEntry>& entry = + _requestGroup->getDownloadContext()->getFileEntries().front(); + // Don't create new command if currently file length is unknown + // and there are no URI left. Because file length is unknown, we + // can assume that there are no in-flight request object. + if(entry->getLength() == 0 && entry->getRemainingUris().size() == 0) { + return; + } + } Commands commands; _requestGroup->createNextCommand(commands, e, 1); e->setNoWait(true); @@ -212,7 +251,18 @@ bool AbstractCommand::prepareForRetry(time_t wait) { if(!_requestGroup->getPieceStorage().isNull()) { _requestGroup->getSegmentMan()->cancelSegment(cuid); } - Command* command = InitiateConnectionCommandFactory::createInitiateConnectionCommand(cuid, req, _requestGroup, e); + if(!req.isNull()) { + _fileEntry->poolRequest(req); + } + if(!_segments.empty()) { + // TODO1.5 subtract 1 from getPositionToWrite() + SharedHandle<FileEntry> fileEntry = _requestGroup->getDownloadContext()->findFileEntryByOffset(_segments.front()->getPositionToWrite()-1); + logger->debug("CUID#%d - Pooling request URI=%s", + cuid, req->getUrl().c_str()); + _requestGroup->getSegmentMan()->recognizeSegmentFor(_fileEntry); + } + + Command* command = new CreateRequestCommand(cuid, _requestGroup, e); if(wait == 0) { e->setNoWait(true); e->commands.push_back(command); @@ -225,16 +275,22 @@ bool AbstractCommand::prepareForRetry(time_t wait) { } void AbstractCommand::onAbort() { - // TODO This might be a problem if the failure is caused by proxy. - e->_requestGroupMan->getOrCreateServerStat(req->getHost(), - req->getProtocol())->setError(); + if(!req.isNull()) { + logger->debug(req->getCurrentUrl().c_str()); + // TODO This might be a problem if the failure is caused by proxy. + e->_requestGroupMan->getOrCreateServerStat(req->getHost(), + req->getProtocol())->setError(); + _requestGroup->removeIdenticalURI(req->getUrl()); + _fileEntry->removeRequest(req); + } logger->debug(MSG_UNREGISTER_CUID, cuid); //_segmentMan->unregisterId(cuid); if(!_requestGroup->getPieceStorage().isNull()) { _requestGroup->getSegmentMan()->cancelSegment(cuid); } - _requestGroup->removeIdenticalURI(req->getUrl()); + // TODO1.5 Should be moved to FileEntry + // _requestGroup->removeIdenticalURI(req->getUrl()); } void AbstractCommand::disableReadCheckSocket() { diff --git a/src/AbstractCommand.h b/src/AbstractCommand.h index fa3781f0..ba7a7aa4 100644 --- a/src/AbstractCommand.h +++ b/src/AbstractCommand.h @@ -38,6 +38,7 @@ #include "Command.h" #include "SharedHandle.h" #include "TimeA2.h" +#include "FileEntry.h" namespace aria2 { @@ -59,6 +60,7 @@ private: protected: RequestGroup* _requestGroup; SharedHandle<Request> req; + SharedHandle<FileEntry> _fileEntry; DownloadEngine* e; SharedHandle<SocketCore> socket; std::deque<SharedHandle<Segment> > _segments; @@ -140,6 +142,11 @@ public: RequestGroup* requestGroup, DownloadEngine* e, const SharedHandle<SocketCore>& s = SharedHandle<SocketCore>()); + AbstractCommand(int32_t cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, + RequestGroup* requestGroup, DownloadEngine* e, + const SharedHandle<SocketCore>& s = SharedHandle<SocketCore>()); + virtual ~AbstractCommand(); bool execute(); }; diff --git a/src/BitfieldMan.cc b/src/BitfieldMan.cc index 1fbfcb8b..2569c691 100644 --- a/src/BitfieldMan.cc +++ b/src/BitfieldMan.cc @@ -308,9 +308,10 @@ bool BitfieldMan::getMissingUnusedIndex(size_t& index) const } } -size_t BitfieldMan::getStartIndex(size_t index) const { - while(index < blocks && (isUseBitSet(index) || isBitSet(index))) { - index++; +template<typename Array> +static size_t getStartIndex(size_t index, const Array& bitfield, const unsigned char* useBitfield, size_t blocks) { + while(index < blocks && (bitfield::test(bitfield, blocks, index) || bitfield::test(useBitfield, blocks, index))) { + ++index; } if(blocks <= index) { return blocks; @@ -319,24 +320,33 @@ size_t BitfieldMan::getStartIndex(size_t index) const { } } -size_t BitfieldMan::getEndIndex(size_t index) const { - while(index < blocks && (!isUseBitSet(index) && !isBitSet(index))) { - index++; +template<typename Array> +static size_t getEndIndex(size_t index, const Array& bitfield, const unsigned char* useBitfield, size_t blocks) { + while(index < blocks && (!bitfield::test(bitfield, blocks, index) && !bitfield::test(useBitfield, blocks, index))) { + ++index; } return index; } -bool BitfieldMan::getSparseMissingUnusedIndex(size_t& index) const { - Range maxRange; - Range currentRange; +template<typename Array> +static bool getSparseMissingUnusedIndex +(size_t& index, + const Array& bitfield, + const unsigned char* useBitfield, + size_t blocks) +{ + BitfieldMan::Range maxRange; + BitfieldMan::Range currentRange; { size_t nextIndex = 0; while(nextIndex < blocks) { - currentRange.startIndex = getStartIndex(nextIndex); + currentRange.startIndex = + getStartIndex(nextIndex, bitfield, useBitfield, blocks); if(currentRange.startIndex == blocks) { break; } - currentRange.endIndex = getEndIndex(currentRange.startIndex); + currentRange.endIndex = + getEndIndex(currentRange.startIndex, bitfield, useBitfield, blocks); if(maxRange < currentRange) { maxRange = currentRange; } @@ -346,7 +356,7 @@ bool BitfieldMan::getSparseMissingUnusedIndex(size_t& index) const { if(maxRange.getSize()) { if(maxRange.startIndex == 0) { index = 0; - } else if(isUseBitSet(maxRange.startIndex-1)) { + } else if(bitfield::test(useBitfield, blocks, maxRange.startIndex-1)) { index = maxRange.getMidIndex(); } else { index = maxRange.startIndex; @@ -357,6 +367,22 @@ bool BitfieldMan::getSparseMissingUnusedIndex(size_t& index) const { } } +bool BitfieldMan::getSparseMissingUnusedIndex +(size_t& index, + const unsigned char* ignoreBitfield, + size_t ignoreBitfieldLength) const +{ + if(filterEnabled) { + return aria2::getSparseMissingUnusedIndex + (index, array(ignoreBitfield)|~array(filterBitfield)|array(bitfield), + useBitfield, blocks); + } else { + return aria2::getSparseMissingUnusedIndex + (index, array(ignoreBitfield)|array(bitfield), + useBitfield, blocks); + } +} + template<typename Array> static bool copyBitfield(unsigned char* dst, const Array& src, size_t blocks) { @@ -571,6 +597,21 @@ void BitfieldMan::addFilter(uint64_t offset, uint64_t length) { updateCache(); } +void BitfieldMan::removeFilter(uint64_t offset, uint64_t length) { + if(!filterBitfield) { + filterBitfield = new unsigned char[bitfieldLength]; + memset(filterBitfield, 0, bitfieldLength); + } + if(length > 0) { + size_t startBlock = offset/blockLength; + size_t endBlock = (offset+length-1)/blockLength; + for(size_t i = startBlock; i <= endBlock && i < blocks; i++) { + setBitInternal(filterBitfield, i, false); + } + } + updateCache(); +} + void BitfieldMan::enableFilter() { if(!filterBitfield) { filterBitfield = new unsigned char[bitfieldLength]; diff --git a/src/BitfieldMan.h b/src/BitfieldMan.h index 3287e2c0..909d82c7 100644 --- a/src/BitfieldMan.h +++ b/src/BitfieldMan.h @@ -83,7 +83,7 @@ private: size_t getEndIndex(size_t index) const; uint64_t getCompletedLength(bool useFilter) const; - +public: // [startIndex, endIndex) class Range { public: @@ -160,7 +160,11 @@ public: /** * affected by filter */ - bool getSparseMissingUnusedIndex(size_t& index) const; + bool getSparseMissingUnusedIndex + (size_t& index, + const unsigned char* ignoreBitfield, + size_t ignoreBitfieldLength) const; + /** * affected by filter */ @@ -243,6 +247,7 @@ public: void setAllUseBit(); void addFilter(uint64_t offset, uint64_t length); + void removeFilter(uint64_t offset, uint64_t length); /** * Clears filter and disables filter */ @@ -306,6 +311,10 @@ public: uint64_t getMissingUnusedLength(size_t startingIndex) const; + const unsigned char* getFilterBitfield() const + { + return filterBitfield; + } }; } // namespace aria2 diff --git a/src/CreateRequestCommand.cc b/src/CreateRequestCommand.cc new file mode 100644 index 00000000..824d26c4 --- /dev/null +++ b/src/CreateRequestCommand.cc @@ -0,0 +1,85 @@ +/* <!-- copyright */ +/* + * aria2 - The high speed download utility + * + * Copyright (C) 2009 Tatsuhiro Tsujikawa + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * In addition, as a special exception, the copyright holders give + * permission to link the code of portions of this program with the + * OpenSSL library under certain conditions as described in each + * individual source file, and distribute linked combinations + * including the two. + * You must obey the GNU General Public License in all respects + * for all of the code used other than OpenSSL. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you + * do not wish to do so, delete this exception statement from your + * version. If you delete this exception statement from all source + * files in the program, then also delete it here. + */ +/* copyright --> */ +#include "CreateRequestCommand.h" + +#include "InitiateConnectionCommandFactory.h" +#include "RequestGroup.h" +#include "Segment.h" +#include "DownloadContext.h" +#include "DlAbortEx.h" +#include "DownloadEngine.h" +#include "SocketCore.h" +#include "SegmentMan.h" + +namespace aria2 { + +CreateRequestCommand::CreateRequestCommand(int32_t cuid, + RequestGroup* requestGroup, + DownloadEngine* e): + AbstractCommand(cuid, SharedHandle<Request>(), requestGroup, e) +{ + setStatus(Command::STATUS_ONESHOT_REALTIME); + disableReadCheckSocket(); + disableWriteCheckSocket(); +} + +bool CreateRequestCommand::executeInternal() +{ + if(_segments.empty()) { + _fileEntry = _requestGroup->getDownloadContext()->findFileEntryByOffset(0); + } else { + // We assume all segments belongs to same file. + _fileEntry = _requestGroup->getDownloadContext()->findFileEntryByOffset + (_segments.front()->getPositionToWrite()); + } + req = _fileEntry->getRequest(_requestGroup->getURISelector()); + if(req.isNull()) { + if(!_requestGroup->getSegmentMan().isNull()) { + _requestGroup->getSegmentMan()->ignoreSegmentFor(_fileEntry); + } + throw DL_ABORT_EX("No URI available."); + } + + Command* command = + InitiateConnectionCommandFactory::createInitiateConnectionCommand + (cuid, req, _fileEntry, _requestGroup, e); + //ServerHostHandle sv(new ServerHost(command->getCuid(), req->getHost())); + //registerServerHost(sv); + e->setNoWait(true); + e->commands.push_back(command); + return true; +} + +} // namespace aria2 diff --git a/src/CreateRequestCommand.h b/src/CreateRequestCommand.h new file mode 100644 index 00000000..d4f5fae3 --- /dev/null +++ b/src/CreateRequestCommand.h @@ -0,0 +1,53 @@ +/* <!-- copyright */ +/* + * aria2 - The high speed download utility + * + * Copyright (C) 2009 Tatsuhiro Tsujikawa + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * In addition, as a special exception, the copyright holders give + * permission to link the code of portions of this program with the + * OpenSSL library under certain conditions as described in each + * individual source file, and distribute linked combinations + * including the two. + * You must obey the GNU General Public License in all respects + * for all of the code used other than OpenSSL. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you + * do not wish to do so, delete this exception statement from your + * version. If you delete this exception statement from all source + * files in the program, then also delete it here. + */ +/* copyright --> */ +#ifndef _D_CREATE_REQUEST_COMMAND_H_ +#define _D_CREATE_REQUEST_COMMAND_H_ + +#include "AbstractCommand.h" + +namespace aria2 { + +class CreateRequestCommand:public AbstractCommand { +public: + CreateRequestCommand(int32_t cuid, + RequestGroup* requestGroup, + DownloadEngine* e); +protected: + virtual bool executeInternal(); +}; + +} // namespace aria2 + +#endif // _D_CREATE_REQUEST_COMMAND_H_ diff --git a/src/DefaultPieceStorage.cc b/src/DefaultPieceStorage.cc index a47fd80e..3a707558 100644 --- a/src/DefaultPieceStorage.cc +++ b/src/DefaultPieceStorage.cc @@ -261,10 +261,11 @@ SharedHandle<Piece> DefaultPieceStorage::getMissingFastPiece #endif // ENABLE_BITTORRENT -PieceHandle DefaultPieceStorage::getMissingPiece() +PieceHandle DefaultPieceStorage::getSparseMissingUnusedPiece +(const unsigned char* ignoreBitfield, size_t length) { size_t index; - if(bitfieldMan->getSparseMissingUnusedIndex(index)) { + if(bitfieldMan->getSparseMissingUnusedIndex(index, ignoreBitfield, length)) { return checkOutPiece(index); } else { return SharedHandle<Piece>(); diff --git a/src/DefaultPieceStorage.h b/src/DefaultPieceStorage.h index 1bf01f62..36e42728 100644 --- a/src/DefaultPieceStorage.h +++ b/src/DefaultPieceStorage.h @@ -133,7 +133,8 @@ public: #endif // ENABLE_BITTORRENT - virtual SharedHandle<Piece> getMissingPiece(); + virtual SharedHandle<Piece> getSparseMissingUnusedPiece + (const unsigned char* ignoreBitfield, size_t length); virtual SharedHandle<Piece> getMissingPiece(size_t index); diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index 7812d375..3c563984 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -66,10 +66,11 @@ namespace aria2 { DownloadCommand::DownloadCommand(int cuid, const RequestHandle& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, DownloadEngine* e, const SocketHandle& s): - AbstractCommand(cuid, req, requestGroup, e, s) + AbstractCommand(cuid, req, fileEntry, requestGroup, e, s) #ifdef ENABLE_MESSAGE_DIGEST , _pieceHashValidationEnabled(false) #endif // ENABLE_MESSAGE_DIGEST @@ -114,8 +115,13 @@ bool DownloadCommand::executeInternal() { size_t BUFSIZE = 16*1024; unsigned char buf[BUFSIZE]; size_t bufSize; - if(segment->getLength() > 0 && segment->getLength()-segment->getWrittenLength() < BUFSIZE) { - bufSize = segment->getLength()-segment->getWrittenLength(); + if(segment->getLength() > 0) { + if(segment->getPosition()+segment->getLength() <= static_cast<uint64_t>(_fileEntry->getLastOffset())) { + bufSize = std::min(segment->getLength()-segment->getWrittenLength(), + BUFSIZE); + } else { + bufSize = std::min(static_cast<size_t>(_fileEntry->getLastOffset()-_fileEntry->gtoloff(segment->getPositionToWrite())), BUFSIZE); + } } else { bufSize = BUFSIZE; } @@ -167,13 +173,15 @@ bool DownloadCommand::executeInternal() { bool segmentComplete = false; // Note that GrowSegment::complete() always returns false. if(_transferEncodingDecoder.isNull() && _contentEncodingDecoder.isNull()) { - if(segment->complete()) { + if(segment->complete() || + segment->getPositionToWrite() == _fileEntry->getLastOffset()) { segmentComplete = true; } else if(segment->getLength() == 0 && bufSize == 0 && !socket->wantRead() && !socket->wantWrite()) { segmentComplete = true; } - } else if(!_transferEncodingDecoder.isNull() && segment->complete()) { + } else if(!_transferEncodingDecoder.isNull() && + (segment->complete() || segment->getPositionToWrite() == _fileEntry->getLastOffset())) { segmentComplete = true; } else if((_transferEncodingDecoder.isNull() || _transferEncodingDecoder->finished()) && @@ -188,36 +196,40 @@ bool DownloadCommand::executeInternal() { } if(segmentComplete) { - logger->info(MSG_SEGMENT_DOWNLOAD_COMPLETED, cuid); + if(segment->complete() || segment->getLength() == 0) { + // If segment->getLength() == 0, the server doesn't provide + // content length, but the client detected that download + // completed. + logger->info(MSG_SEGMENT_DOWNLOAD_COMPLETED, cuid); #ifdef ENABLE_MESSAGE_DIGEST - { - std::string expectedPieceHash = - _requestGroup->getDownloadContext()->getPieceHash(segment->getIndex()); - if(_pieceHashValidationEnabled && !expectedPieceHash.empty()) { - if(segment->isHashCalculated()) { - logger->debug("Hash is available! index=%lu", - static_cast<unsigned long>(segment->getIndex())); - validatePieceHash(segment, expectedPieceHash, segment->getHashString()); + { + std::string expectedPieceHash = + _requestGroup->getDownloadContext()->getPieceHash(segment->getIndex()); + if(_pieceHashValidationEnabled && !expectedPieceHash.empty()) { + if(segment->isHashCalculated()) { + logger->debug("Hash is available! index=%lu", + static_cast<unsigned long>(segment->getIndex())); + validatePieceHash(segment, expectedPieceHash, segment->getHashString()); + } else { + _messageDigestContext->digestReset(); + validatePieceHash(segment, expectedPieceHash, + MessageDigestHelper::digest + (_messageDigestContext.get(), + _requestGroup->getPieceStorage()->getDiskAdaptor(), + segment->getPosition(), + segment->getLength())); + } } else { - _messageDigestContext->digestReset(); - validatePieceHash(segment, expectedPieceHash, - MessageDigestHelper::digest - (_messageDigestContext.get(), - _requestGroup->getPieceStorage()->getDiskAdaptor(), - segment->getPosition(), - segment->getLength())); + _requestGroup->getSegmentMan()->completeSegment(cuid, segment); } - } else { - _requestGroup->getSegmentMan()->completeSegment(cuid, segment); } - } #else // !ENABLE_MESSAGE_DIGEST - - _requestGroup->getSegmentMan()->completeSegment(cuid, segment); - + _requestGroup->getSegmentMan()->completeSegment(cuid, segment); #endif // !ENABLE_MESSAGE_DIGEST + } + checkLowestDownloadSpeed(); // this unit is going to download another segment. @@ -263,6 +275,10 @@ bool DownloadCommand::prepareForNextSegment() { // segment. if(_segments.size() == 1) { SegmentHandle tempSegment = _segments.front(); + if(!tempSegment->complete()) { + return prepareForRetry(0); + } + // TODO1.5 get segment for the same file only SegmentHandle nextSegment = _requestGroup->getSegmentMan()->getSegment(cuid, tempSegment->getIndex()+1); diff --git a/src/DownloadCommand.h b/src/DownloadCommand.h index 05d3b670..ad078746 100644 --- a/src/DownloadCommand.h +++ b/src/DownloadCommand.h @@ -76,6 +76,7 @@ protected: public: DownloadCommand(int cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, DownloadEngine* e, const SharedHandle<SocketCore>& s); diff --git a/src/DownloadContext.cc b/src/DownloadContext.cc index 2d3b888a..a5dda4d2 100644 --- a/src/DownloadContext.cc +++ b/src/DownloadContext.cc @@ -33,6 +33,9 @@ */ /* copyright --> */ #include "DownloadContext.h" + +#include <algorithm> + #include "FileEntry.h" namespace aria2 { @@ -79,4 +82,25 @@ int64_t DownloadContext::calculateSessionTime() const } } +SharedHandle<FileEntry> +DownloadContext::findFileEntryByOffset(off_t offset) const +{ + if(_fileEntries.empty() || + (offset > 0 && + _fileEntries.back()->getOffset()+_fileEntries.back()->getLength() <= + static_cast<uint64_t>(offset))){ + return SharedHandle<FileEntry>(); + } + + SharedHandle<FileEntry> obj(new FileEntry()); + obj->setOffset(offset); + std::deque<SharedHandle<FileEntry> >::const_iterator i = + std::upper_bound(_fileEntries.begin(), _fileEntries.end(), obj); + if(i != _fileEntries.end() && (*i)->getOffset() == offset) { + return *i; + } else { + return *(--i); + } +} + } // namespace aria2 diff --git a/src/DownloadContext.h b/src/DownloadContext.h index e40c9a18..f1aed08e 100644 --- a/src/DownloadContext.h +++ b/src/DownloadContext.h @@ -117,6 +117,10 @@ public: void resetDownloadStopTime(); int64_t calculateSessionTime() const; + + // Returns FileEntry at given offset. SharedHandle<FileEntry>() is + // returned if no such FileEntry is found. + SharedHandle<FileEntry> findFileEntryByOffset(off_t offset) const; }; typedef SharedHandle<DownloadContext> DownloadContextHandle; diff --git a/src/FileEntry.cc b/src/FileEntry.cc index 8abc1a5d..4e23f5d4 100644 --- a/src/FileEntry.cc +++ b/src/FileEntry.cc @@ -33,8 +33,11 @@ */ /* copyright --> */ #include "FileEntry.h" -#include "File.h" + +#include <cassert> + #include "Util.h" +#include "URISelector.h" namespace aria2 { @@ -74,4 +77,66 @@ bool FileEntry::exists() const return File(getPath()).exists(); } +off_t FileEntry::gtoloff(off_t goff) const +{ + assert(offset <= goff); + return goff-offset; +} + +void FileEntry::getUris(std::deque<std::string>& uris) const +{ + uris.insert(uris.end(), _spentUris.begin(), _spentUris.end()); + uris.insert(uris.end(), _uris.begin(), _uris.end()); +} + +std::string FileEntry::selectUri(const SharedHandle<URISelector>& uriSelector) +{ + return uriSelector->select(_uris); +} + +SharedHandle<Request> +FileEntry::getRequest(const SharedHandle<URISelector>& selector) +{ + SharedHandle<Request> req; + if(_requestPool.empty()) { + while(1) { + std::string uri = selector->select(_uris); + if(uri.empty()) { + return req; + } + req.reset(new Request()); + if(req->setUrl(uri)) { + _spentUris.push_back(uri); + _inFlightRequests.push_back(req); + return req; + } else { + req.reset(); + } + } + } else { + req = _requestPool.back(); + _requestPool.pop_back(); + _inFlightRequests.push_back(req); + return req; + } +} + +void FileEntry::poolRequest(const SharedHandle<Request>& request) +{ + removeRequest(request); + _requestPool.push_back(request); +} + +bool FileEntry::removeRequest(const SharedHandle<Request>& request) +{ + for(std::deque<SharedHandle<Request> >::iterator i = + _inFlightRequests.begin(); i != _inFlightRequests.end(); ++i) { + if((*i).get() == request.get()) { + _inFlightRequests.erase(i); + return true; + } + } + return false; +} + } // namespace aria2 diff --git a/src/FileEntry.h b/src/FileEntry.h index 7cc27619..05ba6067 100644 --- a/src/FileEntry.h +++ b/src/FileEntry.h @@ -43,17 +43,23 @@ #include "SharedHandle.h" #include "File.h" +#include "Request.h" namespace aria2 { +class URISelector; + class FileEntry { private: std::string path; std::deque<std::string> _uris; + std::deque<std::string> _spentUris; uint64_t length; off_t offset; bool extracted; bool requested; + std::deque<SharedHandle<Request> > _requestPool; + std::deque<SharedHandle<Request> > _inFlightRequests; public: FileEntry():length(0), offset(0), extracted(false), requested(false) {} @@ -86,6 +92,8 @@ public: void setOffset(off_t offset) { this->offset = offset; } + off_t getLastOffset() { return offset+length; } + bool isExtracted() const { return extracted; } void setExtracted(bool flag) { this->extracted = flag; } @@ -96,14 +104,53 @@ public: void setupDir(); + // TODO1.5 remove this in favor of getRemainingUris() const std::deque<std::string>& getAssociatedUris() const { return _uris; } + const std::deque<std::string>& getRemainingUris() const + { + return _uris; + } + + const std::deque<std::string>& getSpentUris() const + { + return _spentUris; + } + + void setUris(const std::deque<std::string>& uris) + { + _uris = uris; + } + + // Inserts _uris and _spentUris into uris. + void getUris(std::deque<std::string>& uris) const; + + std::string selectUri(const SharedHandle<URISelector>& uriSelector); + + // If pooled Request object is available, one of them is removed + // from the pool and returned. If pool is empty, then select URI + // using selectUri(selector) and construct Request object using it + // and return the Request object. + SharedHandle<Request> getRequest(const SharedHandle<URISelector>& selector); + + void poolRequest(const SharedHandle<Request>& request); + + bool removeRequest(const SharedHandle<Request>& request); + + size_t countInFlightRequest() const + { + return _inFlightRequests.size(); + } + bool operator<(const FileEntry& fileEntry) const; bool exists() const; + + // Translate global offset goff to file local offset. + off_t gtoloff(off_t goff) const; }; typedef SharedHandle<FileEntry> FileEntryHandle; diff --git a/src/FtpDownloadCommand.cc b/src/FtpDownloadCommand.cc index 64929662..49a62b5c 100644 --- a/src/FtpDownloadCommand.cc +++ b/src/FtpDownloadCommand.cc @@ -49,12 +49,13 @@ namespace aria2 { FtpDownloadCommand::FtpDownloadCommand (int cuid, const RequestHandle& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, const SharedHandle<FtpConnection>& ftpConnection, DownloadEngine* e, const SocketHandle& dataSocket, const SocketHandle& ctrlSocket) - :DownloadCommand(cuid, req, requestGroup, e, dataSocket), + :DownloadCommand(cuid, req, fileEntry, requestGroup, e, dataSocket), _ftpConnection(ftpConnection), ctrlSocket(ctrlSocket) {} @@ -64,8 +65,9 @@ FtpDownloadCommand::~FtpDownloadCommand() {} bool FtpDownloadCommand::prepareForNextSegment() { if(getOption()->getAsBool(PREF_FTP_REUSE_CONNECTION) && - (uint64_t)_segments.front()->getPositionToWrite() == _requestGroup->getTotalLength()) { - Command* command = new FtpFinishDownloadCommand(cuid, req, _requestGroup, _ftpConnection, e, ctrlSocket); + static_cast<uint64_t>(_fileEntry->gtoloff(_segments.front()->getPositionToWrite())) == _fileEntry->getLength()) { + Command* command = new FtpFinishDownloadCommand + (cuid, req, _fileEntry, _requestGroup, _ftpConnection, e, ctrlSocket); e->commands.push_back(command); if(_requestGroup->downloadFinished()) { diff --git a/src/FtpDownloadCommand.h b/src/FtpDownloadCommand.h index c4b2829e..1a2b3139 100644 --- a/src/FtpDownloadCommand.h +++ b/src/FtpDownloadCommand.h @@ -51,6 +51,7 @@ protected: public: FtpDownloadCommand(int cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, const SharedHandle<FtpConnection>& ftpConnection, DownloadEngine* e, diff --git a/src/FtpFinishDownloadCommand.cc b/src/FtpFinishDownloadCommand.cc index ae2009d1..bf8af2a2 100644 --- a/src/FtpFinishDownloadCommand.cc +++ b/src/FtpFinishDownloadCommand.cc @@ -53,11 +53,12 @@ namespace aria2 { FtpFinishDownloadCommand::FtpFinishDownloadCommand (int cuid, const RequestHandle& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, const SharedHandle<FtpConnection>& ftpConnection, DownloadEngine* e, const SharedHandle<SocketCore>& socket) - :AbstractCommand(cuid, req, requestGroup, e, socket), + :AbstractCommand(cuid, req, fileEntry, requestGroup, e, socket), _ftpConnection(ftpConnection) { e->addSocketForReadCheck(socket, this); diff --git a/src/FtpFinishDownloadCommand.h b/src/FtpFinishDownloadCommand.h index 2edc6d5b..81adf934 100644 --- a/src/FtpFinishDownloadCommand.h +++ b/src/FtpFinishDownloadCommand.h @@ -51,6 +51,7 @@ protected: public: FtpFinishDownloadCommand(int cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, const SharedHandle<FtpConnection>& ftpConnection, DownloadEngine* e, diff --git a/src/FtpInitiateConnectionCommand.cc b/src/FtpInitiateConnectionCommand.cc index b6c15423..7453337a 100644 --- a/src/FtpInitiateConnectionCommand.cc +++ b/src/FtpInitiateConnectionCommand.cc @@ -56,9 +56,10 @@ namespace aria2 { FtpInitiateConnectionCommand::FtpInitiateConnectionCommand (int cuid, const RequestHandle& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, DownloadEngine* e) - :InitiateConnectionCommand(cuid, req, requestGroup, e) {} + :InitiateConnectionCommand(cuid, req, fileEntry, requestGroup, e) {} FtpInitiateConnectionCommand::~FtpInitiateConnectionCommand() {} @@ -86,7 +87,8 @@ Command* FtpInitiateConnectionCommand::createNextCommand (new HttpConnection(cuid, socket, getOption().get())); HttpRequestCommand* c = - new HttpRequestCommand(cuid, req, _requestGroup, hc, e, socket); + new HttpRequestCommand(cuid, req, _fileEntry, + _requestGroup, hc, e, socket); c->setProxyRequest(proxyRequest); command = c; } else if(proxyMethod == V_TUNNEL) { @@ -99,7 +101,8 @@ Command* FtpInitiateConnectionCommand::createNextCommand } else { if(proxyMethod == V_TUNNEL) { command = - new FtpNegotiationCommand(cuid, req, _requestGroup, e, pooledSocket, + new FtpNegotiationCommand(cuid, req, _fileEntry, + _requestGroup, e, pooledSocket, FtpNegotiationCommand::SEQ_SEND_CWD, options["baseWorkingDir"]); } else if(proxyMethod == V_GET) { @@ -109,7 +112,8 @@ Command* FtpInitiateConnectionCommand::createNextCommand (new HttpConnection(cuid, pooledSocket, getOption().get())); HttpRequestCommand* c = - new HttpRequestCommand(cuid, req, _requestGroup, hc, e, pooledSocket); + new HttpRequestCommand(cuid, req, _fileEntry, + _requestGroup, hc, e, pooledSocket); c->setProxyRequest(proxyRequest); command = c; } else { @@ -126,10 +130,12 @@ Command* FtpInitiateConnectionCommand::createNextCommand req->getPort()); socket.reset(new SocketCore()); socket->establishConnection(resolvedAddresses.front(), req->getPort()); - command = new FtpNegotiationCommand(cuid, req, _requestGroup, e, socket); + command = new FtpNegotiationCommand(cuid, req, _fileEntry, + _requestGroup, e, socket); } else { command = - new FtpNegotiationCommand(cuid, req, _requestGroup, e, pooledSocket, + new FtpNegotiationCommand(cuid, req, _fileEntry, + _requestGroup, e, pooledSocket, FtpNegotiationCommand::SEQ_SEND_CWD, options["baseWorkingDir"]); } diff --git a/src/FtpInitiateConnectionCommand.h b/src/FtpInitiateConnectionCommand.h index e915753d..6a6cded7 100644 --- a/src/FtpInitiateConnectionCommand.h +++ b/src/FtpInitiateConnectionCommand.h @@ -46,6 +46,7 @@ protected: const SharedHandle<Request>& proxyRequest); public: FtpInitiateConnectionCommand(int cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, DownloadEngine* e); virtual ~FtpInitiateConnectionCommand(); diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 1a857c03..cbdea101 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -68,14 +68,16 @@ namespace aria2 { -FtpNegotiationCommand::FtpNegotiationCommand(int32_t cuid, - const RequestHandle& req, - RequestGroup* requestGroup, - DownloadEngine* e, - const SocketHandle& s, - Seq seq, - const std::string& baseWorkingDir): - AbstractCommand(cuid, req, requestGroup, e, s), sequence(seq), +FtpNegotiationCommand::FtpNegotiationCommand +(int32_t cuid, + const RequestHandle& req, + const SharedHandle<FileEntry>& fileEntry, + RequestGroup* requestGroup, + DownloadEngine* e, + const SocketHandle& s, + Seq seq, + const std::string& baseWorkingDir): + AbstractCommand(cuid, req, fileEntry, requestGroup, e, s), sequence(seq), ftp(new FtpConnection(cuid, socket, req, e->getAuthConfigFactory()->createAuthConfig(req), getOption().get())) @@ -96,7 +98,8 @@ bool FtpNegotiationCommand::executeInternal() { return prepareForRetry(0); } else if(sequence == SEQ_NEGOTIATION_COMPLETED) { FtpDownloadCommand* command = - new FtpDownloadCommand(cuid, req, _requestGroup, ftp, e, dataSocket, socket); + new FtpDownloadCommand + (cuid, req, _fileEntry, _requestGroup, ftp, e, dataSocket, socket); command->setStartupIdleTime(getOption()->getAsInt(PREF_STARTUP_IDLE_TIME)); command->setLowestDownloadSpeedLimit(getOption()->getAsInt(PREF_LOWEST_SPEED_LIMIT)); if(!_requestGroup->isSingleHostMultiConnectionEnabled()) { @@ -328,6 +331,7 @@ bool FtpNegotiationCommand::onFileSizeDetermined(uint64_t totalLength) SingleFileDownloadContextHandle dctx = dynamic_pointer_cast<SingleFileDownloadContext>(_requestGroup->getDownloadContext()); dctx->setTotalLength(totalLength); + _fileEntry->setLength(totalLength); dctx->setFilename (strconcat(dctx->getDir(), "/", Util::urldecode(req->getFile()))); _requestGroup->preDownloadProcessing(); @@ -429,7 +433,7 @@ bool FtpNegotiationCommand::recvSize() { return onFileSizeDetermined(size); } else { - _requestGroup->validateTotalLength(size); + _requestGroup->validateTotalLength(_fileEntry->getLength(), size); } } else { diff --git a/src/FtpNegotiationCommand.h b/src/FtpNegotiationCommand.h index 18487033..bdbc2c15 100644 --- a/src/FtpNegotiationCommand.h +++ b/src/FtpNegotiationCommand.h @@ -123,6 +123,7 @@ protected: public: FtpNegotiationCommand(int32_t cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, DownloadEngine* e, const SharedHandle<SocketCore>& s, diff --git a/src/FtpTunnelResponseCommand.cc b/src/FtpTunnelResponseCommand.cc index 121bd24a..293b88c8 100644 --- a/src/FtpTunnelResponseCommand.cc +++ b/src/FtpTunnelResponseCommand.cc @@ -54,7 +54,8 @@ FtpTunnelResponseCommand::~FtpTunnelResponseCommand() {} Command* FtpTunnelResponseCommand::getNextCommand() { - return new FtpNegotiationCommand(cuid, req, _requestGroup, e, socket); + return new FtpNegotiationCommand(cuid, req, _fileEntry, + _requestGroup, e, socket); } } // namespace aria2 diff --git a/src/HttpDownloadCommand.cc b/src/HttpDownloadCommand.cc index 69b09be1..9819efea 100644 --- a/src/HttpDownloadCommand.cc +++ b/src/HttpDownloadCommand.cc @@ -52,12 +52,13 @@ namespace aria2 { HttpDownloadCommand::HttpDownloadCommand (int cuid, const RequestHandle& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, const SharedHandle<HttpResponse>& httpResponse, const HttpConnectionHandle& httpConnection, DownloadEngine* e, const SocketHandle& socket) - :DownloadCommand(cuid, req, requestGroup, e, socket), + :DownloadCommand(cuid, req, fileEntry, requestGroup, e, socket), _httpResponse(httpResponse), _httpConnection(httpConnection) {} @@ -67,7 +68,8 @@ bool HttpDownloadCommand::prepareForNextSegment() { bool downloadFinished = _requestGroup->downloadFinished(); if(req->isPipeliningEnabled() && !downloadFinished) { HttpRequestCommand* command = - new HttpRequestCommand(cuid, req, _requestGroup, _httpConnection, e, + new HttpRequestCommand(cuid, req, _fileEntry, + _requestGroup, _httpConnection, e, socket); // Set proxy request here. aria2 sends the HTTP request specialized for // proxy. @@ -81,8 +83,7 @@ bool HttpDownloadCommand::prepareForNextSegment() { (req->isKeepAliveEnabled() && ((!_transferEncodingDecoder.isNull() && _requestGroup->downloadFinished()) || - (uint64_t)_segments.front()->getPositionToWrite() == - _requestGroup->getTotalLength()))) { + static_cast<uint64_t>(_fileEntry->gtoloff(_segments.front()->getPositionToWrite())) == _fileEntry->getLength()))) { e->poolSocket(req, isProxyDefined(), socket); } // The request was sent assuming that server supported pipelining, but diff --git a/src/HttpDownloadCommand.h b/src/HttpDownloadCommand.h index c4343e31..e1e995da 100644 --- a/src/HttpDownloadCommand.h +++ b/src/HttpDownloadCommand.h @@ -51,6 +51,7 @@ protected: public: HttpDownloadCommand(int cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, const SharedHandle<HttpResponse>& httpResponse, const SharedHandle<HttpConnection>& httpConnection, diff --git a/src/HttpInitiateConnectionCommand.cc b/src/HttpInitiateConnectionCommand.cc index 797b2632..b9552db5 100644 --- a/src/HttpInitiateConnectionCommand.cc +++ b/src/HttpInitiateConnectionCommand.cc @@ -53,9 +53,10 @@ namespace aria2 { HttpInitiateConnectionCommand::HttpInitiateConnectionCommand (int cuid, const RequestHandle& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, DownloadEngine* e): - InitiateConnectionCommand(cuid, req, requestGroup, e) {} + InitiateConnectionCommand(cuid, req, fileEntry, requestGroup, e) {} HttpInitiateConnectionCommand::~HttpInitiateConnectionCommand() {} @@ -81,7 +82,9 @@ Command* HttpInitiateConnectionCommand::createNextCommand } else if(proxyMethod == V_GET) { SharedHandle<HttpConnection> httpConnection (new HttpConnection(cuid, socket, getOption().get())); - HttpRequestCommand* c = new HttpRequestCommand(cuid, req, _requestGroup, + HttpRequestCommand* c = new HttpRequestCommand(cuid, req, + _fileEntry, + _requestGroup, httpConnection, e, socket); c->setProxyRequest(proxyRequest); @@ -93,7 +96,9 @@ Command* HttpInitiateConnectionCommand::createNextCommand } else { SharedHandle<HttpConnection> httpConnection (new HttpConnection(cuid, pooledSocket, getOption().get())); - HttpRequestCommand* c = new HttpRequestCommand(cuid, req, _requestGroup, + HttpRequestCommand* c = new HttpRequestCommand(cuid, req, + _fileEntry, + _requestGroup, httpConnection, e, pooledSocket); if(proxyMethod == V_GET) { @@ -113,8 +118,8 @@ Command* HttpInitiateConnectionCommand::createNextCommand socket = pooledSocket; } SharedHandle<HttpConnection> httpConnection(new HttpConnection(cuid, socket, getOption().get())); - command = new HttpRequestCommand(cuid, req, _requestGroup, httpConnection, - e, socket); + command = new HttpRequestCommand(cuid, req, _fileEntry, _requestGroup, + httpConnection, e, socket); } return command; } diff --git a/src/HttpInitiateConnectionCommand.h b/src/HttpInitiateConnectionCommand.h index cd7ec83c..54f76523 100644 --- a/src/HttpInitiateConnectionCommand.h +++ b/src/HttpInitiateConnectionCommand.h @@ -46,6 +46,7 @@ protected: const SharedHandle<Request>& proxyRequest); public: HttpInitiateConnectionCommand(int cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, DownloadEngine* e); diff --git a/src/HttpProxyResponseCommand.cc b/src/HttpProxyResponseCommand.cc index 65bdff2d..0054d6d9 100644 --- a/src/HttpProxyResponseCommand.cc +++ b/src/HttpProxyResponseCommand.cc @@ -54,7 +54,8 @@ HttpProxyResponseCommand::~HttpProxyResponseCommand() {} Command* HttpProxyResponseCommand::getNextCommand() { - return new HttpRequestCommand(cuid, req, _requestGroup, httpConnection, e, socket); + return new HttpRequestCommand(cuid, req, _fileEntry, + _requestGroup, httpConnection, e, socket); } } // namespace aria2 diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc index 10f243e7..d46f68ac 100644 --- a/src/HttpRequest.cc +++ b/src/HttpRequest.cc @@ -73,7 +73,7 @@ off_t HttpRequest::getStartByte() const if(segment.isNull()) { return 0; } else { - return segment->getPositionToWrite(); + return _fileEntry->gtoloff(segment->getPositionToWrite()); } } @@ -83,7 +83,7 @@ off_t HttpRequest::getEndByte() const return 0; } else { if(request->isPipeliningEnabled()) { - return segment->getPosition()+segment->getLength()-1; + return _fileEntry->gtoloff(segment->getPosition()+segment->getLength()-1); } else { return 0; } diff --git a/src/HttpRequest.h b/src/HttpRequest.h index f84596a5..9ec32171 100644 --- a/src/HttpRequest.h +++ b/src/HttpRequest.h @@ -42,6 +42,7 @@ #include "SharedHandle.h" #include "Request.h" +#include "FileEntry.h" namespace aria2 { @@ -59,6 +60,8 @@ private: SharedHandle<Request> request; + SharedHandle<FileEntry> _fileEntry; + SharedHandle<Segment> segment; uint64_t entityLength; @@ -232,6 +235,16 @@ public: // Returns AuthConfig used in the last invocation of // createRequest(). const SharedHandle<AuthConfig>& getAuthConfig() const; + + void setFileEntry(const SharedHandle<FileEntry>& fileEntry) + { + _fileEntry = fileEntry; + } + + const SharedHandle<FileEntry>& getFileEntry() const + { + return _fileEntry; + } }; typedef SharedHandle<HttpRequest> HttpRequestHandle; diff --git a/src/HttpRequestCommand.cc b/src/HttpRequestCommand.cc index 7acd3fdf..d95e616b 100644 --- a/src/HttpRequestCommand.cc +++ b/src/HttpRequestCommand.cc @@ -58,11 +58,12 @@ namespace aria2 { HttpRequestCommand::HttpRequestCommand (int cuid, const RequestHandle& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, const HttpConnectionHandle& httpConnection, DownloadEngine* e, const SocketHandle& s) - :AbstractCommand(cuid, req, requestGroup, e, s), + :AbstractCommand(cuid, req, fileEntry, requestGroup, e, s), _httpConnection(httpConnection) { setTimeout(getOption()->getAsInt(PREF_CONNECT_TIMEOUT)); @@ -74,6 +75,7 @@ HttpRequestCommand::~HttpRequestCommand() {} static SharedHandle<HttpRequest> createHttpRequest(const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, const SharedHandle<Segment>& segment, uint64_t totalLength, const SharedHandle<Option>& option, @@ -85,8 +87,9 @@ createHttpRequest(const SharedHandle<Request>& req, HttpRequestHandle httpRequest(new HttpRequest()); httpRequest->setUserAgent(option->get(PREF_USER_AGENT)); httpRequest->setRequest(req); + httpRequest->setFileEntry(fileEntry); httpRequest->setSegment(segment); - httpRequest->setEntityLength(totalLength); + httpRequest->setEntityLength(fileEntry->getLength()); httpRequest->addHeader(option->get(PREF_HEADER)); httpRequest->setCookieStorage(cookieStorage); httpRequest->setAuthConfigFactory(authConfigFactory); @@ -113,7 +116,9 @@ bool HttpRequestCommand::executeInternal() { if(_segments.empty()) { HttpRequestHandle httpRequest - (createHttpRequest(req, SharedHandle<Segment>(), + (createHttpRequest(req, + _fileEntry, + SharedHandle<Segment>(), _requestGroup->getTotalLength(), getOption(), _requestGroup, @@ -126,7 +131,9 @@ bool HttpRequestCommand::executeInternal() { const SegmentHandle& segment = *itr; if(!_httpConnection->isIssued(segment)) { HttpRequestHandle httpRequest - (createHttpRequest(req, segment, + (createHttpRequest(req, + _fileEntry, + segment, _requestGroup->getTotalLength(), getOption(), _requestGroup, @@ -141,7 +148,8 @@ bool HttpRequestCommand::executeInternal() { _httpConnection->sendPendingData(); } if(_httpConnection->sendBufferIsEmpty()) { - Command* command = new HttpResponseCommand(cuid, req, _requestGroup, + Command* command = new HttpResponseCommand(cuid, req, _fileEntry, + _requestGroup, _httpConnection, e, socket); e->commands.push_back(command); return true; diff --git a/src/HttpRequestCommand.h b/src/HttpRequestCommand.h index 33ec8c6f..fe0a34de 100644 --- a/src/HttpRequestCommand.h +++ b/src/HttpRequestCommand.h @@ -52,6 +52,7 @@ protected: public: HttpRequestCommand(int cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, const SharedHandle<HttpConnection>& httpConnection, DownloadEngine* e, diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index b337d9cf..30de2641 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -75,14 +75,17 @@ static SharedHandle<Decoder> getTransferEncodingDecoder static SharedHandle<Decoder> getContentEncodingDecoder (const SharedHandle<HttpResponse>& httpResponse); -HttpResponseCommand::HttpResponseCommand(int32_t cuid, - const RequestHandle& req, - RequestGroup* requestGroup, - const HttpConnectionHandle& httpConnection, - DownloadEngine* e, - const SocketHandle& s) - :AbstractCommand(cuid, req, requestGroup, e, s), - httpConnection(httpConnection) {} +HttpResponseCommand::HttpResponseCommand +(int32_t cuid, + const RequestHandle& req, + const SharedHandle<FileEntry>& fileEntry, + RequestGroup* requestGroup, + const HttpConnectionHandle& httpConnection, + DownloadEngine* e, + const SocketHandle& s) + :AbstractCommand(cuid, req, fileEntry, requestGroup, e, s), + httpConnection(httpConnection) +{} HttpResponseCommand::~HttpResponseCommand() {} @@ -136,6 +139,7 @@ bool HttpResponseCommand::executeInternal() SingleFileDownloadContextHandle dctx = dynamic_pointer_cast<SingleFileDownloadContext>(_requestGroup->getDownloadContext()); dctx->setTotalLength(totalLength); + _fileEntry->setLength(totalLength); dctx->setFilename (strconcat(dctx->getDir(), "/", httpResponse->determinFilename())); dctx->setContentType(httpResponse->getContentType()); @@ -166,7 +170,8 @@ bool HttpResponseCommand::executeInternal() } } else { // validate totalsize - _requestGroup->validateTotalLength(httpResponse->getEntityLength()); + _requestGroup->validateTotalLength(_fileEntry->getLength(), + httpResponse->getEntityLength()); // update last modified time updateLastModifiedTime(httpResponse->getLastModifiedTime()); if(_requestGroup->getTotalLength() == 0) { @@ -258,6 +263,7 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpResponseHandle& httpRe command = createHttpDownloadCommand(httpResponse); } else { _requestGroup->getSegmentMan()->cancelSegment(cuid); + _fileEntry->poolRequest(req); } prepareForNextAction(command); if(req->getMethod() == Request::METHOD_HEAD) { @@ -361,7 +367,7 @@ bool HttpResponseCommand::skipResponseBody // thrown away. HttpSkipResponseCommand* command = new HttpSkipResponseCommand - (cuid, req, _requestGroup, httpConnection, httpResponse, e, socket); + (cuid, req, _fileEntry, _requestGroup, httpConnection, httpResponse, e, socket); command->setTransferEncodingDecoder(decoder); // If request method is HEAD or the response body is zero-length, @@ -386,7 +392,7 @@ HttpDownloadCommand* HttpResponseCommand::createHttpDownloadCommand { HttpDownloadCommand* command = - new HttpDownloadCommand(cuid, req, _requestGroup, + new HttpDownloadCommand(cuid, req, _fileEntry, _requestGroup, httpResponse, httpConnection, e, socket); command->setStartupIdleTime(getOption()->getAsInt(PREF_STARTUP_IDLE_TIME)); command->setLowestDownloadSpeedLimit diff --git a/src/HttpResponseCommand.h b/src/HttpResponseCommand.h index be773d2b..ad35dafb 100644 --- a/src/HttpResponseCommand.h +++ b/src/HttpResponseCommand.h @@ -75,6 +75,7 @@ protected: public: HttpResponseCommand(int32_t cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, const SharedHandle<HttpConnection>& httpConnection, DownloadEngine* e, diff --git a/src/HttpSkipResponseCommand.cc b/src/HttpSkipResponseCommand.cc index 224a5e9d..b9edb717 100644 --- a/src/HttpSkipResponseCommand.cc +++ b/src/HttpSkipResponseCommand.cc @@ -59,12 +59,13 @@ namespace aria2 { HttpSkipResponseCommand::HttpSkipResponseCommand (int cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, const SharedHandle<HttpConnection>& httpConnection, const SharedHandle<HttpResponse>& httpResponse, DownloadEngine* e, const SharedHandle<SocketCore>& s): - AbstractCommand(cuid, req, requestGroup, e, s), + AbstractCommand(cuid, req, fileEntry, requestGroup, e, s), _httpConnection(httpConnection), _httpResponse(httpResponse), _totalLength(_httpResponse->getEntityLength()), diff --git a/src/HttpSkipResponseCommand.h b/src/HttpSkipResponseCommand.h index 7ec1a511..d3cbe7bb 100644 --- a/src/HttpSkipResponseCommand.h +++ b/src/HttpSkipResponseCommand.h @@ -63,6 +63,7 @@ protected: public: HttpSkipResponseCommand(int cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, const SharedHandle<HttpConnection>& httpConnection, const SharedHandle<HttpResponse>& httpResponse, diff --git a/src/InitiateConnectionCommand.cc b/src/InitiateConnectionCommand.cc index c5924e76..05c2f9a9 100644 --- a/src/InitiateConnectionCommand.cc +++ b/src/InitiateConnectionCommand.cc @@ -42,15 +42,20 @@ #include "NameResolver.h" #include "DNSCache.h" #include "SocketCore.h" +#include "FileEntry.h" +#include "RequestGroup.h" +#include "DownloadContext.h" +#include "Segment.h" namespace aria2 { InitiateConnectionCommand::InitiateConnectionCommand (int cuid, const RequestHandle& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, DownloadEngine* e): - AbstractCommand(cuid, req, requestGroup, e) + AbstractCommand(cuid, req, fileEntry, requestGroup, e) { setTimeout(getOption()->getAsInt(PREF_DNS_TIMEOUT)); // give a chance to be executed in the next loop in DownloadEngine diff --git a/src/InitiateConnectionCommand.h b/src/InitiateConnectionCommand.h index e7d1c901..ed206581 100644 --- a/src/InitiateConnectionCommand.h +++ b/src/InitiateConnectionCommand.h @@ -54,6 +54,7 @@ protected: const SharedHandle<Request>& proxyRequest) = 0; public: InitiateConnectionCommand(int cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, DownloadEngine* e); diff --git a/src/InitiateConnectionCommandFactory.cc b/src/InitiateConnectionCommandFactory.cc index 2a8ba853..2ad3dbda 100644 --- a/src/InitiateConnectionCommandFactory.cc +++ b/src/InitiateConnectionCommandFactory.cc @@ -47,7 +47,13 @@ namespace aria2 { Command* -InitiateConnectionCommandFactory::createInitiateConnectionCommand(int32_t cuid, const RequestHandle& req, RequestGroup* requestGroup, DownloadEngine* e) { +InitiateConnectionCommandFactory::createInitiateConnectionCommand +(int32_t cuid, + const RequestHandle& req, + const SharedHandle<FileEntry>& fileEntry, + RequestGroup* requestGroup, + DownloadEngine* e) +{ if(req->getProtocol() == Request::PROTO_HTTP #ifdef ENABLE_SSL // for SSL @@ -62,9 +68,11 @@ InitiateConnectionCommandFactory::createInitiateConnectionCommand(int32_t cuid, req->setPipeliningHint(true); } - return new HttpInitiateConnectionCommand(cuid, req, requestGroup, e); + return + new HttpInitiateConnectionCommand(cuid, req, fileEntry, requestGroup, e); } else if(req->getProtocol() == Request::PROTO_FTP) { - return new FtpInitiateConnectionCommand(cuid, req, requestGroup, e); + return + new FtpInitiateConnectionCommand(cuid, req, fileEntry, requestGroup, e); } else { // these protocols are not supported yet throw DL_ABORT_EX diff --git a/src/InitiateConnectionCommandFactory.h b/src/InitiateConnectionCommandFactory.h index 01368001..5ba88e90 100644 --- a/src/InitiateConnectionCommandFactory.h +++ b/src/InitiateConnectionCommandFactory.h @@ -44,12 +44,14 @@ class Request; class RequestGroup; class DownloadEngine; class Command; +class FileEntry; class InitiateConnectionCommandFactory { public: static Command* createInitiateConnectionCommand(int32_t cuid, const SharedHandle<Request>& req, + const SharedHandle<FileEntry>& fileEntry, RequestGroup* requestGroup, DownloadEngine* e); }; diff --git a/src/Makefile.am b/src/Makefile.am index 455c3ce3..5951269f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -201,7 +201,8 @@ SRCS = Socket.h\ PieceSelector.h\ LongestSequencePieceSelector.cc LongestSequencePieceSelector.h\ bitfield.h\ - BDE.cc BDE.h + BDE.cc BDE.h\ + CreateRequestCommand.cc CreateRequestCommand.h if ENABLE_XML_RPC SRCS += XmlRpcRequestParserController.cc XmlRpcRequestParserController.h\ diff --git a/src/Makefile.in b/src/Makefile.in index 90776177..f46e2465 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -425,7 +425,8 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ SelectEventPoll.cc SelectEventPoll.h SequentialPicker.h \ SequentialDispatcherCommand.h PieceSelector.h \ LongestSequencePieceSelector.cc LongestSequencePieceSelector.h \ - bitfield.h BDE.cc BDE.h XmlRpcRequestParserController.cc \ + bitfield.h BDE.cc BDE.h CreateRequestCommand.cc \ + CreateRequestCommand.h XmlRpcRequestParserController.cc \ XmlRpcRequestParserController.h \ XmlRpcRequestParserStateMachine.cc \ XmlRpcRequestParserStateMachine.h XmlRpcRequestParserState.h \ @@ -846,15 +847,15 @@ am__objects_26 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \ SocketBuffer.$(OBJEXT) OptionHandlerException.$(OBJEXT) \ URIResult.$(OBJEXT) SelectEventPoll.$(OBJEXT) \ LongestSequencePieceSelector.$(OBJEXT) BDE.$(OBJEXT) \ - $(am__objects_1) $(am__objects_2) $(am__objects_3) \ - $(am__objects_4) $(am__objects_5) $(am__objects_6) \ - $(am__objects_7) $(am__objects_8) $(am__objects_9) \ - $(am__objects_10) $(am__objects_11) $(am__objects_12) \ - $(am__objects_13) $(am__objects_14) $(am__objects_15) \ - $(am__objects_16) $(am__objects_17) $(am__objects_18) \ - $(am__objects_19) $(am__objects_20) $(am__objects_21) \ - $(am__objects_22) $(am__objects_23) $(am__objects_24) \ - $(am__objects_25) + CreateRequestCommand.$(OBJEXT) $(am__objects_1) \ + $(am__objects_2) $(am__objects_3) $(am__objects_4) \ + $(am__objects_5) $(am__objects_6) $(am__objects_7) \ + $(am__objects_8) $(am__objects_9) $(am__objects_10) \ + $(am__objects_11) $(am__objects_12) $(am__objects_13) \ + $(am__objects_14) $(am__objects_15) $(am__objects_16) \ + $(am__objects_17) $(am__objects_18) $(am__objects_19) \ + $(am__objects_20) $(am__objects_21) $(am__objects_22) \ + $(am__objects_23) $(am__objects_24) $(am__objects_25) am_libaria2c_a_OBJECTS = $(am__objects_26) libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS) am__installdirs = "$(DESTDIR)$(bindir)" @@ -1181,7 +1182,8 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \ SelectEventPoll.cc SelectEventPoll.h SequentialPicker.h \ SequentialDispatcherCommand.h PieceSelector.h \ LongestSequencePieceSelector.cc LongestSequencePieceSelector.h \ - bitfield.h BDE.cc BDE.h $(am__append_1) $(am__append_2) \ + bitfield.h BDE.cc BDE.h CreateRequestCommand.cc \ + CreateRequestCommand.h $(am__append_1) $(am__append_2) \ $(am__append_3) $(am__append_4) $(am__append_5) \ $(am__append_6) $(am__append_7) $(am__append_8) \ $(am__append_9) $(am__append_10) $(am__append_11) \ @@ -1339,6 +1341,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Cookie.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CookieParser.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CookieStorage.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CreateRequestCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DHTAbstractMessage.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DHTAbstractNodeLookupTask.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DHTAbstractTask.Po@am__quote@ diff --git a/src/PieceStorage.h b/src/PieceStorage.h index dc3a72f1..17dd2872 100644 --- a/src/PieceStorage.h +++ b/src/PieceStorage.h @@ -101,8 +101,10 @@ public: /** * Returns a missing piece if available. Otherwise returns 0; + * If ignoreBitfield is set, indexes of true bit are excluded. */ - virtual SharedHandle<Piece> getMissingPiece() = 0; + virtual SharedHandle<Piece> getSparseMissingUnusedPiece + (const unsigned char* ignoreBitfield, size_t length) = 0; /** * Returns a missing piece whose index is index. diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 7035e987..0da471b9 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -44,7 +44,7 @@ #include "NullProgressInfoFile.h" #include "Dependency.h" #include "prefs.h" -#include "InitiateConnectionCommandFactory.h" +#include "CreateRequestCommand.h" #include "File.h" #include "message.h" #include "Util.h" @@ -78,6 +78,7 @@ #include "InOrderURISelector.h" #include "PieceSelector.h" #include "a2functional.h" +#include "SocketCore.h" #ifdef ENABLE_MESSAGE_DIGEST # include "CheckIntegrityCommand.h" #endif // ENABLE_MESSAGE_DIGEST @@ -604,6 +605,7 @@ void RequestGroup::createNextCommand(std::deque<Command*>& commands, unsigned int numCommand, const std::string& method) { + // TODO1.5 The following block should be moved into FileEntry if(_option->getAsBool(PREF_REUSE_URI) && _uris.empty()) { std::deque<std::string> uris = _spentUris; std::sort(uris.begin(), uris.end()); @@ -639,35 +641,45 @@ void RequestGroup::createNextCommand(std::deque<Command*>& commands, } std::deque<std::string> pendingURIs; - for(; numCommand--; ) { - std::string uri = _uriSelector->select(_uris); - if(uri.empty()) - continue; - RequestHandle req(new Request()); - if(req->setUrl(uri)) { - ServerHostHandle sv; - if(!_singleHostMultiConnectionEnabled){ - sv = searchServerHost(req->getHost()); - } - if(sv.isNull()) { - _spentUris.push_back(uri); - req->setReferer(_option->get(PREF_REFERER)); - req->setMethod(method); - - Command* command = - InitiateConnectionCommandFactory::createInitiateConnectionCommand - (e->newCUID(), req, this, e); - ServerHostHandle sv(new ServerHost(command->getCuid(), req->getHost())); - registerServerHost(sv); - commands.push_back(command); - } else { - pendingURIs.push_back(uri); - } - } else { - _logger->error(MSG_UNRECOGNIZED_URI, req->getUrl().c_str()); - } + for(; numCommand--; ) { + Command* command = new CreateRequestCommand(e->newCUID(), this, e); + _logger->debug("filePath=%s", _downloadContext->getFileEntries().front()->getPath().c_str()); + commands.push_back(command); + + // TODO1.5 ServerHost stuff should be moved into FileEntry or + // CreateRequestCommand + +// std::string uri = _uriSelector->select(_uris); +// if(uri.empty()) +// continue; +// RequestHandle req(new Request()); +// if(req->setUrl(uri)) { +// ServerHostHandle sv; +// if(!_singleHostMultiConnectionEnabled){ +// sv = searchServerHost(req->getHost()); +// } +// if(sv.isNull()) { +// _spentUris.push_back(uri); +// req->setReferer(_option->get(PREF_REFERER)); +// req->setMethod(method); + +// Command* command = +// InitiateConnectionCommandFactory::createInitiateConnectionCommand +// (e->newCUID(), req, this, e); +// ServerHostHandle sv(new ServerHost(command->getCuid(), req->getHost())); +// registerServerHost(sv); +// // give a chance to be executed in the next loop in DownloadEngine +// command->setStatus(Command::STATUS_ONESHOT_REALTIME); +// commands.push_back(command); +// } else { +// pendingURIs.push_back(uri); +// } +// } else { +// _logger->error(MSG_UNRECOGNIZED_URI, req->getUrl().c_str()); +// } +// } } - _uris.insert(_uris.begin(), pendingURIs.begin(), pendingURIs.end()); +// _uris.insert(_uris.begin(), pendingURIs.begin(), pendingURIs.end()); if(!commands.empty()) { e->setNoWait(true); } diff --git a/src/RequestGroup.h b/src/RequestGroup.h index 34db9dde..a8c4c388 100644 --- a/src/RequestGroup.h +++ b/src/RequestGroup.h @@ -164,9 +164,6 @@ private: void validateFilename(const std::string& expectedFilename, const std::string& actualFilename) const; - void validateTotalLength(uint64_t expectedTotalLength, - uint64_t actualTotalLength) const; - void initializePreDownloadHandler(); void initializePostDownloadHandler(); @@ -249,6 +246,9 @@ public: */ void validateFilename(const std::string& actualFilename) const; + void validateTotalLength(uint64_t expectedTotalLength, + uint64_t actualTotalLength) const; + void validateTotalLength(uint64_t actualTotalLength) const; void setSegmentManFactory(const SharedHandle<SegmentManFactory>& segmentManFactory); @@ -449,6 +449,11 @@ public: void setURISelector(const SharedHandle<URISelector>& uriSelector); + const SharedHandle<URISelector>& getURISelector() const + { + return _uriSelector; + } + void applyLastModifiedTimeToLocalFiles(); void updateLastModifiedTime(const Time& time); diff --git a/src/RequestGroupMan.cc b/src/RequestGroupMan.cc index 68d029c6..41a0f515 100644 --- a/src/RequestGroupMan.cc +++ b/src/RequestGroupMan.cc @@ -486,6 +486,7 @@ void RequestGroupMan::getInitialCommands(std::deque<Command*>& commands, if((*itr)->isDependencyResolved()) { configureRequestGroup(*itr); createInitialCommand(*itr, commands, e, + // TODO1.5 We need PREF_DRY_RUN here? _option->getAsBool(PREF_USE_HEAD)); executeStartHook(*itr, e->option); ++itr; diff --git a/src/SegmentMan.cc b/src/SegmentMan.cc index 33ed1709..4598e3d3 100644 --- a/src/SegmentMan.cc +++ b/src/SegmentMan.cc @@ -50,6 +50,7 @@ #include "Option.h" #include "DownloadContext.h" #include "Piece.h" +#include "FileEntry.h" namespace aria2 { @@ -66,8 +67,12 @@ SegmentMan::SegmentMan(const Option* option, _downloadContext(downloadContext), _pieceStorage(pieceStorage), _lastPeerStatDlspdMapUpdated(0), - _cachedDlspd(0) -{} + _cachedDlspd(0), + _ignoreBitfield(downloadContext->getPieceLength(), + downloadContext->getTotalLength()) +{ + _ignoreBitfield.enableFilter(); +} SegmentMan::~SegmentMan() {} @@ -126,6 +131,22 @@ SegmentHandle SegmentMan::checkoutSegment(cuid_t cuid, segment->getLength(), segment->getSegmentLength(), segment->getWrittenLength()); + if(piece->getLength() > 0) { + std::map<size_t, size_t>::iterator positr = + _segmentWrittenLengthMemo.find(segment->getIndex()); + if(positr != _segmentWrittenLengthMemo.end()) { + const size_t writtenLength = (*positr).second; + logger->debug("writtenLength(in memo)=%d, writtenLength=%d", + writtenLength, segment->getWrittenLength()); + // If the difference between cached writtenLength and segment's + // writtenLength is less than one block, we assume that these + // missing bytes are already downloaded. + if(segment->getWrittenLength() < writtenLength && + writtenLength-segment->getWrittenLength() < piece->getBlockLength()) { + segment->updateWrittenLength(writtenLength-segment->getWrittenLength()); + } + } + } return segment; } @@ -178,7 +199,9 @@ void SegmentMan::getInFlightSegment(std::deque<SharedHandle<Segment> >& segments } SegmentHandle SegmentMan::getSegment(cuid_t cuid) { - PieceHandle piece = _pieceStorage->getMissingPiece(); + PieceHandle piece = + _pieceStorage->getSparseMissingUnusedPiece + (_ignoreBitfield.getFilterBitfield(),_ignoreBitfield.getBitfieldLength()); if(piece.isNull()) { PeerStatHandle myPeerStat = getPeerStat(cuid); if(myPeerStat.isNull()) { @@ -219,6 +242,11 @@ void SegmentMan::cancelSegment(cuid_t cuid) { itr != usedSegmentEntries.end();) { if((*itr)->cuid == cuid) { _pieceStorage->cancelPiece((*itr)->segment->getPiece()); + _segmentWrittenLengthMemo[(*itr)->segment->getIndex()] = + (*itr)->segment->getWrittenLength(); + logger->debug("Memorized segment index=%u, writtenLength=%u", + (*itr)->segment->getIndex(), + (*itr)->segment->getWrittenLength()); itr = usedSegmentEntries.erase(itr); } else { ++itr; @@ -356,4 +384,14 @@ size_t SegmentMan::countFreePieceFrom(size_t index) const return _downloadContext->getNumPieces()-index; } +void SegmentMan::ignoreSegmentFor(const SharedHandle<FileEntry>& fileEntry) +{ + _ignoreBitfield.addFilter(fileEntry->getOffset(), fileEntry->getLength()); +} + +void SegmentMan::recognizeSegmentFor(const SharedHandle<FileEntry>& fileEntry) +{ + _ignoreBitfield.removeFilter(fileEntry->getOffset(), fileEntry->getLength()); +} + } // namespace aria2 diff --git a/src/SegmentMan.h b/src/SegmentMan.h index db9f68e9..ccf77d50 100644 --- a/src/SegmentMan.h +++ b/src/SegmentMan.h @@ -43,6 +43,7 @@ #include "SharedHandle.h" #include "TimeA2.h" #include "Command.h" +#include "BitfieldMan.h" namespace aria2 { @@ -53,6 +54,7 @@ class PeerStat; class DownloadContext; class PieceStorage; class Piece; +class FileEntry; class SegmentEntry { public: @@ -82,6 +84,10 @@ private: SegmentEntries usedSegmentEntries; + // Remember writtenLength for each segment. The key is an index of a + // segment. The value is writtenLength for that segment. + std::map<size_t, size_t> _segmentWrittenLengthMemo; + std::deque<SharedHandle<PeerStat> > peerStats; // key: PeerStat's cuid, value: its download speed @@ -91,6 +97,8 @@ private: unsigned int _cachedDlspd; + BitfieldMan _ignoreBitfield; + SharedHandle<Segment> checkoutSegment(cuid_t cuid, const SharedHandle<Piece>& piece); @@ -201,6 +209,12 @@ public: uint64_t calculateSessionDownloadLength() const; size_t countFreePieceFrom(size_t index) const; + + // Excludes segments that fileEntry covers from segment selection. + void ignoreSegmentFor(const SharedHandle<FileEntry>& fileEntry); + + // Includes segments that fileEntry covers in segment selection. + void recognizeSegmentFor(const SharedHandle<FileEntry>& fileEntry); }; typedef SharedHandle<SegmentMan> SegmentManHandle; diff --git a/src/SingleFileDownloadContext.h b/src/SingleFileDownloadContext.h index 1375ba5c..7f07d9a9 100644 --- a/src/SingleFileDownloadContext.h +++ b/src/SingleFileDownloadContext.h @@ -93,7 +93,11 @@ public: virtual FILE_MODE getFileMode() const { - return SINGLE; + if(_fileEntries.size() == 1) { + return SINGLE; + } else { + return MULTI; + } } virtual size_t getPieceLength() const diff --git a/src/StreamFileAllocationEntry.cc b/src/StreamFileAllocationEntry.cc index e514457b..5caf81d5 100644 --- a/src/StreamFileAllocationEntry.cc +++ b/src/StreamFileAllocationEntry.cc @@ -41,7 +41,6 @@ #include "Request.h" #include "prefs.h" #include "RequestGroup.h" -#include "InitiateConnectionCommandFactory.h" #include "DownloadContext.h" #include "Command.h" @@ -68,16 +67,7 @@ void StreamFileAllocationEntry::prepareForNextAction(std::deque<Command*>& comma // try remaining uris _requestGroup->createNextCommandWithAdj(commands, e, -1); } else { - if(_currentRequest.isNull()) { - _requestGroup->createNextCommandWithAdj(commands, e, 0); - } else { - Command* command = - InitiateConnectionCommandFactory::createInitiateConnectionCommand - (e->newCUID(), _currentRequest, _requestGroup, e); - e->setNoWait(true); - commands.push_back(command); - _requestGroup->createNextCommandWithAdj(commands, e, -1); - } + _requestGroup->createNextCommandWithAdj(commands, e, 0); } } diff --git a/src/TrackerWatcherCommand.cc b/src/TrackerWatcherCommand.cc index 701bb108..87a51c58 100644 --- a/src/TrackerWatcherCommand.cc +++ b/src/TrackerWatcherCommand.cc @@ -223,6 +223,7 @@ TrackerWatcherCommand::createRequestGroup(const std::string& uri) A2STR::NIL, TRACKER_ANNOUNCE_FILE)); dctx->setDir(A2STR::NIL); + dctx->getFileEntries().front()->setUris(uris); rg->setDownloadContext(dctx); SharedHandle<DiskWriterFactory> dwf(new ByteArrayDiskWriterFactory()); rg->setDiskWriterFactory(dwf); diff --git a/src/UnknownLengthPieceStorage.cc b/src/UnknownLengthPieceStorage.cc index 2bbc2061..369767c7 100644 --- a/src/UnknownLengthPieceStorage.cc +++ b/src/UnknownLengthPieceStorage.cc @@ -100,7 +100,8 @@ SharedHandle<Piece> UnknownLengthPieceStorage::getMissingFastPiece #endif // ENABLE_BITTORRENT -PieceHandle UnknownLengthPieceStorage::getMissingPiece() +SharedHandle<Piece> UnknownLengthPieceStorage::getSparseMissingUnusedPiece +(const unsigned char* ignoreBitfield, size_t length) { if(_downloadFinished) { return SharedHandle<Piece>(); @@ -116,7 +117,7 @@ PieceHandle UnknownLengthPieceStorage::getMissingPiece() PieceHandle UnknownLengthPieceStorage::getMissingPiece(size_t index) { if(index == 0) { - return getMissingPiece(); + return getSparseMissingUnusedPiece(0, 0); } else { return SharedHandle<Piece>(); } diff --git a/src/UnknownLengthPieceStorage.h b/src/UnknownLengthPieceStorage.h index f71e195b..dcf05130 100644 --- a/src/UnknownLengthPieceStorage.h +++ b/src/UnknownLengthPieceStorage.h @@ -101,7 +101,8 @@ public: /** * Returns a missing piece if available. Otherwise returns 0; */ - virtual SharedHandle<Piece> getMissingPiece(); + virtual SharedHandle<Piece> getSparseMissingUnusedPiece + (const unsigned char* ignoreBitfield, size_t length); /** * Returns a missing piece whose index is index. diff --git a/src/array_fun.h b/src/array_fun.h index a8990b9d..c6d53475 100644 --- a/src/array_fun.h +++ b/src/array_fun.h @@ -155,6 +155,13 @@ struct And }; template<typename T> +struct Or +{ + typedef T returnType; + static inline returnType apply(T lhs, T rhs) { return lhs|rhs; } +}; + +template<typename T> struct Negate { typedef T returnType; @@ -184,6 +191,13 @@ operator&(const L& l, const R& r) return BinExpr<L, And<typename L::returnType>, R>(l, r); } +template<typename L, typename R> +BinExpr<L, Or<typename L::returnType>, R> +operator|(const L& l, const R& r) +{ + return BinExpr<L, Or<typename L::returnType>, R>(l, r); +} + template<typename A> UnExpr<Negate<typename A::returnType>, A> operator~(const A& a) diff --git a/src/bitfield.h b/src/bitfield.h index e1b0cfaa..23eb6bb9 100644 --- a/src/bitfield.h +++ b/src/bitfield.h @@ -54,7 +54,8 @@ inline unsigned char lastByteMask(size_t nbits) } // Returns true if index-th bits is set. Otherwise returns false. -inline bool test(const unsigned char* bitfield, size_t nbits, size_t index) +template<typename Array> +inline bool test(const Array& bitfield, size_t nbits, size_t index) { assert(index < nbits); unsigned char mask = 128 >> (index%8); diff --git a/src/download_helper.cc b/src/download_helper.cc index d3612662..b232f33d 100644 --- a/src/download_helper.cc +++ b/src/download_helper.cc @@ -187,6 +187,7 @@ static SharedHandle<RequestGroup> createRequestGroup strconcat(option->get(PREF_DIR), "/", option->get(PREF_OUT)):A2STR::NIL)); dctx->setDir(option->get(PREF_DIR)); + dctx->getFileEntries().front()->setUris(uris); rg->setDownloadContext(dctx); return rg; } diff --git a/test/DownloadContextTest.cc b/test/DownloadContextTest.cc new file mode 100644 index 00000000..bfe88400 --- /dev/null +++ b/test/DownloadContextTest.cc @@ -0,0 +1,44 @@ +#include "DownloadContext.h" + +#include <cppunit/extensions/HelperMacros.h> + +#include "FileEntry.h" +#include "MockDownloadContext.h" + +namespace aria2 { + +class DownloadContextTest:public CppUnit::TestFixture { + + CPPUNIT_TEST_SUITE(DownloadContextTest); + CPPUNIT_TEST(testFindFileEntryByOffset); + CPPUNIT_TEST_SUITE_END(); +public: + void testFindFileEntryByOffset(); +}; + + +CPPUNIT_TEST_SUITE_REGISTRATION(DownloadContextTest); + +void DownloadContextTest::testFindFileEntryByOffset() +{ + MockDownloadContext ctx; + + CPPUNIT_ASSERT(ctx.findFileEntryByOffset(0).isNull()); + + ctx.addFileEntry(SharedHandle<FileEntry>(new FileEntry("file1",1000,0))); + ctx.addFileEntry(SharedHandle<FileEntry>(new FileEntry("file2",0,1000))); + ctx.addFileEntry(SharedHandle<FileEntry>(new FileEntry("file3",0,1000))); + ctx.addFileEntry(SharedHandle<FileEntry>(new FileEntry("file4",2000,1000))); + ctx.addFileEntry(SharedHandle<FileEntry>(new FileEntry("file5",3000,3000))); + ctx.addFileEntry(SharedHandle<FileEntry>(new FileEntry("file6",0,6000))); + + CPPUNIT_ASSERT_EQUAL(std::string("file1"), + ctx.findFileEntryByOffset(0)->getPath()); + CPPUNIT_ASSERT_EQUAL(std::string("file4"), + ctx.findFileEntryByOffset(1500)->getPath()); + CPPUNIT_ASSERT_EQUAL(std::string("file5"), + ctx.findFileEntryByOffset(5999)->getPath()); + CPPUNIT_ASSERT(ctx.findFileEntryByOffset(6000).isNull()); +} + +} // namespace aria2 diff --git a/test/Makefile.am b/test/Makefile.am index a9805744..18e03245 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -70,7 +70,9 @@ aria2c_SOURCES = AllTest.cc\ LongestSequencePieceSelectorTest.cc\ a2algoTest.cc\ bitfieldTest.cc\ - BDETest.cc + BDETest.cc\ + DownloadContextTest.cc\ + MockDownloadContext.h if ENABLE_XML_RPC aria2c_SOURCES += XmlRpcRequestParserControllerTest.cc\ diff --git a/test/Makefile.in b/test/Makefile.in index 142c96eb..a11a66bb 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -200,8 +200,8 @@ am__aria2c_SOURCES_DIST = AllTest.cc TestUtil.cc TestUtil.h \ SequentialPickerTest.cc RarestPieceSelectorTest.cc \ PieceStatManTest.cc InOrderPieceSelector.h \ LongestSequencePieceSelectorTest.cc a2algoTest.cc \ - bitfieldTest.cc BDETest.cc \ - XmlRpcRequestParserControllerTest.cc \ + bitfieldTest.cc BDETest.cc DownloadContextTest.cc \ + MockDownloadContext.h XmlRpcRequestParserControllerTest.cc \ XmlRpcRequestProcessorTest.cc XmlRpcMethodTest.cc \ FallocFileAllocationIteratorTest.cc GZipDecoderTest.cc \ Sqlite3MozCookieParserTest.cc MessageDigestHelperTest.cc \ @@ -377,9 +377,9 @@ am_aria2c_OBJECTS = AllTest.$(OBJEXT) TestUtil.$(OBJEXT) \ RarestPieceSelectorTest.$(OBJEXT) PieceStatManTest.$(OBJEXT) \ LongestSequencePieceSelectorTest.$(OBJEXT) \ a2algoTest.$(OBJEXT) bitfieldTest.$(OBJEXT) BDETest.$(OBJEXT) \ - $(am__objects_1) $(am__objects_2) $(am__objects_3) \ - $(am__objects_4) $(am__objects_5) $(am__objects_6) \ - $(am__objects_7) + DownloadContextTest.$(OBJEXT) $(am__objects_1) \ + $(am__objects_2) $(am__objects_3) $(am__objects_4) \ + $(am__objects_5) $(am__objects_6) $(am__objects_7) aria2c_OBJECTS = $(am_aria2c_OBJECTS) am__DEPENDENCIES_1 = aria2c_DEPENDENCIES = ../src/libaria2c.a ../src/download_helper.o \ @@ -607,7 +607,8 @@ aria2c_SOURCES = AllTest.cc TestUtil.cc TestUtil.h SocketCoreTest.cc \ SequentialPickerTest.cc RarestPieceSelectorTest.cc \ PieceStatManTest.cc InOrderPieceSelector.h \ LongestSequencePieceSelectorTest.cc a2algoTest.cc \ - bitfieldTest.cc BDETest.cc $(am__append_1) $(am__append_2) \ + bitfieldTest.cc BDETest.cc DownloadContextTest.cc \ + MockDownloadContext.h $(am__append_1) $(am__append_2) \ $(am__append_3) $(am__append_4) $(am__append_5) \ $(am__append_6) $(am__append_7) @@ -773,6 +774,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPeerStorageTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DefaultPieceStorageTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DirectDiskAdaptorTest.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadContextTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadHandlerFactoryTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadHelperTest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ExceptionTest.Po@am__quote@ diff --git a/test/MockDownloadContext.h b/test/MockDownloadContext.h new file mode 100644 index 00000000..b01fc8c0 --- /dev/null +++ b/test/MockDownloadContext.h @@ -0,0 +1,67 @@ +#ifndef _D_MOCK_DOWNLOAD_CONTEXT_H_ +#define _D_MOCK_DOWNLOAD_CONTEXT_H_ + +#include "DownloadContext.h" +#include "A2STR.h" + +namespace aria2 { + +class MockDownloadContext:public DownloadContext +{ +private: + std::deque<std::string> _pieceHashes; +public: + virtual const std::string& getPieceHash(size_t index) const + { + return A2STR::NIL; + } + + virtual const std::deque<std::string>& getPieceHashes() const + { + return _pieceHashes; + } + + virtual uint64_t getTotalLength() const + { + return 0; + } + + virtual bool knowsTotalLength() const + { + return false; + } + + virtual FILE_MODE getFileMode() const + { + return MULTI; + } + + virtual size_t getPieceLength() const + { + return 0; + } + + virtual size_t getNumPieces() const + { + return 0; + } + + virtual const std::string& getPieceHashAlgo() const + { + return A2STR::NIL; + } + + virtual std::string getActualBasePath() const + { + return A2STR::NIL; + } + + void addFileEntry(const SharedHandle<FileEntry>& fileEntry) + { + _fileEntries.push_back(fileEntry); + } +}; + +} // namespace aria2 + +#endif // _D_MOCK_DOWNLOAD_CONTEXT_H_ |