diff options
Diffstat (limited to 'src')
54 files changed, 438 insertions, 313 deletions
diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index 2c05a555..7e1a43cc 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -174,7 +174,7 @@ bool AbstractCommand::execute() { req->getProtocol()); ss->setError(); - throw DL_RETRY_EX2(EX_TIME_OUT, DownloadResult::TIME_OUT); + throw DL_RETRY_EX2(EX_TIME_OUT, downloadresultcode::TIME_OUT); } e->commands.push_back(this); return false; @@ -187,7 +187,8 @@ bool AbstractCommand::execute() { DL_ABORT_EX2(StringFormat ("URI=%s", req->getCurrentUrl().c_str()).str(),err), cuid, req->getUrl().c_str()); - _requestGroup->addURIResult(req->getUrl(), err.getCode()); + _fileEntry->addURIResult(req->getUrl(), err.getCode()); + _requestGroup->setLastUriResult(req->getUrl(), err.getCode()); } onAbort(); // TODO Do we need this? @@ -195,6 +196,7 @@ bool AbstractCommand::execute() { tryReserved(); return true; } catch(DlRetryEx& err) { + // TODO1.5 Consider the case when req is null logger->info(MSG_RESTARTING_DOWNLOAD, DL_RETRY_EX2(StringFormat ("URI=%s", req->getCurrentUrl().c_str()).str(),err), @@ -214,15 +216,18 @@ bool AbstractCommand::execute() { if(isAbort) { logger->info(MSG_MAX_TRY, cuid, req->getTryCount()); logger->error(MSG_DOWNLOAD_ABORTED, err, cuid, req->getUrl().c_str()); - _requestGroup->addURIResult(req->getUrl(), err.getCode()); + _fileEntry->addURIResult(req->getUrl(), err.getCode()); + _requestGroup->setLastUriResult(req->getUrl(), err.getCode()); tryReserved(); return true; } else { return prepareForRetry(getOption()->getAsInt(PREF_RETRY_WAIT)); } } catch(DownloadFailureException& err) { + // TODO1.5 Consider the case when req is null logger->error(EX_EXCEPTION_CAUGHT, err); - _requestGroup->addURIResult(req->getUrl(), err.getCode()); + _fileEntry->addURIResult(req->getUrl(), err.getCode()); + _requestGroup->setLastUriResult(req->getUrl(), err.getCode()); _requestGroup->setHaltRequested(true); return true; } @@ -279,7 +284,7 @@ void AbstractCommand::onAbort() { // TODO This might be a problem if the failure is caused by proxy. e->_requestGroupMan->getOrCreateServerStat(req->getHost(), req->getProtocol())->setError(); - _requestGroup->removeIdenticalURI(req->getUrl()); + _fileEntry->removeIdenticalURI(req->getUrl()); _fileEntry->removeRequest(req); } diff --git a/src/AdaptiveURISelector.cc b/src/AdaptiveURISelector.cc index ef355b15..7ceb09fa 100644 --- a/src/AdaptiveURISelector.cc +++ b/src/AdaptiveURISelector.cc @@ -50,6 +50,7 @@ #include "Option.h" #include "SimpleRandomizer.h" #include "SocketCore.h" +#include "FileEntry.h" namespace aria2 { @@ -71,14 +72,15 @@ AdaptiveURISelector::AdaptiveURISelector AdaptiveURISelector::~AdaptiveURISelector() {} -std::string AdaptiveURISelector::select(std::deque<std::string>& uris) +std::string AdaptiveURISelector::select(FileEntry* fileEntry) { _logger->debug("AdaptiveURISelector: called %d", _requestGroup->getNumConnection()); + std::deque<std::string>& uris = fileEntry->getRemainingUris(); if (uris.empty() && _requestGroup->getNumConnection() <= 1) { // here we know the download will fail, trying to find previously // failed uris that may succeed with more permissive values - mayRetryWithIncreasedTimeout(uris); + mayRetryWithIncreasedTimeout(fileEntry); } std::string selected = selectOne(uris); @@ -89,15 +91,15 @@ std::string AdaptiveURISelector::select(std::deque<std::string>& uris) return selected; } -void AdaptiveURISelector::mayRetryWithIncreasedTimeout -(std::deque<std::string>& uris) +void AdaptiveURISelector::mayRetryWithIncreasedTimeout(FileEntry* fileEntry) { if (_requestGroup->getTimeout()*2 >= MAX_TIMEOUT) return; _requestGroup->setTimeout(_requestGroup->getTimeout()*2); + std::deque<std::string>& uris = fileEntry->getRemainingUris(); // looking for retries std::deque<URIResult> timeouts; - _requestGroup->extractURIResult(timeouts, DownloadResult::TIME_OUT); + fileEntry->extractURIResult(timeouts, downloadresultcode::TIME_OUT); std::transform(timeouts.begin(), timeouts.end(), std::back_inserter(uris), std::mem_fun_ref(&URIResult::getURI)); diff --git a/src/AdaptiveURISelector.h b/src/AdaptiveURISelector.h index 6804b9cc..1e3c0869 100644 --- a/src/AdaptiveURISelector.h +++ b/src/AdaptiveURISelector.h @@ -57,7 +57,7 @@ private: Logger* _logger; - void mayRetryWithIncreasedTimeout(std::deque<std::string>& uris); + void mayRetryWithIncreasedTimeout(FileEntry* fileEntry); std::string selectOne(const std::deque<std::string>& uris); void adjustLowestSpeedLimit(const std::deque<std::string>& uris, @@ -78,7 +78,7 @@ public: virtual ~AdaptiveURISelector(); - virtual std::string select(std::deque<std::string>& uris); + virtual std::string select(FileEntry* fileEntry); virtual void tuneDownloadCommand(const std::deque<std::string>& uris, DownloadCommand* command); diff --git a/src/AutoSaveCommand.cc b/src/AutoSaveCommand.cc index ad5001bc..4202c68c 100644 --- a/src/AutoSaveCommand.cc +++ b/src/AutoSaveCommand.cc @@ -35,6 +35,7 @@ #include "AutoSaveCommand.h" #include "DownloadEngine.h" #include "RequestGroupMan.h" +#include "FileEntry.h" namespace aria2 { diff --git a/src/BtPostDownloadHandler.cc b/src/BtPostDownloadHandler.cc index 5e733d75..14d3c4ae 100644 --- a/src/BtPostDownloadHandler.cc +++ b/src/BtPostDownloadHandler.cc @@ -66,7 +66,7 @@ void BtPostDownloadHandler::getNextRequestGroups const SharedHandle<Option>& op = requestGroup->getOption(); _logger->debug("Generating RequestGroups for Torrent file %s", requestGroup->getFirstFilePath().c_str()); - RequestGroupHandle rg(new RequestGroup(op, std::deque<std::string>())); + RequestGroupHandle rg(new RequestGroup(op)); std::string content; try { diff --git a/src/CheckIntegrityDispatcherCommand.cc b/src/CheckIntegrityDispatcherCommand.cc index fff00638..ceec1f4a 100644 --- a/src/CheckIntegrityDispatcherCommand.cc +++ b/src/CheckIntegrityDispatcherCommand.cc @@ -37,6 +37,7 @@ #include "CheckIntegrityCommand.h" #include "message.h" #include "Logger.h" +#include "FileEntry.h" namespace aria2 { diff --git a/src/CookieStorage.cc b/src/CookieStorage.cc index 3e838657..8c3d6ac1 100644 --- a/src/CookieStorage.cc +++ b/src/CookieStorage.cc @@ -44,6 +44,7 @@ #include "DlAbortEx.h" #include "StringFormat.h" #include "NsCookieParser.h" +#include "File.h" #ifdef HAVE_SQLITE3 # include "Sqlite3MozCookieParser.h" #endif // HAVE_SQLITE3 diff --git a/src/DHTAutoSaveCommand.cc b/src/DHTAutoSaveCommand.cc index 026fc445..aa4b34e4 100644 --- a/src/DHTAutoSaveCommand.cc +++ b/src/DHTAutoSaveCommand.cc @@ -52,6 +52,7 @@ #include "message.h" #include "Logger.h" #include "a2functional.h" +#include "FileEntry.h" namespace aria2 { diff --git a/src/DHTBucketRefreshCommand.cc b/src/DHTBucketRefreshCommand.cc index 30d62cdb..e4923c10 100644 --- a/src/DHTBucketRefreshCommand.cc +++ b/src/DHTBucketRefreshCommand.cc @@ -39,6 +39,7 @@ #include "DHTTask.h" #include "DownloadEngine.h" #include "RequestGroupMan.h" +#include "FileEntry.h" namespace aria2 { diff --git a/src/DHTEntryPointNameResolveCommand.cc b/src/DHTEntryPointNameResolveCommand.cc index 0e09d746..dbe9ae9a 100644 --- a/src/DHTEntryPointNameResolveCommand.cc +++ b/src/DHTEntryPointNameResolveCommand.cc @@ -51,6 +51,7 @@ #include "RequestGroupMan.h" #include "Logger.h" #include "StringFormat.h" +#include "FileEntry.h" namespace aria2 { diff --git a/src/DHTInteractionCommand.cc b/src/DHTInteractionCommand.cc index ec86d7ae..a5fd28e8 100644 --- a/src/DHTInteractionCommand.cc +++ b/src/DHTInteractionCommand.cc @@ -45,6 +45,7 @@ #include "Logger.h" #include "DHTMessageCallback.h" #include "DHTNode.h" +#include "FileEntry.h" namespace aria2 { diff --git a/src/DHTPeerAnnounceCommand.cc b/src/DHTPeerAnnounceCommand.cc index 1de3fead..2b8ddc4c 100644 --- a/src/DHTPeerAnnounceCommand.cc +++ b/src/DHTPeerAnnounceCommand.cc @@ -39,6 +39,7 @@ #include "RecoverableException.h" #include "message.h" #include "Logger.h" +#include "FileEntry.h" namespace aria2 { diff --git a/src/DHTTokenUpdateCommand.cc b/src/DHTTokenUpdateCommand.cc index a699f4ca..e29dcebb 100644 --- a/src/DHTTokenUpdateCommand.cc +++ b/src/DHTTokenUpdateCommand.cc @@ -39,6 +39,7 @@ #include "RecoverableException.h" #include "message.h" #include "Logger.h" +#include "FileEntry.h" namespace aria2 { diff --git a/src/DlAbortEx.h b/src/DlAbortEx.h index d6bc4d4e..4d6e9281 100644 --- a/src/DlAbortEx.h +++ b/src/DlAbortEx.h @@ -54,7 +54,7 @@ public: DlAbortEx(const char* file, int line, const RecoverableException& e): RecoverableException(file, line, e) {} DlAbortEx(const char* file, int line, const std::string& msg, - DownloadResult::RESULT code): + downloadresultcode::RESULT code): RecoverableException(file, line, msg, code) {} }; diff --git a/src/DlRetryEx.h b/src/DlRetryEx.h index 8c88a0fd..0b861e97 100644 --- a/src/DlRetryEx.h +++ b/src/DlRetryEx.h @@ -54,7 +54,7 @@ public: DlRetryEx(const char* file, int line, const DlRetryEx& e): RecoverableException(file, line, e) {} DlRetryEx(const char* file, int line, const std::string& msg, - DownloadResult::RESULT code): + downloadresultcode::RESULT code): RecoverableException(file, line, msg, code) {} }; diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index 3c563984..48783817 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -252,7 +252,7 @@ void DownloadCommand::checkLowestDownloadSpeed() const nowSpeed, lowestDownloadSpeedLimit, req->getHost().c_str()).str(), - DownloadResult::TOO_SLOW_DOWNLOAD_SPEED); + downloadresultcode::TOO_SLOW_DOWNLOAD_SPEED); } } } diff --git a/src/DownloadContext.h b/src/DownloadContext.h index ef55b035..479efab5 100644 --- a/src/DownloadContext.h +++ b/src/DownloadContext.h @@ -47,10 +47,10 @@ #include "A2STR.h" #include "BDE.h" #include "IntSequence.h" +#include "FileEntry.h" namespace aria2 { -class FileEntry; class RequestGroup; class DownloadContext diff --git a/src/DownloadFailureException.h b/src/DownloadFailureException.h index aee71046..b9fb1132 100644 --- a/src/DownloadFailureException.h +++ b/src/DownloadFailureException.h @@ -59,7 +59,8 @@ public: const DownloadFailureException& e): RecoverableException(file, line, e) {} DownloadFailureException(const char* file, int line, - const std::string& msg, DownloadResult::RESULT code): + const std::string& msg, + downloadresultcode::RESULT code): RecoverableException(file, line, msg, code) {} }; diff --git a/src/DownloadResult.h b/src/DownloadResult.h index d52c79dc..093154da 100644 --- a/src/DownloadResult.h +++ b/src/DownloadResult.h @@ -43,59 +43,37 @@ #include <vector> #include "SharedHandle.h" -#include "FileEntry.h" +#include "DownloadResultCode.h" namespace aria2 { +class FileEntry; + class DownloadResult { public: - // RESULT is used as an exit status code. - enum RESULT { - FINISHED = 0, - UNKNOWN_ERROR = 1, - TIME_OUT = 2, - RESOURCE_NOT_FOUND = 3, - MAX_FILE_NOT_FOUND = 4, - TOO_SLOW_DOWNLOAD_SPEED = 5, - NETWORK_PROBLEM = 6, - IN_PROGRESS = 7, - }; - int32_t gid; std::vector<SharedHandle<FileEntry> > fileEntries; bool inMemoryDownload; - uint64_t totalLength; - - std::string uri; - - size_t numUri; - uint64_t sessionDownloadLength; // milliseconds int64_t sessionTime; - RESULT result; + downloadresultcode::RESULT result; DownloadResult(int32_t gid, const std::vector<SharedHandle<FileEntry> >& fileEntries, bool inMemoryDownload, - uint64_t totalLength, - const std::string& uri, - size_t numUri, uint64_t sessionDownloadLength, int64_t sessionTime, - RESULT result): + downloadresultcode::RESULT result): gid(gid), fileEntries(fileEntries), inMemoryDownload(inMemoryDownload), - totalLength(totalLength), - uri(uri), - numUri(numUri), sessionDownloadLength(sessionDownloadLength), sessionTime(sessionTime), result(result) {} diff --git a/src/DownloadResultCode.h b/src/DownloadResultCode.h new file mode 100644 index 00000000..8a99f730 --- /dev/null +++ b/src/DownloadResultCode.h @@ -0,0 +1,59 @@ +/* <!-- copyright */ +/* + * aria2 - The high speed download utility + * + * Copyright (C) 2009 Tatsuhiro Tsujikawa + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * In addition, as a special exception, the copyright holders give + * permission to link the code of portions of this program with the + * OpenSSL library under certain conditions as described in each + * individual source file, and distribute linked combinations + * including the two. + * You must obey the GNU General Public License in all respects + * for all of the code used other than OpenSSL. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you + * do not wish to do so, delete this exception statement from your + * version. If you delete this exception statement from all source + * files in the program, then also delete it here. + */ +/* copyright --> */ +#ifndef _D_DOWNLOAD_RESULT_CODE_H_ +#define _D_DOWNLOAD_RESULT_CODE_H_ + +#include "common.h" + +namespace aria2 { + +namespace downloadresultcode { + +enum RESULT { + FINISHED = 0, + UNKNOWN_ERROR = 1, + TIME_OUT = 2, + RESOURCE_NOT_FOUND = 3, + MAX_FILE_NOT_FOUND = 4, + TOO_SLOW_DOWNLOAD_SPEED = 5, + NETWORK_PROBLEM = 6, + IN_PROGRESS = 7, +}; + +} // namespace downloadresultcode + +} // namespace aria2 + +#endif // _D_DOWNLOAD_RESULT_CODE_H_ diff --git a/src/FeedbackURISelector.cc b/src/FeedbackURISelector.cc index fe96f080..f20d4286 100644 --- a/src/FeedbackURISelector.cc +++ b/src/FeedbackURISelector.cc @@ -40,6 +40,7 @@ #include "ServerStat.h" #include "Request.h" #include "A2STR.h" +#include "FileEntry.h" namespace aria2 { @@ -59,8 +60,9 @@ public: } }; -std::string FeedbackURISelector::select(std::deque<std::string>& uris) +std::string FeedbackURISelector::select(FileEntry* fileEntry) { + std::deque<std::string>& uris = fileEntry->getRemainingUris(); if(uris.empty()) { return A2STR::NIL; } diff --git a/src/FeedbackURISelector.h b/src/FeedbackURISelector.h index 10ed6f88..46e5c8fa 100644 --- a/src/FeedbackURISelector.h +++ b/src/FeedbackURISelector.h @@ -4,7 +4,7 @@ * * Copyright (C) 2006 Tatsuhiro Tsujikawa * - * This program is free software; you can redistribute it and/or modify +nn * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. @@ -50,7 +50,7 @@ public: virtual ~FeedbackURISelector(); - virtual std::string select(std::deque<std::string>& uris); + virtual std::string select(FileEntry* fileEntry); }; } // namespace aria2 diff --git a/src/FileEntry.cc b/src/FileEntry.cc index 4e23f5d4..c07ecbb8 100644 --- a/src/FileEntry.cc +++ b/src/FileEntry.cc @@ -35,9 +35,11 @@ #include "FileEntry.h" #include <cassert> +#include <algorithm> #include "Util.h" #include "URISelector.h" +#include "LogFactory.h" namespace aria2 { @@ -46,7 +48,12 @@ FileEntry::FileEntry(const std::string& path, off_t offset, const std::deque<std::string>& uris): path(path), _uris(uris), length(length), offset(offset), - extracted(false), requested(true) {} + extracted(false), requested(true), + _logger(LogFactory::getInstance()) {} + +FileEntry::FileEntry(): + length(0), offset(0), extracted(false), requested(false), + _logger(LogFactory::getInstance()) {} FileEntry::~FileEntry() {} @@ -91,7 +98,7 @@ void FileEntry::getUris(std::deque<std::string>& uris) const std::string FileEntry::selectUri(const SharedHandle<URISelector>& uriSelector) { - return uriSelector->select(_uris); + return uriSelector->select(this); } SharedHandle<Request> @@ -100,7 +107,7 @@ FileEntry::getRequest(const SharedHandle<URISelector>& selector) SharedHandle<Request> req; if(_requestPool.empty()) { while(1) { - std::string uri = selector->select(_uris); + std::string uri = selector->select(this); if(uri.empty()) { return req; } @@ -139,4 +146,51 @@ bool FileEntry::removeRequest(const SharedHandle<Request>& request) return false; } +void FileEntry::removeURIWhoseHostnameIs(const std::string& hostname) +{ + std::deque<std::string> newURIs; + Request req; + for(std::deque<std::string>::const_iterator itr = _uris.begin(); itr != _uris.end(); ++itr) { + if(((*itr).find(hostname) == std::string::npos) || + (req.setUrl(*itr) && (req.getHost() != hostname))) { + newURIs.push_back(*itr); + } + } + _logger->debug("Removed %d duplicate hostname URIs for path=%s", + _uris.size()-newURIs.size(), getPath().c_str()); + _uris = newURIs; +} + +void FileEntry::removeIdenticalURI(const std::string& uri) +{ + _uris.erase(std::remove(_uris.begin(), _uris.end(), uri), _uris.end()); +} + +void FileEntry::addURIResult(std::string uri, downloadresultcode::RESULT result) +{ + _uriResults.push_back(URIResult(uri, result)); +} + +class FindURIResultByResult { +private: + downloadresultcode::RESULT _r; +public: + FindURIResultByResult(downloadresultcode::RESULT r):_r(r) {} + + bool operator()(const URIResult& uriResult) const + { + return uriResult.getResult() == _r; + } +}; + +void FileEntry::extractURIResult +(std::deque<URIResult>& res, downloadresultcode::RESULT r) +{ + std::deque<URIResult>::iterator i = + std::stable_partition(_uriResults.begin(), _uriResults.end(), + FindURIResultByResult(r)); + std::copy(_uriResults.begin(), i, std::back_inserter(res)); + _uriResults.erase(_uriResults.begin(), i); +} + } // namespace aria2 diff --git a/src/FileEntry.h b/src/FileEntry.h index a40be12e..3d9106e9 100644 --- a/src/FileEntry.h +++ b/src/FileEntry.h @@ -44,11 +44,15 @@ #include "SharedHandle.h" #include "File.h" #include "Request.h" +#include "URIResult.h" +#include "DownloadResultCode.h" namespace aria2 { class URISelector; +class Logger; + class FileEntry { private: std::string path; @@ -61,8 +65,13 @@ private: std::deque<SharedHandle<Request> > _requestPool; std::deque<SharedHandle<Request> > _inFlightRequests; std::string _contentType; + // URIResult is stored in the ascending order of the time when its result is + // available. + std::deque<URIResult> _uriResults; + + Logger* _logger; public: - FileEntry():length(0), offset(0), extracted(false), requested(false) {} + FileEntry(); FileEntry(const std::string& path, uint64_t length, off_t offset, const std::deque<std::string>& uris = std::deque<std::string>()); @@ -116,6 +125,11 @@ public: return _uris; } + std::deque<std::string>& getRemainingUris() + { + return _uris; + } + const std::deque<std::string>& getSpentUris() const { return _spentUris; @@ -159,6 +173,22 @@ public: // Translate global offset goff to file local offset. off_t gtoloff(off_t goff) const; + + void removeURIWhoseHostnameIs(const std::string& hostname); + + void removeIdenticalURI(const std::string& uri); + + void addURIResult(std::string uri, downloadresultcode::RESULT result); + + const std::deque<URIResult>& getURIResults() const + { + return _uriResults; + } + + // Extracts URIResult whose _result is r and stores them into res. + // The extracted URIResults are removed from _uriResults. + void extractURIResult + (std::deque<URIResult>& res, downloadresultcode::RESULT r); }; typedef SharedHandle<FileEntry> FileEntryHandle; diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index 555ea605..9abdaa34 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -65,6 +65,7 @@ #include "AuthConfigFactory.h" #include "AuthConfig.h" #include "a2functional.h" +#include "URISelector.h" namespace aria2 { @@ -106,10 +107,11 @@ bool FtpNegotiationCommand::executeInternal() { SharedHandle<ServerHost> sv = _requestGroup->searchServerHost(req->getHost()); if(!sv.isNull()) { - _requestGroup->removeURIWhoseHostnameIs(sv->getHostname()); + _fileEntry->removeURIWhoseHostnameIs(sv->getHostname()); } } - _requestGroup->tuneDownloadCommand(command); + _requestGroup->getURISelector()->tuneDownloadCommand + (_fileEntry->getRemainingUris(), command); e->commands.push_back(command); return true; } else if(sequence == SEQ_HEAD_OK || sequence == SEQ_DOWNLOAD_ALREADY_COMPLETED) { @@ -265,7 +267,8 @@ bool FtpNegotiationCommand::recvCwd() { poolConnection(); _requestGroup->increaseAndValidateFileNotFoundCount(); if (status == 550) - throw DL_ABORT_EX2(MSG_RESOURCE_NOT_FOUND, DownloadResult::RESOURCE_NOT_FOUND); + throw DL_ABORT_EX2(MSG_RESOURCE_NOT_FOUND, + downloadresultcode::RESOURCE_NOT_FOUND); else throw DL_ABORT_EX(StringFormat(EX_BAD_STATUS, status).str()); } @@ -579,7 +582,8 @@ bool FtpNegotiationCommand::recvRetr() { if(status != 150 && status != 125) { _requestGroup->increaseAndValidateFileNotFoundCount(); if (status == 550) - throw DL_ABORT_EX2(MSG_RESOURCE_NOT_FOUND, DownloadResult::RESOURCE_NOT_FOUND); + throw DL_ABORT_EX2(MSG_RESOURCE_NOT_FOUND, + downloadresultcode::RESOURCE_NOT_FOUND); else throw DL_ABORT_EX(StringFormat(EX_BAD_STATUS, status).str()); } diff --git a/src/HttpListenCommand.cc b/src/HttpListenCommand.cc index 55cec76f..381aa1fa 100644 --- a/src/HttpListenCommand.cc +++ b/src/HttpListenCommand.cc @@ -41,6 +41,7 @@ #include "HttpServerCommand.h" #include "CUIDCounter.h" #include "RequestGroupMan.h" +#include "FileEntry.h" namespace aria2 { diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index 9dbf9ffe..4b4d8080 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -66,6 +66,7 @@ #include "AuthConfigFactory.h" #include "AuthConfig.h" #include "a2functional.h" +#include "URISelector.h" namespace aria2 { @@ -131,7 +132,7 @@ bool HttpResponseCommand::executeInternal() SharedHandle<ServerHost> sv = _requestGroup->searchServerHost(req->getHost()); if(!sv.isNull()) { - _requestGroup->removeURIWhoseHostnameIs(sv->getHostname()); + _fileEntry->removeURIWhoseHostnameIs(sv->getHostname()); } } if(_requestGroup->getPieceStorage().isNull()) { @@ -409,7 +410,8 @@ HttpDownloadCommand* HttpResponseCommand::createHttpDownloadCommand _requestGroup->setFileAllocationEnabled(false); } - _requestGroup->tuneDownloadCommand(command); + _requestGroup->getURISelector()->tuneDownloadCommand + (_fileEntry->getRemainingUris(), command); return command; } diff --git a/src/HttpServerResponseCommand.cc b/src/HttpServerResponseCommand.cc index 054f88be..42e04945 100644 --- a/src/HttpServerResponseCommand.cc +++ b/src/HttpServerResponseCommand.cc @@ -40,6 +40,7 @@ #include "HttpServerCommand.h" #include "RequestGroupMan.h" #include "RecoverableException.h" +#include "FileEntry.h" namespace aria2 { diff --git a/src/HttpSkipResponseCommand.cc b/src/HttpSkipResponseCommand.cc index ff1bf174..685c785c 100644 --- a/src/HttpSkipResponseCommand.cc +++ b/src/HttpSkipResponseCommand.cc @@ -171,7 +171,7 @@ bool HttpSkipResponseCommand::processResponse() } }else if(_httpResponse->getResponseStatus() == HttpHeader::S404) { throw DL_ABORT_EX2(MSG_RESOURCE_NOT_FOUND, - DownloadResult::RESOURCE_NOT_FOUND); + downloadresultcode::RESOURCE_NOT_FOUND); } else { throw DL_ABORT_EX(StringFormat(EX_BAD_STATUS, Util::parseUInt(_httpResponse->getResponseStatus())).str()); } diff --git a/src/InOrderURISelector.cc b/src/InOrderURISelector.cc index 0f89e863..b08d74e7 100644 --- a/src/InOrderURISelector.cc +++ b/src/InOrderURISelector.cc @@ -34,6 +34,7 @@ /* copyright --> */ #include "InOrderURISelector.h" #include "A2STR.h" +#include "FileEntry.h" namespace aria2 { @@ -41,8 +42,9 @@ InOrderURISelector::InOrderURISelector() {} InOrderURISelector::~InOrderURISelector() {} -std::string InOrderURISelector::select(std::deque<std::string>& uris) +std::string InOrderURISelector::select(FileEntry* fileEntry) { + std::deque<std::string>& uris = fileEntry->getRemainingUris(); if(uris.empty()) { return A2STR::NIL; } else { diff --git a/src/InOrderURISelector.h b/src/InOrderURISelector.h index 43496f8f..98a29700 100644 --- a/src/InOrderURISelector.h +++ b/src/InOrderURISelector.h @@ -44,7 +44,7 @@ public: virtual ~InOrderURISelector(); - virtual std::string select(std::deque<std::string>& uris); + virtual std::string select(FileEntry* fileEntry); }; } // namespace aria2 diff --git a/src/Makefile.am b/src/Makefile.am index 235befee..043b29e4 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -201,7 +201,8 @@ SRCS = Socket.h\ LongestSequencePieceSelector.cc LongestSequencePieceSelector.h\ bitfield.h\ BDE.cc BDE.h\ - CreateRequestCommand.cc CreateRequestCommand.h + CreateRequestCommand.cc CreateRequestCommand.h\ + DownloadResultCode.h if ENABLE_XML_RPC SRCS += XmlRpcRequestParserController.cc XmlRpcRequestParserController.h\ diff --git a/src/Makefile.in b/src/Makefile.in index 09e34803..348a3088 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -424,7 +424,8 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \ SequentialDispatcherCommand.h PieceSelector.h \ LongestSequencePieceSelector.cc LongestSequencePieceSelector.h \ bitfield.h BDE.cc BDE.h CreateRequestCommand.cc \ - CreateRequestCommand.h XmlRpcRequestParserController.cc \ + CreateRequestCommand.h DownloadResultCode.h \ + XmlRpcRequestParserController.cc \ XmlRpcRequestParserController.h \ XmlRpcRequestParserStateMachine.cc \ XmlRpcRequestParserStateMachine.h XmlRpcRequestParserState.h \ @@ -1178,15 +1179,15 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \ SequentialDispatcherCommand.h PieceSelector.h \ LongestSequencePieceSelector.cc LongestSequencePieceSelector.h \ bitfield.h BDE.cc BDE.h CreateRequestCommand.cc \ - CreateRequestCommand.h $(am__append_1) $(am__append_2) \ - $(am__append_3) $(am__append_4) $(am__append_5) \ - $(am__append_6) $(am__append_7) $(am__append_8) \ - $(am__append_9) $(am__append_10) $(am__append_11) \ - $(am__append_12) $(am__append_13) $(am__append_14) \ - $(am__append_15) $(am__append_16) $(am__append_17) \ - $(am__append_18) $(am__append_19) $(am__append_20) \ - $(am__append_21) $(am__append_22) $(am__append_23) \ - $(am__append_24) $(am__append_25) + CreateRequestCommand.h DownloadResultCode.h $(am__append_1) \ + $(am__append_2) $(am__append_3) $(am__append_4) \ + $(am__append_5) $(am__append_6) $(am__append_7) \ + $(am__append_8) $(am__append_9) $(am__append_10) \ + $(am__append_11) $(am__append_12) $(am__append_13) \ + $(am__append_14) $(am__append_15) $(am__append_16) \ + $(am__append_17) $(am__append_18) $(am__append_19) \ + $(am__append_20) $(am__append_21) $(am__append_22) \ + $(am__append_23) $(am__append_24) $(am__append_25) noinst_LIBRARIES = libaria2c.a libaria2c_a_SOURCES = $(SRCS) aria2c_LDADD = libaria2c.a @LIBINTL@ @ALLOCA@ @LIBGNUTLS_LIBS@\ diff --git a/src/Metalink2RequestGroup.cc b/src/Metalink2RequestGroup.cc index 1b9a36a4..dcad7912 100644 --- a/src/Metalink2RequestGroup.cc +++ b/src/Metalink2RequestGroup.cc @@ -169,13 +169,14 @@ Metalink2RequestGroup::createRequestGroup if(itr != entry->resources.end()) { std::deque<std::string> uris; uris.push_back((*itr)->url); - torrentRg.reset(new RequestGroup(option, uris)); + torrentRg.reset(new RequestGroup(option)); SharedHandle<DownloadContext> dctx (new DownloadContext(option->getAsInt(PREF_SEGMENT_SIZE), 0, A2STR::NIL)); // Since torrent is downloaded in memory, setting dir has no effect. //dctx->setDir(_option->get(PREF_DIR)); + dctx->getFirstFileEntry()->setUris(uris); torrentRg->setDownloadContext(dctx); torrentRg->clearPreDowloadHandler(); torrentRg->clearPostDowloadHandler(); @@ -196,7 +197,7 @@ Metalink2RequestGroup::createRequestGroup std::deque<std::string> uris; std::for_each(entry->resources.begin(), entry->resources.end(), AccumulateNonP2PUrl(uris)); - SharedHandle<RequestGroup> rg(new RequestGroup(option, uris)); + SharedHandle<RequestGroup> rg(new RequestGroup(option)); // If piece hash is specified in the metalink, // make segment size equal to piece hash size. size_t pieceLength; @@ -215,6 +216,7 @@ Metalink2RequestGroup::createRequestGroup entry->getLength(), strconcat(option->get(PREF_DIR), "/", entry->file->getPath()))); dctx->setDir(option->get(PREF_DIR)); + dctx->getFirstFileEntry()->setUris(uris); #ifdef ENABLE_MESSAGE_DIGEST if(entry->chunkChecksum.isNull()) { if(!entry->checksum.isNull()) { diff --git a/src/MultiUrlRequestInfo.cc b/src/MultiUrlRequestInfo.cc index 11351ad3..bec4265a 100644 --- a/src/MultiUrlRequestInfo.cc +++ b/src/MultiUrlRequestInfo.cc @@ -100,9 +100,9 @@ void MultiUrlRequestInfo::printMessageForContinue() << "\n"; } -DownloadResult::RESULT MultiUrlRequestInfo::execute() +downloadresultcode::RESULT MultiUrlRequestInfo::execute() { - DownloadResult::RESULT returnValue = DownloadResult::FINISHED; + downloadresultcode::RESULT returnValue = downloadresultcode::FINISHED; try { DownloadEngineHandle e = DownloadEngineFactory().newDownloadEngine(_option.get(), _requestGroups); @@ -190,9 +190,9 @@ DownloadResult::RESULT MultiUrlRequestInfo::execute() RequestGroupMan::DownloadStat s = e->_requestGroupMan->getDownloadStat(); if(!s.allCompleted()) { printMessageForContinue(); - if(s.getLastErrorResult() == DownloadResult::FINISHED && + if(s.getLastErrorResult() == downloadresultcode::FINISHED && s.getInProgress() > 0) { - returnValue = DownloadResult::IN_PROGRESS; + returnValue = downloadresultcode::IN_PROGRESS; } else { returnValue = s.getLastErrorResult(); } diff --git a/src/MultiUrlRequestInfo.h b/src/MultiUrlRequestInfo.h index aaa9b05f..91706f3d 100644 --- a/src/MultiUrlRequestInfo.h +++ b/src/MultiUrlRequestInfo.h @@ -77,7 +77,7 @@ public: * Returns FINISHED if all downloads have completed, otherwise returns the * last download result. */ - DownloadResult::RESULT execute(); + downloadresultcode::RESULT execute(); }; typedef SharedHandle<MultiUrlRequestInfo> MultiUrlRequestInfoHandle; diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc index b55c8941..fcbee87c 100644 --- a/src/OptionHandlerFactory.cc +++ b/src/OptionHandlerFactory.cc @@ -41,6 +41,7 @@ #include "Util.h" #include "help_tags.h" #include "StringFormat.h" +#include "File.h" namespace aria2 { diff --git a/src/PeerListenCommand.cc b/src/PeerListenCommand.cc index ae9be4b4..70adcf93 100644 --- a/src/PeerListenCommand.cc +++ b/src/PeerListenCommand.cc @@ -47,6 +47,7 @@ #include "Logger.h" #include "Socket.h" #include "SimpleRandomizer.h" +#include "FileEntry.h" namespace aria2 { diff --git a/src/RecoverableException.h b/src/RecoverableException.h index a91f2a54..cffe2ac4 100644 --- a/src/RecoverableException.h +++ b/src/RecoverableException.h @@ -35,13 +35,13 @@ #ifndef _D_RECOVERABLE_EXCEPTION_H_ #define _D_RECOVERABLE_EXCEPTION_H_ #include "Exception.h" -#include "DownloadResult.h" +#include "DownloadResultCode.h" namespace aria2 { class RecoverableException:public Exception { private: - DownloadResult::RESULT _code; + downloadresultcode::RESULT _code; protected: virtual SharedHandle<Exception> copy() const @@ -52,23 +52,23 @@ protected: public: RecoverableException(const char* file, int line, const std::string& msg): Exception(file, line, msg), - _code(DownloadResult::UNKNOWN_ERROR) {} + _code(downloadresultcode::UNKNOWN_ERROR) {} RecoverableException(const char* file, int line, const std::string& msg, const Exception& cause): Exception(file, line, msg, cause), - _code(DownloadResult::UNKNOWN_ERROR) {} + _code(downloadresultcode::UNKNOWN_ERROR) {} RecoverableException(const char* file, int line, const RecoverableException& e): Exception(file, line, e), - _code(DownloadResult::UNKNOWN_ERROR) {} + _code(downloadresultcode::UNKNOWN_ERROR) {} RecoverableException(const char* file, int line, const std::string& msg, - DownloadResult::RESULT result): + downloadresultcode::RESULT result): Exception(file, line, msg), _code(result) {} - DownloadResult::RESULT getCode() const { return _code; } + downloadresultcode::RESULT getCode() const { return _code; } }; } // namespace aria2 diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc index 19432816..3dfa75f1 100644 --- a/src/RequestGroup.cc +++ b/src/RequestGroup.cc @@ -112,11 +112,9 @@ int32_t RequestGroup::_gidCounter = 0; const std::string RequestGroup::ACCEPT_METALINK = "application/metalink+xml"; -RequestGroup::RequestGroup(const SharedHandle<Option>& option, - const std::deque<std::string>& uris): +RequestGroup::RequestGroup(const SharedHandle<Option>& option): _gid(++_gidCounter), _option(new Option(*option.get())), - _uris(uris), _numConcurrentCommand(option->getAsInt(PREF_SPLIT)), _numStreamConnection(0), _numCommand(0), @@ -175,19 +173,19 @@ bool RequestGroup::allDownloadFinished() const } } -DownloadResult::RESULT RequestGroup::downloadResult() const +downloadresultcode::RESULT RequestGroup::downloadResult() const { if (downloadFinished()) - return DownloadResult::FINISHED; + return downloadresultcode::FINISHED; else { - if (_uriResults.empty()) { + if (_lastUriResult.isNull()) { if(_haltReason == RequestGroup::USER_REQUEST) { - return DownloadResult::IN_PROGRESS; + return downloadresultcode::IN_PROGRESS; } else { - return DownloadResult::UNKNOWN_ERROR; + return downloadresultcode::UNKNOWN_ERROR; } } else { - return _uriResults.back().getResult(); + return _lastUriResult->getResult(); } } } @@ -403,6 +401,17 @@ void RequestGroup::processCheckIntegrityEntry(std::deque<Command*>& commands, } } +template<typename InputIterator> +static bool hasAssociatedUri(InputIterator first, InputIterator last) +{ + for(; first != last; ++first) { + if((*first)->isRequested() && !(*first)->getRemainingUris().empty()) { + return true; + } + } + return false; +} + void RequestGroup::initPieceStorage() { if(_downloadContext->knowsTotalLength()) { @@ -411,9 +420,9 @@ void RequestGroup::initPieceStorage() (new DefaultPieceStorage(_downloadContext, _option.get())); // Use LongestSequencePieceSelector when HTTP/FTP/BitTorrent integrated // downloads. Currently multi-file integrated download is not supported. - if(!_uris.empty() && - _downloadContext->getFileEntries().size() == 1 && - _downloadContext->hasAttribute(bittorrent::BITTORRENT)) { + if(_downloadContext->hasAttribute(bittorrent::BITTORRENT) && + hasAssociatedUri(_downloadContext->getFileEntries().begin(), + _downloadContext->getFileEntries().end())) { _logger->debug("Using LongestSequencePieceSelector"); ps->setPieceSelector (SharedHandle<PieceSelector>(new LongestSequencePieceSelector())); @@ -578,14 +587,8 @@ void RequestGroup::createNextCommandWithAdj(std::deque<Command*>& commands, if(getTotalLength() == 0) { numCommand = 1+numAdj; } else { - if(_numConcurrentCommand == 0) { - // TODO remove _uris.size() support - numCommand = _uris.size(); - } else { - numCommand = _numConcurrentCommand; - } - numCommand = std::min(static_cast<int>(_downloadContext->getNumPieces()), - numCommand); + numCommand = std::min(_downloadContext->getNumPieces(), + _numConcurrentCommand); numCommand += numAdj; } if(numCommand > 0) { @@ -599,40 +602,40 @@ void RequestGroup::createNextCommand(std::deque<Command*>& commands, const std::string& method) { // TODO1.5 The following block should be moved into FileEntry - if(_option->getAsBool(PREF_REUSE_URI) && _uris.empty()) { - std::deque<std::string> uris = _spentUris; - std::sort(uris.begin(), uris.end()); - uris.erase(std::unique(uris.begin(), uris.end()), uris.end()); - - std::deque<std::string> errorUris(_uriResults.size()); - std::transform(_uriResults.begin(), _uriResults.end(), - errorUris.begin(), std::mem_fun_ref(&URIResult::getURI)); - std::sort(errorUris.begin(), errorUris.end()); - errorUris.erase(std::unique(errorUris.begin(), errorUris.end()), - errorUris.end()); +// if(_option->getAsBool(PREF_REUSE_URI) && _uris.empty()) { +// std::deque<std::string> uris = _spentUris; +// std::sort(uris.begin(), uris.end()); +// uris.erase(std::unique(uris.begin(), uris.end()), uris.end()); + +// std::deque<std::string> errorUris(_uriResults.size()); +// std::transform(_uriResults.begin(), _uriResults.end(), +// errorUris.begin(), std::mem_fun_ref(&URIResult::getURI)); +// std::sort(errorUris.begin(), errorUris.end()); +// errorUris.erase(std::unique(errorUris.begin(), errorUris.end()), +// errorUris.end()); - std::deque<std::string> reusableURIs; - std::set_difference(uris.begin(), uris.end(), - errorUris.begin(), errorUris.end(), - std::back_inserter(reusableURIs)); - size_t ininum = reusableURIs.size(); - _logger->debug("Found %u reusable URIs", - static_cast<unsigned int>(ininum)); - // Reuse at least _numConcurrentCommand URIs here to avoid to - // run this process repeatedly. - if(ininum > 0 && ininum < _numConcurrentCommand) { - _logger->debug("fewer than _numConcurrentCommand=%u", - _numConcurrentCommand); - for(size_t i = 0; i < _numConcurrentCommand/ininum; ++i) { - _uris.insert(_uris.end(), reusableURIs.begin(), reusableURIs.end()); - } - _uris.insert(_uris.end(), reusableURIs.begin(), - reusableURIs.begin()+(_numConcurrentCommand%ininum)); - _logger->debug("Duplication complete: now %u URIs for reuse", - static_cast<unsigned int>(_uris.size())); - } - } - std::deque<std::string> pendingURIs; +// std::deque<std::string> reusableURIs; +// std::set_difference(uris.begin(), uris.end(), +// errorUris.begin(), errorUris.end(), +// std::back_inserter(reusableURIs)); +// size_t ininum = reusableURIs.size(); +// _logger->debug("Found %u reusable URIs", +// static_cast<unsigned int>(ininum)); +// // Reuse at least _numConcurrentCommand URIs here to avoid to +// // run this process repeatedly. +// if(ininum > 0 && ininum < _numConcurrentCommand) { +// _logger->debug("fewer than _numConcurrentCommand=%u", +// _numConcurrentCommand); +// for(size_t i = 0; i < _numConcurrentCommand/ininum; ++i) { +// _uris.insert(_uris.end(), reusableURIs.begin(), reusableURIs.end()); +// } +// _uris.insert(_uris.end(), reusableURIs.begin(), +// reusableURIs.begin()+(_numConcurrentCommand%ininum)); +// _logger->debug("Duplication complete: now %u URIs for reuse", +// static_cast<unsigned int>(_uris.size())); +// } +// } +// std::deque<std::string> pendingURIs; for(; numCommand--; ) { Command* command = new CreateRequestCommand(e->newCUID(), this, e); @@ -917,12 +920,6 @@ void RequestGroup::initializePostDownloadHandler() #endif // ENABLE_METALINK } -void RequestGroup::getURIs(std::deque<std::string>& uris) const -{ - uris.insert(uris.end(), _spentUris.begin(), _spentUris.end()); - uris.insert(uris.end(), _uris.begin(), _uris.end()); -} - bool RequestGroup::isDependencyResolved() { if(_dependency.isNull()) { @@ -985,9 +982,6 @@ bool RequestGroup::needsFileAllocation() const DownloadResultHandle RequestGroup::createDownloadResult() const { - std::deque<std::string> uris; - getURIs(uris); - uint64_t sessionDownloadLength = 0; #ifdef ENABLE_BITTORRENT @@ -1001,14 +995,12 @@ DownloadResultHandle RequestGroup::createDownloadResult() const _segmentMan->calculateSessionDownloadLength(); } + // TODO1.5 Purge unnecessary data in FileEntry here. return SharedHandle<DownloadResult> (new DownloadResult(_gid, _downloadContext->getFileEntries(), _inMemoryDownload, - getTotalLength(), - uris.empty() ? A2STR::NIL:uris.front(), - uris.size(), sessionDownloadLength, _downloadContext->calculateSessionTime(), downloadResult())); @@ -1072,26 +1064,6 @@ void RequestGroup::removeServerHost(int32_t cuid) _serverHosts.erase(std::remove_if(_serverHosts.begin(), _serverHosts.end(), FindServerHostByCUID(cuid)), _serverHosts.end()); } -void RequestGroup::removeURIWhoseHostnameIs(const std::string& hostname) -{ - std::deque<std::string> newURIs; - Request req; - for(std::deque<std::string>::const_iterator itr = _uris.begin(); itr != _uris.end(); ++itr) { - if(((*itr).find(hostname) == std::string::npos) || - (req.setUrl(*itr) && (req.getHost() != hostname))) { - newURIs.push_back(*itr); - } - } - _logger->debug("GUID#%d - Removed %d duplicate hostname URIs", - _gid, _uris.size()-newURIs.size()); - _uris = newURIs; -} - -void RequestGroup::removeIdenticalURI(const std::string& uri) -{ - _uris.erase(std::remove(_uris.begin(), _uris.end(), uri), _uris.end()); -} - void RequestGroup::reportDownloadFinished() { _logger->notice(MSG_FILE_DOWNLOAD_COMPLETED, @@ -1155,7 +1127,7 @@ void RequestGroup::increaseAndValidateFileNotFoundCount() _segmentMan->calculateSessionDownloadLength() == 0) { throw DOWNLOAD_FAILURE_EXCEPTION2 (StringFormat("Reached max-file-not-found count=%u", maxCount).str(), - DownloadResult::MAX_FILE_NOT_FOUND); + downloadresultcode::MAX_FILE_NOT_FOUND); } } @@ -1164,38 +1136,6 @@ void RequestGroup::markInMemoryDownload() _inMemoryDownload = true; } -void RequestGroup::tuneDownloadCommand(DownloadCommand* command) -{ - _uriSelector->tuneDownloadCommand(_uris, command); -} - -void RequestGroup::addURIResult(std::string uri, DownloadResult::RESULT result) -{ - _uriResults.push_back(URIResult(uri, result)); -} - -class FindURIResultByResult { -private: - DownloadResult::RESULT _r; -public: - FindURIResultByResult(DownloadResult::RESULT r):_r(r) {} - - bool operator()(const URIResult& uriResult) const - { - return uriResult.getResult() == _r; - } -}; - -void RequestGroup::extractURIResult -(std::deque<URIResult>& res, DownloadResult::RESULT r) -{ - std::deque<URIResult>::iterator i = - std::stable_partition(_uriResults.begin(), _uriResults.end(), - FindURIResultByResult(r)); - std::copy(_uriResults.begin(), i, std::back_inserter(res)); - _uriResults.erase(_uriResults.begin(), i); -} - void RequestGroup::setTimeout(time_t timeout) { _timeout = timeout; @@ -1213,4 +1153,10 @@ bool RequestGroup::doesUploadSpeedExceed() _maxUploadSpeedLimit < calculateStat().getUploadSpeed(); } +void RequestGroup::setLastUriResult +(const std::string uri, downloadresultcode::RESULT result) +{ + _lastUriResult.reset(new URIResult(uri, result)); +} + } // namespace aria2 diff --git a/src/RequestGroup.h b/src/RequestGroup.h index 3d579929..889cfc7c 100644 --- a/src/RequestGroup.h +++ b/src/RequestGroup.h @@ -44,8 +44,7 @@ #include "TransferStat.h" #include "TimeA2.h" #include "Request.h" -#include "DownloadResult.h" -#include "URIResult.h" +#include "DownloadResultCode.h" namespace aria2 { @@ -68,6 +67,7 @@ class CheckIntegrityEntry; class DownloadResult; class ServerHost; class URISelector; +class URIResult; #ifdef ENABLE_BITTORRENT class BtRuntime; class PeerStorage; @@ -87,10 +87,7 @@ private: SharedHandle<Option> _option; - std::deque<std::string> _uris; - std::deque<std::string> _spentUris; - - unsigned int _numConcurrentCommand; + size_t _numConcurrentCommand; /** * This is the number of connections used in streaming protocol(http/ftp) @@ -124,10 +121,6 @@ private: HaltReason _haltReason; - // URIResult is stored in the ascending order of the time when its result is - // available. - std::deque<URIResult> _uriResults; - bool _singleHostMultiConnectionEnabled; std::deque<SharedHandle<PreDownloadHandler> > _preDownloadHandlers; @@ -159,6 +152,8 @@ private: unsigned int _maxUploadSpeedLimit; + SharedHandle<URIResult> _lastUriResult; + Logger* _logger; void validateFilename(const std::string& expectedFilename, @@ -170,15 +165,14 @@ private: bool tryAutoFileRenaming(); - // Returns the result code of this RequestGroup. - // If the download finished, then returns DownloadResult::FINISHED. - // If the download didn't finish and error result is available in _uriResults, - // then last result code is returned. - // Otherwise returns DownloadResult::UNKNOWN_ERROR. - DownloadResult::RESULT downloadResult() const; + // Returns the result code of this RequestGroup. If the download + // finished, then returns downloadresultcode::FINISHED. If the + // download didn't finish and error result is available in + // _uriResults, then last result code is returned. Otherwise + // returns downloadresultcode::UNKNOWN_ERROR. + downloadresultcode::RESULT downloadResult() const; public: - RequestGroup(const SharedHandle<Option>& option, - const std::deque<std::string>& uris); + RequestGroup(const SharedHandle<Option>& option); ~RequestGroup(); /** @@ -210,11 +204,6 @@ public: DownloadEngine* e, unsigned int numCommand, const std::string& method = Request::METHOD_GET); - void addURI(const std::string& uri) - { - _uris.push_back(uri); - } - bool downloadFinished() const; bool allDownloadFinished() const; @@ -227,18 +216,6 @@ public: uint64_t getCompletedLength() const; - const std::deque<std::string>& getRemainingUris() const - { - return _uris; - } - - const std::deque<std::string>& getSpentUris() const - { - return _spentUris; - } - - void getURIs(std::deque<std::string>& uris) const; - /** * Compares expected filename with specified actualFilename. * The expected filename refers to FileEntry::getBasename() of the first @@ -358,17 +335,6 @@ public: return _forceHaltRequested; } - void addURIResult(std::string uri, DownloadResult::RESULT result); - - const std::deque<URIResult>& getURIResults() const - { - return _uriResults; - } - - // Extracts URIResult whose _result is r and stores them into res. - // The extracted URIResults are removed from _uriResults. - void extractURIResult(std::deque<URIResult>& res, DownloadResult::RESULT r); - void dependsOn(const SharedHandle<Dependency>& dep); bool isDependencyResolved(); @@ -433,10 +399,6 @@ public: void removeServerHost(int32_t cuid); - void removeURIWhoseHostnameIs(const std::string& hostname); - - void removeIdenticalURI(const std::string& uri); - void reportDownloadFinished(); const std::deque<std::string>& getAcceptTypes() const @@ -472,8 +434,6 @@ public: return _inMemoryDownload; } - void tuneDownloadCommand(DownloadCommand* command); - void setTimeout(time_t timeout); time_t getTimeout() const @@ -511,6 +471,8 @@ public: _maxUploadSpeedLimit = speed; } + void setLastUriResult(std::string uri, downloadresultcode::RESULT result); + static void resetGIDCounter() { _gidCounter = 0; } }; diff --git a/src/RequestGroupMan.cc b/src/RequestGroupMan.cc index 9069a4dd..ff3aaee4 100644 --- a/src/RequestGroupMan.cc +++ b/src/RequestGroupMan.cc @@ -249,10 +249,10 @@ static void executeStartHook static void executeStopHook (const SharedHandle<DownloadResult>& result, const Option* option) { - if(result->result == DownloadResult::FINISHED && + if(result->result == downloadresultcode::FINISHED && !option->blank(PREF_ON_DOWNLOAD_COMPLETE)) { executeHook(option->get(PREF_ON_DOWNLOAD_COMPLETE), result->gid); - } else if(result->result != DownloadResult::IN_PROGRESS && + } else if(result->result != downloadresultcode::IN_PROGRESS && !option->blank(PREF_ON_DOWNLOAD_ERROR)) { executeHook(option->get(PREF_ON_DOWNLOAD_ERROR), result->gid); } else if(!option->blank(PREF_ON_DOWNLOAD_STOP)) { @@ -531,10 +531,10 @@ RequestGroupMan::DownloadStat RequestGroupMan::getDownloadStat() const size_t finished = 0; size_t error = 0; size_t inprogress = 0; - DownloadResult::RESULT lastError = DownloadResult::FINISHED; + downloadresultcode::RESULT lastError = downloadresultcode::FINISHED; for(std::deque<SharedHandle<DownloadResult> >::const_iterator itr = _downloadResults.begin(); itr != _downloadResults.end(); ++itr) { - if((*itr)->result == DownloadResult::FINISHED) { + if((*itr)->result == downloadresultcode::FINISHED) { ++finished; } else { ++error; @@ -544,7 +544,7 @@ RequestGroupMan::DownloadStat RequestGroupMan::getDownloadStat() const for(RequestGroups::const_iterator itr = _requestGroups.begin(); itr != _requestGroups.end(); ++itr) { DownloadResultHandle result = (*itr)->createDownloadResult(); - if(result->result == DownloadResult::FINISHED) { + if(result->result == downloadresultcode::FINISHED) { ++finished; } else { ++inprogress; @@ -575,10 +575,10 @@ void RequestGroupMan::showDownloadResults(std::ostream& o) const for(std::deque<SharedHandle<DownloadResult> >::const_iterator itr = _downloadResults.begin(); itr != _downloadResults.end(); ++itr) { std::string status; - if((*itr)->result == DownloadResult::FINISHED) { + if((*itr)->result == downloadresultcode::FINISHED) { status = MARK_OK; ++ok; - } else if((*itr)->result == DownloadResult::IN_PROGRESS) { + } else if((*itr)->result == downloadresultcode::IN_PROGRESS) { status = MARK_INPR; ++inpr; } else { @@ -591,7 +591,7 @@ void RequestGroupMan::showDownloadResults(std::ostream& o) const itr != _requestGroups.end(); ++itr) { DownloadResultHandle result = (*itr)->createDownloadResult(); std::string status; - if(result->result == DownloadResult::FINISHED) { + if(result->result == downloadresultcode::FINISHED) { status = MARK_OK; ++ok; } else { @@ -639,16 +639,8 @@ std::string RequestGroupMan::formatDownloadResult(const std::string& status, con o << "|"; const std::vector<SharedHandle<FileEntry> >& fileEntries = downloadResult->fileEntries; - if(downloadResult->result == DownloadResult::FINISHED || - downloadResult->numUri == 0) { - writeFilePath(fileEntries.begin(), fileEntries.end(), o, - downloadResult->inMemoryDownload); - } else { - o << downloadResult->uri; - if(downloadResult->numUri > 1) { - o << " (" << downloadResult->numUri-1 << "more)"; - } - } + writeFilePath(fileEntries.begin(), fileEntries.end(), o, + downloadResult->inMemoryDownload); return o.str(); } diff --git a/src/RequestGroupMan.h b/src/RequestGroupMan.h index 58032cd6..1652ae08 100644 --- a/src/RequestGroupMan.h +++ b/src/RequestGroupMan.h @@ -146,21 +146,21 @@ public: size_t _error; size_t _inProgress; size_t _waiting; - DownloadResult::RESULT _lastErrorResult; + downloadresultcode::RESULT _lastErrorResult; public: DownloadStat(size_t completed, size_t error, size_t inProgress, size_t waiting, - DownloadResult::RESULT lastErrorResult = - DownloadResult::FINISHED): + downloadresultcode::RESULT lastErrorResult = + downloadresultcode::FINISHED): _completed(completed), _error(error), _inProgress(inProgress), _waiting(waiting), _lastErrorResult(lastErrorResult) {} - DownloadResult::RESULT getLastErrorResult() const + downloadresultcode::RESULT getLastErrorResult() const { return _lastErrorResult; } diff --git a/src/TimedHaltCommand.cc b/src/TimedHaltCommand.cc index 48e1170f..6fc0a4e6 100644 --- a/src/TimedHaltCommand.cc +++ b/src/TimedHaltCommand.cc @@ -37,6 +37,7 @@ #include "RequestGroupMan.h" #include "Logger.h" #include "message.h" +#include "FileEntry.h" namespace aria2 { diff --git a/src/TrackerWatcherCommand.cc b/src/TrackerWatcherCommand.cc index dddf163e..3ad1110e 100644 --- a/src/TrackerWatcherCommand.cc +++ b/src/TrackerWatcherCommand.cc @@ -200,7 +200,7 @@ TrackerWatcherCommand::createRequestGroup(const std::string& uri) { std::deque<std::string> uris; uris.push_back(uri); - RequestGroupHandle rg(new RequestGroup(getOption(), uris)); + RequestGroupHandle rg(new RequestGroup(getOption())); // If backup tracker is available, only try 2 times for each tracker // and if they all fails, then try next one. if(backupTrackerIsAvailable(_requestGroup->getDownloadContext())) { diff --git a/src/URIResult.cc b/src/URIResult.cc index e848d70f..ecdcbef5 100644 --- a/src/URIResult.cc +++ b/src/URIResult.cc @@ -36,7 +36,7 @@ namespace aria2 { -URIResult::URIResult(const std::string& uri, DownloadResult::RESULT result): +URIResult::URIResult(const std::string& uri, downloadresultcode::RESULT result): _uri(uri), _result(result) {} } // namespace aria2 diff --git a/src/URIResult.h b/src/URIResult.h index 45380690..f5d956ba 100644 --- a/src/URIResult.h +++ b/src/URIResult.h @@ -39,7 +39,7 @@ #include <string> -#include "DownloadResult.h" +#include "DownloadResultCode.h" namespace aria2 { @@ -48,16 +48,16 @@ class URIResult { private: std::string _uri; - DownloadResult::RESULT _result; + downloadresultcode::RESULT _result; public: - URIResult(const std::string& uri, DownloadResult::RESULT result); + URIResult(const std::string& uri, downloadresultcode::RESULT result); const std::string& getURI() const { return _uri; } - DownloadResult::RESULT getResult() const + downloadresultcode::RESULT getResult() const { return _result; } diff --git a/src/URISelector.h b/src/URISelector.h index d00aeb54..76c259b2 100644 --- a/src/URISelector.h +++ b/src/URISelector.h @@ -41,14 +41,15 @@ namespace aria2 { class DownloadCommand; +class FileEntry; class URISelector { public: virtual ~URISelector() {} - virtual std::string select(std::deque<std::string>& uris) = 0; + virtual std::string select(FileEntry* fileEntry) = 0; - virtual void tuneDownloadCommand(std::deque<std::string>& uris, + virtual void tuneDownloadCommand(const std::deque<std::string>& uris, DownloadCommand* command) {}; virtual void resetCounters() { return; }; diff --git a/src/XmlRpcMethodImpl.cc b/src/XmlRpcMethodImpl.cc index 954369c2..d3f0d78e 100644 --- a/src/XmlRpcMethodImpl.cc +++ b/src/XmlRpcMethodImpl.cc @@ -344,9 +344,9 @@ static void gatherStoppedDownload (BDE& entryDict, const SharedHandle<DownloadResult>& ds) { entryDict["gid"] = Util::itos(ds->gid); - if(ds->result == DownloadResult::IN_PROGRESS) { + if(ds->result == downloadresultcode::IN_PROGRESS) { entryDict["status"] = BDE_REMOVED; - } else if(ds->result == DownloadResult::FINISHED) { + } else if(ds->result == downloadresultcode::FINISHED) { entryDict["status"] = BDE_COMPLETE; } else { entryDict["status"] = BDE_ERROR; @@ -428,7 +428,9 @@ BDE GetUrisXmlRpcMethod::process } BDE uriList = BDE::list(); std::deque<std::string> uris; - group->getURIs(uris); + // TODO1.5 getUris should return list of URIs attached to each FileEntry. + // Current implementation just returns first FileEntry's URIs. + group->getDownloadContext()->getFirstFileEntry()->getUris(uris); for(std::deque<std::string>::const_iterator i = uris.begin(); i != uris.end(); ++i) { BDE entry = BDE::dict(); diff --git a/src/bittorrent_helper.cc b/src/bittorrent_helper.cc index cb3be528..b21d9cc8 100644 --- a/src/bittorrent_helper.cc +++ b/src/bittorrent_helper.cc @@ -301,7 +301,8 @@ static void processRootDictionary (const SharedHandle<DownloadContext>& ctx, const BDE& rootDict, const std::string& defaultName, - const std::string& overrideName) + const std::string& overrideName, + const std::deque<std::string>& uris) { if(!rootDict.isDict()) { throw DL_ABORT_EX("torrent file does not contain a root dictionary."); @@ -359,6 +360,9 @@ static void processRootDictionary // see http://www.getright.com/seedtorrent.html std::vector<std::string> urlList; extractUrlList(urlList, rootDict[C_URL_LIST]); + urlList.insert(urlList.end(), uris.begin(), uris.end()); + std::sort(urlList.begin(), urlList.end()); + urlList.erase(std::unique(urlList.begin(), urlList.end()), urlList.end()); // retrieve file entries extractFileEntries(ctx, torrent, infoDict, defaultName, overrideName, urlList); @@ -380,8 +384,20 @@ void load(const std::string& torrentFile, processRootDictionary(ctx, bencode::decodeFromFile(torrentFile), torrentFile, - overrideName); + overrideName, + std::deque<std::string>()); +} +void load(const std::string& torrentFile, + const SharedHandle<DownloadContext>& ctx, + const std::deque<std::string>& uris, + const std::string& overrideName) +{ + processRootDictionary(ctx, + bencode::decodeFromFile(torrentFile), + torrentFile, + overrideName, + uris); } void loadFromMemory(const unsigned char* content, @@ -393,11 +409,40 @@ void loadFromMemory(const unsigned char* content, processRootDictionary(ctx, bencode::decode(content, length), defaultName, - overrideName); + overrideName, + std::deque<std::string>()); +} + +void loadFromMemory(const unsigned char* content, + size_t length, + const SharedHandle<DownloadContext>& ctx, + const std::deque<std::string>& uris, + const std::string& defaultName, + const std::string& overrideName) +{ + processRootDictionary(ctx, + bencode::decode(content, length), + defaultName, + overrideName, + uris); +} + +void loadFromMemory(const std::string& context, + const SharedHandle<DownloadContext>& ctx, + const std::string& defaultName, + const std::string& overrideName) +{ + processRootDictionary + (ctx, + bencode::decode(reinterpret_cast<const unsigned char*>(context.c_str()), + context.size()), + defaultName, overrideName, + std::deque<std::string>()); } void loadFromMemory(const std::string& context, const SharedHandle<DownloadContext>& ctx, + const std::deque<std::string>& uris, const std::string& defaultName, const std::string& overrideName) { @@ -405,7 +450,8 @@ void loadFromMemory(const std::string& context, (ctx, bencode::decode(reinterpret_cast<const unsigned char*>(context.c_str()), context.size()), - defaultName, overrideName); + defaultName, overrideName, + uris); } const unsigned char* diff --git a/src/bittorrent_helper.h b/src/bittorrent_helper.h index 99db7d9a..a53a222e 100644 --- a/src/bittorrent_helper.h +++ b/src/bittorrent_helper.h @@ -39,6 +39,7 @@ #include <string> #include <vector> +#include <deque> #include "SharedHandle.h" #include "AnnounceTier.h" @@ -78,13 +79,30 @@ void load(const std::string& torrentFile, const SharedHandle<DownloadContext>& ctx, const std::string& overrideName = ""); +void load(const std::string& torrentFile, + const SharedHandle<DownloadContext>& ctx, + const std::deque<std::string>& uris, + const std::string& overrideName = ""); + +void loadFromMemory(const unsigned char* content, size_t length, + const SharedHandle<DownloadContext>& ctx, + const std::string& defaultName, + const std::string& overrideName = ""); + void loadFromMemory(const unsigned char* content, size_t length, const SharedHandle<DownloadContext>& ctx, + const std::deque<std::string>& uris, + const std::string& defaultName, + const std::string& overrideName = ""); + +void loadFromMemory(const std::string& context, + const SharedHandle<DownloadContext>& ctx, const std::string& defaultName, const std::string& overrideName = ""); void loadFromMemory(const std::string& context, const SharedHandle<DownloadContext>& ctx, + const std::deque<std::string>& uris, const std::string& defaultName, const std::string& overrideName = ""); diff --git a/src/download_helper.cc b/src/download_helper.cc index baabf94e..a0adb72c 100644 --- a/src/download_helper.cc +++ b/src/download_helper.cc @@ -179,7 +179,7 @@ static SharedHandle<RequestGroup> createRequestGroup (const SharedHandle<Option>& option, const std::deque<std::string>& uris, bool useOutOption = false) { - SharedHandle<RequestGroup> rg(new RequestGroup(option, uris)); + SharedHandle<RequestGroup> rg(new RequestGroup(option)); SharedHandle<DownloadContext> dctx (new DownloadContext (option->getAsInt(PREF_SEGMENT_SIZE), @@ -201,15 +201,16 @@ createBtRequestGroup(const std::string& torrentFilePath, const std::deque<std::string>& auxUris, const std::string& torrentData = "") { - SharedHandle<RequestGroup> rg(new RequestGroup(option, auxUris)); + SharedHandle<RequestGroup> rg(new RequestGroup(option)); SharedHandle<DownloadContext> dctx(new DownloadContext()); dctx->setDir(option->get(PREF_DIR)); + // TODO1.5 Add additional argument auxUris for bittorrent::load* functions. if(torrentData.empty()) { - bittorrent::load(torrentFilePath, dctx);// may throw exception + bittorrent::load(torrentFilePath, dctx, auxUris);// may throw exception } else { - bittorrent::loadFromMemory(torrentData, dctx, "default"); // may - // throw - // exception + bittorrent::loadFromMemory(torrentData, dctx, auxUris, "default"); // may + // throw + // exception } dctx->setFileFilter(Util::parseIntRange(option->get(PREF_SELECT_FILE))); std::istringstream indexOutIn(option->get(PREF_INDEX_OUT)); diff --git a/src/main.cc b/src/main.cc index a3a13ffb..5b990c63 100644 --- a/src/main.cc +++ b/src/main.cc @@ -165,7 +165,7 @@ static void showFiles extern void option_processing(Option& option, std::deque<std::string>& uris, int argc, char* const argv[]); -DownloadResult::RESULT main(int argc, char* argv[]) +downloadresultcode::RESULT main(int argc, char* argv[]) { std::deque<std::string> args; SharedHandle<Option> op(new Option()); @@ -192,7 +192,7 @@ DownloadResult::RESULT main(int argc, char* argv[]) if(op->get(PREF_EVENT_POLL) == V_SELECT) { SocketCore::useSelect(); } - DownloadResult::RESULT exitStatus = DownloadResult::FINISHED; + downloadresultcode::RESULT exitStatus = downloadresultcode::FINISHED; try { Logger* logger = LogFactory::getInstance(); logger->info("<<--- --- --- ---"); @@ -253,7 +253,7 @@ DownloadResult::RESULT main(int argc, char* argv[]) } } catch(Exception& ex) { std::cerr << EX_EXCEPTION_CAUGHT << "\n" << ex.stackTrace() << std::endl; - exitStatus = DownloadResult::UNKNOWN_ERROR; + exitStatus = downloadresultcode::UNKNOWN_ERROR; } LogFactory::release(); return exitStatus; @@ -264,7 +264,7 @@ DownloadResult::RESULT main(int argc, char* argv[]) int main(int argc, char* argv[]) { aria2::Platform platform; - aria2::DownloadResult::RESULT r = aria2::main(argc, argv); + aria2::downloadresultcode::RESULT r = aria2::main(argc, argv); return r; } diff --git a/src/option_processing.cc b/src/option_processing.cc index 240f2bac..9741e4ff 100644 --- a/src/option_processing.cc +++ b/src/option_processing.cc @@ -53,7 +53,7 @@ #include "File.h" #include "StringFormat.h" #include "OptionHandlerException.h" -#include "DownloadResult.h" +#include "DownloadResultCode.h" #include "SimpleRandomizer.h" #include "bittorrent_helper.h" @@ -98,7 +98,7 @@ void option_processing(Option& op, std::deque<std::string>& uris, if(op.defined("version")) { showVersion(); - exit(DownloadResult::FINISHED); + exit(downloadresultcode::FINISHED); } if(op.defined("help")) { std::string keyword; @@ -115,7 +115,7 @@ void option_processing(Option& op, std::deque<std::string>& uris, } } showUsage(keyword, oparser); - exit(DownloadResult::FINISHED); + exit(downloadresultcode::FINISHED); } } @@ -137,18 +137,18 @@ void option_processing(Option& op, std::deque<std::string>& uris, << "Usage:" << "\n" << oparser.findByName(e.getOptionName())->getDescription() << std::endl; - exit(DownloadResult::UNKNOWN_ERROR); + exit(downloadresultcode::UNKNOWN_ERROR); } catch(Exception& e) { std::cerr << "Parse error in " << cfname << "\n" << e.stackTrace() << std::endl; - exit(DownloadResult::UNKNOWN_ERROR); + exit(downloadresultcode::UNKNOWN_ERROR); } } else if(!ucfname.empty()) { std::cerr << StringFormat("Configuration file %s is not found.", cfname.c_str()) << "\n"; showUsage(TAG_HELP, oparser); - exit(DownloadResult::UNKNOWN_ERROR); + exit(downloadresultcode::UNKNOWN_ERROR); } } // Override configuration with environment variables. @@ -168,11 +168,11 @@ void option_processing(Option& op, std::deque<std::string>& uris, << "Usage:" << "\n" << oparser.findByName(e.getOptionName()) << std::endl; - exit(DownloadResult::UNKNOWN_ERROR); + exit(downloadresultcode::UNKNOWN_ERROR); } catch(Exception& e) { std::cerr << e.stackTrace() << std::endl; showUsage(TAG_HELP, oparser); - exit(DownloadResult::UNKNOWN_ERROR); + exit(downloadresultcode::UNKNOWN_ERROR); } if( #ifdef ENABLE_XML_RPC @@ -188,7 +188,7 @@ void option_processing(Option& op, std::deque<std::string>& uris, if(uris.empty()) { std::cerr << MSG_URI_REQUIRED << std::endl; showUsage(TAG_HELP, oparser); - exit(DownloadResult::UNKNOWN_ERROR); + exit(downloadresultcode::UNKNOWN_ERROR); } } #ifdef HAVE_DAEMON @@ -200,7 +200,7 @@ void option_processing(Option& op, std::deque<std::string>& uris, } if(daemon(0, 0) < 0) { perror(MSG_DAEMON_FAILED); - exit(DownloadResult::UNKNOWN_ERROR); + exit(downloadresultcode::UNKNOWN_ERROR); } } #endif // HAVE_DAEMON |