summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--ChangeLog23
-rw-r--r--src/BtDependency.cc62
-rw-r--r--src/FileEntry.h11
-rw-r--r--src/Metalink2RequestGroup.cc199
-rw-r--r--src/Metalink2RequestGroup.h2
-rw-r--r--src/MetalinkEntry.cc4
-rw-r--r--src/MetalinkEntry.h4
-rw-r--r--src/MetalinkHelper.cc38
-rw-r--r--src/MetalinkHelper.h10
-rw-r--r--src/MetalinkMetaurl.cc5
-rw-r--r--src/MetalinkMetaurl.h5
-rw-r--r--src/MetalinkParserController.cc1
-rw-r--r--src/RequestGroup.cc125
-rw-r--r--src/RequestGroup.h3
-rw-r--r--src/bittorrent_helper.cc2
-rw-r--r--test/BittorrentHelperTest.cc4
-rw-r--r--test/BtDependencyTest.cc70
-rw-r--r--test/MetalinkHelperTest.cc77
18 files changed, 494 insertions, 151 deletions
diff --git a/ChangeLog b/ChangeLog
index 91f67b01..4511ac3b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,26 @@
+2010-02-27 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
+
+ Supported name attribute of metalink::metaurl element and
+ multi-file torrent with Metalink4. Files with same metaurl are
+ grouped and downloaded in one RequestGroup.
+ * src/BtDependency.cc
+ * src/FileEntry.h
+ * src/Metalink2RequestGroup.cc
+ * src/Metalink2RequestGroup.h
+ * src/MetalinkEntry.cc
+ * src/MetalinkEntry.h
+ * src/MetalinkHelper.cc
+ * src/MetalinkHelper.h
+ * src/MetalinkMetaurl.cc
+ * src/MetalinkMetaurl.h
+ * src/MetalinkParserController.cc
+ * src/RequestGroup.cc
+ * src/RequestGroup.h
+ * src/bittorrent_helper.cc
+ * test/BittorrentHelperTest.cc
+ * test/BtDependencyTest.cc
+ * test/MetalinkHelperTest.cc
+
2010-02-26 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>
Store name attribute of metalink:metaurl element in MetalinkMetaurl.
diff --git a/src/BtDependency.cc b/src/BtDependency.cc
index a2009087..3c70804c 100644
--- a/src/BtDependency.cc
+++ b/src/BtDependency.cc
@@ -47,6 +47,7 @@
#include "File.h"
#include "bittorrent_helper.h"
#include "DlAbortEx.h"
+#include "StringFormat.h"
namespace aria2 {
@@ -58,6 +59,18 @@ BtDependency::BtDependency(const RequestGroupWeakHandle& dependant,
BtDependency::~BtDependency() {}
+static void copyValues(const SharedHandle<FileEntry>& d,
+ const SharedHandle<FileEntry>& s)
+{
+ d->setRequested(true);
+ d->setPath(s->getPath());
+ d->addUris(s->getRemainingUris().begin(),
+ s->getRemainingUris().end());
+ if(!s->isSingleHostMultiConnectionEnabled()) {
+ d->disableSingleHostMultiConnection();
+ }
+}
+
bool BtDependency::resolve()
{
if(_dependee->getNumCommand() == 0 && _dependee->downloadFinished()) {
@@ -80,23 +93,38 @@ bool BtDependency::resolve()
bittorrent::loadFromMemory
(content, context, File(dependee->getFirstFilePath()).getBasename());
}
- if(context->getFileEntries().size() !=
- _dependant->getDownloadContext()->getFileEntries().size()) {
- throw DL_ABORT_EX("The number of file in torrent doesn't match to"
- " the dependent.");
- }
- // Copy file path in _dependant's FileEntries to newly created
- // context's FileEntries to endorse the path structure of
- // _dependant. URIs and singleHostMultiConnection are also copied.
- for(std::vector<SharedHandle<FileEntry> >::const_iterator s =
- _dependant->getDownloadContext()->getFileEntries().begin(),
- d = context->getFileEntries().begin();
- d != context->getFileEntries().end(); ++s, ++d) {
- (*d)->setPath((*s)->getPath());
- (*d)->addUris((*s)->getRemainingUris().begin(),
- (*s)->getRemainingUris().end());
- if(!(*s)->isSingleHostMultiConnectionEnabled()) {
- (*d)->disableSingleHostMultiConnection();
+ const std::vector<SharedHandle<FileEntry> >& fileEntries =
+ context->getFileEntries();
+ const std::vector<SharedHandle<FileEntry> >& dependantFileEntries =
+ _dependant->getDownloadContext()->getFileEntries();
+ // If dependant's FileEntry::getOriginalName() is empty, we
+ // assume that torrent is single file. In Metalink3, this is
+ // always assumed.
+ if(fileEntries.size() == 1 && dependantFileEntries.size() == 1 &&
+ dependantFileEntries[0]->getOriginalName().empty()) {
+ copyValues(fileEntries[0], dependantFileEntries[0]);
+ } else {
+ std::for_each(fileEntries.begin(), fileEntries.end(),
+ std::bind2nd(mem_fun_sh(&FileEntry::setRequested),false));
+ // Copy file path in _dependant's FileEntries to newly created
+ // context's FileEntries to endorse the path structure of
+ // _dependant. URIs and singleHostMultiConnection are also copied.
+ for(std::vector<SharedHandle<FileEntry> >::const_iterator s =
+ dependantFileEntries.begin(); s != dependantFileEntries.end();
+ ++s){
+ std::vector<SharedHandle<FileEntry> >::const_iterator d =
+ context->getFileEntries().begin();
+ for(; d != context->getFileEntries().end(); ++d) {
+ if((*d)->getOriginalName() == (*s)->getOriginalName()) {
+ break;
+ }
+ }
+ if(d == context->getFileEntries().end()) {
+ throw DL_ABORT_EX
+ (StringFormat("No entry %s in torrent file",
+ (*s)->getOriginalName().c_str()).str());
+ }
+ copyValues(*d, *s);
}
}
} catch(RecoverableException& e) {
diff --git a/src/FileEntry.h b/src/FileEntry.h
index 2dfe7eda..0199ba28 100644
--- a/src/FileEntry.h
+++ b/src/FileEntry.h
@@ -70,6 +70,7 @@ private:
// available.
std::deque<URIResult> _uriResults;
bool _singleHostMultiConnection;
+ std::string _originalName;
Logger* _logger;
void storePool(const SharedHandle<Request>& request);
@@ -227,6 +228,16 @@ public:
void reuseUri(size_t num);
void releaseRuntimeResource();
+
+ void setOriginalName(const std::string& originalName)
+ {
+ _originalName = originalName;
+ }
+
+ const std::string& getOriginalName() const
+ {
+ return _originalName;
+ }
};
typedef SharedHandle<FileEntry> FileEntryHandle;
diff --git a/src/Metalink2RequestGroup.cc b/src/Metalink2RequestGroup.cc
index 74e1b2d5..4ccd924e 100644
--- a/src/Metalink2RequestGroup.cc
+++ b/src/Metalink2RequestGroup.cc
@@ -136,57 +136,66 @@ void removeMetalinkContentTypes(const SharedHandle<RequestGroup>& group)
void
Metalink2RequestGroup::createRequestGroup
(std::deque<SharedHandle<RequestGroup> >& groups,
- std::deque<SharedHandle<MetalinkEntry> > entries,
+ const std::deque<SharedHandle<MetalinkEntry> >& entries,
const SharedHandle<Option>& option)
{
- if(entries.size() == 0) {
+ if(entries.empty()) {
_logger->notice(EX_NO_RESULT_WITH_YOUR_PREFS);
return;
}
std::deque<int32_t> selectIndexes =
util::parseIntRange(option->get(PREF_SELECT_FILE)).flush();
- bool useIndex;
- if(selectIndexes.size()) {
- useIndex = true;
- } else {
- useIndex = false;
+ std::sort(selectIndexes.begin(), selectIndexes.end());
+ std::vector<SharedHandle<MetalinkEntry> > selectedEntries;
+ selectedEntries.reserve(entries.size());
+
+ std::deque<std::string> locations;
+ if(option->defined(PREF_METALINK_LOCATION)) {
+ util::split(option->get(PREF_METALINK_LOCATION),
+ std::back_inserter(locations), ",", true);
+ std::transform
+ (locations.begin(), locations.end(), locations.begin(), util::toLower);
}
- int32_t count = 0;
- for(std::deque<SharedHandle<MetalinkEntry> >::iterator itr = entries.begin(); itr != entries.end();
- ++itr, ++count) {
- SharedHandle<MetalinkEntry>& entry = *itr;
- if(option->defined(PREF_METALINK_LOCATION)) {
- std::deque<std::string> locations;
- util::split(option->get(PREF_METALINK_LOCATION),
- std::back_inserter(locations), ",", true);
- std::transform
- (locations.begin(), locations.end(), locations.begin(), util::toLower);
- entry->setLocationPriority
- (locations, -MetalinkResource::getLowestPriority());
- }
- if(option->get(PREF_METALINK_PREFERRED_PROTOCOL) != V_NONE) {
- entry->setProtocolPriority
- (option->get(PREF_METALINK_PREFERRED_PROTOCOL),
- -MetalinkResource::getLowestPriority());
- }
- if(useIndex) {
- if(std::find(selectIndexes.begin(), selectIndexes.end(), count+1) ==
- selectIndexes.end()) {
+ std::string preferredProtocol;
+ if(option->get(PREF_METALINK_PREFERRED_PROTOCOL) != V_NONE) {
+ preferredProtocol = option->get(PREF_METALINK_PREFERRED_PROTOCOL);
+ }
+ {
+ int32_t count = 1;
+ for(std::deque<SharedHandle<MetalinkEntry> >::const_iterator i =
+ entries.begin(); i != entries.end(); ++i, ++count) {
+ (*i)->dropUnsupportedResource();
+ if((*i)->resources.empty() && (*i)->metaurls.empty()) {
continue;
}
+ (*i)->setLocationPriority
+ (locations, -MetalinkResource::getLowestPriority());
+ if(!preferredProtocol.empty()) {
+ (*i)->setProtocolPriority
+ (preferredProtocol, -MetalinkResource::getLowestPriority());
+ }
+ if(selectIndexes.empty() ||
+ std::binary_search(selectIndexes.begin(), selectIndexes.end(), count)){
+ selectedEntries.push_back(*i);
+ }
}
- entry->dropUnsupportedResource();
- if(entry->resources.empty() && entry->metaurls.empty()) {
- continue;
- }
- _logger->info(MSG_METALINK_QUEUEING, entry->getPath().c_str());
+ }
+ std::for_each(entries.begin(), entries.end(),
+ mem_fun_sh(&MetalinkEntry::reorderMetaurlsByPriority));
+ std::vector<std::pair<std::string,
+ std::vector<SharedHandle<MetalinkEntry> > > > entryGroups;
+ MetalinkHelper::groupEntryByMetaurlName(entryGroups, selectedEntries);
+ for(std::vector<std::pair<std::string,
+ std::vector<SharedHandle<MetalinkEntry> > > >::const_iterator itr =
+ entryGroups.begin(); itr != entryGroups.end(); ++itr) {
+ const std::string& metaurl = (*itr).first;
+ const std::vector<SharedHandle<MetalinkEntry> >& mes = (*itr).second;
+ _logger->info("Processing metaurl group metaurl=%s", metaurl.c_str());
#ifdef ENABLE_BITTORRENT
SharedHandle<RequestGroup> torrentRg;
- if(!entry->metaurls.empty()) {
- entry->reorderMetaurlsByPriority();
- // there is torrent entry
+ if(!metaurl.empty()) {
std::deque<std::string> uris;
- uris.push_back(entry->metaurls[0]->url);
+ uris.push_back(metaurl);
{
std::deque<SharedHandle<RequestGroup> > result;
createRequestGroupForUri(result, option, uris,
@@ -213,62 +222,94 @@ Metalink2RequestGroup::createRequestGroup
}
}
#endif // ENABLE_BITTORRENT
- entry->reorderResourcesByPriority();
- std::deque<std::string> uris;
- std::for_each(entry->resources.begin(), entry->resources.end(),
- AccumulateNonP2PUrl(uris));
SharedHandle<RequestGroup> rg(new RequestGroup(option));
- // If piece hash is specified in the metalink,
- // make segment size equal to piece hash size.
- size_t pieceLength;
+ SharedHandle<DownloadContext> dctx;
+ if(mes.size() == 1) {
+ SharedHandle<MetalinkEntry> entry = mes[0];
+ _logger->info(MSG_METALINK_QUEUEING, entry->getPath().c_str());
+ entry->reorderResourcesByPriority();
+ std::deque<std::string> uris;
+ std::for_each(entry->resources.begin(), entry->resources.end(),
+ AccumulateNonP2PUrl(uris));
+ // If piece hash is specified in the metalink,
+ // make segment size equal to piece hash size.
+ size_t pieceLength;
#ifdef ENABLE_MESSAGE_DIGEST
- if(entry->chunkChecksum.isNull()) {
- pieceLength = option->getAsInt(PREF_SEGMENT_SIZE);
- } else {
- pieceLength = entry->chunkChecksum->getChecksumLength();
- }
+ if(entry->chunkChecksum.isNull()) {
+ pieceLength = option->getAsInt(PREF_SEGMENT_SIZE);
+ } else {
+ pieceLength = entry->chunkChecksum->getChecksumLength();
+ }
#else
- pieceLength = option->getAsInt(PREF_SEGMENT_SIZE);
+ pieceLength = option->getAsInt(PREF_SEGMENT_SIZE);
#endif // ENABLE_MESSAGE_DIGEST
- SharedHandle<DownloadContext> dctx
- (new DownloadContext
- (pieceLength,
- entry->getLength(),
- util::applyDir(option->get(PREF_DIR), entry->file->getPath())));
- dctx->setDir(option->get(PREF_DIR));
- dctx->getFirstFileEntry()->setUris(uris);
- if(option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)) {
- dctx->getFirstFileEntry()->disableSingleHostMultiConnection();
- }
+ dctx.reset(new DownloadContext
+ (pieceLength,
+ entry->getLength(),
+ util::applyDir(option->get(PREF_DIR),
+ entry->file->getPath())));
+ dctx->getFirstFileEntry()->setUris(uris);
+ if(option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)) {
+ dctx->getFirstFileEntry()->disableSingleHostMultiConnection();
+ }
#ifdef ENABLE_MESSAGE_DIGEST
- if(entry->chunkChecksum.isNull()) {
- if(!entry->checksum.isNull()) {
- dctx->setChecksum(entry->checksum->getMessageDigest());
- dctx->setChecksumHashAlgo(entry->checksum->getAlgo());
+ if(entry->chunkChecksum.isNull()) {
+ if(!entry->checksum.isNull()) {
+ dctx->setChecksum(entry->checksum->getMessageDigest());
+ dctx->setChecksumHashAlgo(entry->checksum->getAlgo());
+ }
+ } else {
+ dctx->setPieceHashes(entry->chunkChecksum->getChecksums().begin(),
+ entry->chunkChecksum->getChecksums().end());
+ dctx->setPieceHashAlgo(entry->chunkChecksum->getAlgo());
}
+#endif // ENABLE_MESSAGE_DIGEST
+ dctx->setSignature(entry->getSignature());
+ rg->setNumConcurrentCommand
+ (entry->maxConnections < 0 ?
+ option->getAsInt(PREF_METALINK_SERVERS) :
+ std::min(option->getAsInt(PREF_METALINK_SERVERS),
+ static_cast<int32_t>(entry->maxConnections)));
} else {
- dctx->setPieceHashes(entry->chunkChecksum->getChecksums().begin(),
- entry->chunkChecksum->getChecksums().end());
- dctx->setPieceHashAlgo(entry->chunkChecksum->getAlgo());
+ dctx.reset(new DownloadContext());
+ // piece length is overridden by the one in torrent file.
+ dctx->setPieceLength(option->getAsInt(PREF_SEGMENT_SIZE));
+ std::vector<SharedHandle<FileEntry> > fileEntries;
+ off_t offset = 0;
+ for(std::deque<SharedHandle<MetalinkEntry> >::const_iterator i =
+ entries.begin(); i != entries.end(); ++i) {
+ _logger->info("Metalink: Queueing %s for download as a member.",
+ (*i)->getPath().c_str());
+ _logger->debug("originalName = %s", (*i)->metaurls[0]->name.c_str());
+ (*i)->reorderResourcesByPriority();
+ std::deque<std::string> uris;
+ std::for_each((*i)->resources.begin(), (*i)->resources.end(),
+ AccumulateNonP2PUrl(uris));
+ SharedHandle<FileEntry> fe
+ (new FileEntry
+ (util::applyDir(option->get(PREF_DIR), (*i)->file->getPath()),
+ (*i)->file->getLength(), offset, uris));
+ if(option->getAsBool(PREF_METALINK_ENABLE_UNIQUE_PROTOCOL)) {
+ fe->disableSingleHostMultiConnection();
+ }
+ fe->setOriginalName((*i)->metaurls[0]->name);
+ fileEntries.push_back(fe);
+ offset += (*i)->file->getLength();
+ }
+ dctx->setFileEntries(fileEntries.begin(), fileEntries.end());
+ rg->setNumConcurrentCommand(option->getAsInt(PREF_METALINK_SERVERS));
}
-#endif // ENABLE_MESSAGE_DIGEST
- dctx->setSignature(entry->getSignature());
+ dctx->setDir(option->get(PREF_DIR));
rg->setDownloadContext(dctx);
- rg->setNumConcurrentCommand
- (entry->maxConnections < 0 ?
- option->getAsInt(PREF_METALINK_SERVERS) :
- std::min(option->getAsInt(PREF_METALINK_SERVERS),
- static_cast<int32_t>(entry->maxConnections)));
- // remove "metalink" from Accept Type list to avoid loop in tranparent
- // metalink
+ // remove "metalink" from Accept Type list to avoid loop in
+ // tranparent metalink
removeMetalinkContentTypes(rg);
-
#ifdef ENABLE_BITTORRENT
- // Inject depenency between rg and torrentRg here if torrentRg.isNull() == false
+ // Inject depenency between rg and torrentRg here if
+ // torrentRg.isNull() == false
if(!torrentRg.isNull()) {
SharedHandle<Dependency> dep(new BtDependency(rg, torrentRg));
rg->dependsOn(dep);
-
torrentRg->belongsTo(rg->getGID());
}
#endif // ENABLE_BITTORRENT
diff --git a/src/Metalink2RequestGroup.h b/src/Metalink2RequestGroup.h
index cacbb333..a3d84011 100644
--- a/src/Metalink2RequestGroup.h
+++ b/src/Metalink2RequestGroup.h
@@ -54,7 +54,7 @@ private:
void
createRequestGroup(std::deque<SharedHandle<RequestGroup> >& groups,
- std::deque<SharedHandle<MetalinkEntry> > entries,
+ const std::deque<SharedHandle<MetalinkEntry> >& entries,
const SharedHandle<Option>& option);
public:
Metalink2RequestGroup();
diff --git a/src/MetalinkEntry.cc b/src/MetalinkEntry.cc
index 218f8ba5..75e5dc36 100644
--- a/src/MetalinkEntry.cc
+++ b/src/MetalinkEntry.cc
@@ -51,7 +51,7 @@
namespace aria2 {
MetalinkEntry::MetalinkEntry():
- file(0),
+ sizeKnown(false),
maxConnections(-1)
{}
@@ -94,7 +94,7 @@ MetalinkEntry& MetalinkEntry::operator=(const MetalinkEntry& metalinkEntry)
return *this;
}
-std::string MetalinkEntry::getPath() const
+const std::string& MetalinkEntry::getPath() const
{
return file->getPath();
}
diff --git a/src/MetalinkEntry.h b/src/MetalinkEntry.h
index 4b10ec82..2fbb6795 100644
--- a/src/MetalinkEntry.h
+++ b/src/MetalinkEntry.h
@@ -61,6 +61,8 @@ public:
std::string version;
std::vector<std::string> languages;
std::vector<std::string> oses;
+ // True if size is specified in Metalink document.
+ bool sizeKnown;
std::deque<SharedHandle<MetalinkResource> > resources;
std::vector<SharedHandle<MetalinkMetaurl> > metaurls;
int maxConnections; // Metalink3Spec
@@ -77,7 +79,7 @@ public:
MetalinkEntry& operator=(const MetalinkEntry& metalinkEntry);
- std::string getPath() const;
+ const std::string& getPath() const;
uint64_t getLength() const;
diff --git a/src/MetalinkHelper.cc b/src/MetalinkHelper.cc
index c26d1d03..abfbc99a 100644
--- a/src/MetalinkHelper.cc
+++ b/src/MetalinkHelper.cc
@@ -41,6 +41,7 @@
#include "prefs.h"
#include "DlAbortEx.h"
#include "BinaryStream.h"
+#include "MetalinkMetaurl.h"
namespace aria2 {
@@ -79,4 +80,41 @@ void MetalinkHelper::query
option->get(PREF_METALINK_OS));
}
+void MetalinkHelper::groupEntryByMetaurlName
+(std::vector<
+ std::pair<std::string, std::vector<SharedHandle<MetalinkEntry> > > >& result,
+ const std::vector<SharedHandle<MetalinkEntry> >& entries)
+{
+ for(std::vector<SharedHandle<MetalinkEntry> >::const_iterator eiter =
+ entries.begin(); eiter != entries.end(); ++eiter) {
+ if((*eiter)->metaurls.empty()) {
+ std::pair<std::string, std::vector<SharedHandle<MetalinkEntry> > > p;
+ p.second.push_back(*eiter);
+ result.push_back(p);
+ } else {
+ std::vector<
+ std::pair<std::string,
+ std::vector<SharedHandle<MetalinkEntry> > > >::iterator i =
+ result.begin();
+ if((*eiter)->metaurls[0]->name.empty() ||
+ !(*eiter)->sizeKnown) {
+ i = result.end();
+ }
+ for(; i != result.end(); ++i) {
+ if((*i).first == (*eiter)->metaurls[0]->url &&
+ !(*i).second[0]->metaurls[0]->name.empty()) {
+ (*i).second.push_back(*eiter);
+ break;
+ }
+ }
+ if(i == result.end()) {
+ std::pair<std::string, std::vector<SharedHandle<MetalinkEntry> > > p;
+ p.first = (*eiter)->metaurls[0]->url;
+ p.second.push_back(*eiter);
+ result.push_back(p);
+ }
+ }
+ }
+}
+
} // namespace aria2
diff --git a/src/MetalinkHelper.h b/src/MetalinkHelper.h
index 0b329576..10586fc5 100644
--- a/src/MetalinkHelper.h
+++ b/src/MetalinkHelper.h
@@ -36,9 +36,12 @@
#define _D_METALINK_HELPER_H_
#include "common.h"
-#include "SharedHandle.h"
+
#include <string>
#include <deque>
+#include <vector>
+
+#include "SharedHandle.h"
namespace aria2 {
@@ -65,6 +68,11 @@ public:
static void parseAndQuery
(std::deque<SharedHandle<MetalinkEntry> >& result,
const SharedHandle<BinaryStream>& binaryStream, const Option* option);
+
+ static void groupEntryByMetaurlName
+ (std::vector<
+ std::pair<std::string, std::vector<SharedHandle<MetalinkEntry> > > >& result,
+ const std::vector<SharedHandle<MetalinkEntry> >& entries);
};
} // namespace aria2
diff --git a/src/MetalinkMetaurl.cc b/src/MetalinkMetaurl.cc
index cdd47d08..2058fcc1 100644
--- a/src/MetalinkMetaurl.cc
+++ b/src/MetalinkMetaurl.cc
@@ -42,4 +42,9 @@ const std::string MetalinkMetaurl::MEDIATYPE_TORRENT("torrent");
MetalinkMetaurl::MetalinkMetaurl():
priority(MetalinkResource::getLowestPriority()) {}
+MetalinkMetaurl::MetalinkMetaurl
+(const std::string& url, const std::string& mediatype,
+ const std::string& name, int priority):
+ url(url), mediatype(mediatype), name(name), priority(priority) {}
+
} // namespace aria2
diff --git a/src/MetalinkMetaurl.h b/src/MetalinkMetaurl.h
index 748ee872..b237d2fc 100644
--- a/src/MetalinkMetaurl.h
+++ b/src/MetalinkMetaurl.h
@@ -45,11 +45,14 @@ class MetalinkMetaurl {
public:
std::string url;
std::string mediatype;
- int priority;
std::string name;
+ int priority;
MetalinkMetaurl();
+ MetalinkMetaurl(const std::string& url, const std::string& mediatype,
+ const std::string& name, int priority);
+
static const std::string MEDIATYPE_TORRENT;
};
diff --git a/src/MetalinkParserController.cc b/src/MetalinkParserController.cc
index 5a1be233..cdb97d32 100644
--- a/src/MetalinkParserController.cc
+++ b/src/MetalinkParserController.cc
@@ -100,6 +100,7 @@ void MetalinkParserController::setFileLengthOfEntry(uint64_t length)
} else {
_tEntry->file->setLength(length);
}
+ _tEntry->sizeKnown = true;
}
void MetalinkParserController::setVersionOfEntry(const std::string& version)
diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc
index dcc8af43..05fc74dd 100644
--- a/src/RequestGroup.cc
+++ b/src/RequestGroup.cc
@@ -205,6 +205,8 @@ void RequestGroup::createInitialCommand
SharedHandle<BtRegistry> btRegistry = e->getBtRegistry();
if(!btRegistry->getDownloadContext
(torrentAttrs[bittorrent::INFO_HASH].s()).isNull()) {
+ // TODO If metadataGetMode == false and each FileEntry has
+ // URI, then go without BT.
throw DOWNLOAD_FAILURE_EXCEPTION
(StringFormat
("InfoHash %s is already registered.",
@@ -284,14 +286,7 @@ void RequestGroup::createInitialCommand
return;
}
-
- // Remove the control file if download file doesn't exist
- if(progressInfoFile->exists() && !_pieceStorage->getDiskAdaptor()->fileExists()) {
- progressInfoFile->removeFile();
- _logger->notice(MSG_REMOVED_DEFUNCT_CONTROL_FILE,
- progressInfoFile->getFilename().c_str(),
- _downloadContext->getBasePath().c_str());
- }
+ removeDefunctControlFile(progressInfoFile);
{
uint64_t actualFileSize = _pieceStorage->getDiskAdaptor()->size();
if(actualFileSize == _downloadContext->getTotalLength()) {
@@ -372,42 +367,80 @@ void RequestGroup::createInitialCommand
}
}
#endif // ENABLE_BITTORRENT
- // TODO Currently, BitTorrent+WEB-Seeding is only way to download
- // multiple files in one RequestGroup. In this context, we don't
- // have BitTorrent, so add assertion here. This situation will be
- // changed if Metalink spec is formalized to support multi-file
- // torrent.
- assert(_downloadContext->getFileEntries().size() == 1);
- // TODO I assume here when totallength is set to DownloadContext and it is
- // not 0, then filepath is also set DownloadContext correctly....
- if(_option->getAsBool(PREF_DRY_RUN) ||
- _downloadContext->getTotalLength() == 0) {
- createNextCommand(commands, e, 1);
- }else {
+ if(_downloadContext->getFileEntries().size() == 1) {
+ // TODO I assume here when totallength is set to DownloadContext and it is
+ // not 0, then filepath is also set DownloadContext correctly....
+ if(_option->getAsBool(PREF_DRY_RUN) ||
+ _downloadContext->getTotalLength() == 0) {
+ createNextCommand(commands, e, 1);
+ }else {
+ if(e->_requestGroupMan->isSameFileBeingDownloaded(this)) {
+ throw DOWNLOAD_FAILURE_EXCEPTION
+ (StringFormat(EX_DUPLICATE_FILE_DOWNLOAD,
+ _downloadContext->getBasePath().c_str()).str());
+ }
+ adjustFilename
+ (SharedHandle<BtProgressInfoFile>(new DefaultBtProgressInfoFile
+ (_downloadContext,
+ SharedHandle<PieceStorage>(),
+ _option.get())));
+ initPieceStorage();
+ BtProgressInfoFileHandle infoFile
+ (new DefaultBtProgressInfoFile(_downloadContext, _pieceStorage,
+ _option.get()));
+ if(!infoFile->exists() && downloadFinishedByFileLength()) {
+ _pieceStorage->markAllPiecesDone();
+ _logger->notice(MSG_DOWNLOAD_ALREADY_COMPLETED,
+ _gid, _downloadContext->getBasePath().c_str());
+ } else {
+ loadAndOpenFile(infoFile);
+ SharedHandle<CheckIntegrityEntry> checkIntegrityEntry
+ (new StreamCheckIntegrityEntry(this));
+ processCheckIntegrityEntry(commands, checkIntegrityEntry, e);
+ }
+ }
+ } else {
+ // In this context, multiple FileEntry objects are in
+ // DownloadContext.
if(e->_requestGroupMan->isSameFileBeingDownloaded(this)) {
throw DOWNLOAD_FAILURE_EXCEPTION
(StringFormat(EX_DUPLICATE_FILE_DOWNLOAD,
_downloadContext->getBasePath().c_str()).str());
}
- adjustFilename
- (SharedHandle<BtProgressInfoFile>(new DefaultBtProgressInfoFile
- (_downloadContext,
- SharedHandle<PieceStorage>(),
- _option.get())));
initPieceStorage();
- BtProgressInfoFileHandle infoFile
- (new DefaultBtProgressInfoFile(_downloadContext, _pieceStorage,
+ if(_downloadContext->getFileEntries().size() > 1) {
+ _pieceStorage->setupFileFilter();
+ }
+ SharedHandle<DefaultBtProgressInfoFile> progressInfoFile
+ (new DefaultBtProgressInfoFile(_downloadContext,
+ _pieceStorage,
_option.get()));
- if(!infoFile->exists() && downloadFinishedByFileLength()) {
- _pieceStorage->markAllPiecesDone();
- _logger->notice(MSG_DOWNLOAD_ALREADY_COMPLETED,
- _gid, _downloadContext->getBasePath().c_str());
+ removeDefunctControlFile(progressInfoFile);
+ // Call Load, Save and file allocation command here
+ if(progressInfoFile->exists()) {
+ // load .aria2 file if it exists.
+ progressInfoFile->load();
+ _pieceStorage->getDiskAdaptor()->openFile();
} else {
- loadAndOpenFile(infoFile);
- SharedHandle<CheckIntegrityEntry> checkIntegrityEntry
- (new StreamCheckIntegrityEntry(this));
- processCheckIntegrityEntry(commands, checkIntegrityEntry, e);
+ if(_pieceStorage->getDiskAdaptor()->fileExists()) {
+ if(!_option->getAsBool(PREF_CHECK_INTEGRITY) &&
+ !_option->getAsBool(PREF_ALLOW_OVERWRITE)) {
+ // TODO we need this->haltRequested = true?
+ throw DOWNLOAD_FAILURE_EXCEPTION
+ (StringFormat
+ (MSG_FILE_ALREADY_EXISTS,
+ _downloadContext->getBasePath().c_str()).str());
+ } else {
+ _pieceStorage->getDiskAdaptor()->openFile();
+ }
+ } else {
+ _pieceStorage->getDiskAdaptor()->openFile();
+ }
}
+ _progressInfoFile = progressInfoFile;
+ SharedHandle<CheckIntegrityEntry> checkIntegrityEntry
+ (new StreamCheckIntegrityEntry(this));
+ processCheckIntegrityEntry(commands, checkIntegrityEntry, e);
}
}
@@ -543,6 +576,19 @@ void RequestGroup::adjustFilename
}
}
+void RequestGroup::removeDefunctControlFile
+(const SharedHandle<BtProgressInfoFile>& progressInfoFile)
+{
+ // Remove the control file if download file doesn't exist
+ if(progressInfoFile->exists() &&
+ !_pieceStorage->getDiskAdaptor()->fileExists()) {
+ progressInfoFile->removeFile();
+ _logger->notice(MSG_REMOVED_DEFUNCT_CONTROL_FILE,
+ progressInfoFile->getFilename().c_str(),
+ _downloadContext->getBasePath().c_str());
+ }
+}
+
void RequestGroup::loadAndOpenFile(const BtProgressInfoFileHandle& progressInfoFile)
{
try {
@@ -550,14 +596,7 @@ void RequestGroup::loadAndOpenFile(const BtProgressInfoFileHandle& progressInfoF
_pieceStorage->getDiskAdaptor()->initAndOpenFile();
return;
}
- // Remove the control file if download file doesn't exist
- if(progressInfoFile->exists() && !_pieceStorage->getDiskAdaptor()->fileExists()) {
- progressInfoFile->removeFile();
- _logger->notice(MSG_REMOVED_DEFUNCT_CONTROL_FILE,
- progressInfoFile->getFilename().c_str(),
- _downloadContext->getBasePath().c_str());
- }
-
+ removeDefunctControlFile(progressInfoFile);
if(progressInfoFile->exists()) {
progressInfoFile->load();
_pieceStorage->getDiskAdaptor()->openExistingFile();
diff --git a/src/RequestGroup.h b/src/RequestGroup.h
index 3d33dd04..4d85ec40 100644
--- a/src/RequestGroup.h
+++ b/src/RequestGroup.h
@@ -180,6 +180,9 @@ private:
// _uriResults, then last result code is returned. Otherwise
// returns downloadresultcode::UNKNOWN_ERROR.
downloadresultcode::RESULT downloadResult() const;
+
+ void removeDefunctControlFile
+ (const SharedHandle<BtProgressInfoFile>& progressInfoFile);
public:
// The copy of option is stored in RequestGroup object.
RequestGroup(const SharedHandle<Option>& option);
diff --git a/src/bittorrent_helper.cc b/src/bittorrent_helper.cc
index 52b9f78f..fc8a3b49 100644
--- a/src/bittorrent_helper.cc
+++ b/src/bittorrent_helper.cc
@@ -263,6 +263,7 @@ static void extractFileEntries
(new FileEntry(util::applyDir(ctx->getDir(), path),
fileLengthData.i(),
offset, uris));
+ fileEntry->setOriginalName(path);
fileEntries.push_back(fileEntry);
offset += fileEntry->getLength();
}
@@ -291,6 +292,7 @@ static void extractFileEntries
SharedHandle<FileEntry> fileEntry
(new FileEntry(util::applyDir(ctx->getDir(), name),totalLength, 0,
uris));
+ fileEntry->setOriginalName(name);
fileEntries.push_back(fileEntry);
}
ctx->setFileEntries(fileEntries.begin(), fileEntries.end());
diff --git a/test/BittorrentHelperTest.cc b/test/BittorrentHelperTest.cc
index ab1a2c97..45cbd19a 100644
--- a/test/BittorrentHelperTest.cc
+++ b/test/BittorrentHelperTest.cc
@@ -168,6 +168,8 @@ void BittorrentHelperTest::testGetFileEntries() {
SharedHandle<FileEntry> fileEntry1 = *itr;
CPPUNIT_ASSERT_EQUAL(std::string("./aria2-test/aria2/src/aria2c"),
fileEntry1->getPath());
+ CPPUNIT_ASSERT_EQUAL(std::string("aria2-test/aria2/src/aria2c"),
+ fileEntry1->getOriginalName());
itr++;
SharedHandle<FileEntry> fileEntry2 = *itr;
CPPUNIT_ASSERT_EQUAL(std::string("./aria2-test/aria2-0.2.2.tar.bz2"),
@@ -186,6 +188,8 @@ void BittorrentHelperTest::testGetFileEntriesSingle() {
SharedHandle<FileEntry> fileEntry1 = *itr;
CPPUNIT_ASSERT_EQUAL(std::string("./aria2-0.8.2.tar.bz2"),
fileEntry1->getPath());
+ CPPUNIT_ASSERT_EQUAL(std::string("aria2-0.8.2.tar.bz2"),
+ fileEntry1->getOriginalName());
}
void BittorrentHelperTest::testGetTotalLength() {
diff --git a/test/BtDependencyTest.cc b/test/BtDependencyTest.cc
index 0fe912e2..312fa54b 100644
--- a/test/BtDependencyTest.cc
+++ b/test/BtDependencyTest.cc
@@ -23,6 +23,9 @@ class BtDependencyTest:public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE(BtDependencyTest);
CPPUNIT_TEST(testResolve);
+ CPPUNIT_TEST(testResolve_originalNameNoMatch);
+ CPPUNIT_TEST(testResolve_singleFileWithoutOriginalName);
+ CPPUNIT_TEST(testResolve_multiFile);
CPPUNIT_TEST(testResolve_metadata);
CPPUNIT_TEST(testResolve_loadError);
CPPUNIT_TEST(testResolve_dependeeFailure);
@@ -37,7 +40,9 @@ class BtDependencyTest:public CppUnit::TestFixture {
dctx->setDir("/tmp");
std::deque<std::string> uris;
uris.push_back("http://localhost/outfile.path");
- dctx->getFirstFileEntry()->setUris(uris);
+ SharedHandle<FileEntry> fileEntry = dctx->getFirstFileEntry();
+ fileEntry->setUris(uris);
+ fileEntry->setOriginalName("aria2-0.8.2.tar.bz2");
dependant->setDownloadContext(dctx);
return dependant;
}
@@ -65,6 +70,9 @@ public:
}
void testResolve();
+ void testResolve_originalNameNoMatch();
+ void testResolve_singleFileWithoutOriginalName();
+ void testResolve_multiFile();
void testResolve_metadata();
void testResolve_loadError();
void testResolve_dependeeFailure();
@@ -93,6 +101,64 @@ void BtDependencyTest::testResolve()
CPPUNIT_ASSERT_EQUAL(std::string("/tmp/outfile.path"),
firstFileEntry->getPath());
CPPUNIT_ASSERT_EQUAL((size_t)1, firstFileEntry->getRemainingUris().size());
+ CPPUNIT_ASSERT(firstFileEntry->isRequested());
+}
+
+void BtDependencyTest::testResolve_originalNameNoMatch()
+{
+ std::string filename = "single.torrent";
+ SharedHandle<RequestGroup> dependant = createDependant(_option);
+ dependant->getDownloadContext()->getFirstFileEntry()->setOriginalName
+ ("aria2-1.1.0.tar.bz2");
+ SharedHandle<RequestGroup> dependee =
+ createDependee(_option, filename, File(filename).size());
+ dependee->getPieceStorage()->markAllPiecesDone();
+
+ BtDependency dep(dependant, dependee);
+ CPPUNIT_ASSERT(dep.resolve());
+
+ CPPUNIT_ASSERT(!dependant->getDownloadContext()->hasAttribute
+ (bittorrent::BITTORRENT));
+}
+
+void BtDependencyTest::testResolve_singleFileWithoutOriginalName()
+{
+ std::string filename = "single.torrent";
+ SharedHandle<RequestGroup> dependant = createDependant(_option);
+ dependant->getDownloadContext()->getFirstFileEntry()->setOriginalName("");
+ SharedHandle<RequestGroup> dependee =
+ createDependee(_option, filename, File(filename).size());
+ dependee->getPieceStorage()->markAllPiecesDone();
+ BtDependency dep(dependant, dependee);
+ CPPUNIT_ASSERT(dep.resolve());
+ CPPUNIT_ASSERT(dependant->getDownloadContext()->hasAttribute
+ (bittorrent::BITTORRENT));
+}
+
+void BtDependencyTest::testResolve_multiFile()
+{
+ std::string filename = "test.torrent";
+ SharedHandle<RequestGroup> dependant = createDependant(_option);
+ dependant->getDownloadContext()->getFirstFileEntry()->setOriginalName
+ ("aria2-test/aria2/src/aria2c");
+ SharedHandle<RequestGroup> dependee =
+ createDependee(_option, filename, File(filename).size());
+ dependee->getPieceStorage()->markAllPiecesDone();
+
+ BtDependency dep(dependant, dependee);
+ CPPUNIT_ASSERT(dep.resolve());
+
+ CPPUNIT_ASSERT(dependant->getDownloadContext()->hasAttribute
+ (bittorrent::BITTORRENT));
+
+ const std::vector<SharedHandle<FileEntry> >& fileEntries =
+ dependant->getDownloadContext()->getFileEntries();
+ CPPUNIT_ASSERT_EQUAL(std::string("/tmp/outfile.path"),
+ fileEntries[0]->getPath());
+ CPPUNIT_ASSERT(fileEntries[0]->isRequested());
+ CPPUNIT_ASSERT_EQUAL(std::string("/tmp/aria2-test/aria2-0.2.2.tar.bz2"),
+ fileEntries[1]->getPath());
+ CPPUNIT_ASSERT(!fileEntries[1]->isRequested());
}
void BtDependencyTest::testResolve_metadata()
@@ -121,6 +187,8 @@ void BtDependencyTest::testResolve_metadata()
CPPUNIT_ASSERT_EQUAL
(std::string("cd41c7fdddfd034a15a04d7ff881216e01c4ceaf"),
bittorrent::getInfoHashString(dependant->getDownloadContext()));
+ CPPUNIT_ASSERT
+ (dependant->getDownloadContext()->getFirstFileEntry()->isRequested());
}
void BtDependencyTest::testResolve_loadError()
diff --git a/test/MetalinkHelperTest.cc b/test/MetalinkHelperTest.cc
index 902e654f..3886ac9b 100644
--- a/test/MetalinkHelperTest.cc
+++ b/test/MetalinkHelperTest.cc
@@ -1,8 +1,11 @@
#include "MetalinkHelper.h"
+
+#include <cppunit/extensions/HelperMacros.h>
+
#include "MetalinkEntry.h"
#include "Option.h"
#include "prefs.h"
-#include <cppunit/extensions/HelperMacros.h>
+#include "MetalinkMetaurl.h"
namespace aria2 {
@@ -11,16 +14,14 @@ class MetalinkHelperTest:public CppUnit::TestFixture {
CPPUNIT_TEST_SUITE(MetalinkHelperTest);
CPPUNIT_TEST(testParseAndQuery);
CPPUNIT_TEST(testParseAndQuery_version);
+ CPPUNIT_TEST(testGroupEntryByMetaurlName);
CPPUNIT_TEST_SUITE_END();
private:
public:
- void setUp() {}
-
- void tearDown() {}
-
void testParseAndQuery();
void testParseAndQuery_version();
+ void testGroupEntryByMetaurlName();
};
@@ -45,4 +46,70 @@ void MetalinkHelperTest::testParseAndQuery_version()
CPPUNIT_ASSERT_EQUAL(std::string("aria2-0.5.1.tar.bz2"), entry->getPath());
}
+void MetalinkHelperTest::testGroupEntryByMetaurlName()
+{
+ std::vector<SharedHandle<MetalinkEntry> > entries;
+
+ SharedHandle<MetalinkEntry> e1(new MetalinkEntry());
+ e1->version = "1";
+ e1->sizeKnown = true;
+ // no name
+ e1->metaurls.push_back
+ (SharedHandle<MetalinkMetaurl>
+ (new MetalinkMetaurl("http://meta1", "torrent", "", 1)));
+
+ SharedHandle<MetalinkEntry> e2(new MetalinkEntry());
+ e2->version = "2";
+ e2->sizeKnown = true;
+
+ SharedHandle<MetalinkEntry> e3(new MetalinkEntry());
+ e3->version = "3";
+ e3->sizeKnown = true;
+ e3->metaurls.push_back
+ (SharedHandle<MetalinkMetaurl>
+ (new MetalinkMetaurl("http://meta2", "torrent", "f3", 1)));
+
+ SharedHandle<MetalinkEntry> e4(new MetalinkEntry());
+ e4->version = "4";
+ e4->sizeKnown = true;
+ e4->metaurls.push_back
+ (SharedHandle<MetalinkMetaurl>
+ (new MetalinkMetaurl("http://meta1", "torrent", "f4", 1)));
+
+ SharedHandle<MetalinkEntry> e5(new MetalinkEntry());
+ e5->version = "5";
+ // no size
+ e5->metaurls.push_back
+ (SharedHandle<MetalinkMetaurl>
+ (new MetalinkMetaurl("http://meta1", "torrent", "f5", 1)));
+
+ SharedHandle<MetalinkEntry> e6(new MetalinkEntry());
+ e6->version = "6";
+ e6->sizeKnown = true;
+ e6->metaurls.push_back
+ (SharedHandle<MetalinkMetaurl>
+ (new MetalinkMetaurl("http://meta1", "torrent", "f6", 1)));
+
+ entries.push_back(e1);
+ entries.push_back(e2);
+ entries.push_back(e3);
+ entries.push_back(e4);
+ entries.push_back(e5);
+ entries.push_back(e6);
+
+ std::vector<std::pair<std::string,
+ std::vector<SharedHandle<MetalinkEntry> > > > result;
+ MetalinkHelper::groupEntryByMetaurlName(result, entries);
+
+ CPPUNIT_ASSERT_EQUAL(std::string("http://meta1"), result[0].first);
+ CPPUNIT_ASSERT_EQUAL(std::string("1"), result[0].second[0]->version);
+ CPPUNIT_ASSERT_EQUAL(std::string(""), result[1].first);
+ CPPUNIT_ASSERT_EQUAL(std::string("2"), result[1].second[0]->version);
+ CPPUNIT_ASSERT_EQUAL(std::string("http://meta2"), result[2].first);
+ CPPUNIT_ASSERT_EQUAL(std::string("3"), result[2].second[0]->version);
+ CPPUNIT_ASSERT_EQUAL(std::string("http://meta1"), result[3].first);
+ CPPUNIT_ASSERT_EQUAL(std::string("4"), result[3].second[0]->version);
+ CPPUNIT_ASSERT_EQUAL(std::string("6"), result[3].second[1]->version);
+}
+
} // namespace aria2
generated by cgit (git 2.46.0) at 2026-10-12 06:27:41 +0000