2008-09-10 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

	Added --max-file-not-found=NUM option. If aria2 receives `file 
not
	found' error from HTTP/FTP server NUM times without getting 
single
	bytes of data, then force the download to fail.
	Basically aria2 tris all URIs available. So if long list of URIs 
are
	specified in Metalink, then aria2 will take a long time to fail
	eventually if all URIs are outdated. This option eases this 
situation
	to limit the number of servers to try.
	* src/FtpNegotiationCommand.cc
	* src/HelpItemFactory.cc
	* src/HttpResponseCommand.cc
	* src/OptionHandlerFactory.cc
	* src/PeerStat.h
	* src/RequestGroup.cc
	* src/RequestGroup.h
	* src/SegmentMan.cc
	* src/SegmentMan.h
	* src/option_processing.cc
	* src/prefs.cc
	* src/prefs.h
	* src/usage_text.h

13 files changed, 89 insertions, 3 deletions

diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc
index 7380019c..c160292e 100644
--- a/src/FtpNegotiationCommand.cc
+++ b/src/FtpNegotiationCommand.cc
@@ -204,6 +204,7 @@ bool FtpNegotiationCommand::recvCwd() {
   }
   if(status != 250) {
     poolConnection();
+    _requestGroup->increaseAndValidateFileNotFoundCount();
     throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str());
   }
   if(e->option->getAsBool(PREF_REMOTE_TIME)) {
@@ -444,6 +445,7 @@ bool FtpNegotiationCommand::recvRetr() {
     return false;
   }
   if(status != 150 && status != 125) {
+    _requestGroup->increaseAndValidateFileNotFoundCount();
     throw DlAbortEx(StringFormat(EX_BAD_STATUS, status).str());
   }
   if(e->option->getAsBool(PREF_FTP_PASV)) {
diff --git a/src/HelpItemFactory.cc b/src/HelpItemFactory.cc
index 6ad324c5..22d951ad 100644
--- a/src/HelpItemFactory.cc
+++ b/src/HelpItemFactory.cc
@@ -527,6 +527,14 @@ TagContainerHandle HelpItemFactory::createHelpItems(const Option* op)
     tc->addItem(item);
   }
   {
+    HelpItemHandle item(new HelpItem(PREF_MAX_FILE_NOT_FOUND,
+				     TEXT_MAX_FILE_NOT_FOUND,
+				     op->get(PREF_MAX_FILE_NOT_FOUND)));
+    item->addTag(TAG_HTTP);
+    item->addTag(TAG_FTP);
+    tc->addItem(item);	     
+  }
+  {
     HelpItemHandle item(new HelpItem("help", TEXT_HELP, TAG_BASIC));
     item->setAvailableValues
       (StringFormat("%s,%s,%s,%s,%s,%s,%s,all", TAG_BASIC, TAG_ADVANCED, TAG_HTTP, TAG_FTP, TAG_METALINK, TAG_BITTORRENT,TAG_HELP).str());
diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc
index 847a2d8f..0ccd7d2f 100644
--- a/src/HttpResponseCommand.cc
+++ b/src/HttpResponseCommand.cc
@@ -91,9 +91,11 @@ bool HttpResponseCommand::executeInternal()
   httpResponse->retrieveCookie();
 
   if(httpResponse->getResponseStatus() >= HttpHeader::S300) {
+    if(httpResponse->getResponseStatus() == HttpHeader::S404) {
+      _requestGroup->increaseAndValidateFileNotFoundCount();
+    }
     return skipResponseBody(httpResponse);
   }
-
   if(!_requestGroup->isSingleHostMultiConnectionEnabled()) {
     // Query by hostname. Searching by CUID may returns NULL.
     // In case when resuming download, ServerHost is registered with CUID A.
diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc
index fcbdd157..5049abb1 100644
--- a/src/OptionHandlerFactory.cc
+++ b/src/OptionHandlerFactory.cc
@@ -156,6 +156,7 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers()
   handlers.push_back(SH(new DefaultOptionHandler(PREF_SERVER_STAT_OF)));
   handlers.push_back(SH(new BooleanOptionHandler(PREF_REMOTE_TIME)));
   handlers.push_back(SH(new NumberOptionHandler(PREF_CONNECT_TIMEOUT, 1, 600)));
+  handlers.push_back(SH(new NumberOptionHandler(PREF_MAX_FILE_NOT_FOUND, 0)));
 
   return handlers;
 }
diff --git a/src/PeerStat.h b/src/PeerStat.h
index fa66eab0..6b3ad8b2 100644
--- a/src/PeerStat.h
+++ b/src/PeerStat.h
@@ -59,6 +59,8 @@ private:
   PeerStat::STATUS status;
   unsigned int _avgDownloadSpeed;
   unsigned int _avgUploadSpeed;
+  uint64_t _sessionDownloadLength;
+  uint64_t _sessionUploadLength;
 public:
 
   PeerStat(int32_t cuid, const std::string& hostname,
@@ -68,11 +70,15 @@ public:
     _protocol(protocol),
     status(PeerStat::IDLE),
     _avgDownloadSpeed(0),
-    _avgUploadSpeed(0) {}
+    _avgUploadSpeed(0),
+    _sessionDownloadLength(0),
+    _sessionUploadLength(0) {}
 
   PeerStat(int32_t cuid = 0):cuid(cuid), status(PeerStat::IDLE),
 			     _avgDownloadSpeed(0),
-			     _avgUploadSpeed(0) {}
+			     _avgUploadSpeed(0),
+			     _sessionDownloadLength(0),
+			     _sessionUploadLength(0) {}
 
   ~PeerStat() {}
 
@@ -107,10 +113,12 @@ public:
 
   void updateDownloadLength(size_t bytes) {
     downloadSpeed.update(bytes);
+    _sessionDownloadLength += bytes;
   }
 
   void updateUploadLength(size_t bytes) {
     uploadSpeed.update(bytes);
+    _sessionUploadLength += bytes;
   }
 
   unsigned int getMaxDownloadSpeed() const {
@@ -172,6 +180,16 @@ public:
   {
     return _protocol;
   }
+
+  uint64_t getSessionDownloadLength() const
+  {
+    return _sessionDownloadLength;
+  }
+
+  uint64_t getSessionUploadLength() const
+  {
+    return _sessionUploadLength;
+  }
 };
 
 typedef SharedHandle<PeerStat> PeerStatHandle;
diff --git a/src/RequestGroup.cc b/src/RequestGroup.cc
index f6662346..ed30d8e6 100644
--- a/src/RequestGroup.cc
+++ b/src/RequestGroup.cc
@@ -124,6 +124,7 @@ RequestGroup::RequestGroup(const Option* option,
   _singleHostMultiConnectionEnabled(true),
   _uriSelector(new InOrderURISelector()),
   _lastModifiedTime(-1),
+  _fileNotFoundCount(0),
   _option(option),
   _logger(LogFactory::getInstance())
 {
@@ -1037,6 +1038,7 @@ void RequestGroup::applyLastModifiedTimeToLocalFiles()
     _logger->info("Last-Modified attrs of %zu files were updated.", n);
   }
 }
+
 void RequestGroup::updateLastModifiedTime(const Time& time)
 {
   if(time.good() && _lastModifiedTime < time) {
@@ -1044,4 +1046,15 @@ void RequestGroup::updateLastModifiedTime(const Time& time)
   }
 }
 
+void RequestGroup::increaseAndValidateFileNotFoundCount()
+{
+  ++_fileNotFoundCount;
+  const unsigned int maxCount = _option->getAsInt(PREF_MAX_FILE_NOT_FOUND);
+  if(maxCount > 0 && _fileNotFoundCount >= maxCount &&
+     _segmentMan->calculateSessionDownloadLength() == 0) {
+    throw DownloadFailureException
+      (StringFormat("Reached max-file-not-found count=%u", maxCount).str());
+  }
+}
+
 } // namespace aria2
diff --git a/src/RequestGroup.h b/src/RequestGroup.h
index 1b2c213d..12dc0fb5 100644
--- a/src/RequestGroup.h
+++ b/src/RequestGroup.h
@@ -118,6 +118,8 @@ private:
 
   Time _lastModifiedTime;
 
+  unsigned int _fileNotFoundCount;
+
   const Option* _option;
 
   Logger* _logger;
@@ -368,6 +370,8 @@ public:
   void applyLastModifiedTimeToLocalFiles();
 
   void updateLastModifiedTime(const Time& time);
+
+  void increaseAndValidateFileNotFoundCount();
 };
 
 typedef SharedHandle<RequestGroup> RequestGroupHandle;
diff --git a/src/SegmentMan.cc b/src/SegmentMan.cc
index 9383ec0a..eccf4d71 100644
--- a/src/SegmentMan.cc
+++ b/src/SegmentMan.cc
@@ -46,6 +46,7 @@
 #include "DownloadContext.h"
 #include "Piece.h"
 #include <algorithm>
+#include <numeric>
 #include <cassert>
 
 namespace aria2 {
@@ -306,6 +307,21 @@ unsigned int SegmentMan::calculateDownloadSpeed() const {
   return speed;
 }
 
+class PeerStatDownloadLengthOperator {
+public:
+  uint64_t operator()(uint64_t total, const SharedHandle<PeerStat>& ps)
+  {
+    return ps->getSessionDownloadLength()+total;
+  }
+};
+
+uint64_t SegmentMan::calculateSessionDownloadLength() const
+{
+  return std::accumulate(peerStats.begin(), peerStats.end(), 0LL,
+			 PeerStatDownloadLengthOperator());
+ 
+}
+
 size_t SegmentMan::countFreePieceFrom(size_t index) const
 {
   size_t numPieces = _downloadContext->getNumPieces();
diff --git a/src/SegmentMan.h b/src/SegmentMan.h
index 1b3b6993..5edee31d 100644
--- a/src/SegmentMan.h
+++ b/src/SegmentMan.h
@@ -179,6 +179,11 @@ public:
    */
   unsigned int calculateDownloadSpeed() const;
 
+  /**
+   * Returns the downloaded bytes in this session.
+   */
+  uint64_t calculateSessionDownloadLength() const;
+
   size_t countFreePieceFrom(size_t index) const;
 };
 
diff --git a/src/option_processing.cc b/src/option_processing.cc
index 6fa374a0..e54d5632 100644
--- a/src/option_processing.cc
+++ b/src/option_processing.cc
@@ -160,6 +160,7 @@ Option* createDefaultOption()
   op->put(PREF_URI_SELECTOR, V_INORDER);
   op->put(PREF_SERVER_STAT_TIMEOUT, "86400");// 1day
   op->put(PREF_REMOTE_TIME, V_FALSE);
+  op->put(PREF_MAX_FILE_NOT_FOUND, "0");
   return op;
 }
 
@@ -243,6 +244,7 @@ Option* option_processing(int argc, char* const argv[])
       { PREF_SERVER_STAT_TIMEOUT.c_str(), required_argument, &lopt, 223 },
       { PREF_REMOTE_TIME.c_str(), optional_argument, 0, 'R' },
       { PREF_CONNECT_TIMEOUT.c_str(), required_argument, &lopt, 224 },
+      { PREF_MAX_FILE_NOT_FOUND.c_str(), required_argument, &lopt, 225 },
 #if defined ENABLE_BITTORRENT || defined ENABLE_METALINK
       { PREF_SHOW_FILES.c_str(), no_argument, NULL, 'S' },
       { PREF_SELECT_FILE.c_str(), required_argument, &lopt, 21 },
@@ -488,6 +490,9 @@ Option* option_processing(int argc, char* const argv[])
       case 224:
 	cmdstream << PREF_CONNECT_TIMEOUT << "=" << optarg << "\n";
 	break;
+      case 225:
+	cmdstream << PREF_MAX_FILE_NOT_FOUND << "=" << optarg << "\n";
+	break;
       }
       break;
     }
diff --git a/src/prefs.cc b/src/prefs.cc
index 15a7d79a..d20a025d 100644
--- a/src/prefs.cc
+++ b/src/prefs.cc
@@ -149,6 +149,8 @@ const std::string PREF_SERVER_STAT_IF("server-stat-if");
 const std::string PREF_SERVER_STAT_OF("server-stat-of");
 // value: true | false
 const std::string PREF_REMOTE_TIME("remote-time");
+// value: 1*digit
+const std::string PREF_MAX_FILE_NOT_FOUND("max-file-not-found");
 
 /**
  * FTP related preferences
diff --git a/src/prefs.h b/src/prefs.h
index a7667fc8..22e8b3a7 100644
--- a/src/prefs.h
+++ b/src/prefs.h
@@ -153,6 +153,8 @@ extern const std::string PREF_SERVER_STAT_IF;
 extern const std::string PREF_SERVER_STAT_OF;
 // value: true | false
 extern const std::string PREF_REMOTE_TIME;
+// value: 1*digit
+extern const std::string PREF_MAX_FILE_NOT_FOUND;
 
 /**
  * FTP related preferences
diff --git a/src/usage_text.h b/src/usage_text.h
index b59af147..96cc4606 100644
--- a/src/usage_text.h
+++ b/src/usage_text.h
@@ -365,3 +365,11 @@ _(" --connect-timeout=SEC        Set the connect timeout in seconds to establish
   "                              connection to HTTP/FTP/proxy server. After the\n"\
   "                              connection is established, this option makes no\n"\
   "                              effect and --timeout option is used instead.")
+#define TEXT_MAX_FILE_NOT_FOUND \
+_(" --max-file-not-found=NUM     If aria2 recieves `file not found' status from the\n"\
+  "                              remote HTTP/FTP servers NUM times without getting\n"\
+  "                              a single byte, then force the download to fail.\n"\
+  "                              Specify 0 to disable this option.\n"\
+  "                              This options is only effective only when using\n"\
+  "                              HTTP/FTP servers.")
+