Added --retry-wait option.

This option was once existed in aria2 but erased on 2009-09-20.  Now
it is resurrected once again.  We choose 2 as default value, but there
is no good theory behind it.  Now we retry HTTP download when remote
server returns 503 Service Unavailable if --retry-wait > 0. We also
added error code 29: HTTP_SERVICE_UNAVAILABLE.

13 files changed, 77 insertions, 7 deletions

diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc
index b4e5ee2f..9d573ab1 100644
--- a/src/AbstractCommand.cc
+++ b/src/AbstractCommand.cc
@@ -306,6 +306,9 @@ bool AbstractCommand::execute() {
       tryReserved();
       return true;
     } else {
+      Timer wakeTime(global::wallclock);
+      wakeTime.advance(getOption()->getAsInt(PREF_RETRY_WAIT));
+      req_->setWakeTime(wakeTime);
       return prepareForRetry(0);
     }
   } catch(DownloadFailureException& err) {
diff --git a/src/CreateRequestCommand.cc b/src/CreateRequestCommand.cc
index 3130b741..f731d91a 100644
--- a/src/CreateRequestCommand.cc
+++ b/src/CreateRequestCommand.cc
@@ -46,6 +46,8 @@
 #include "RequestGroupMan.h"
 #include "FileEntry.h"
 #include "SocketRecvBuffer.h"
+#include "LogFactory.h"
+#include "wallclock.h"
 
 namespace aria2 {
 
@@ -93,6 +95,11 @@ bool CreateRequestCommand::executeInternal()
       getSegmentMan()->ignoreSegmentFor(getFileEntry());
     }
     throw DL_ABORT_EX("No URI available.");
+  } else if(getRequest()->getWakeTime() > global::wallclock) {
+    A2_LOG_DEBUG("This request object is still sleeping.");
+    getFileEntry()->poolRequest(getRequest());
+    getDownloadEngine()->addCommand(this);
+    return false;
   }
 
   Command* command =
diff --git a/src/FileEntry.cc b/src/FileEntry.cc
index 9fe560e9..262ae890 100644
--- a/src/FileEntry.cc
+++ b/src/FileEntry.cc
@@ -170,9 +170,23 @@ FileEntry::getRequest
         break;
       }
     }
-  } else {    
-    req = requestPool_.front();
-    requestPool_.pop_front();
+  } else {
+    // Skip Request object if it is still
+    // sleeping(Request::getWakeTime() < global::wallclock).  If all
+    // pooled objects are sleeping, return first one.  Caller should
+    // inspect returned object's getWakeTime().
+    std::deque<SharedHandle<Request> >::iterator i = requestPool_.begin();
+    std::deque<SharedHandle<Request> >::iterator eoi = requestPool_.end();
+    for(; i != eoi; ++i) {
+      if((*i)->getWakeTime() <= global::wallclock) {
+        break;
+      }
+    }
+    if(i == eoi) {
+      i = requestPool_.begin();
+    }
+    req = *i;
+    requestPool_.erase(i);
     inFlightRequests_.push_back(req);
     A2_LOG_DEBUG(fmt("Picked up from pool: %s", req->getUri().c_str()));
   }
diff --git a/src/HttpSkipResponseCommand.cc b/src/HttpSkipResponseCommand.cc
index 60b600d3..bf6db6e3 100644
--- a/src/HttpSkipResponseCommand.cc
+++ b/src/HttpSkipResponseCommand.cc
@@ -207,6 +207,16 @@ bool HttpSkipResponseCommand::processResponse()
     } else if(statusCode == 404) {
       throw DL_ABORT_EX2(MSG_RESOURCE_NOT_FOUND,
                          error_code::RESOURCE_NOT_FOUND);
+    } else if(statusCode == 503) {
+      // Only retry if pretry-wait > 0. Hammering 'busy' server is not
+      // a good idea.
+      if(getOption()->getAsInt(PREF_RETRY_WAIT) > 0) {
+        throw DL_RETRY_EX2(fmt(EX_BAD_STATUS, statusCode),
+                           error_code::HTTP_SERVICE_UNAVAILABLE);
+      } else {
+        throw DL_ABORT_EX2(fmt(EX_BAD_STATUS, statusCode),
+                           error_code::HTTP_SERVICE_UNAVAILABLE);
+      }
     } else {
       throw DL_ABORT_EX2(fmt(EX_BAD_STATUS, statusCode),
                          error_code::HTTP_PROTOCOL_ERROR);
diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc
index 38a48603..e342bca4 100644
--- a/src/OptionHandlerFactory.cc
+++ b/src/OptionHandlerFactory.cc
@@ -690,6 +690,16 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers()
     handlers.push_back(op);
   }
   {
+    SharedHandle<OptionHandler> op(new NumberOptionHandler
+                                   (PREF_RETRY_WAIT,
+                                    TEXT_RETRY_WAIT,
+                                    "2",
+                                    0, 600));
+    op->addTag(TAG_FTP);
+    op->addTag(TAG_HTTP);
+    handlers.push_back(op);
+  }
+  {
     SharedHandle<OptionHandler> op(new BooleanOptionHandler
                                    (PREF_REUSE_URI,
                                     TEXT_REUSE_URI,
diff --git a/src/Request.cc b/src/Request.cc
index 0c68aa90..2fad4b19 100644
--- a/src/Request.cc
+++ b/src/Request.cc
@@ -42,6 +42,7 @@
 #include "A2STR.h"
 #include "uri.h"
 #include "PeerStat.h"
+#include "wallclock.h"
 
 namespace aria2 {
 
@@ -66,7 +67,8 @@ Request::Request():
   hasPassword_(false),
   ipv6LiteralAddress_(false),
   removalRequested_(false),
-  connectedPort_(0)
+  connectedPort_(0),
+  wakeTime_(global::wallclock)
 {}
 
 Request::~Request() {}
diff --git a/src/Request.h b/src/Request.h
index e1889ad2..c4653968 100644
--- a/src/Request.h
+++ b/src/Request.h
@@ -39,6 +39,7 @@
 #include <string>
 
 #include "SharedHandle.h"
+#include "TimerA2.h"
 
 namespace aria2 {
 
@@ -96,6 +97,8 @@ private:
 
   uint16_t connectedPort_;
 
+  Timer wakeTime_;
+
   bool parseUri(const std::string& uri);
 public:
   Request();
@@ -229,6 +232,16 @@ public:
     return connectedPort_;
   }
 
+  void setWakeTime(Timer timer)
+  {
+    wakeTime_ = timer;
+  }
+
+  const Timer& getWakeTime()
+  {
+    return wakeTime_;
+  }
+
   static const std::string METHOD_GET;
   static const std::string METHOD_HEAD;
 
diff --git a/src/TimerA2.cc b/src/TimerA2.cc
index 973c830e..3ed6b564 100644
--- a/src/TimerA2.cc
+++ b/src/TimerA2.cc
@@ -75,6 +75,11 @@ bool Timer::operator<(const Timer& timer) const
   return util::difftv(timer.tv_, tv_) > 0;
 }
 
+bool Timer::operator<=(const Timer& timer) const
+{
+  return util::difftv(timer.tv_, tv_) >= 0;
+}
+
 bool Timer::operator>(const Timer& timer) const
 {
   return util::difftv(tv_, timer.tv_) > 0;
diff --git a/src/TimerA2.h b/src/TimerA2.h
index 42235fac..83b68faf 100644
--- a/src/TimerA2.h
+++ b/src/TimerA2.h
@@ -59,6 +59,8 @@ public:
 
   bool operator<(const Timer& timer) const;
 
+  bool operator<=(const Timer& timer) const;
+
   bool operator>(const Timer& timer) const;
 
   void reset();
diff --git a/src/error_code.h b/src/error_code.h
index e9670a11..18abc6f0 100644
--- a/src/error_code.h
+++ b/src/error_code.h
@@ -71,7 +71,8 @@ enum Value {
   BENCODE_PARSE_ERROR = 25,
   BITTORRENT_PARSE_ERROR = 26,
   MAGNET_PARSE_ERROR = 27,
-  OPTION_ERROR = 28
+  OPTION_ERROR = 28,
+  HTTP_SERVICE_UNAVAILABLE = 29
 };
 
 } // namespace error_code
diff --git a/src/prefs.cc b/src/prefs.cc
index 3be820d7..2694b8ba 100644
--- a/src/prefs.cc
+++ b/src/prefs.cc
@@ -202,6 +202,8 @@ const std::string PREF_SELECT_LEAST_USED_HOST("select-least-used-host");
 const std::string PREF_ENABLE_ASYNC_DNS6("enable-async-dns6");
 // value: 1*digit
 const std::string PREF_MAX_DOWNLOAD_RESULT("max-download-result");
+// value: 1*digit
+const std::string PREF_RETRY_WAIT("retry-wait");
 
 /**
  * FTP related preferences
diff --git a/src/prefs.h b/src/prefs.h
index f20a80e2..b92ffd68 100644
--- a/src/prefs.h
+++ b/src/prefs.h
@@ -206,6 +206,8 @@ extern const std::string PREF_SELECT_LEAST_USED_HOST;
 extern const std::string PREF_ENABLE_ASYNC_DNS6;
 // value: 1*digit
 extern const std::string PREF_MAX_DOWNLOAD_RESULT;
+// value: 1*digit
+extern const std::string PREF_RETRY_WAIT;
 
 /**
  * FTP related preferences
diff --git a/src/usage_text.h b/src/usage_text.h
index 1595c7a0..34368e38 100644
--- a/src/usage_text.h
+++ b/src/usage_text.h
@@ -59,8 +59,7 @@
     "                              Please note that in Metalink download, this\n" \
     "                              option has no effect and use -C option instead.")
 #define TEXT_RETRY_WAIT                                                 \
-  _(" --retry-wait=SEC             Set the seconds to wait to retry after an error\n" \
-    "                              has occured.")
+  _(" --retry-wait=SEC             Set the seconds to wait between retries.")
 #define TEXT_TIMEOUT                                            \
   _(" -t, --timeout=SEC            Set timeout in seconds.")
 #define TEXT_MAX_TRIES                                                  \