Add --keep-unfinished-download-result option

This option keeps unfinished download results even if doing so exceeds
--max-download-result. This is useful if all unfinished downloads must
be saved in session file (see --save-session option). Please keep in
mind that there is no upper bound to the number of unfinished download
result to keep.  User should use this option only when they know the
total number of downloads in advance.

9 files changed, 89 insertions, 10 deletions

diff --git a/doc/manual-src/en/aria2c.rst b/doc/manual-src/en/aria2c.rst
index 0cb21fac..ef709e20 100644
--- a/doc/manual-src/en/aria2c.rst
+++ b/doc/manual-src/en/aria2c.rst
@@ -1391,6 +1391,16 @@ Advanced Options
     system doesn't have :manpage:`getifaddrs(3)`, this option doesn't accept interface
     name.
 
+.. option:: --keep-unfinished-download-result[=true|false]
+
+  Keep unfinished download results even if doing so exceeds
+  :option:`--max-download-result`.  This is useful if all unfinished
+  downloads must be saved in session file (see
+  :option:`--save-session` option).  Please keep in mind that there is
+  no upper bound to the number of unfinished download result to keep.
+  User should use this option only when they know the total number of
+  downloads in advance.  Default: ``false``
+
 .. option:: --max-download-result=<NUM>
 
   Set maximum number of download result kept in memory. The download
@@ -3240,6 +3250,7 @@ For information on the *secret* parameter, see :ref:`rpc_auth`.
 
   * :option:`bt-max-open-files <--bt-max-open-files>`
   * :option:`download-result <--download-result>`
+  * :option:`keep-unfinished-download-result <--keep-unfinished-download-result>`
   * :option:`log <-l>`
   * :option:`log-level <--log-level>`
   * :option:`max-concurrent-downloads <-j>`
diff --git a/doc/xmlrpc/aria2rpc b/doc/xmlrpc/aria2rpc
index 934fbdbc..f52b7f6a 100755
--- a/doc/xmlrpc/aria2rpc
+++ b/doc/xmlrpc/aria2rpc
@@ -228,6 +228,9 @@ OptionParser.new do |opt|
   opt.on("-l","--log FILE"){|val| options["log"]=val}
   opt.on("--max-download-result NUM"){|val| options["max-download-result"]=val}
   opt.on("--download-result OPT"){|val| options["download-result"]=val}
+  opt.on("--keep-unfinished-download-result [BOOL]",["true","false"]){|val|
+    options["keep-unfinished-download-result"]=val||"true"
+  }
   opt.on("--save-session FILE"){|val| options["save-session"]=val}
   opt.on("--server-stat-of FILE"){|val| options["server-stat-of"]=val}
   opt.on("--save-cookies FILE"){|val| options["save-cookies"]=val}
diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc
index 842ae56f..e718382b 100644
--- a/src/OptionHandlerFactory.cc
+++ b/src/OptionHandlerFactory.cc
@@ -404,6 +404,15 @@ std::vector<OptionHandler*> OptionHandlerFactory::createOptionHandlers()
     handlers.push_back(op);
   }
   {
+    OptionHandler* op(
+        new BooleanOptionHandler(PREF_KEEP_UNFINISHED_DOWNLOAD_RESULT,
+                                 TEXT_KEEP_UNFINISHED_DOWNLOAD_RESULT,
+                                 A2_V_FALSE, OptionHandler::OPT_ARG));
+    op->addTag(TAG_ADVANCED);
+    op->setChangeGlobalOption(true);
+    handlers.push_back(op);
+  }
+  {
     OptionHandler* op(new DefaultOptionHandler(
         PREF_LOG, TEXT_LOG, NO_DEFAULT_VALUE, PATH_TO_FILE_STDOUT,
         OptionHandler::REQ_ARG, 'l'));
diff --git a/src/RequestGroupMan.cc b/src/RequestGroupMan.cc
index 19ee63ad..169a6041 100644
--- a/src/RequestGroupMan.cc
+++ b/src/RequestGroupMan.cc
@@ -920,13 +920,21 @@ void RequestGroupMan::addDownloadResult(
   bool rv = downloadResults_.push_back(dr->gid->getNumericId(), dr);
   assert(rv);
   while (downloadResults_.size() > maxDownloadResult_) {
-    DownloadResultList::iterator i = downloadResults_.begin();
     // Save last encountered error code so that we can report it
     // later.
-    const std::shared_ptr<DownloadResult>& dr = *i;
+    const auto& dr = downloadResults_[0];
     if (dr->belongsTo == 0 && dr->result != error_code::FINISHED) {
       removedLastErrorResult_ = dr->result;
       ++removedErrorResult_;
+
+      // Keep unfinished download result, so that we can save them by
+      // SessionSerializer.
+      if (option_->getAsBool(PREF_KEEP_UNFINISHED_DOWNLOAD_RESULT)) {
+        if (dr->result != error_code::REMOVED ||
+            dr->option->getAsBool(PREF_FORCE_SAVE)) {
+          unfinishedDownloadResults_.push_back(dr);
+        }
+      }
     }
     downloadResults_.pop_front();
   }
diff --git a/src/RequestGroupMan.h b/src/RequestGroupMan.h
index 566b77d3..b8aa8d4f 100644
--- a/src/RequestGroupMan.h
+++ b/src/RequestGroupMan.h
@@ -71,6 +71,10 @@ private:
   RequestGroupList requestGroups_;
   RequestGroupList reservedGroups_;
   DownloadResultList downloadResults_;
+  // This includes download result which did not finish, and deleted
+  // from downloadResults_.  This is used to save them in
+  // SessionSerializer.
+  std::vector<std::shared_ptr<DownloadResult>> unfinishedDownloadResults_;
 
   int maxConcurrentDownloads_;
 
@@ -261,6 +265,12 @@ public:
 
   void addDownloadResult(const std::shared_ptr<DownloadResult>& downloadResult);
 
+  const std::vector<std::shared_ptr<DownloadResult>>&
+  getUnfinishedDownloadResult() const
+  {
+    return unfinishedDownloadResults_;
+  }
+
   std::shared_ptr<ServerStat> findServerStat(const std::string& hostname,
                                              const std::string& protocol) const;
 
diff --git a/src/SessionSerializer.cc b/src/SessionSerializer.cc
index 0d260a3f..8dd9008a 100644
--- a/src/SessionSerializer.cc
+++ b/src/SessionSerializer.cc
@@ -272,11 +272,14 @@ bool writeDownloadResult(IOFile& fp, std::set<a2_gid_t>& metainfoCache,
 }
 } // namespace
 
-bool SessionSerializer::save(IOFile& fp) const
+namespace {
+template <typename InputIt>
+bool saveDownloadResult(IOFile& fp, std::set<a2_gid_t>& metainfoCache,
+                        InputIt first, InputIt last, bool saveInProgress,
+                        bool saveError)
 {
-  std::set<a2_gid_t> metainfoCache;
-  const DownloadResultList& results = rgman_->getDownloadResults();
-  for (const auto& dr : results) {
+  for (; first != last; ++first) {
+    const auto& dr = *first;
     auto save = false;
     switch (dr->result) {
     case error_code::FINISHED:
@@ -284,20 +287,41 @@ bool SessionSerializer::save(IOFile& fp) const
       save = dr->option->getAsBool(PREF_FORCE_SAVE);
       break;
     case error_code::IN_PROGRESS:
-      save = saveInProgress_;
+      save = saveInProgress;
       break;
     case error_code::RESOURCE_NOT_FOUND:
     case error_code::MAX_FILE_NOT_FOUND:
-      save = saveError_ && dr->option->getAsBool(PREF_SAVE_NOT_FOUND);
+      save = saveError && dr->option->getAsBool(PREF_SAVE_NOT_FOUND);
       break;
     default:
-      save = saveError_;
+      save = saveError;
       break;
     }
     if (save && !writeDownloadResult(fp, metainfoCache, dr, false)) {
       return false;
     }
   }
+  return true;
+}
+} // namespace
+
+bool SessionSerializer::save(IOFile& fp) const
+{
+  std::set<a2_gid_t> metainfoCache;
+
+  const auto& unfinishedResults = rgman_->getUnfinishedDownloadResult();
+  if (!saveDownloadResult(fp, metainfoCache, std::begin(unfinishedResults),
+                          std::end(unfinishedResults), saveInProgress_,
+                          saveError_)) {
+    return false;
+  }
+
+  const auto& results = rgman_->getDownloadResults();
+  if (!saveDownloadResult(fp, metainfoCache, std::begin(results),
+                          std::end(results), saveInProgress_, saveError_)) {
+    return false;
+  }
+
   {
     // Save active downloads.
     const RequestGroupList& groups = rgman_->getRequestGroups();
diff --git a/src/prefs.cc b/src/prefs.cc
index 8ec80569..c0e1af6d 100644
--- a/src/prefs.cc
+++ b/src/prefs.cc
@@ -374,6 +374,9 @@ PrefPtr PREF_SOCKET_RECV_BUFFER_SIZE = makePref("socket-recv-buffer-size");
 PrefPtr PREF_MAX_MMAP_LIMIT = makePref("max-mmap-limit");
 // value: true | false
 PrefPtr PREF_STDERR = makePref("stderr");
+// value: true | false
+PrefPtr PREF_KEEP_UNFINISHED_DOWNLOAD_RESULT =
+    makePref("keep-unfinished-download-result");
 
 /**
  * FTP related preferences
diff --git a/src/prefs.h b/src/prefs.h
index cc759f28..fb064727 100644
--- a/src/prefs.h
+++ b/src/prefs.h
@@ -328,6 +328,8 @@ extern PrefPtr PREF_SOCKET_RECV_BUFFER_SIZE;
 extern PrefPtr PREF_MAX_MMAP_LIMIT;
 // value: true | false
 extern PrefPtr PREF_STDERR;
+// value: true | false
+extern PrefPtr PREF_KEEP_UNFINISHED_DOWNLOAD_RESULT;
 
 /**
  * FTP related preferences
diff --git a/src/usage_text.h b/src/usage_text.h
index 4a4bde97..a9240732 100644
--- a/src/usage_text.h
+++ b/src/usage_text.h
@@ -1099,5 +1099,14 @@
 #define TEXT_STDERR \
   _(" --stderr[=true|false]        Redirect all console output that would be\n" \
     "                              otherwise printed in stdout to stderr.")
-
+#define TEXT_KEEP_UNFINISHED_DOWNLOAD_RESULT \
+  _(" --keep-unfinished-download-result[=true|false]\n" \
+    "                              Keep unfinished download results even if doing\n" \
+    "                              so exceeds --max-download-result. This is useful\n" \
+    "                              if all unfinished downloads must be saved in\n" \
+    "                              session file (see --save-session option). Please\n" \
+    "                              keep in mind that there is no upper bound to the\n" \
+    "                              number of unfinished download result to keep.\n" \
+    "                              User should use this option only when they know\n" \
+    "                              the total number of downloads in advance.")
 // clang-format on