2010-07-12 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

	Set end byte in Range header if start byte > 0 to get more chance
	to pool socket.
	* src/DefaultPieceStorage.cc
	* src/DefaultPieceStorage.h
	* src/DownloadCommand.cc
	* src/HttpDownloadCommand.cc
	* src/HttpRequest.cc
	* src/HttpRequest.h
	* src/HttpRequestCommand.cc
	* src/PieceStorage.h
	* src/UnknownLengthPieceStorage.h
	* test/DefaultPieceStorageTest.cc
	* test/HttpRequestTest.cc
	* test/MockPieceStorage.h

9 files changed, 53 insertions, 6 deletions

diff --git a/src/DefaultPieceStorage.cc b/src/DefaultPieceStorage.cc
index 7ae7f799..901e5a42 100644
--- a/src/DefaultPieceStorage.cc
+++ b/src/DefaultPieceStorage.cc
@@ -684,4 +684,14 @@ void DefaultPieceStorage::addPieceStats(size_t index)
   pieceStatMan_->addPieceStats(index);
 }
 
+size_t DefaultPieceStorage::getNextUsedIndex(size_t index)
+{
+  for(size_t i = index+1; i < bitfieldMan_->countBlock(); ++i) {
+    if(bitfieldMan_->isUseBitSet(i) || bitfieldMan_->isBitSet(i)) {
+      return i;
+    }
+  }
+  return bitfieldMan_->countBlock();
+}
+
 } // namespace aria2
diff --git a/src/DefaultPieceStorage.h b/src/DefaultPieceStorage.h
index f282deb4..06fc64ae 100644
--- a/src/DefaultPieceStorage.h
+++ b/src/DefaultPieceStorage.h
@@ -227,6 +227,8 @@ public:
                                 size_t newBitfieldLength,
                                 const unsigned char* oldBitfield);
 
+  virtual size_t getNextUsedIndex(size_t index);
+
   /**
    * This method is made private for test purpose only.
    */
diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc
index fe82521c..4b08183c 100644
--- a/src/DownloadCommand.cc
+++ b/src/DownloadCommand.cc
@@ -336,6 +336,11 @@ bool DownloadCommand::prepareForNextSegment() {
       if(!tempSegment->complete()) {
         return prepareForRetry(0);
       }
+      if(getRequestEndOffset() ==
+         getFileEntry()->gtoloff
+         (tempSegment->getPosition()+tempSegment->getLength())) {
+        return prepareForRetry(0);
+      }
       SharedHandle<Segment> nextSegment = getSegmentMan()->getSegment
         (getCuid(), tempSegment->getIndex()+1);
       if(nextSegment.isNull()) {
diff --git a/src/HttpDownloadCommand.cc b/src/HttpDownloadCommand.cc
index deed69d7..9e4798e4 100644
--- a/src/HttpDownloadCommand.cc
+++ b/src/HttpDownloadCommand.cc
@@ -52,7 +52,7 @@
 #include "FileAllocationEntry.h"
 #include "CheckIntegrityEntry.h"
 #include "ServerStatMan.h"
-
+#include "Logger.h"
 namespace aria2 {
 
 HttpDownloadCommand::HttpDownloadCommand
@@ -88,12 +88,13 @@ bool HttpDownloadCommand::prepareForNextSegment() {
     if(getRequest()->isPipeliningEnabled() ||
        (getRequest()->isKeepAliveEnabled() &&
         (
+         // TODO make sure that all decoder is finished to pool socket
          ((!getTransferEncodingDecoder().isNull() &&
            getTransferEncodingDecoder()->finished()) ||
           (!getContentEncodingDecoder().isNull() &&
            getContentEncodingDecoder()->finished())) ||
-         getFileEntry()->getLastOffset() ==
-         getSegments().front()->getPositionToWrite()
+         getRequestEndOffset() ==
+         getFileEntry()->gtoloff(getSegments().front()->getPositionToWrite())
          )
         )
        ) {
diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc
index 4fa090aa..476295fd 100644
--- a/src/HttpRequest.cc
+++ b/src/HttpRequest.cc
@@ -58,7 +58,8 @@ const std::string HttpRequest::USER_AGENT("aria2");
 HttpRequest::HttpRequest():contentEncodingEnabled_(true),
                            userAgent_(USER_AGENT),
                            noCache_(true),
-                           acceptGzip_(false)
+                           acceptGzip_(false),
+                           endOffsetOverride_(false)
 {}
 
 void HttpRequest::setSegment(const SharedHandle<Segment>& segment)
@@ -196,6 +197,9 @@ std::string HttpRequest::createRequest()
     rangeHeader += "-";
     if(request_->isPipeliningEnabled()) {
       rangeHeader += util::itos(getEndByte());
+    } else if(getProtocol() != Request::PROTO_FTP && endOffsetOverride_ > 0) {
+      // FTP via http proxy does not support endbytes 
+      rangeHeader += util::itos(endOffsetOverride_-1);
     }
     builtinHds.push_back(std::make_pair("Range:", rangeHeader));
   }
diff --git a/src/HttpRequest.h b/src/HttpRequest.h
index 1bc2089e..091b5b35 100644
--- a/src/HttpRequest.h
+++ b/src/HttpRequest.h
@@ -87,6 +87,8 @@ private:
 
   bool acceptGzip_;
 
+  off_t endOffsetOverride_;
+
   std::pair<std::string, std::string> getProxyAuthString() const;
 public:
   HttpRequest();
@@ -272,6 +274,11 @@ public:
   {
     return acceptGzip_;
   }
+
+  void setEndOffsetOverride(off_t offset)
+  {
+    endOffsetOverride_ = offset;
+  }
 };
 
 } // namespace aria2
diff --git a/src/HttpRequestCommand.cc b/src/HttpRequestCommand.cc
index 9432b6a8..e0e39a07 100644
--- a/src/HttpRequestCommand.cc
+++ b/src/HttpRequestCommand.cc
@@ -57,6 +57,7 @@
 #include "FileAllocationEntry.h"
 #include "CheckIntegrityEntry.h"
 #include "ServerStatMan.h"
+#include "PieceStorage.h"
 
 namespace aria2 {
 
@@ -87,7 +88,8 @@ createHttpRequest(const SharedHandle<Request>& req,
                   const RequestGroup* rg,
                   const SharedHandle<CookieStorage>& cookieStorage,
                   const SharedHandle<AuthConfigFactory>& authConfigFactory,
-                  const SharedHandle<Request>& proxyRequest)
+                  const SharedHandle<Request>& proxyRequest,
+                  off_t endOffset = 0)
 {
   SharedHandle<HttpRequest> httpRequest(new HttpRequest());
   httpRequest->setUserAgent(option->get(PREF_USER_AGENT));
@@ -110,6 +112,7 @@ createHttpRequest(const SharedHandle<Request>& req,
   } else {
     httpRequest->disableNoCache();
   }
+  httpRequest->setEndOffsetOverride(endOffset);
   return httpRequest;
 }
 
@@ -148,6 +151,15 @@ bool HttpRequestCommand::executeInternal() {
           itr != eoi; ++itr) {
         const SharedHandle<Segment>& segment = *itr;
         if(!httpConnection_->isIssued(segment)) {
+          off_t endOffset = 0;
+          if(getRequestGroup()->getTotalLength() > 0 &&
+             !getPieceStorage().isNull()) {
+            size_t nextIndex =
+              getPieceStorage()->getNextUsedIndex(segment->getIndex());
+            endOffset = std::min
+              (static_cast<off_t>(getFileEntry()->getLength()),
+               getFileEntry()->gtoloff(segment->getSegmentLength()*nextIndex));
+          }
           SharedHandle<HttpRequest> httpRequest
             (createHttpRequest(getRequest(),
                                getFileEntry(),
@@ -157,7 +169,8 @@ bool HttpRequestCommand::executeInternal() {
                                getRequestGroup(),
                                getDownloadEngine()->getCookieStorage(),
                                getDownloadEngine()->getAuthConfigFactory(),
-                               proxyRequest_));
+                               proxyRequest_,
+                               endOffset));
           httpConnection_->sendRequest(httpRequest);
         }
       }
diff --git a/src/PieceStorage.h b/src/PieceStorage.h
index 62734dd0..cc14bed7 100644
--- a/src/PieceStorage.h
+++ b/src/PieceStorage.h
@@ -244,6 +244,10 @@ public:
                                 size_t newBitfieldLength,
                                 const unsigned char* oldBitfield) = 0;
 
+  // Returns index x where all pieces in [index+1, x-1], inclusive,
+  // are not used and not completed. If all pieces after index+1 are
+  // used or completed, returns the number of pieces.
+  virtual size_t getNextUsedIndex(size_t index) = 0;
 };
 
 typedef SharedHandle<PieceStorage> PieceStorageHandle;
diff --git a/src/UnknownLengthPieceStorage.h b/src/UnknownLengthPieceStorage.h
index 1ecfa09f..d90f5644 100644
--- a/src/UnknownLengthPieceStorage.h
+++ b/src/UnknownLengthPieceStorage.h
@@ -273,6 +273,7 @@ public:
                                 size_t newBitfieldLength,
                                 const unsigned char* oldBitfield) {}
 
+  virtual size_t getNextUsedIndex(size_t index) { return 0; }
 
   void setDiskWriterFactory(const SharedHandle<DiskWriterFactory>& diskWriterFactory);
 };