2008-06-29 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

	Added gzip, deflate decoding support in HTTP using libz. If 
compiled
	with this feature, aria2 sends "Accept-Encoding: deflate, gzip" 
header
	to a HTTP server. If a server returns "Content-Encoding: gzip" 
or
	"Content-Encoding: deflate" then, aria2 decodes the response 
body on the
	fly and writes decoded data to a local disk.
	* README
	* README.html
	* configure.ac
	* m4/aria2_arg.m4: Added ARIA2_ARG_WITH and ARIA2_ARG_ENABLE, 
they are
	wrapper function for AC_ARG_WITH and AC_ARG_ENABLE respectively.
	* m4/libz.m4
	* src/Decoder.h
	* src/DownloadCommand.cc
	* src/DownloadCommand.h
	* src/Exception.h
	* src/GZipDecoder.cc
	* src/GZipDecoder.h
	* src/HttpHeader.cc
	* src/HttpHeader.h
	* src/HttpRequest.cc
	* src/HttpRequest.h
	* src/HttpResponse.cc
	* src/HttpResponse.h
	* src/HttpResponseCommand.cc
	* src/Makefile.am
	* test/GZipDecoderTest.cc
	* test/HttpRequestTest.cc
	* test/HttpResponseTest.cc
	* test/Makefile.am
	* test/Makefile.in
	* test/gzip_decode_test.gz

1 files changed, 42 insertions, 27 deletions

diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc
index e15023e7..2b4cfa7b 100644
--- a/src/DownloadCommand.cc
+++ b/src/DownloadCommand.cc
@@ -55,6 +55,7 @@
 #include "message.h"
 #include "prefs.h"
 #include "StringFormat.h"
+#include "Decoder.h"
 #ifdef ENABLE_MESSAGE_DIGEST
 # include "MessageDigestHelper.h"
 #endif // ENABLE_MESSAGE_DIGEST
@@ -119,46 +120,47 @@ bool DownloadCommand::executeInternal() {
   }
   socket->readData(buf, bufSize);
 
-  if(transferDecoder.isNull()) {
-    _requestGroup->getPieceStorage()->getDiskAdaptor()->writeData(buf, bufSize,
-								  segment->getPositionToWrite());
-    //logger->debug("bufSize = %d, posToWrite = %lld", bufSize, segment->getPositionToWrite());
-#ifdef ENABLE_MESSAGE_DIGEST
-
-    if(_pieceHashValidationEnabled) {
-      segment->updateHash(segment->getWrittenLength(), buf, bufSize);
-    }
-
-#endif // ENABLE_MESSAGE_DIGEST
+  const SharedHandle<DiskAdaptor>& diskAdaptor =
+    _requestGroup->getPieceStorage()->getDiskAdaptor();
 
-    segment->updateWrittenLength(bufSize);
+  unsigned char* bufFinal;
+  size_t bufSizeFinal;
 
-
-    //logger->debug("overflow length = %d, next posToWrite = %lld", segment->getOverflowLength(), segment->getPositionToWrite());
-    //logger->debug("%s", Util::toHex(segment->getPiece()->getBitfield(),
-    //segment->getPiece()->getBitfieldLength()).c_str());
-    //segment->writtenLength += bufSize;
-    peerStat->updateDownloadLength(bufSize);
+  if(transferDecoder.isNull()) {
+    bufFinal = buf;
+    bufSizeFinal = bufSize;
   } else {
     size_t infbufSize = 16*1024;
     unsigned char infbuf[infbufSize];
     transferDecoder->inflate(infbuf, infbufSize, buf, bufSize);
-    _requestGroup->getPieceStorage()->getDiskAdaptor()->writeData(infbuf, infbufSize,
-								  segment->getPositionToWrite());
+
+    bufFinal = infbuf;
+    bufSizeFinal = infbufSize;
+  }
+
+  if(_contentEncodingDecoder.isNull()) {
+    diskAdaptor->writeData(bufFinal, bufSizeFinal,
+			   segment->getPositionToWrite());
+  } else {
+    std::string out = _contentEncodingDecoder->decode(bufFinal, bufSizeFinal);
+    diskAdaptor->writeData(reinterpret_cast<const unsigned char*>(out.data()),
+			   out.size(),
+			   segment->getPositionToWrite());
+    bufSizeFinal = out.size();
+  }
 
 #ifdef ENABLE_MESSAGE_DIGEST
 
-    if(_pieceHashValidationEnabled) {
-      segment->updateHash(segment->getWrittenLength(), infbuf, infbufSize);
-    }
+  if(_pieceHashValidationEnabled) {
+    segment->updateHash(segment->getWrittenLength(), bufFinal, bufSizeFinal);
+  }
 
 #endif // ENABLE_MESSAGE_DIGEST
 
-    segment->updateWrittenLength(infbufSize);
+  segment->updateWrittenLength(bufSizeFinal);
+  
+  peerStat->updateDownloadLength(bufSize);
 
-    //segment->writtenLength += infbufSize;
-    peerStat->updateDownloadLength(infbufSize);
-  }
   if(_requestGroup->getTotalLength() != 0 && bufSize == 0) {
     throw DlRetryEx(EX_GOT_EOF);
   }
@@ -168,6 +170,13 @@ bool DownloadCommand::executeInternal() {
     if(!transferDecoder.isNull()) transferDecoder->end();
     logger->info(MSG_SEGMENT_DOWNLOAD_COMPLETED, cuid);
 
+    if(!_contentEncodingDecoder.isNull() &&
+       !_contentEncodingDecoder->finished()) {
+      logger->warn("CUID#%d - Transfer was completed, but inflate operation"
+		   " have not finished. Maybe the file is broken in the server"
+		   " side.", cuid);
+    }
+
 #ifdef ENABLE_MESSAGE_DIGEST
 
     {
@@ -280,4 +289,10 @@ void DownloadCommand::setTransferDecoder(const TransferEncodingHandle& transferD
   this->transferDecoder = transferDecoder;
 }
 
+void DownloadCommand::setContentEncodingDecoder
+(const SharedHandle<Decoder>& decoder)
+{
+  _contentEncodingDecoder = decoder;
+}
+
 } // namespace aria2