2008-06-29 Tatsuhiro Tsujikawa <tujikawa at rednoah dot com>

	Added gzip, deflate decoding support in HTTP using libz. If 
compiled
	with this feature, aria2 sends "Accept-Encoding: deflate, gzip" 
header
	to a HTTP server. If a server returns "Content-Encoding: gzip" 
or
	"Content-Encoding: deflate" then, aria2 decodes the response 
body on the
	fly and writes decoded data to a local disk.
	* README
	* README.html
	* configure.ac
	* m4/aria2_arg.m4: Added ARIA2_ARG_WITH and ARIA2_ARG_ENABLE, 
they are
	wrapper function for AC_ARG_WITH and AC_ARG_ENABLE respectively.
	* m4/libz.m4
	* src/Decoder.h
	* src/DownloadCommand.cc
	* src/DownloadCommand.h
	* src/Exception.h
	* src/GZipDecoder.cc
	* src/GZipDecoder.h
	* src/HttpHeader.cc
	* src/HttpHeader.h
	* src/HttpRequest.cc
	* src/HttpRequest.h
	* src/HttpResponse.cc
	* src/HttpResponse.h
	* src/HttpResponseCommand.cc
	* src/Makefile.am
	* test/GZipDecoderTest.cc
	* test/HttpRequestTest.cc
	* test/HttpResponseTest.cc
	* test/Makefile.am
	* test/Makefile.in
	* test/gzip_decode_test.gz

15 files changed, 473 insertions, 85 deletions

diff --git a/src/Decoder.h b/src/Decoder.h
new file mode 100644
index 00000000..3b5c0df0
--- /dev/null
+++ b/src/Decoder.h
@@ -0,0 +1,60 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#ifndef _D_DECODER_H_
+#define _D_DECODER_H_
+
+#include "common.h"
+#include <string>
+
+namespace aria2 {
+
+class Decoder {
+public:
+  virtual ~Decoder() {}
+
+  virtual void init() = 0;
+
+  virtual std::string decode(const unsigned char* inbuf, size_t inlen) = 0;
+
+  virtual bool finished() = 0;
+
+  virtual void release() = 0;
+
+  virtual const std::string& getName() const = 0;
+};
+
+} // namespace aria2
+
+#endif // _D_DECODER_H_
diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc
index e15023e7..2b4cfa7b 100644
--- a/src/DownloadCommand.cc
+++ b/src/DownloadCommand.cc
@@ -55,6 +55,7 @@
 #include "message.h"
 #include "prefs.h"
 #include "StringFormat.h"
+#include "Decoder.h"
 #ifdef ENABLE_MESSAGE_DIGEST
 # include "MessageDigestHelper.h"
 #endif // ENABLE_MESSAGE_DIGEST
@@ -119,46 +120,47 @@ bool DownloadCommand::executeInternal() {
   }
   socket->readData(buf, bufSize);
 
-  if(transferDecoder.isNull()) {
-    _requestGroup->getPieceStorage()->getDiskAdaptor()->writeData(buf, bufSize,
-								  segment->getPositionToWrite());
-    //logger->debug("bufSize = %d, posToWrite = %lld", bufSize, segment->getPositionToWrite());
-#ifdef ENABLE_MESSAGE_DIGEST
-
-    if(_pieceHashValidationEnabled) {
-      segment->updateHash(segment->getWrittenLength(), buf, bufSize);
-    }
-
-#endif // ENABLE_MESSAGE_DIGEST
+  const SharedHandle<DiskAdaptor>& diskAdaptor =
+    _requestGroup->getPieceStorage()->getDiskAdaptor();
 
-    segment->updateWrittenLength(bufSize);
+  unsigned char* bufFinal;
+  size_t bufSizeFinal;
 
-
-    //logger->debug("overflow length = %d, next posToWrite = %lld", segment->getOverflowLength(), segment->getPositionToWrite());
-    //logger->debug("%s", Util::toHex(segment->getPiece()->getBitfield(),
-    //segment->getPiece()->getBitfieldLength()).c_str());
-    //segment->writtenLength += bufSize;
-    peerStat->updateDownloadLength(bufSize);
+  if(transferDecoder.isNull()) {
+    bufFinal = buf;
+    bufSizeFinal = bufSize;
   } else {
     size_t infbufSize = 16*1024;
     unsigned char infbuf[infbufSize];
     transferDecoder->inflate(infbuf, infbufSize, buf, bufSize);
-    _requestGroup->getPieceStorage()->getDiskAdaptor()->writeData(infbuf, infbufSize,
-								  segment->getPositionToWrite());
+
+    bufFinal = infbuf;
+    bufSizeFinal = infbufSize;
+  }
+
+  if(_contentEncodingDecoder.isNull()) {
+    diskAdaptor->writeData(bufFinal, bufSizeFinal,
+			   segment->getPositionToWrite());
+  } else {
+    std::string out = _contentEncodingDecoder->decode(bufFinal, bufSizeFinal);
+    diskAdaptor->writeData(reinterpret_cast<const unsigned char*>(out.data()),
+			   out.size(),
+			   segment->getPositionToWrite());
+    bufSizeFinal = out.size();
+  }
 
 #ifdef ENABLE_MESSAGE_DIGEST
 
-    if(_pieceHashValidationEnabled) {
-      segment->updateHash(segment->getWrittenLength(), infbuf, infbufSize);
-    }
+  if(_pieceHashValidationEnabled) {
+    segment->updateHash(segment->getWrittenLength(), bufFinal, bufSizeFinal);
+  }
 
 #endif // ENABLE_MESSAGE_DIGEST
 
-    segment->updateWrittenLength(infbufSize);
+  segment->updateWrittenLength(bufSizeFinal);
+  
+  peerStat->updateDownloadLength(bufSize);
 
-    //segment->writtenLength += infbufSize;
-    peerStat->updateDownloadLength(infbufSize);
-  }
   if(_requestGroup->getTotalLength() != 0 && bufSize == 0) {
     throw DlRetryEx(EX_GOT_EOF);
   }
@@ -168,6 +170,13 @@ bool DownloadCommand::executeInternal() {
     if(!transferDecoder.isNull()) transferDecoder->end();
     logger->info(MSG_SEGMENT_DOWNLOAD_COMPLETED, cuid);
 
+    if(!_contentEncodingDecoder.isNull() &&
+       !_contentEncodingDecoder->finished()) {
+      logger->warn("CUID#%d - Transfer was completed, but inflate operation"
+		   " have not finished. Maybe the file is broken in the server"
+		   " side.", cuid);
+    }
+
 #ifdef ENABLE_MESSAGE_DIGEST
 
     {
@@ -280,4 +289,10 @@ void DownloadCommand::setTransferDecoder(const TransferEncodingHandle& transferD
   this->transferDecoder = transferDecoder;
 }
 
+void DownloadCommand::setContentEncodingDecoder
+(const SharedHandle<Decoder>& decoder)
+{
+  _contentEncodingDecoder = decoder;
+}
+
 } // namespace aria2
diff --git a/src/DownloadCommand.h b/src/DownloadCommand.h
index a87b62ee..e4388e5a 100644
--- a/src/DownloadCommand.h
+++ b/src/DownloadCommand.h
@@ -40,6 +40,7 @@
 namespace aria2 {
 
 class TransferEncoding;
+class Decoder;
 class PeerStat;
 #ifdef ENABLE_MESSAGE_DIGEST
 class MessageDigestContext;
@@ -68,6 +69,8 @@ private:
 protected:
   SharedHandle<TransferEncoding> transferDecoder;
 
+  SharedHandle<Decoder> _contentEncodingDecoder;
+
   virtual bool executeInternal();
 
   virtual bool prepareForNextSegment();
@@ -82,6 +85,8 @@ public:
 
   void setTransferDecoder(const SharedHandle<TransferEncoding>& transferDecoder);
 
+  void setContentEncodingDecoder(const SharedHandle<Decoder>& decoder);
+
   void setMaxDownloadSpeedLimit(unsigned int maxDownloadSpeedLimit) {
     this->maxDownloadSpeedLimit = maxDownloadSpeedLimit;
   }
diff --git a/src/Exception.h b/src/Exception.h
index f788483d..a0fedb39 100644
--- a/src/Exception.h
+++ b/src/Exception.h
@@ -51,7 +51,7 @@ protected:
   virtual SharedHandle<Exception> copy() const = 0;
 
 public:
-  Exception(const std::string& msg);
+  explicit Exception(const std::string& msg);
 
   Exception(const std::string& msg, const Exception& cause);
 
diff --git a/src/GZipDecoder.cc b/src/GZipDecoder.cc
new file mode 100644
index 00000000..81f46d2c
--- /dev/null
+++ b/src/GZipDecoder.cc
@@ -0,0 +1,119 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#include "GZipDecoder.h"
+#include "StringFormat.h"
+#include "DlAbortEx.h"
+
+namespace aria2 {
+
+const std::string GZipDecoder::NAME("GZipDecoder");
+
+GZipDecoder::GZipDecoder():_strm(0), _finished(false) {}
+
+GZipDecoder::~GZipDecoder()
+{
+  release();
+}
+
+void GZipDecoder::init()
+{
+  _strm = new z_stream();
+  _strm->zalloc = Z_NULL;
+  _strm->zfree = Z_NULL;
+  _strm->opaque = Z_NULL;
+  _strm->avail_in = 0;
+  _strm->next_in = Z_NULL;
+
+  // initalize z_stream with gzip/zlib format auto detection enabled.
+  if(Z_OK != inflateInit2(_strm, 47)) {
+    throw DlAbortEx("Initializing z_stream failed.");
+  }
+}
+
+void GZipDecoder::release()
+{
+  if(_strm) {
+    inflateEnd(_strm);
+    _strm = 0;
+  }
+}
+
+std::string GZipDecoder::decode(const unsigned char* in, size_t length)
+{
+  std::string out;
+
+  if(length == 0) {
+    return out;
+  }
+
+  _strm->avail_in = length;
+  _strm->next_in = const_cast<unsigned char*>(in);
+
+  unsigned char outbuf[OUTBUF_LENGTH];
+  while(1) {
+    _strm->avail_out = OUTBUF_LENGTH;
+    _strm->next_out = outbuf;
+
+    int ret = ::inflate(_strm, Z_NO_FLUSH);
+
+    if(ret == Z_STREAM_END) {
+      _finished = true;
+    } else if(ret != Z_OK) {
+      throw DlAbortEx(StringFormat("libz::inflate() failed. cause:%s",
+				   _strm->msg).str());
+    }
+
+    size_t produced = OUTBUF_LENGTH-_strm->avail_out;
+
+    out.insert(out.end(), &outbuf[0], &outbuf[produced]);
+
+    if(_strm->avail_out > 0) {
+      break;
+    }
+  }
+  return out;
+}
+
+bool GZipDecoder::finished()
+{
+  return _finished;
+}
+
+const std::string& GZipDecoder::getName() const
+{
+  return NAME;
+}
+
+} // namespace aria2
diff --git a/src/GZipDecoder.h b/src/GZipDecoder.h
new file mode 100644
index 00000000..79151d5a
--- /dev/null
+++ b/src/GZipDecoder.h
@@ -0,0 +1,72 @@
+/* <!-- copyright */
+/*
+ * aria2 - The high speed download utility
+ *
+ * Copyright (C) 2006 Tatsuhiro Tsujikawa
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * In addition, as a special exception, the copyright holders give
+ * permission to link the code of portions of this program with the
+ * OpenSSL library under certain conditions as described in each
+ * individual source file, and distribute linked combinations
+ * including the two.
+ * You must obey the GNU General Public License in all respects
+ * for all of the code used other than OpenSSL.  If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so.  If you
+ * do not wish to do so, delete this exception statement from your
+ * version.  If you delete this exception statement from all source
+ * files in the program, then also delete it here.
+ */
+/* copyright --> */
+#ifndef _D_GZIP_DECODER_H_
+#define _D_GZIP_DECODER_H_
+
+#include "Decoder.h"
+#include <zlib.h>
+
+namespace aria2 {
+
+// GZipDecoder can decode both gzip and deflate format.
+class GZipDecoder : public Decoder {
+private:
+  z_stream* _strm;
+
+  bool _finished;
+
+  static const size_t OUTBUF_LENGTH = 16*1024;
+  
+  static const std::string NAME;
+public:
+  GZipDecoder();
+
+  virtual ~GZipDecoder();
+
+  virtual void init();
+
+  virtual std::string decode(const unsigned char* inbuf, size_t inlen);
+
+  virtual bool finished();
+
+  virtual void release();
+
+  virtual const std::string& getName() const;
+
+};
+
+} // namespace aria2
+
+#endif // _D_GZIP_DECODER_H_
diff --git a/src/HttpHeader.cc b/src/HttpHeader.cc
index 9194c808..2fcfed9e 100644
--- a/src/HttpHeader.cc
+++ b/src/HttpHeader.cc
@@ -44,12 +44,18 @@ const std::string HttpHeader::LOCATION("Location");
 
 const std::string HttpHeader::TRANSFER_ENCODING("Transfer-Encoding");
 
+const std::string HttpHeader::CONTENT_ENCODING("Content-Encoding");
+
 const std::string HttpHeader::CONTENT_DISPOSITION("Content-Disposition");
 
 const std::string HttpHeader::SET_COOKIE("Set-Cookie");
 
 const std::string HttpHeader::CHUNKED("chunked");
 
+const std::string HttpHeader::GZIP("gzip");
+
+const std::string HttpHeader::DEFLATE("deflate");
+
 const std::string HttpHeader::CONTENT_TYPE("Content-Type");
 
 const std::string HttpHeader::RETRY_AFTER("Retry-After");
diff --git a/src/HttpHeader.h b/src/HttpHeader.h
index c11862e7..286e6eb7 100644
--- a/src/HttpHeader.h
+++ b/src/HttpHeader.h
@@ -83,12 +83,18 @@ public:
 
   static const std::string TRANSFER_ENCODING;
   
+  static const std::string CONTENT_ENCODING;
+
   static const std::string CONTENT_DISPOSITION;
   
   static const std::string SET_COOKIE;
   
   static const std::string CHUNKED;
   
+  static const std::string GZIP;
+
+  static const std::string DEFLATE;
+
   static const std::string CONTENT_TYPE;
   
   static const std::string RETRY_AFTER;
diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc
index 7940315d..f0b4d89d 100644
--- a/src/HttpRequest.cc
+++ b/src/HttpRequest.cc
@@ -55,6 +55,7 @@ HttpRequest::HttpRequest():entityLength(0),
 			   authEnabled(false),
 			   proxyEnabled(false),
 			   proxyAuthEnabled(false),
+			   _contentEncodingEnabled(true),
 			   userAgent(USER_AGENT)
 {}
 
@@ -155,6 +156,16 @@ std::string HttpRequest::createRequest() const
   }
   requestLine += "\r\n";
 
+  if(_contentEncodingEnabled) {
+    std::string acceptableEncodings;
+#ifdef HAVE_LIBZ
+    acceptableEncodings += "deflate, gzip";
+#endif // HAVE_LIBZ
+    if(!acceptableEncodings.empty()) {
+      requestLine += "Accept-Encoding: "+acceptableEncodings+"\r\n";
+    }
+  }
+
   requestLine +=
     "Host: "+getHostText(getHost(), getPort())+"\r\n"+
     "Pragma: no-cache\r\n"+
@@ -235,6 +246,16 @@ std::string HttpRequest::getProxyAuthString() const {
     Base64::encode(AuthConfigFactorySingleton::instance()->createAuthConfigForHttpProxy(request)->getAuthText())+"\r\n";
 }
 
+void HttpRequest::enableContentEncoding()
+{
+  _contentEncodingEnabled = true;
+}
+
+void HttpRequest::disableContentEncoding()
+{
+  _contentEncodingEnabled = false;
+}
+
 void HttpRequest::addHeader(const std::string& headersString)
 {
   std::deque<std::string> headers;
diff --git a/src/HttpRequest.h b/src/HttpRequest.h
index ba5c55a9..2dce737b 100644
--- a/src/HttpRequest.h
+++ b/src/HttpRequest.h
@@ -64,6 +64,8 @@ private:
 
   bool proxyAuthEnabled;
 
+  bool _contentEncodingEnabled;
+
   std::string userAgent;
 
   std::deque<std::string> _headers;
@@ -166,6 +168,10 @@ public:
     this->authEnabled = authEnabled;
   }
 
+  void enableContentEncoding();
+
+  void disableContentEncoding();
+
   void setUserAgent(const std::string& userAgent)
   {
     this->userAgent = userAgent;
diff --git a/src/HttpResponse.cc b/src/HttpResponse.cc
index d51a806c..5c9c2df4 100644
--- a/src/HttpResponse.cc
+++ b/src/HttpResponse.cc
@@ -47,6 +47,10 @@
 #include "DlAbortEx.h"
 #include "StringFormat.h"
 #include "A2STR.h"
+#include "Decoder.h"
+#ifdef HAVE_LIBZ
+# include "GZipDecoder.h"
+#endif // HAVE_LIBZ
 #include <deque>
 
 namespace aria2 {
@@ -150,6 +154,27 @@ TransferEncodingHandle HttpResponse::getTransferDecoder() const
   return SharedHandle<TransferEncoding>();
 }
 
+bool HttpResponse::isContentEncodingSpecified() const
+{
+  return httpHeader->defined(HttpHeader::CONTENT_ENCODING);
+}
+
+const std::string& HttpResponse::getContentEncoding() const
+{
+  return httpHeader->getFirst(HttpHeader::CONTENT_ENCODING);
+}
+
+SharedHandle<Decoder> HttpResponse::getContentEncodingDecoder() const
+{
+#ifdef HAVE_LIBZ
+  if(getContentEncoding() == HttpHeader::GZIP ||
+     getContentEncoding() == HttpHeader::DEFLATE) {
+    return SharedHandle<Decoder>(new GZipDecoder());
+  }
+#endif // HAVE_LIBZ
+  return SharedHandle<Decoder>();
+}
+
 uint64_t HttpResponse::getContentLength() const
 {
   if(httpHeader.isNull()) {
diff --git a/src/HttpResponse.h b/src/HttpResponse.h
index 3aeac22a..71d72c43 100644
--- a/src/HttpResponse.h
+++ b/src/HttpResponse.h
@@ -46,6 +46,7 @@ class HttpRequest;
 class HttpHeader;
 class TransferEncoding;
 class Logger;
+class Decoder;
 
 class HttpResponse {
 private:
@@ -85,6 +86,12 @@ public:
 
   SharedHandle<TransferEncoding> getTransferDecoder() const;
 
+  bool isContentEncodingSpecified() const;
+
+  const std::string& getContentEncoding() const;
+
+  SharedHandle<Decoder> getContentEncodingDecoder() const;
+
   uint64_t getContentLength() const;
 
   uint64_t getEntityLength() const;
diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc
index b7522bd6..0cdffcc5 100644
--- a/src/HttpResponseCommand.cc
+++ b/src/HttpResponseCommand.cc
@@ -62,6 +62,8 @@
 #include "StringFormat.h"
 #include "HttpSkipResponseCommand.h"
 #include "HttpHeader.h"
+#include "Decoder.h"
+#include "LogFactory.h"
 
 namespace aria2 {
 
@@ -194,10 +196,32 @@ static SharedHandle<TransferEncoding> getTransferEncoding
   return enc;
 }
 
+static SharedHandle<Decoder> getContentEncodingDecoder
+(const SharedHandle<HttpResponse>& httpResponse)
+{
+  SharedHandle<Decoder> decoder;
+  if(httpResponse->isContentEncodingSpecified()) {
+    decoder = httpResponse->getContentEncodingDecoder();
+    if(decoder.isNull()) {
+      LogFactory::getInstance()->info
+	("Content-Encoding %s is specified, but the current implementation"
+	 "doesn't support it. The decoding process is skipped and the"
+	 "downloaded content will be still encoded.",
+	 httpResponse->getContentEncoding().c_str());
+    } else {
+      decoder->init();
+    }
+  }
+  return decoder;
+}
+
 bool HttpResponseCommand::skipResponseBody
 (const SharedHandle<HttpResponse>& httpResponse)
 {
   SharedHandle<TransferEncoding> enc(getTransferEncoding(httpResponse));
+  // We don't use Content-Encoding here because this response body is just
+  // thrown away.
+
   HttpSkipResponseCommand* command = new HttpSkipResponseCommand
     (cuid, req, _requestGroup, httpConnection, httpResponse, e, socket);
   command->setTransferDecoder(enc);
@@ -216,7 +240,10 @@ bool HttpResponseCommand::skipResponseBody
 
 HttpDownloadCommand* HttpResponseCommand::createHttpDownloadCommand(const HttpResponseHandle& httpResponse)
 {
-  TransferEncodingHandle enc(getTransferEncoding(httpResponse));
+  TransferEncodingHandle enc = getTransferEncoding(httpResponse);
+  SharedHandle<Decoder> contentEncodingDecoder =
+    getContentEncodingDecoder(httpResponse);
+
   HttpDownloadCommand* command =
     new HttpDownloadCommand(cuid, req, _requestGroup, httpConnection, e, socket);
   command->setMaxDownloadSpeedLimit(e->option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT));
@@ -224,6 +251,14 @@ HttpDownloadCommand* HttpResponseCommand::createHttpDownloadCommand(const HttpRe
   command->setLowestDownloadSpeedLimit(e->option->getAsInt(PREF_LOWEST_SPEED_LIMIT));
   command->setTransferDecoder(enc);
 
+  if(!contentEncodingDecoder.isNull()) {
+    command->setContentEncodingDecoder(contentEncodingDecoder);
+    // Since the compressed file's length are returned in the response header
+    // and the decompressed file size is unknown at this point, disable file
+    // allocation here.
+    _requestGroup->setFileAllocationEnabled(false);
+  }
+
   return command;
 }
 
diff --git a/src/Makefile.am b/src/Makefile.am
index 7fb7f5ed..937424bc 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -188,7 +188,12 @@ SRCS =  Socket.h\
 	InitiateConnectionCommand.cc InitiateConnectionCommand.h\
 	FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h\
 	A2STR.cc A2STR.h\
-	RarestPieceSelector.cc RarestPieceSelector.h
+	RarestPieceSelector.cc RarestPieceSelector.h\
+	Decoder.h
+
+if HAVE_LIBZ
+SRCS += GZipDecoder.cc GZipDecoder.h
+endif # HAVE_LIBZ
 
 if ENABLE_ASYNC_DNS
 SRCS += AsyncNameResolver.cc AsyncNameResolver.h
@@ -462,10 +467,10 @@ noinst_LIBRARIES = libaria2c.a
 libaria2c_a_SOURCES = $(SRCS)
 aria2c_LDADD = libaria2c.a @LIBINTL@ @ALLOCA@ @LIBGNUTLS_LIBS@\
 	@LIBGCRYPT_LIBS@ @OPENSSL_LIBS@ @XML_LIBS@ @LIBARES_LIBS@\
-	@LIBCARES_LIBS@ @WINSOCK_LIBS@ @LIBEXPAT_LIBS@ #-lprofiler
+	@LIBCARES_LIBS@ @WINSOCK_LIBS@ @LIBEXPAT_LIBS@ @LIBZ_LIBS@ #-lprofiler
 #aria2c_LDFLAGS =  -pg
 AM_CPPFLAGS =  -Wall\
 	-I../lib -I../intl -I$(top_srcdir)/intl\
 	@LIBGNUTLS_CFLAGS@ @LIBGCRYPT_CFLAGS@ @OPENSSL_CFLAGS@ @XML_CPPFLAGS@\
 	@LIBARES_CPPFLAGS@ @LIBCARES_CPPFLAGS@ @LIBEXPAT_CPPFLAGS@\
-	-DLOCALEDIR=\"$(localedir)\" @DEFS@ #-pg
\ No newline at end of file
+	@LIBZ_CPPFLAGS@	-DLOCALEDIR=\"$(localedir)\" @DEFS@ #-pg
\ No newline at end of file
diff --git a/src/Makefile.in b/src/Makefile.in
index 38bd3d7b..d15845e4 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -35,8 +35,9 @@ build_triplet = @build@
 host_triplet = @host@
 target_triplet = @target@
 bin_PROGRAMS = aria2c$(EXEEXT)
-@ENABLE_ASYNC_DNS_TRUE@am__append_1 = AsyncNameResolver.cc AsyncNameResolver.h
-@ENABLE_MESSAGE_DIGEST_TRUE@am__append_2 = IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\
+@HAVE_LIBZ_TRUE@am__append_1 = GZipDecoder.cc GZipDecoder.h
+@ENABLE_ASYNC_DNS_TRUE@am__append_2 = AsyncNameResolver.cc AsyncNameResolver.h
+@ENABLE_MESSAGE_DIGEST_TRUE@am__append_3 = IteratableChunkChecksumValidator.cc IteratableChunkChecksumValidator.h\
 @ENABLE_MESSAGE_DIGEST_TRUE@	IteratableChecksumValidator.cc IteratableChecksumValidator.h\
 @ENABLE_MESSAGE_DIGEST_TRUE@	CheckIntegrityCommand.cc CheckIntegrityCommand.h\
 @ENABLE_MESSAGE_DIGEST_TRUE@	ChecksumCheckIntegrityEntry.cc ChecksumCheckIntegrityEntry.h\
@@ -45,7 +46,7 @@ bin_PROGRAMS = aria2c$(EXEEXT)
 @ENABLE_MESSAGE_DIGEST_TRUE@	Checksum.h\
 @ENABLE_MESSAGE_DIGEST_TRUE@	ChunkChecksum.h
 
-@ENABLE_BITTORRENT_TRUE@am__append_3 = MetaEntry.h\
+@ENABLE_BITTORRENT_TRUE@am__append_4 = MetaEntry.h\
 @ENABLE_BITTORRENT_TRUE@	Data.cc Data.h\
 @ENABLE_BITTORRENT_TRUE@	Dictionary.cc Dictionary.h\
 @ENABLE_BITTORRENT_TRUE@	List.cc List.h\
@@ -222,7 +223,7 @@ bin_PROGRAMS = aria2c$(EXEEXT)
 @ENABLE_BITTORRENT_TRUE@	BtLeecherStateChoke.cc BtLeecherStateChoke.h\
 @ENABLE_BITTORRENT_TRUE@	BtSeederStateChoke.cc BtSeederStateChoke.h
 
-@ENABLE_METALINK_TRUE@am__append_4 = Metalinker.cc Metalinker.h\
+@ENABLE_METALINK_TRUE@am__append_5 = Metalinker.cc Metalinker.h\
 @ENABLE_METALINK_TRUE@	MetalinkEntry.cc MetalinkEntry.h\
 @ENABLE_METALINK_TRUE@	MetalinkResource.cc MetalinkResource.h\
 @ENABLE_METALINK_TRUE@	MetalinkProcessor.h\
@@ -250,38 +251,38 @@ bin_PROGRAMS = aria2c$(EXEEXT)
 @ENABLE_METALINK_TRUE@	MetalinkPostDownloadHandler.cc MetalinkPostDownloadHandler.h\
 @ENABLE_METALINK_TRUE@	MetalinkHelper.cc MetalinkHelper.h
 
-@ENABLE_LIBXML2_TRUE@am__append_5 = XML2SAXMetalinkProcessor.cc XML2SAXMetalinkProcessor.h
-@ENABLE_LIBEXPAT_TRUE@am__append_6 = ExpatMetalinkProcessor.cc ExpatMetalinkProcessor.h
-@HAVE_ASCTIME_R_FALSE@am__append_7 = asctime_r.c asctime_r.h
-@HAVE_BASENAME_FALSE@am__append_8 = libgen.c libgen.h
-@HAVE_GETADDRINFO_FALSE@am__append_9 = getaddrinfo.c getaddrinfo.h
-@HAVE_GAI_STRERROR_FALSE@am__append_10 = gai_strerror.c gai_strerror.h
-@HAVE_GETTIMEOFDAY_FALSE@am__append_11 = gettimeofday.c gettimeofday.h
-@HAVE_INET_ATON_FALSE@am__append_12 = inet_aton.c inet_aton.h
-@HAVE_LOCALTIME_R_FALSE@am__append_13 = localtime_r.c localtime_r.h
-@HAVE_STRPTIME_FALSE@am__append_14 = strptime.c strptime.h
-@HAVE_TIMEGM_FALSE@am__append_15 = timegm.c timegm.h
+@ENABLE_LIBXML2_TRUE@am__append_6 = XML2SAXMetalinkProcessor.cc XML2SAXMetalinkProcessor.h
+@ENABLE_LIBEXPAT_TRUE@am__append_7 = ExpatMetalinkProcessor.cc ExpatMetalinkProcessor.h
+@HAVE_ASCTIME_R_FALSE@am__append_8 = asctime_r.c asctime_r.h
+@HAVE_BASENAME_FALSE@am__append_9 = libgen.c libgen.h
+@HAVE_GETADDRINFO_FALSE@am__append_10 = getaddrinfo.c getaddrinfo.h
+@HAVE_GAI_STRERROR_FALSE@am__append_11 = gai_strerror.c gai_strerror.h
+@HAVE_GETTIMEOFDAY_FALSE@am__append_12 = gettimeofday.c gettimeofday.h
+@HAVE_INET_ATON_FALSE@am__append_13 = inet_aton.c inet_aton.h
+@HAVE_LOCALTIME_R_FALSE@am__append_14 = localtime_r.c localtime_r.h
+@HAVE_STRPTIME_FALSE@am__append_15 = strptime.c strptime.h
+@HAVE_TIMEGM_FALSE@am__append_16 = timegm.c timegm.h
 subdir = src
 DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in alloca.c
 ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
-am__aclocal_m4_deps = $(top_srcdir)/m4/codeset.m4 \
-	$(top_srcdir)/m4/gettext.m4 $(top_srcdir)/m4/glibc2.m4 \
-	$(top_srcdir)/m4/glibc21.m4 $(top_srcdir)/m4/iconv.m4 \
-	$(top_srcdir)/m4/intdiv0.m4 $(top_srcdir)/m4/intl.m4 \
-	$(top_srcdir)/m4/intlmacosx.m4 $(top_srcdir)/m4/intmax.m4 \
-	$(top_srcdir)/m4/inttypes-pri.m4 \
+am__aclocal_m4_deps = $(top_srcdir)/m4/aria2_arg.m4 \
+	$(top_srcdir)/m4/codeset.m4 $(top_srcdir)/m4/gettext.m4 \
+	$(top_srcdir)/m4/glibc2.m4 $(top_srcdir)/m4/glibc21.m4 \
+	$(top_srcdir)/m4/iconv.m4 $(top_srcdir)/m4/intdiv0.m4 \
+	$(top_srcdir)/m4/intl.m4 $(top_srcdir)/m4/intlmacosx.m4 \
+	$(top_srcdir)/m4/intmax.m4 $(top_srcdir)/m4/inttypes-pri.m4 \
 	$(top_srcdir)/m4/inttypes_h.m4 $(top_srcdir)/m4/lcmessage.m4 \
 	$(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \
 	$(top_srcdir)/m4/lib-prefix.m4 $(top_srcdir)/m4/libares.m4 \
 	$(top_srcdir)/m4/libcares.m4 $(top_srcdir)/m4/libexpat.m4 \
-	$(top_srcdir)/m4/lock.m4 $(top_srcdir)/m4/longlong.m4 \
-	$(top_srcdir)/m4/nls.m4 $(top_srcdir)/m4/openssl.m4 \
-	$(top_srcdir)/m4/po.m4 $(top_srcdir)/m4/printf-posix.m4 \
-	$(top_srcdir)/m4/progtest.m4 $(top_srcdir)/m4/size_max.m4 \
-	$(top_srcdir)/m4/stdint_h.m4 $(top_srcdir)/m4/uintmax_t.m4 \
-	$(top_srcdir)/m4/visibility.m4 $(top_srcdir)/m4/wchar_t.m4 \
-	$(top_srcdir)/m4/wint_t.m4 $(top_srcdir)/m4/xsize.m4 \
-	$(top_srcdir)/configure.ac
+	$(top_srcdir)/m4/libz.m4 $(top_srcdir)/m4/lock.m4 \
+	$(top_srcdir)/m4/longlong.m4 $(top_srcdir)/m4/nls.m4 \
+	$(top_srcdir)/m4/openssl.m4 $(top_srcdir)/m4/po.m4 \
+	$(top_srcdir)/m4/printf-posix.m4 $(top_srcdir)/m4/progtest.m4 \
+	$(top_srcdir)/m4/size_max.m4 $(top_srcdir)/m4/stdint_h.m4 \
+	$(top_srcdir)/m4/uintmax_t.m4 $(top_srcdir)/m4/visibility.m4 \
+	$(top_srcdir)/m4/wchar_t.m4 $(top_srcdir)/m4/wint_t.m4 \
+	$(top_srcdir)/m4/xsize.m4 $(top_srcdir)/configure.ac
 am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
 	$(ACLOCAL_M4)
 mkinstalldirs = $(SHELL) $(top_srcdir)/mkinstalldirs
@@ -406,8 +407,8 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
 	InitiateConnectionCommand.cc InitiateConnectionCommand.h \
 	FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h \
 	A2STR.cc A2STR.h RarestPieceSelector.cc RarestPieceSelector.h \
-	AsyncNameResolver.cc AsyncNameResolver.h \
-	IteratableChunkChecksumValidator.cc \
+	Decoder.h GZipDecoder.cc GZipDecoder.h AsyncNameResolver.cc \
+	AsyncNameResolver.h IteratableChunkChecksumValidator.cc \
 	IteratableChunkChecksumValidator.h \
 	IteratableChecksumValidator.cc IteratableChecksumValidator.h \
 	CheckIntegrityCommand.cc CheckIntegrityCommand.h \
@@ -563,14 +564,15 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
 	gai_strerror.h gettimeofday.c gettimeofday.h inet_aton.c \
 	inet_aton.h localtime_r.c localtime_r.h strptime.c strptime.h \
 	timegm.c timegm.h
-@ENABLE_ASYNC_DNS_TRUE@am__objects_1 = AsyncNameResolver.$(OBJEXT)
-@ENABLE_MESSAGE_DIGEST_TRUE@am__objects_2 = IteratableChunkChecksumValidator.$(OBJEXT) \
+@HAVE_LIBZ_TRUE@am__objects_1 = GZipDecoder.$(OBJEXT)
+@ENABLE_ASYNC_DNS_TRUE@am__objects_2 = AsyncNameResolver.$(OBJEXT)
+@ENABLE_MESSAGE_DIGEST_TRUE@am__objects_3 = IteratableChunkChecksumValidator.$(OBJEXT) \
 @ENABLE_MESSAGE_DIGEST_TRUE@	IteratableChecksumValidator.$(OBJEXT) \
 @ENABLE_MESSAGE_DIGEST_TRUE@	CheckIntegrityCommand.$(OBJEXT) \
 @ENABLE_MESSAGE_DIGEST_TRUE@	ChecksumCheckIntegrityEntry.$(OBJEXT) \
 @ENABLE_MESSAGE_DIGEST_TRUE@	messageDigest.$(OBJEXT) \
 @ENABLE_MESSAGE_DIGEST_TRUE@	MessageDigestHelper.$(OBJEXT)
-@ENABLE_BITTORRENT_TRUE@am__objects_3 = Data.$(OBJEXT) \
+@ENABLE_BITTORRENT_TRUE@am__objects_4 = Data.$(OBJEXT) \
 @ENABLE_BITTORRENT_TRUE@	Dictionary.$(OBJEXT) List.$(OBJEXT) \
 @ENABLE_BITTORRENT_TRUE@	MetaFileUtil.$(OBJEXT) \
 @ENABLE_BITTORRENT_TRUE@	BencodeVisitor.$(OBJEXT) \
@@ -683,7 +685,7 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
 @ENABLE_BITTORRENT_TRUE@	MSEHandshake.$(OBJEXT) \
 @ENABLE_BITTORRENT_TRUE@	BtLeecherStateChoke.$(OBJEXT) \
 @ENABLE_BITTORRENT_TRUE@	BtSeederStateChoke.$(OBJEXT)
-@ENABLE_METALINK_TRUE@am__objects_4 = Metalinker.$(OBJEXT) \
+@ENABLE_METALINK_TRUE@am__objects_5 = Metalinker.$(OBJEXT) \
 @ENABLE_METALINK_TRUE@	MetalinkEntry.$(OBJEXT) \
 @ENABLE_METALINK_TRUE@	MetalinkResource.$(OBJEXT) \
 @ENABLE_METALINK_TRUE@	MetalinkProcessorFactory.$(OBJEXT) \
@@ -708,20 +710,20 @@ am__libaria2c_a_SOURCES_DIST = Socket.h SocketCore.cc SocketCore.h \
 @ENABLE_METALINK_TRUE@	Metalink2RequestGroup.$(OBJEXT) \
 @ENABLE_METALINK_TRUE@	MetalinkPostDownloadHandler.$(OBJEXT) \
 @ENABLE_METALINK_TRUE@	MetalinkHelper.$(OBJEXT)
-@ENABLE_LIBXML2_TRUE@am__objects_5 =  \
+@ENABLE_LIBXML2_TRUE@am__objects_6 =  \
 @ENABLE_LIBXML2_TRUE@	XML2SAXMetalinkProcessor.$(OBJEXT)
-@ENABLE_LIBEXPAT_TRUE@am__objects_6 =  \
+@ENABLE_LIBEXPAT_TRUE@am__objects_7 =  \
 @ENABLE_LIBEXPAT_TRUE@	ExpatMetalinkProcessor.$(OBJEXT)
-@HAVE_ASCTIME_R_FALSE@am__objects_7 = asctime_r.$(OBJEXT)
-@HAVE_BASENAME_FALSE@am__objects_8 = libgen.$(OBJEXT)
-@HAVE_GETADDRINFO_FALSE@am__objects_9 = getaddrinfo.$(OBJEXT)
-@HAVE_GAI_STRERROR_FALSE@am__objects_10 = gai_strerror.$(OBJEXT)
-@HAVE_GETTIMEOFDAY_FALSE@am__objects_11 = gettimeofday.$(OBJEXT)
-@HAVE_INET_ATON_FALSE@am__objects_12 = inet_aton.$(OBJEXT)
-@HAVE_LOCALTIME_R_FALSE@am__objects_13 = localtime_r.$(OBJEXT)
-@HAVE_STRPTIME_FALSE@am__objects_14 = strptime.$(OBJEXT)
-@HAVE_TIMEGM_FALSE@am__objects_15 = timegm.$(OBJEXT)
-am__objects_16 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \
+@HAVE_ASCTIME_R_FALSE@am__objects_8 = asctime_r.$(OBJEXT)
+@HAVE_BASENAME_FALSE@am__objects_9 = libgen.$(OBJEXT)
+@HAVE_GETADDRINFO_FALSE@am__objects_10 = getaddrinfo.$(OBJEXT)
+@HAVE_GAI_STRERROR_FALSE@am__objects_11 = gai_strerror.$(OBJEXT)
+@HAVE_GETTIMEOFDAY_FALSE@am__objects_12 = gettimeofday.$(OBJEXT)
+@HAVE_INET_ATON_FALSE@am__objects_13 = inet_aton.$(OBJEXT)
+@HAVE_LOCALTIME_R_FALSE@am__objects_14 = localtime_r.$(OBJEXT)
+@HAVE_STRPTIME_FALSE@am__objects_15 = strptime.$(OBJEXT)
+@HAVE_TIMEGM_FALSE@am__objects_16 = timegm.$(OBJEXT)
+am__objects_17 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \
 	AbstractCommand.$(OBJEXT) \
 	InitiateConnectionCommandFactory.$(OBJEXT) \
 	DownloadCommand.$(OBJEXT) \
@@ -797,8 +799,8 @@ am__objects_16 = SocketCore.$(OBJEXT) Command.$(OBJEXT) \
 	$(am__objects_5) $(am__objects_6) $(am__objects_7) \
 	$(am__objects_8) $(am__objects_9) $(am__objects_10) \
 	$(am__objects_11) $(am__objects_12) $(am__objects_13) \
-	$(am__objects_14) $(am__objects_15)
-am_libaria2c_a_OBJECTS = $(am__objects_16)
+	$(am__objects_14) $(am__objects_15) $(am__objects_16)
+am_libaria2c_a_OBJECTS = $(am__objects_17)
 libaria2c_a_OBJECTS = $(am_libaria2c_a_OBJECTS)
 am__installdirs = "$(DESTDIR)$(bindir)"
 binPROGRAMS_INSTALL = $(INSTALL_PROGRAM)
@@ -898,6 +900,8 @@ LIBPTH = @LIBPTH@
 LIBPTH_PREFIX = @LIBPTH_PREFIX@
 LIBS = @LIBS@
 LIBTHREAD = @LIBTHREAD@
+LIBZ_CPPFLAGS = @LIBZ_CPPFLAGS@
+LIBZ_LIBS = @LIBZ_LIBS@
 LTLIBC = @LTLIBC@
 LTLIBICONV = @LTLIBICONV@
 LTLIBINTL = @LTLIBINTL@
@@ -1113,23 +1117,24 @@ SRCS = Socket.h SocketCore.cc SocketCore.h BinaryStream.h Command.cc \
 	InitiateConnectionCommand.cc InitiateConnectionCommand.h \
 	FtpFinishDownloadCommand.cc FtpFinishDownloadCommand.h \
 	A2STR.cc A2STR.h RarestPieceSelector.cc RarestPieceSelector.h \
-	$(am__append_1) $(am__append_2) $(am__append_3) \
+	Decoder.h $(am__append_1) $(am__append_2) $(am__append_3) \
 	$(am__append_4) $(am__append_5) $(am__append_6) \
 	$(am__append_7) $(am__append_8) $(am__append_9) \
 	$(am__append_10) $(am__append_11) $(am__append_12) \
-	$(am__append_13) $(am__append_14) $(am__append_15)
+	$(am__append_13) $(am__append_14) $(am__append_15) \
+	$(am__append_16)
 noinst_LIBRARIES = libaria2c.a
 libaria2c_a_SOURCES = $(SRCS)
 aria2c_LDADD = libaria2c.a @LIBINTL@ @ALLOCA@ @LIBGNUTLS_LIBS@\
 	@LIBGCRYPT_LIBS@ @OPENSSL_LIBS@ @XML_LIBS@ @LIBARES_LIBS@\
-	@LIBCARES_LIBS@ @WINSOCK_LIBS@ @LIBEXPAT_LIBS@ #-lprofiler
+	@LIBCARES_LIBS@ @WINSOCK_LIBS@ @LIBEXPAT_LIBS@ @LIBZ_LIBS@ #-lprofiler
 
 #aria2c_LDFLAGS =  -pg
 AM_CPPFLAGS = -Wall\
 	-I../lib -I../intl -I$(top_srcdir)/intl\
 	@LIBGNUTLS_CFLAGS@ @LIBGCRYPT_CFLAGS@ @OPENSSL_CFLAGS@ @XML_CPPFLAGS@\
 	@LIBARES_CPPFLAGS@ @LIBCARES_CPPFLAGS@ @LIBEXPAT_CPPFLAGS@\
-	-DLOCALEDIR=\"$(localedir)\" @DEFS@ #-pg
+	@LIBZ_CPPFLAGS@	-DLOCALEDIR=\"$(localedir)\" @DEFS@ #-pg
 
 all: all-am
 
@@ -1365,6 +1370,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpNegotiationCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpTunnelRequestCommand.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FtpTunnelResponseCommand.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GZipDecoder.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/GrowSegment.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HandshakeExtensionMessage.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/HashMetalinkParserState.Po@am__quote@