2010-04-02 Tatsuhiro Tsujikawa <t-tujikawa@users.sourceforge.net>

	Don't send "Accept: default, gzip" by default. This is because
	some server responds with "Content-Encoding: gzip" for files which
	itself is gzipped file and aria2 inflates them. This is a problem
	if user don't want to inflate the file. Apparently this is server
	configuration error, but I cannot do anything about this. So turn
	this off.  Added --http-accept-gzip option. If true is given to
	this option, aria2 sends 'Accept: deflate, gzip' request header
	and inflates response if remote server responds with
	'Content-Encoding: gzip' or 'Content-Encoding: deflate'.  This
	indicates we removed extension tgz hack in order not to inflate
	files with tgz extensions.
	* doc/aria2c.1.txt
	* src/HttpRequest.cc
	* src/HttpRequest.h
	* src/HttpRequestCommand.cc
	* src/HttpResponseCommand.cc
	* src/OptionHandlerFactory.cc
	* src/prefs.cc
	* src/prefs.h
	* src/usage_text.h
	* test/HttpRequestTest.cc

8 files changed, 55 insertions, 18 deletions

diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc
index e3123eb6..c6c0afbf 100644
--- a/src/HttpRequest.cc
+++ b/src/HttpRequest.cc
@@ -57,7 +57,8 @@ const std::string HttpRequest::USER_AGENT("aria2");
 
 HttpRequest::HttpRequest():_contentEncodingEnabled(true),
                            userAgent(USER_AGENT),
-                           _noCache(true)
+                           _noCache(true),
+                           _acceptGzip(false)
 {}
 
 void HttpRequest::setSegment(const SharedHandle<Segment>& segment)
@@ -169,7 +170,9 @@ std::string HttpRequest::createRequest()
   if(_contentEncodingEnabled) {
     std::string acceptableEncodings;
 #ifdef HAVE_LIBZ
-    acceptableEncodings += "deflate, gzip";
+    if(_acceptGzip) {
+      acceptableEncodings += "deflate, gzip";
+    }
 #endif // HAVE_LIBZ
     if(!acceptableEncodings.empty()) {
       builtinHds.push_back
diff --git a/src/HttpRequest.h b/src/HttpRequest.h
index 95afbb33..af3765e7 100644
--- a/src/HttpRequest.h
+++ b/src/HttpRequest.h
@@ -85,6 +85,8 @@ private:
 
   bool _noCache;
 
+  bool _acceptGzip;
+
   std::pair<std::string, std::string> getProxyAuthString() const;
 public:
   HttpRequest();
@@ -255,6 +257,21 @@ public:
   {
     _noCache = false;
   }
+
+  void enableAcceptGZip()
+  {
+    _acceptGzip = true;
+  }
+
+  void disableAcceptGZip()
+  {
+    _acceptGzip = false;
+  }
+
+  bool acceptGZip() const
+  {
+    return _acceptGzip;
+  }
 };
 
 } // namespace aria2
diff --git a/src/HttpRequestCommand.cc b/src/HttpRequestCommand.cc
index 407b0c38..4a7d3c8b 100644
--- a/src/HttpRequestCommand.cc
+++ b/src/HttpRequestCommand.cc
@@ -96,6 +96,11 @@ createHttpRequest(const SharedHandle<Request>& req,
   httpRequest->setProxyRequest(proxyRequest);
   httpRequest->addAcceptType(rg->getAcceptTypes().begin(),
                              rg->getAcceptTypes().end());
+  if(option->getAsBool(PREF_HTTP_ACCEPT_GZIP)) {
+    httpRequest->enableAcceptGZip();
+  } else {
+    httpRequest->disableAcceptGZip();
+  }
   if(option->getAsBool(PREF_HTTP_NO_CACHE)) {
     httpRequest->enableNoCache();
   } else {
diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc
index f410adfe..5a49881d 100644
--- a/src/HttpResponseCommand.cc
+++ b/src/HttpResponseCommand.cc
@@ -192,25 +192,19 @@ void HttpResponseCommand::updateLastModifiedTime(const Time& lastModified)
   }
 }
 
-static bool fileIsGzipped(const SharedHandle<HttpResponse>& httpResponse)
-{
-  std::string filename =
-    util::toLower(httpResponse->getHttpRequest()->getRequest()->getFile());
-  return util::endsWith(filename, ".gz") || util::endsWith(filename, ".tgz");
-}
-
 bool HttpResponseCommand::shouldInflateContentEncoding
 (const SharedHandle<HttpResponse>& httpResponse)
 {
-  // Basically, on the fly inflation cannot be made with segment download,
-  // because in each segment we don't know where the date should be written.
-  // So turn off segmented downloading.
-  // Meanwhile, Some server returns content-encoding: gzip for .tgz files.
-  // I think those files should not be inflated by clients, because it is the
-  // original format of those files. So I made filename ending ".gz" or ".tgz"
-  // (case-insensitive) not inflated.
-  return httpResponse->isContentEncodingSpecified() &&
-    !fileIsGzipped(httpResponse);
+  // Basically, on the fly inflation cannot be made with segment
+  // download, because in each segment we don't know where the date
+  // should be written.  So turn off segmented downloading.
+  // Meanwhile, Some server returns content-encoding: gzip for .tgz
+  // files.  I think those files should not be inflated by clients,
+  // because it is the original format of those files. Current
+  // implementation just inflates these files nonetheless.
+  const std::string& ce = httpResponse->getContentEncoding();
+  return httpResponse->getHttpRequest()->acceptGZip() &&
+    (ce == "gzip" || ce == "deflate");
 }
 
 bool HttpResponseCommand::handleDefaultEncoding
diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc
index bc7dc63a..2dabd7c6 100644
--- a/src/OptionHandlerFactory.cc
+++ b/src/OptionHandlerFactory.cc
@@ -741,6 +741,15 @@ OptionHandlers OptionHandlerFactory::createOptionHandlers()
   }
   {
     SharedHandle<OptionHandler> op(new BooleanOptionHandler
+                                   (PREF_HTTP_ACCEPT_GZIP,
+                                    TEXT_HTTP_ACCEPT_GZIP,
+                                    V_FALSE,
+                                    OptionHandler::OPT_ARG));
+    op->addTag(TAG_HTTP);
+    handlers.push_back(op);
+  }
+  {
+    SharedHandle<OptionHandler> op(new BooleanOptionHandler
                                    (PREF_HTTP_AUTH_CHALLENGE,
                                     TEXT_HTTP_AUTH_CHALLENGE,
                                     V_FALSE,
diff --git a/src/prefs.cc b/src/prefs.cc
index 3f80f62b..8d82f024 100644
--- a/src/prefs.cc
+++ b/src/prefs.cc
@@ -184,6 +184,8 @@ const std::string PREF_REMOVE_CONTROL_FILE("remove-control-file");
 const std::string PREF_ALWAYS_RESUME("always-resume");
 // value: 1*digit
 const std::string PREF_MAX_RESUME_FAILURE_TRIES("max-resume-failure-tries");
+// value: true | false
+const std::string PREF_HTTP_ACCEPT_GZIP("http-accept-gzip");
 
 /**
  * FTP related preferences
diff --git a/src/prefs.h b/src/prefs.h
index 08b09970..35a289ed 100644
--- a/src/prefs.h
+++ b/src/prefs.h
@@ -188,6 +188,8 @@ extern const std::string PREF_REMOVE_CONTROL_FILE;
 extern const std::string PREF_ALWAYS_RESUME;
 // value: 1*digit
 extern const std::string PREF_MAX_RESUME_FAILURE_TRIES;
+// value: true | false
+extern const std::string PREF_HTTP_ACCEPT_GZIP;
 
 /**
  * FTP related preferences
diff --git a/src/usage_text.h b/src/usage_text.h
index d4d93310..deb21ba0 100644
--- a/src/usage_text.h
+++ b/src/usage_text.h
@@ -667,3 +667,8 @@
     "                              instead.")
 #define TEXT_DHT_MESSAGE_TIMEOUT                \
   _(" --dht-message-timeout=SEC    Set timeout in seconds.")
+#define TEXT_HTTP_ACCEPT_GZIP                   \
+  _(" --http-accept-gzip[=true|false] Send 'Accept: deflate, gzip' request header\n" \
+    "                              and inflate response if remote server responds\n" \
+    "                              with 'Content-Encoding: gzip' or\n"  \
+    "                              'Content-Encoding: deflate'.")