new option content-disposition-default-utf8

9 files changed, 57 insertions, 15 deletions

diff --git a/src/HttpResponse.cc b/src/HttpResponse.cc
index f009ecae..230041d9 100644
--- a/src/HttpResponse.cc
+++ b/src/HttpResponse.cc
@@ -112,10 +112,11 @@ void HttpResponse::validateResponse() const
                      error_code::HTTP_PROTOCOL_ERROR);
 }
 
-std::string HttpResponse::determineFilename() const
+std::string HttpResponse::determineFilename(bool contentDispositionUTF8) const
 {
   std::string contentDisposition = util::getContentDispositionFilename(
-      httpHeader_->find(HttpHeader::CONTENT_DISPOSITION));
+      httpHeader_->find(HttpHeader::CONTENT_DISPOSITION),
+      contentDispositionUTF8);
   if (contentDisposition.empty()) {
     auto file = httpRequest_->getFile();
     file = util::percentDecode(file.begin(), file.end());
diff --git a/src/HttpResponse.h b/src/HttpResponse.h
index 38bfa342..2caffaf1 100644
--- a/src/HttpResponse.h
+++ b/src/HttpResponse.h
@@ -70,7 +70,7 @@ public:
    * this function returns the filename from it.
    * If it is not there, returns the part of filename from the request URL.
    */
-  std::string determineFilename() const;
+  std::string determineFilename(bool contentDispositionUTF8) const;
 
   void retrieveCookie();
 
diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc
index 18972575..d0e44803 100644
--- a/src/HttpResponseCommand.cc
+++ b/src/HttpResponseCommand.cc
@@ -261,8 +261,8 @@ bool HttpResponseCommand::executeInternal()
     int64_t totalLength = httpResponse->getEntityLength();
     fe->setLength(totalLength);
     if (fe->getPath().empty()) {
-      auto suffixPath = util::createSafePath(httpResponse->determineFilename());
-
+      auto suffixPath = util::createSafePath(httpResponse->determineFilename(
+          getOption()->getAsBool(PREF_CONTENT_DISPOSITION_DEFAULT_UTF8)));
       fe->setPath(util::applyDir(getOption()->get(PREF_DIR), suffixPath));
       fe->setSuffixPath(suffixPath);
     }
diff --git a/src/OptionHandlerFactory.cc b/src/OptionHandlerFactory.cc
index fd963613..b8f07770 100644
--- a/src/OptionHandlerFactory.cc
+++ b/src/OptionHandlerFactory.cc
@@ -1055,6 +1055,18 @@ std::vector<OptionHandler*> OptionHandlerFactory::createOptionHandlers()
     handlers.push_back(op);
   }
   {
+    OptionHandler* op(
+        new BooleanOptionHandler(PREF_CONTENT_DISPOSITION_DEFAULT_UTF8,
+                                 TEXT_CONTENT_DISPOSITION_DEFAULT_UTF8,
+                                 A2_V_FALSE, OptionHandler::OPT_ARG));
+    op->addTag(TAG_ADVANCED);
+    op->addTag(TAG_HTTP);
+    op->setInitialOption(true);
+    op->setChangeGlobalOption(true);
+    op->setChangeOptionForReserved(true);
+    handlers.push_back(op);
+  }
+  {
     OptionHandler* op(new BooleanOptionHandler(
         PREF_ENABLE_HTTP_KEEP_ALIVE, TEXT_ENABLE_HTTP_KEEP_ALIVE, A2_V_TRUE,
         OptionHandler::OPT_ARG));
diff --git a/src/prefs.cc b/src/prefs.cc
index c0e1af6d..4f213f2c 100644
--- a/src/prefs.cc
+++ b/src/prefs.cc
@@ -427,6 +427,9 @@ PrefPtr PREF_HTTP_AUTH_CHALLENGE = makePref("http-auth-challenge");
 PrefPtr PREF_HTTP_NO_CACHE = makePref("http-no-cache");
 // value: true | false
 PrefPtr PREF_HTTP_ACCEPT_GZIP = makePref("http-accept-gzip");
+// value: true | false
+PrefPtr PREF_CONTENT_DISPOSITION_DEFAULT_UTF8 =
+    makePref("content-disposition-default-utf8");
 
 /**
  * Proxy related preferences
diff --git a/src/prefs.h b/src/prefs.h
index fb064727..91fa8769 100644
--- a/src/prefs.h
+++ b/src/prefs.h
@@ -380,6 +380,8 @@ extern PrefPtr PREF_HTTP_AUTH_CHALLENGE;
 extern PrefPtr PREF_HTTP_NO_CACHE;
 // value: true | false
 extern PrefPtr PREF_HTTP_ACCEPT_GZIP;
+// value: true | false
+extern PrefPtr PREF_CONTENT_DISPOSITION_DEFAULT_UTF8;
 
 /**;
  * Proxy related preferences
diff --git a/src/usage_text.h b/src/usage_text.h
index 3bd835eb..19817bcd 100644
--- a/src/usage_text.h
+++ b/src/usage_text.h
@@ -537,6 +537,11 @@
 #define TEXT_USE_HEAD                                                   \
   _(" --use-head[=true|false]      Use HEAD method for the first request to the HTTP\n" \
     "                              server.")
+#define TEXT_CONTENT_DISPOSITION_DEFAULT_UTF8                          \
+  _(" --content-disposition-default-utf8[=true|false] Handle quoted string in\n" \
+    "                              Content-Disposition header as UTF-8 instead of\n" \
+    "                              ISO-8859-1, for example, the filename parameter,\n" \
+    "                              but not the extended version filename*.")
 #define TEXT_EVENT_POLL                                                 \
   _(" --event-poll=POLL            Specify the method for polling events.")
 #define TEXT_BT_EXTERNAL_IP                                             \
diff --git a/src/util.cc b/src/util.cc
index 8229f98b..4d9b5fd0 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -879,7 +879,7 @@ typedef enum {
 
 ssize_t parse_content_disposition(char* dest, size_t destlen,
                                   const char** charsetp, size_t* charsetlenp,
-                                  const char* in, size_t len)
+                                  const char* in, size_t len, bool defaultUTF8)
 {
   const char *p = in, *eop = in + len, *mark_first = nullptr,
              *mark_last = nullptr;
@@ -891,7 +891,7 @@ ssize_t parse_content_disposition(char* dest, size_t destlen,
   /* To suppress warnings */
   char* dp = dest;
   size_t dlen = destlen;
-  uint32_t dfa_state = 0;
+  uint32_t dfa_state = UTF8_ACCEPT;
   uint32_t dfa_code = 0;
   uint8_t pctval = 0;
 
@@ -981,6 +981,10 @@ ssize_t parse_content_disposition(char* dest, size_t destlen,
       if (*p == '"') {
         quoted_seen = 0;
         state = CD_QUOTED_STRING;
+        if (defaultUTF8) {
+          dfa_state = UTF8_ACCEPT;
+          dfa_code = 0;
+        }
       }
       else if (inRFC2616HttpToken(*p)) {
         if (in_file_parm) {
@@ -1011,16 +1015,25 @@ ssize_t parse_content_disposition(char* dest, size_t destlen,
         quoted_seen = 1;
       }
       else if (*p == '"' && quoted_seen == 0) {
+        if (defaultUTF8 && dfa_state != UTF8_ACCEPT) {
+          return -1;
+        }
         if (in_file_parm) {
           flags |= CD_FILENAME_FOUND;
         }
         state = CD_AFTER_VALUE;
       }
       else {
-        /* TEXT which is OCTET except CTLs, but including LWS. We only
-           accept ISO-8859-1 chars. */
+        /* TEXT which is OCTET except CTLs, but including LWS. Accept
+           ISO-8859-1 chars, or UTF-8 if defaultUTF8 is set */
         quoted_seen = 0;
-        if (!isIso8859p1(*p)) {
+        if (defaultUTF8) {
+          if (utf8dfa(&dfa_state, &dfa_code, (unsigned char)*p) ==
+              UTF8_REJECT) {
+            return -1;
+          }
+        }
+        else if (!isIso8859p1(*p)) {
           return -1;
         }
         if (in_file_parm) {
@@ -1204,7 +1217,8 @@ ssize_t parse_content_disposition(char* dest, size_t destlen,
   }
 }
 
-std::string getContentDispositionFilename(const std::string& header)
+std::string getContentDispositionFilename(const std::string& header,
+                                          bool defaultUTF8)
 {
   std::array<char, 1_k> cdval;
   size_t cdvallen = cdval.size();
@@ -1212,13 +1226,14 @@ std::string getContentDispositionFilename(const std::string& header)
   size_t charsetlen;
   ssize_t rv =
       parse_content_disposition(cdval.data(), cdvallen, &charset, &charsetlen,
-                                header.c_str(), header.size());
+                                header.c_str(), header.size(), defaultUTF8);
   if (rv == -1) {
     return "";
   }
 
   std::string res;
-  if (!charset || strieq(charset, charset + charsetlen, "iso-8859-1")) {
+  if ((charset && strieq(charset, charset + charsetlen, "iso-8859-1")) ||
+      (!charset && !defaultUTF8)) {
     res = iso8859p1ToUtf8(cdval.data(), rv);
   }
   else {
diff --git a/src/util.h b/src/util.h
index 76ecb549..2fc8ca40 100644
--- a/src/util.h
+++ b/src/util.h
@@ -322,11 +322,15 @@ std::string iso8859p1ToUtf8(const std::string& src);
 // succeeds, or -1. If there is enough room to store filename in
 // |dest|, this function returns -1. If this function returns -1, the
 // |dest|, |*charsetp| and |*charsetlenp| are undefined.
+//
+// It will handle quoted string(as in RFC 7230 section 3.2.6) as
+// ISO-8859-1 by default, or UTF-8 if defaultUTF8 == true.
 ssize_t parse_content_disposition(char* dest, size_t destlen,
                                   const char** charsetp, size_t* charsetlenp,
-                                  const char* in, size_t len);
+                                  const char* in, size_t len, bool defaultUTF8);
 
-std::string getContentDispositionFilename(const std::string& header);
+std::string getContentDispositionFilename(const std::string& header,
+                                          bool defaultUTF8);
 
 std::string toUpper(std::string src);