From efbfe4c006cfb6e0bf3ee4ed57d0657c0cd85c19 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Mon, 6 Sep 2010 14:29:36 +0000 Subject: 2010-09-06 Tatsuhiro Tsujikawa Data from remote server in HTTP/FTP download are now written to the disk(or memory) through StreamFilter. Decoding chunked and gziped streams are done cascading StreamFilter. Removed inefficient 1byte read code. * src/ChunkedDecodingStreamFilter.cc * src/ChunkedDecodingStreamFilter.h * src/DownloadCommand.cc * src/DownloadCommand.h * src/GZipDecodingStreamFilter.cc * src/GZipDecodingStreamFilter.h * src/HttpConnection.cc * src/HttpDownloadCommand.cc * src/HttpResponse.cc * src/HttpResponse.h * src/HttpResponseCommand.cc * src/HttpResponseCommand.h * src/HttpSkipResponseCommand.cc * src/HttpSkipResponseCommand.h * src/Makefile.am * src/NullSinkStreamFilter.cc * src/NullSinkStreamFilter.h * src/RequestGroup.cc * src/SinkStreamFilter.cc * src/SinkStreamFilter.h * src/StreamFilter.cc * src/StreamFilter.h * test/ChunkedDecodingStreamFilterTest.cc * test/GZipDecodingStreamFilterTest.cc * test/HttpResponseTest.cc * test/Makefile.am * test/MockSegment.h --- src/HttpResponseCommand.cc | 120 +++++++++++++++++++++++++++++---------------- 1 file changed, 78 insertions(+), 42 deletions(-) (limited to 'src/HttpResponseCommand.cc') diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index a1c8c2aa..bfa67430 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -69,14 +69,20 @@ #include "ServerStatMan.h" #include "FileAllocationEntry.h" #include "CheckIntegrityEntry.h" +#include "StreamFilter.h" +#include "SinkStreamFilter.h" +#include "ChunkedDecodingStreamFilter.h" +#include "GZipDecodingStreamFilter.h" namespace aria2 { -static SharedHandle getTransferEncodingDecoder -(const SharedHandle& httpResponse); +static SharedHandle getTransferEncodingStreamFilter +(const SharedHandle& httpResponse, + const SharedHandle& delegate = SharedHandle()); -static SharedHandle getContentEncodingDecoder -(const SharedHandle& httpResponse); +static SharedHandle getContentEncodingStreamFilter +(const SharedHandle& httpResponse, + const SharedHandle& delegate = SharedHandle()); HttpResponseCommand::HttpResponseCommand (cuid_t cuid, @@ -198,12 +204,16 @@ bool HttpResponseCommand::executeInternal() // anyway. getPieceStorage()->getDiskAdaptor()->truncate(0); getDownloadEngine()->addCommand - (createHttpDownloadCommand(httpResponse, - getTransferEncodingDecoder(httpResponse), - getContentEncodingDecoder(httpResponse))); + (createHttpDownloadCommand + (httpResponse, + getTransferEncodingStreamFilter + (httpResponse, + getContentEncodingStreamFilter(httpResponse)))); } else { - getDownloadEngine()->addCommand(createHttpDownloadCommand - (httpResponse, getTransferEncodingDecoder(httpResponse))); + getDownloadEngine()->addCommand + (createHttpDownloadCommand + (httpResponse, + getTransferEncodingStreamFilter(httpResponse))); } return true; } @@ -275,7 +285,8 @@ bool HttpResponseCommand::handleDefaultEncoding !segment.isNull() && segment->getPositionToWrite() == 0 && !getRequest()->isPipeliningEnabled()) { command = createHttpDownloadCommand - (httpResponse, getTransferEncodingDecoder(httpResponse)); + (httpResponse, + getTransferEncodingStreamFilter(httpResponse)); } else { getSegmentMan()->cancelSegment(getCuid()); getFileEntry()->poolRequest(getRequest()); @@ -291,39 +302,49 @@ bool HttpResponseCommand::handleDefaultEncoding return true; } -static SharedHandle getTransferEncodingDecoder -(const SharedHandle& httpResponse) +static SharedHandle getTransferEncodingStreamFilter +(const SharedHandle& httpResponse, + const SharedHandle& delegate) { - SharedHandle decoder; + SharedHandle filter; if(httpResponse->isTransferEncodingSpecified()) { - decoder = httpResponse->getTransferEncodingDecoder(); - if(decoder.isNull()) { + filter = httpResponse->getTransferEncodingStreamFilter(); + if(filter.isNull()) { throw DL_ABORT_EX (StringFormat(EX_TRANSFER_ENCODING_NOT_SUPPORTED, httpResponse->getTransferEncoding().c_str()).str()); } - decoder->init(); + filter->init(); + filter->installDelegate(delegate); + } + if(filter.isNull()) { + filter = delegate; } - return decoder; + return filter; } -static SharedHandle getContentEncodingDecoder -(const SharedHandle& httpResponse) +static SharedHandle getContentEncodingStreamFilter +(const SharedHandle& httpResponse, + const SharedHandle& delegate) { - SharedHandle decoder; + SharedHandle filter; if(httpResponse->isContentEncodingSpecified()) { - decoder = httpResponse->getContentEncodingDecoder(); - if(decoder.isNull()) { + filter = httpResponse->getContentEncodingStreamFilter(); + if(filter.isNull()) { LogFactory::getInstance()->info ("Content-Encoding %s is specified, but the current implementation" "doesn't support it. The decoding process is skipped and the" "downloaded content will be still encoded.", httpResponse->getContentEncoding().c_str()); } else { - decoder->init(); + filter->init(); + filter->installDelegate(delegate); } } - return decoder; + if(filter.isNull()) { + filter = delegate; + } + return filter; } bool HttpResponseCommand::handleOtherEncoding @@ -357,9 +378,20 @@ bool HttpResponseCommand::handleOtherEncoding getRequestGroup()->shouldCancelDownloadForSafety(); getRequestGroup()->initPieceStorage(); getPieceStorage()->getDiskAdaptor()->initAndOpenFile(); + + SharedHandle streamFilter = + getTransferEncodingStreamFilter + (httpResponse, + getContentEncodingStreamFilter(httpResponse)); + // In this context, knowsTotalLength() is true only when the file is // really zero-length. - if(getDownloadContext()->knowsTotalLength()) { + if(getDownloadContext()->knowsTotalLength() && + (streamFilter.isNull() || + streamFilter->getName() != ChunkedDecodingStreamFilter::NAME)) { + // If chunked transfer-encoding is specified, we have to read end + // of chunk markers(0\r\n\r\n, for example), so cannot pool + // connection here. poolConnection(); return true; } @@ -369,16 +401,15 @@ bool HttpResponseCommand::handleOtherEncoding getSegmentMan()->getSegmentWithIndex(getCuid(), 0); getDownloadEngine()->addCommand - (createHttpDownloadCommand(httpResponse, - getTransferEncodingDecoder(httpResponse), - getContentEncodingDecoder(httpResponse))); + (createHttpDownloadCommand(httpResponse, streamFilter)); return true; } bool HttpResponseCommand::skipResponseBody (const SharedHandle& httpResponse) { - SharedHandle decoder = getTransferEncodingDecoder(httpResponse); + SharedHandle filter = + getTransferEncodingStreamFilter(httpResponse); // We don't use Content-Encoding here because this response body is just // thrown away. @@ -386,7 +417,7 @@ bool HttpResponseCommand::skipResponseBody (getCuid(), getRequest(), getFileEntry(), getRequestGroup(), httpConnection_, httpResponse, getDownloadEngine(), getSocket()); - command->setTransferEncodingDecoder(decoder); + command->installStreamFilter(filter); // If request method is HEAD or the response body is zero-length, // set command's status to real time so that avoid read check blocking @@ -403,10 +434,23 @@ bool HttpResponseCommand::skipResponseBody return true; } +static bool decideFileAllocation +(const SharedHandle& filter) +{ + for(SharedHandle f = filter; !f.isNull(); f = f->getDelegate()){ + // Since the compressed file's length are returned in the response header + // and the decompressed file size is unknown at this point, disable file + // allocation here. + if(f->getName() == GZipDecodingStreamFilter::NAME) { + return false; + } + } + return true; +} + HttpDownloadCommand* HttpResponseCommand::createHttpDownloadCommand (const SharedHandle& httpResponse, - const SharedHandle& transferEncodingDecoder, - const SharedHandle& contentEncodingDecoder) + const SharedHandle& filter) { HttpDownloadCommand* command = @@ -417,16 +461,8 @@ HttpDownloadCommand* HttpResponseCommand::createHttpDownloadCommand command->setStartupIdleTime(getOption()->getAsInt(PREF_STARTUP_IDLE_TIME)); command->setLowestDownloadSpeedLimit (getOption()->getAsInt(PREF_LOWEST_SPEED_LIMIT)); - command->setTransferEncodingDecoder(transferEncodingDecoder); - - if(!contentEncodingDecoder.isNull()) { - command->setContentEncodingDecoder(contentEncodingDecoder); - // Since the compressed file's length are returned in the response header - // and the decompressed file size is unknown at this point, disable file - // allocation here. - getRequestGroup()->setFileAllocationEnabled(false); - } - + command->installStreamFilter(filter); + getRequestGroup()->setFileAllocationEnabled(decideFileAllocation(filter)); getRequestGroup()->getURISelector()->tuneDownloadCommand (getFileEntry()->getRemainingUris(), command); -- cgit