From 5a8710676d58be3b3f3606993e802c6a075dbd89 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 21 Feb 2006 12:27:17 +0000 Subject: --- src/AbstractCommand.cc | 68 ++++++++++++++---- src/AbstractCommand.h | 13 +++- src/DownloadCommand.cc | 9 ++- src/DownloadCommand.h | 2 +- src/DownloadEngine.cc | 44 ++++++------ src/HttpConnection.cc | 88 ++++++++++------------- src/HttpConnection.h | 25 +++---- src/HttpDownloadCommand.h | 3 - src/HttpProxyRequestCommand.cc | 11 ++- src/HttpProxyResponseCommand.cc | 15 ++-- src/HttpProxyResponseCommand.h | 3 + src/HttpRequestCommand.cc | 11 ++- src/HttpResponseCommand.cc | 50 ++++++++------ src/HttpResponseCommand.h | 1 + src/InitiateConnectionCommandFactory.cc | 3 + src/Logger.h | 12 ++-- src/Makefile.am | 7 ++ src/Makefile.in | 119 ++++++++++++++++++++++++++++++++ src/Request.cc | 17 +---- src/Request.h | 13 ++-- src/SegmentMan.cc | 28 +++++--- src/SegmentMan.h | 20 +++--- src/SimpleLogger.cc | 15 ++-- src/SimpleLogger.h | 14 ++-- src/Socket.cc | 40 ++++++++--- src/Socket.h | 26 ++++--- src/SocketCore.cc | 113 ++++++++++++++++++++++++------ src/SocketCore.h | 22 +++--- src/Util.cc | 11 +++ src/Util.h | 2 + src/main.cc | 9 +++ src/message.h | 6 ++ 32 files changed, 561 insertions(+), 259 deletions(-) (limited to 'src') diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index 0a2f78d3..89683a70 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -28,13 +28,14 @@ #include "message.h" #include "SleepCommand.h" -#define TIMEOUT_SEC 5 +#define TIMEOUT_SEC 60 AbstractCommand::AbstractCommand(int cuid, Request* req, DownloadEngine* e, Socket* s): Command(cuid), req(req), e(e), checkSocketIsReadable(false), checkSocketIsWritable(false) { + if(s != NULL) { socket = new Socket(*s); - e->addSocketForReadCheck(socket); + setReadCheckSocket(socket); } else { socket = NULL; } @@ -43,8 +44,8 @@ AbstractCommand::AbstractCommand(int cuid, Request* req, DownloadEngine* e, Sock } AbstractCommand::~AbstractCommand() { - e->deleteSocketForReadCheck(socket); - e->deleteSocketForWriteCheck(socket); + setReadCheckSocket(NULL); + setWriteCheckSocket(NULL); if(socket != NULL) { delete(socket); } @@ -58,6 +59,7 @@ bool AbstractCommand::isTimeoutDetected() { struct timeval now; gettimeofday(&now, NULL); if(checkPoint.tv_sec == 0 && checkPoint.tv_usec == 0) { + checkPoint = now; return false; } else { long long int elapsed = Util::difftv(now, checkPoint); @@ -71,12 +73,11 @@ bool AbstractCommand::isTimeoutDetected() { bool AbstractCommand::execute() { try { - if(checkSocketIsReadable && !socket->isReadable(0) - || checkSocketIsWritable && !socket->isWritable(0)) { + if(checkSocketIsReadable && !readCheckTarget->isReadable(0) + || checkSocketIsWritable && !writeCheckTarget->isWritable(0)) { if(isTimeoutDetected()) { throw new DlRetryEx(EX_TIME_OUT); } - updateCheckPoint(); e->commands.push(this); return false; } @@ -93,17 +94,16 @@ bool AbstractCommand::execute() { return executeInternal(seg); } catch(DlAbortEx* err) { e->logger->error(MSG_DOWNLOAD_ABORTED, err, cuid); - onError(err); + onAbort(err); delete(err); req->resetUrl(); return true; } catch(DlRetryEx* err) { e->logger->error(MSG_RESTARTING_DOWNLOAD, err, cuid); - onError(err); delete(err); //req->resetUrl(); - req->addRetryCount(); - if(req->noMoreRetry()) { + req->addTryCount(); + if(req->noMoreTry()) { e->logger->error(MSG_MAX_RETRY, cuid); return true; } else { @@ -123,5 +123,49 @@ bool AbstractCommand::prepareForRetry(int wait) { return true; } -void AbstractCommand::onError(Exception* e) { +void AbstractCommand::onAbort(Exception* e) { +} + +void AbstractCommand::setReadCheckSocket(Socket* socket) { + if(socket == NULL) { + if(checkSocketIsReadable) { + e->deleteSocketForReadCheck(readCheckTarget); + checkSocketIsReadable = false; + readCheckTarget = NULL; + } + } else { + if(checkSocketIsReadable) { + if(readCheckTarget != socket) { + e->deleteSocketForReadCheck(readCheckTarget); + e->addSocketForReadCheck(socket); + readCheckTarget = socket; + } + } else { + e->addSocketForReadCheck(socket); + checkSocketIsReadable = true; + readCheckTarget = socket; + } + } +} + +void AbstractCommand::setWriteCheckSocket(Socket* socket) { + if(socket == NULL) { + if(checkSocketIsWritable) { + e->deleteSocketForWriteCheck(writeCheckTarget); + checkSocketIsWritable = false; + writeCheckTarget = NULL; + } + } else { + if(checkSocketIsWritable) { + if(writeCheckTarget != socket) { + e->deleteSocketForWriteCheck(writeCheckTarget); + e->addSocketForWriteCheck(socket); + writeCheckTarget = socket; + } + } else { + e->addSocketForWriteCheck(socket); + checkSocketIsWritable = true; + writeCheckTarget = socket; + } + } } diff --git a/src/AbstractCommand.h b/src/AbstractCommand.h index 3d94511b..10b36462 100644 --- a/src/AbstractCommand.h +++ b/src/AbstractCommand.h @@ -37,11 +37,18 @@ protected: Request* req; DownloadEngine* e; Socket* socket; - bool checkSocketIsReadable; - bool checkSocketIsWritable; + virtual bool prepareForRetry(int wait); - virtual void onError(Exception* e); + virtual void onAbort(Exception* e); virtual bool executeInternal(Segment segment) = 0; + + void setReadCheckSocket(Socket* socket); + void setWriteCheckSocket(Socket* socket); +private: + bool checkSocketIsReadable; + bool checkSocketIsWritable; + Socket* readCheckTarget; + Socket* writeCheckTarget; public: AbstractCommand(int cuid, Request* req, DownloadEngine* e, Socket* s= NULL); virtual ~AbstractCommand(); diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index 39391963..0cc15661 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -27,9 +27,7 @@ #include "InitiateConnectionCommandFactory.h" #include "message.h" -DownloadCommand::DownloadCommand(int cuid, Request* req, DownloadEngine* e, Socket* s):AbstractCommand(cuid, req, e, s) { - AbstractCommand::checkSocketIsReadable = true; -} +DownloadCommand::DownloadCommand(int cuid, Request* req, DownloadEngine* e, Socket* s):AbstractCommand(cuid, req, e, s) {} DownloadCommand::~DownloadCommand() {} @@ -84,3 +82,8 @@ bool DownloadCommand::prepareForNextSegment() { return true; } } + +void DownloadCommand::onAbort(Exception* ex) { + e->logger->debug(MSG_UNREGISTER_CUID, cuid); + e->segmentMan->unregisterId(cuid); +} diff --git a/src/DownloadCommand.h b/src/DownloadCommand.h index 0ea9d680..af44901c 100644 --- a/src/DownloadCommand.h +++ b/src/DownloadCommand.h @@ -34,7 +34,7 @@ protected: bool prepareForRetry(int wait); bool prepareForNextSegment(); - + void onAbort(Exception* ex); public: DownloadCommand(int cuid, Request* req, DownloadEngine* e, Socket* s); virtual ~DownloadCommand(); diff --git a/src/DownloadEngine.cc b/src/DownloadEngine.cc index 0621e66b..337e221c 100644 --- a/src/DownloadEngine.cc +++ b/src/DownloadEngine.cc @@ -51,30 +51,31 @@ void DownloadEngine::run() { } if(!noWait && !commands.empty()) { waitData(); - long long int dlSize = segmentMan->getDownloadedSize(); + } + noWait = false; - struct timeval now; - gettimeofday(&now, NULL); - if(cp.tv_sec == 0 && cp.tv_usec == 0) { + long long int dlSize = segmentMan->getDownloadedSize(); + struct timeval now; + gettimeofday(&now, NULL); + if(cp.tv_sec == 0 && cp.tv_usec == 0) { + cp = now; + psize = dlSize; + } else { + long long int elapsed = Util::difftv(now, cp); + if(elapsed >= 500000) { + int nspeed = (int)((dlSize-psize)/(elapsed/1000000.0)); + speed = (nspeed+speed)/2; cp = now; psize = dlSize; - } else { - long long int elapsed = Util::difftv(now, cp); - if(elapsed >= 500000) { - int nspeed = (int)((dlSize-psize)/(elapsed/1000000.0)); - speed = (nspeed+speed)/2; - cp = now; - psize = dlSize; - cout << "\r "; - cout << "\rProgress " << Util::llitos(dlSize, true) << " Bytes/" << - Util::llitos(segmentMan->totalSize, true) << " Bytes " << - (segmentMan->totalSize == 0 ? 0 : (dlSize*100)/segmentMan->totalSize) << "% " << - speed/1000.0 << "KB/s " << - "(" << commands.size() << " connections)" << flush; - } + cout << "\r "; + cout << "\rProgress " << Util::llitos(dlSize, true) << " Bytes/" << + Util::llitos(segmentMan->totalSize, true) << " Bytes " << + (segmentMan->totalSize == 0 ? 0 : (dlSize*100)/segmentMan->totalSize) << "% " << + speed/1000.0 << "KB/s " << + "(" << commands.size() << " connections)" << flush; } } - noWait = false; + } segmentMan->removeIfFinished(); diskWriter->closeFile(); @@ -99,7 +100,7 @@ void DownloadEngine::waitData() { fd_set wfds; struct timeval tv; int retval; - + FD_ZERO(&rfds); FD_ZERO(&wfds); int max = 0; @@ -110,6 +111,7 @@ void DownloadEngine::waitData() { } } for(vector::iterator itr = wsockets.begin(); itr != wsockets.end(); itr++) { + FD_SET((*itr)->getSockfd(), &wfds); if(max < (*itr)->getSockfd()) { max = (*itr)->getSockfd(); @@ -118,7 +120,7 @@ void DownloadEngine::waitData() { tv.tv_sec = 1; tv.tv_usec = 0; - retval = select(max+1, &rfds, &wfds, NULL, &tv); + retval = select(max+1, &rfds, /*&wfds*/NULL, NULL, &tv); } bool DownloadEngine::addSocket(vector& sockets, Socket* socket) { diff --git a/src/HttpConnection.cc b/src/HttpConnection.cc index b22d9126..9f373d33 100644 --- a/src/HttpConnection.cc +++ b/src/HttpConnection.cc @@ -26,15 +26,16 @@ #include "Base64.h" #include "message.h" -HttpConnection::HttpConnection(int cuid, Socket* socket, const Option* op, Logger* logger):cuid(cuid), socket(socket),option(op),logger(logger) {} +HttpConnection::HttpConnection(int cuid, const Socket* socket, const Request* req, const Option* op, const Logger* logger): + cuid(cuid), socket(socket), req(req), option(op), logger(logger) {} -void HttpConnection::sendRequest(const Request* req, const Segment& segment) { - string request = createRequest(req, segment); +void HttpConnection::sendRequest(const Segment& segment) const { + string request = createRequest(segment); logger->info(MSG_SENDING_HTTP_REQUEST, cuid, request.c_str()); socket->writeData(request.c_str(), request.size()); } -void HttpConnection::sendProxyRequest(const Request* req) { +void HttpConnection::sendProxyRequest() const { string request = string("CONNECT ")+req->getHost()+":"+Util::llitos(req->getPort())+ string(" HTTP/1.1\r\n")+ "Host: "+getHost(req->getHost(), req->getPort())+"\r\n"; @@ -47,19 +48,19 @@ void HttpConnection::sendProxyRequest(const Request* req) { socket->writeData(request.c_str(), request.size()); } -string HttpConnection::getHost(const string& host, int port) { +string HttpConnection::getHost(const string& host, int port) const { return host+(port == 80 || port == 443 ? "" : ":"+Util::llitos(port)); } -string HttpConnection::createRequest(const Request* req, const Segment& segment) { +string HttpConnection::createRequest(const Segment& segment) const { string request = string("GET ")+ - // some servers cannot permit absolute URI as requet URI. - //req->getCurrentUrl()+ - (req->getDir() == "/" ? "/" : req->getDir()+"/")+req->getFile()+ + (req->getProtocol() == "ftp" ? + req->getCurrentUrl() : + ((req->getDir() == "/" ? "/" : req->getDir()+"/")+req->getFile()))+ string(" HTTP/1.1\r\n")+ "User-Agent: aria2\r\n"+ "Connection: close\r\n"+ - "Accept: */*\r\n"+ + "Accept: */*\r\n"+ /* */ "Host: "+getHost(req->getHost(), req->getPort())+"\r\n"+ "Pragma: no-cache\r\n"+ "Cache-Control: no-cache\r\n"; @@ -87,48 +88,31 @@ string HttpConnection::createRequest(const Request* req, const Segment& segment) } int HttpConnection::receiveResponse(HttpHeader& headers) { - string header; - char* buf = NULL; - try { - // read a line of the header - int bufSize = 256; - // TODO limit iteration count - while(1) { - bufSize += 256; - if(bufSize > 2048) { - throw new DlAbortEx(EX_INVALID_HEADER); - } - buf = new char[bufSize]; - int tbufSize = bufSize-1; - socket->peekData(buf, tbufSize); - if(tbufSize > 0) { - buf[tbufSize] = '\0'; - } - header = buf; - char* p; - if((p = strstr(buf, "\r\n")) == buf) { - throw new DlAbortEx(EX_NO_HEADER); - } - if((p = strstr(buf, "\r\n\r\n")) != NULL) { - *(p+4) = '\0'; - header = buf; - tbufSize = header.size(); - socket->readData(buf, tbufSize); - delete [] buf; - buf = NULL; - break; + char buf[512]; + while(socket->isReadable(0)) { + int size = sizeof(buf)-1; + socket->peekData(buf, size); + buf[size] = '\0'; + int hlenTemp = header.size(); + header += buf; + string::size_type p; + if((p = header.find("\r\n\r\n")) == string::npos) { + socket->readData(buf, size); + } else { + if(Util::endsWith(header, "\r\n\r\n")) { + socket->readData(buf, size); } else { - delete [] buf; - buf = NULL; + header.erase(p+4); + size = p+4-hlenTemp; + socket->readData(buf, size); } + break; } - } catch(Exception* e) { - if(buf != NULL) { - delete [] buf; - } - throw; } - // OK, i got all headers. + if(!Util::endsWith(header, "\r\n\r\n")) { + return 0; + } + // OK, we got all headers. logger->info(MSG_RECEIVE_RESPONSE, cuid, header.c_str()); string::size_type p, np; p = np = 0; @@ -145,19 +129,17 @@ int HttpConnection::receiveResponse(HttpHeader& headers) { p = np+2; pair hp; Util::split(hp, line, ':'); - HttpHeader::value_type nh(hp.first, hp.second); - headers.insert(nh); + headers.put(hp.first, hp.second); } - // TODO rewrite this using strtoul return (int)strtol(status.c_str(), NULL, 10); } -bool HttpConnection::useProxy() { +bool HttpConnection::useProxy() const { return option->defined("http_proxy_enabled") && option->get("http_proxy_enabled") == "true"; } -bool HttpConnection::useProxyAuth() { +bool HttpConnection::useProxyAuth() const { return option->defined("http_proxy_auth_enabled") && option->get("http_proxy_auth_enabled") == "true"; } diff --git a/src/HttpConnection.h b/src/HttpConnection.h index c918d21d..a5a5eceb 100644 --- a/src/HttpConnection.h +++ b/src/HttpConnection.h @@ -27,30 +27,31 @@ #include "Request.h" #include "Option.h" #include "Logger.h" +#include "HttpHeader.h" #include #include using namespace std; -typedef multimap HttpHeader; +//typedef multimap HttpHeader; class HttpConnection { private: - string getHost(const string& host, int port); - string createRequest(const Request* req, const Segment& segment); - bool useProxy(); - bool useProxyAuth(); - bool useBasicAuth(); + string getHost(const string& host, int port) const; + string createRequest(const Segment& segment) const; + bool useProxy() const; + bool useProxyAuth() const; int cuid; - Socket* socket; + const Socket* socket; + const Request* req; const Option* option; - Logger* logger; + const Logger* logger; + string header; public: + HttpConnection(int cuid, const Socket* socket, const Request* req, const Option* op, const Logger* logger); - HttpConnection(int cuid, Socket* socket, const Option* op, Logger* logger); - - void sendRequest(const Request* req, const Segment& segment); - void sendProxyRequest(const Request* req); + void sendRequest(const Segment& segment) const; + void sendProxyRequest() const; int receiveResponse(HttpHeader& headers); }; diff --git a/src/HttpDownloadCommand.h b/src/HttpDownloadCommand.h index dbd50fa6..81a537d0 100644 --- a/src/HttpDownloadCommand.h +++ b/src/HttpDownloadCommand.h @@ -37,13 +37,10 @@ class HttpDownloadCommand:public DownloadCommand { private: map transferEncodings; public: - int cuid; - HttpDownloadCommand(int cuid, Request* req, DownloadEngine* e, Socket* s); ~HttpDownloadCommand(); TransferEncoding* getTransferEncoding(string transferEncoding); - }; #endif // _D_HTTP_DOWNLOAD_COMMAND_H_ diff --git a/src/HttpProxyRequestCommand.cc b/src/HttpProxyRequestCommand.cc index bbff980a..f1a5a8c9 100644 --- a/src/HttpProxyRequestCommand.cc +++ b/src/HttpProxyRequestCommand.cc @@ -24,17 +24,16 @@ #include "HttpProxyResponseCommand.h" HttpProxyRequestCommand::HttpProxyRequestCommand(int cuid, Request* req, DownloadEngine* e, Socket* s):AbstractCommand(cuid, req, e, s) { - AbstractCommand::checkSocketIsWritable = true; - e->deleteSocketForReadCheck(socket); - e->addSocketForWriteCheck(socket); + setReadCheckSocket(NULL); + setWriteCheckSocket(socket); } HttpProxyRequestCommand::~HttpProxyRequestCommand() {} bool HttpProxyRequestCommand::executeInternal(Segment segment) { - socket->setNonBlockingMode(); - HttpConnection httpConnection(cuid, socket, e->option, e->logger); - httpConnection.sendProxyRequest(req); + socket->setBlockingMode(); + HttpConnection httpConnection(cuid, socket, req, e->option, e->logger); + httpConnection.sendProxyRequest(); HttpProxyResponseCommand* command = new HttpProxyResponseCommand(cuid, req, e, socket); e->commands.push(command); diff --git a/src/HttpProxyResponseCommand.cc b/src/HttpProxyResponseCommand.cc index e62d3939..cb2764b9 100644 --- a/src/HttpProxyResponseCommand.cc +++ b/src/HttpProxyResponseCommand.cc @@ -22,19 +22,24 @@ #include "HttpProxyResponseCommand.h" #include "HttpRequestCommand.h" #include "DlRetryEx.h" -#include "HttpConnection.h" #include "message.h" HttpProxyResponseCommand::HttpProxyResponseCommand(int cuid, Request* req, DownloadEngine* e, Socket* s):AbstractCommand(cuid, req, e, s) { - AbstractCommand::checkSocketIsReadable = true; + http = new HttpConnection(cuid, socket, req, e->option, e->logger); } -HttpProxyResponseCommand::~HttpProxyResponseCommand() {} +HttpProxyResponseCommand::~HttpProxyResponseCommand() { + delete http; +} bool HttpProxyResponseCommand::executeInternal(Segment segment) { HttpHeader headers; - HttpConnection httpConnection(cuid, socket, e->option, e->logger); - int status = httpConnection.receiveResponse(headers); + int status = http->receiveResponse(headers); + if(status == 0) { + // we didn't receive all of headers yet. + e->commands.push(this); + return false; + } if(status != 200) { throw new DlRetryEx(EX_PROXY_CONNECTION_FAILED); } diff --git a/src/HttpProxyResponseCommand.h b/src/HttpProxyResponseCommand.h index 6ff0f2ab..370066e4 100644 --- a/src/HttpProxyResponseCommand.h +++ b/src/HttpProxyResponseCommand.h @@ -23,8 +23,11 @@ #define _D_HTTP_PROXY_RESPONSE_COMMAND_H_ #include "AbstractCommand.h" +#include "HttpConnection.h" class HttpProxyResponseCommand : public AbstractCommand { +private: + HttpConnection* http; protected: bool executeInternal(Segment segment); public: diff --git a/src/HttpRequestCommand.cc b/src/HttpRequestCommand.cc index caa8413a..42090bef 100644 --- a/src/HttpRequestCommand.cc +++ b/src/HttpRequestCommand.cc @@ -27,22 +27,21 @@ #include "HttpConnection.h" HttpRequestCommand::HttpRequestCommand(int cuid, Request* req, DownloadEngine* e, Socket* s):AbstractCommand(cuid, req, e, s) { - AbstractCommand::checkSocketIsWritable = true; - e->deleteSocketForReadCheck(socket); - e->addSocketForWriteCheck(socket); + setReadCheckSocket(NULL); + setWriteCheckSocket(socket); } HttpRequestCommand::~HttpRequestCommand() {} bool HttpRequestCommand::executeInternal(Segment seg) { - socket->setNonBlockingMode(); + socket->setBlockingMode(); if(req->getProtocol() == "https") { socket->initiateSecureConnection(); } - HttpConnection httpConnection(cuid, socket, e->option, e->logger); + HttpConnection http(cuid, socket, req, e->option, e->logger); // set seg to request in order to remember the request range req->seg = seg; - httpConnection.sendRequest(req, seg); + http.sendRequest(seg); HttpResponseCommand* command = new HttpResponseCommand(cuid, req, e, socket); e->commands.push(command); diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index 22c7fa71..1fd273bb 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -28,10 +28,12 @@ HttpResponseCommand::HttpResponseCommand(int cuid, Request* req, DownloadEngine* e, Socket* s): AbstractCommand(cuid, req, e, s) { - AbstractCommand::checkSocketIsReadable = true; + http = new HttpConnection(cuid, socket, req, e->option, e->logger); } -HttpResponseCommand::~HttpResponseCommand() {} +HttpResponseCommand::~HttpResponseCommand() { + delete http; +} bool HttpResponseCommand::executeInternal(Segment seg) { if(SEGMENT_EQUAL(req->seg, seg) == false) { @@ -39,26 +41,30 @@ bool HttpResponseCommand::executeInternal(Segment seg) { return prepareForRetry(0); } HttpHeader headers; - HttpConnection httpConnection(cuid, socket, e->option, e->logger); - int status = httpConnection.receiveResponse(headers); + int status = http->receiveResponse(headers); + if(status == 0) { + // didn't receive header fully + e->commands.push(this); + return false; + } // check HTTP status number checkResponse(status, seg); retrieveCookie(headers); // check whether Location header exists. If it does, update request object // with redirected URL. // then establish a connection to the new host and port - if(headers.count("Location")) { - return handleRedirect((*headers.find("Location")).second, headers); + if(headers.defined("Location")) { + return handleRedirect(headers.getFirst("Location"), headers); } if(!e->segmentMan->downloadStarted) { string transferEncoding; - headers.find("Transfer-Encoding"); - if(headers.count("Transfer-Encoding")) { - return handleOtherEncoding((*headers.find("Transfer-Encoding")).second, headers); + if(headers.defined("Transfer-Encoding")) { + return handleOtherEncoding(headers.getFirst("Transfer-Encoding"), headers); } else { return handleDefaultEncoding(headers); } } else { + // TODO we must check headers["size"] == e->segmentMan->totalSize here if(req->getFile() != e->segmentMan->filename) { throw new DlAbortEx(EX_FILENAME_MISMATCH, req->getFile().c_str(), e->segmentMan->filename.c_str()); } @@ -83,28 +89,25 @@ bool HttpResponseCommand::handleRedirect(string url, const HttpHeader& headers) } bool HttpResponseCommand::handleDefaultEncoding(const HttpHeader& headers) { - long long int size; - if(headers.count("Content-Length") == 0) { - size = 0; - } else { - size = STRTOLL((*headers.find("Content-Length")).second.c_str()); - if(size == LONG_LONG_MAX || size == LONG_LONG_MIN) { - throw new DlAbortEx(EX_TOO_LARGE_FILE, size); - } + long long int size = headers.getFirstAsLLInt("Content-Length"); + if(size == LONG_LONG_MAX || size == LONG_LONG_MIN || size < 0) { + throw new DlAbortEx(EX_TOO_LARGE_FILE, size); } e->segmentMan->isSplittable = !(size == 0); e->segmentMan->filename = req->getFile(); - e->segmentMan->totalSize = size; bool segFileExists = e->segmentMan->segmentFileExists(); e->segmentMan->downloadStarted = true; if(segFileExists) { - e->segmentMan->load(); + e->diskWriter->openExistingFile(e->segmentMan->getFilePath()); + // we must check headers["size"] == e->segmentMan->totalSize here + if(e->segmentMan->totalSize != size) { + return new DlAbortEx(EX_SIZE_MISMATCH, e->segmentMan->totalSize, size); + } // send request again to the server with Range header return prepareForRetry(0); } else { - Segment seg; - e->segmentMan->getSegment(seg, cuid); + e->segmentMan->totalSize = size; e->diskWriter->initAndOpenFile(e->segmentMan->getFilePath()); createHttpDownloadCommand(); return true; @@ -140,9 +143,10 @@ void HttpResponseCommand::createHttpDownloadCommand(string transferEncoding) { } void HttpResponseCommand::retrieveCookie(const HttpHeader& headers) { - for(HttpHeader::const_iterator itr = headers.find("Set-Cookie"); itr != headers.end(); itr++) { + vector v = headers.get("Set-Cookie"); + for(vector::const_iterator itr = v.begin(); itr != v.end(); itr++) { Cookie c; - req->cookieBox->parse(c, (*itr).second); + req->cookieBox->parse(c, *itr); req->cookieBox->add(c); } } diff --git a/src/HttpResponseCommand.h b/src/HttpResponseCommand.h index 8300bdc2..c5269235 100644 --- a/src/HttpResponseCommand.h +++ b/src/HttpResponseCommand.h @@ -33,6 +33,7 @@ private: bool handleOtherEncoding(string transferEncoding, const HttpHeader& headers); void createHttpDownloadCommand(string transferEncoding = ""); void retrieveCookie(const HttpHeader& headers); + HttpConnection* http; protected: bool executeInternal(Segment segment); public: diff --git a/src/InitiateConnectionCommandFactory.cc b/src/InitiateConnectionCommandFactory.cc index 3ba177bd..e72c6020 100644 --- a/src/InitiateConnectionCommandFactory.cc +++ b/src/InitiateConnectionCommandFactory.cc @@ -21,6 +21,7 @@ /* copyright --> */ #include "InitiateConnectionCommandFactory.h" #include "HttpInitiateConnectionCommand.h" +#include "FtpInitiateConnectionCommand.h" Command* InitiateConnectionCommandFactory::createInitiateConnectionCommand(int cuid, Request* req, DownloadEngine* e) { if(req->getProtocol() == "http" @@ -30,6 +31,8 @@ Command* InitiateConnectionCommandFactory::createInitiateConnectionCommand(int c #endif // HAVE_LIBSSL ) { return new HttpInitiateConnectionCommand(cuid, req, e); + } else if(req->getProtocol() == "ftp") { + return new FtpInitiateConnectionCommand(cuid, req, e); } else { // these protocols are not supported yet return NULL; diff --git a/src/Logger.h b/src/Logger.h index 620544ea..7e842b76 100644 --- a/src/Logger.h +++ b/src/Logger.h @@ -29,12 +29,12 @@ using namespace std; class Logger { public: virtual ~Logger() {} - virtual void debug(string msg, ...) = 0; - virtual void debug(string msg, Exception* ex, ...) = 0; - virtual void info(string msg, ...) = 0; - virtual void info(string msg, Exception* ex, ...) = 0; - virtual void error(string msg, ...) = 0; - virtual void error(string msg, Exception* ex, ...) = 0; + virtual void debug(string msg, ...) const = 0; + virtual void debug(string msg, Exception* ex, ...) const = 0; + virtual void info(string msg, ...) const = 0; + virtual void info(string msg, Exception* ex, ...) const = 0; + virtual void error(string msg, ...) const = 0; + virtual void error(string msg, Exception* ex, ...) const = 0; }; #endif // _D_LOGGER_H_ diff --git a/src/Makefile.am b/src/Makefile.am index 6b5994ea..4ae30cca 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -12,7 +12,14 @@ SRCS = Socket.cc Socket.h\ HttpProxyRequestCommand.cc HttpProxyRequestCommand.h\ HttpProxyResponseCommand.cc HttpProxyResponseCommand.h\ HttpDownloadCommand.cc HttpDownloadCommand.h\ + HttpHeader.cc HttpHeader.h\ HttpConnection.cc HttpConnection.h\ + FtpConnection.cc FtpConnection.h\ + FtpInitiateConnectionCommand.cc FtpInitiateConnectionCommand.h\ + FtpNegotiationCommand.cc FtpNegotiationCommand.h\ + FtpDownloadCommand.cc FtpDownloadCommand.h\ + FtpTunnelRequestCommand.cc FtpTunnelRequestCommand.h\ + FtpTunnelResponseCommand.cc FtpTunnelResponseCommand.h\ SleepCommand.cc SleepCommand.h\ DownloadEngine.cc DownloadEngine.h\ Segment.h\ diff --git a/src/Makefile.in b/src/Makefile.in index 006717df..760a3bce 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -61,7 +61,14 @@ am__objects_1 = libaria2c_a-Socket.$(OBJEXT) \ libaria2c_a-HttpProxyRequestCommand.$(OBJEXT) \ libaria2c_a-HttpProxyResponseCommand.$(OBJEXT) \ libaria2c_a-HttpDownloadCommand.$(OBJEXT) \ + libaria2c_a-HttpHeader.$(OBJEXT) \ libaria2c_a-HttpConnection.$(OBJEXT) \ + libaria2c_a-FtpConnection.$(OBJEXT) \ + libaria2c_a-FtpInitiateConnectionCommand.$(OBJEXT) \ + libaria2c_a-FtpNegotiationCommand.$(OBJEXT) \ + libaria2c_a-FtpDownloadCommand.$(OBJEXT) \ + libaria2c_a-FtpTunnelRequestCommand.$(OBJEXT) \ + libaria2c_a-FtpTunnelResponseCommand.$(OBJEXT) \ libaria2c_a-SleepCommand.$(OBJEXT) \ libaria2c_a-DownloadEngine.$(OBJEXT) \ libaria2c_a-SegmentMan.$(OBJEXT) libaria2c_a-Util.$(OBJEXT) \ @@ -192,7 +199,14 @@ SRCS = Socket.cc Socket.h\ HttpProxyRequestCommand.cc HttpProxyRequestCommand.h\ HttpProxyResponseCommand.cc HttpProxyResponseCommand.h\ HttpDownloadCommand.cc HttpDownloadCommand.h\ + HttpHeader.cc HttpHeader.h\ HttpConnection.cc HttpConnection.h\ + FtpConnection.cc FtpConnection.h\ + FtpInitiateConnectionCommand.cc FtpInitiateConnectionCommand.h\ + FtpNegotiationCommand.cc FtpNegotiationCommand.h\ + FtpDownloadCommand.cc FtpDownloadCommand.h\ + FtpTunnelRequestCommand.cc FtpTunnelRequestCommand.h\ + FtpTunnelResponseCommand.cc FtpTunnelResponseCommand.h\ SleepCommand.cc SleepCommand.h\ DownloadEngine.cc DownloadEngine.h\ Segment.h\ @@ -305,8 +319,15 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-DownloadCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-DownloadEngine.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-File.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-FtpConnection.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-FtpDownloadCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-FtpInitiateConnectionCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-FtpNegotiationCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-FtpTunnelRequestCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-FtpTunnelResponseCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-HttpConnection.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-HttpDownloadCommand.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-HttpHeader.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-HttpInitiateConnectionCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-HttpProxyRequestCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-HttpProxyResponseCommand.Po@am__quote@ @@ -490,6 +511,20 @@ libaria2c_a-HttpDownloadCommand.obj: HttpDownloadCommand.cc @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-HttpDownloadCommand.obj `if test -f 'HttpDownloadCommand.cc'; then $(CYGPATH_W) 'HttpDownloadCommand.cc'; else $(CYGPATH_W) '$(srcdir)/HttpDownloadCommand.cc'; fi` +libaria2c_a-HttpHeader.o: HttpHeader.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-HttpHeader.o -MD -MP -MF "$(DEPDIR)/libaria2c_a-HttpHeader.Tpo" -c -o libaria2c_a-HttpHeader.o `test -f 'HttpHeader.cc' || echo '$(srcdir)/'`HttpHeader.cc; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-HttpHeader.Tpo" "$(DEPDIR)/libaria2c_a-HttpHeader.Po"; else rm -f "$(DEPDIR)/libaria2c_a-HttpHeader.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='HttpHeader.cc' object='libaria2c_a-HttpHeader.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-HttpHeader.o `test -f 'HttpHeader.cc' || echo '$(srcdir)/'`HttpHeader.cc + +libaria2c_a-HttpHeader.obj: HttpHeader.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-HttpHeader.obj -MD -MP -MF "$(DEPDIR)/libaria2c_a-HttpHeader.Tpo" -c -o libaria2c_a-HttpHeader.obj `if test -f 'HttpHeader.cc'; then $(CYGPATH_W) 'HttpHeader.cc'; else $(CYGPATH_W) '$(srcdir)/HttpHeader.cc'; fi`; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-HttpHeader.Tpo" "$(DEPDIR)/libaria2c_a-HttpHeader.Po"; else rm -f "$(DEPDIR)/libaria2c_a-HttpHeader.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='HttpHeader.cc' object='libaria2c_a-HttpHeader.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-HttpHeader.obj `if test -f 'HttpHeader.cc'; then $(CYGPATH_W) 'HttpHeader.cc'; else $(CYGPATH_W) '$(srcdir)/HttpHeader.cc'; fi` + libaria2c_a-HttpConnection.o: HttpConnection.cc @am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-HttpConnection.o -MD -MP -MF "$(DEPDIR)/libaria2c_a-HttpConnection.Tpo" -c -o libaria2c_a-HttpConnection.o `test -f 'HttpConnection.cc' || echo '$(srcdir)/'`HttpConnection.cc; \ @am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-HttpConnection.Tpo" "$(DEPDIR)/libaria2c_a-HttpConnection.Po"; else rm -f "$(DEPDIR)/libaria2c_a-HttpConnection.Tpo"; exit 1; fi @@ -504,6 +539,90 @@ libaria2c_a-HttpConnection.obj: HttpConnection.cc @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-HttpConnection.obj `if test -f 'HttpConnection.cc'; then $(CYGPATH_W) 'HttpConnection.cc'; else $(CYGPATH_W) '$(srcdir)/HttpConnection.cc'; fi` +libaria2c_a-FtpConnection.o: FtpConnection.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-FtpConnection.o -MD -MP -MF "$(DEPDIR)/libaria2c_a-FtpConnection.Tpo" -c -o libaria2c_a-FtpConnection.o `test -f 'FtpConnection.cc' || echo '$(srcdir)/'`FtpConnection.cc; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-FtpConnection.Tpo" "$(DEPDIR)/libaria2c_a-FtpConnection.Po"; else rm -f "$(DEPDIR)/libaria2c_a-FtpConnection.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='FtpConnection.cc' object='libaria2c_a-FtpConnection.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-FtpConnection.o `test -f 'FtpConnection.cc' || echo '$(srcdir)/'`FtpConnection.cc + +libaria2c_a-FtpConnection.obj: FtpConnection.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-FtpConnection.obj -MD -MP -MF "$(DEPDIR)/libaria2c_a-FtpConnection.Tpo" -c -o libaria2c_a-FtpConnection.obj `if test -f 'FtpConnection.cc'; then $(CYGPATH_W) 'FtpConnection.cc'; else $(CYGPATH_W) '$(srcdir)/FtpConnection.cc'; fi`; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-FtpConnection.Tpo" "$(DEPDIR)/libaria2c_a-FtpConnection.Po"; else rm -f "$(DEPDIR)/libaria2c_a-FtpConnection.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='FtpConnection.cc' object='libaria2c_a-FtpConnection.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-FtpConnection.obj `if test -f 'FtpConnection.cc'; then $(CYGPATH_W) 'FtpConnection.cc'; else $(CYGPATH_W) '$(srcdir)/FtpConnection.cc'; fi` + +libaria2c_a-FtpInitiateConnectionCommand.o: FtpInitiateConnectionCommand.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-FtpInitiateConnectionCommand.o -MD -MP -MF "$(DEPDIR)/libaria2c_a-FtpInitiateConnectionCommand.Tpo" -c -o libaria2c_a-FtpInitiateConnectionCommand.o `test -f 'FtpInitiateConnectionCommand.cc' || echo '$(srcdir)/'`FtpInitiateConnectionCommand.cc; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-FtpInitiateConnectionCommand.Tpo" "$(DEPDIR)/libaria2c_a-FtpInitiateConnectionCommand.Po"; else rm -f "$(DEPDIR)/libaria2c_a-FtpInitiateConnectionCommand.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='FtpInitiateConnectionCommand.cc' object='libaria2c_a-FtpInitiateConnectionCommand.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-FtpInitiateConnectionCommand.o `test -f 'FtpInitiateConnectionCommand.cc' || echo '$(srcdir)/'`FtpInitiateConnectionCommand.cc + +libaria2c_a-FtpInitiateConnectionCommand.obj: FtpInitiateConnectionCommand.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-FtpInitiateConnectionCommand.obj -MD -MP -MF "$(DEPDIR)/libaria2c_a-FtpInitiateConnectionCommand.Tpo" -c -o libaria2c_a-FtpInitiateConnectionCommand.obj `if test -f 'FtpInitiateConnectionCommand.cc'; then $(CYGPATH_W) 'FtpInitiateConnectionCommand.cc'; else $(CYGPATH_W) '$(srcdir)/FtpInitiateConnectionCommand.cc'; fi`; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-FtpInitiateConnectionCommand.Tpo" "$(DEPDIR)/libaria2c_a-FtpInitiateConnectionCommand.Po"; else rm -f "$(DEPDIR)/libaria2c_a-FtpInitiateConnectionCommand.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='FtpInitiateConnectionCommand.cc' object='libaria2c_a-FtpInitiateConnectionCommand.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-FtpInitiateConnectionCommand.obj `if test -f 'FtpInitiateConnectionCommand.cc'; then $(CYGPATH_W) 'FtpInitiateConnectionCommand.cc'; else $(CYGPATH_W) '$(srcdir)/FtpInitiateConnectionCommand.cc'; fi` + +libaria2c_a-FtpNegotiationCommand.o: FtpNegotiationCommand.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-FtpNegotiationCommand.o -MD -MP -MF "$(DEPDIR)/libaria2c_a-FtpNegotiationCommand.Tpo" -c -o libaria2c_a-FtpNegotiationCommand.o `test -f 'FtpNegotiationCommand.cc' || echo '$(srcdir)/'`FtpNegotiationCommand.cc; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-FtpNegotiationCommand.Tpo" "$(DEPDIR)/libaria2c_a-FtpNegotiationCommand.Po"; else rm -f "$(DEPDIR)/libaria2c_a-FtpNegotiationCommand.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='FtpNegotiationCommand.cc' object='libaria2c_a-FtpNegotiationCommand.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-FtpNegotiationCommand.o `test -f 'FtpNegotiationCommand.cc' || echo '$(srcdir)/'`FtpNegotiationCommand.cc + +libaria2c_a-FtpNegotiationCommand.obj: FtpNegotiationCommand.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-FtpNegotiationCommand.obj -MD -MP -MF "$(DEPDIR)/libaria2c_a-FtpNegotiationCommand.Tpo" -c -o libaria2c_a-FtpNegotiationCommand.obj `if test -f 'FtpNegotiationCommand.cc'; then $(CYGPATH_W) 'FtpNegotiationCommand.cc'; else $(CYGPATH_W) '$(srcdir)/FtpNegotiationCommand.cc'; fi`; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-FtpNegotiationCommand.Tpo" "$(DEPDIR)/libaria2c_a-FtpNegotiationCommand.Po"; else rm -f "$(DEPDIR)/libaria2c_a-FtpNegotiationCommand.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='FtpNegotiationCommand.cc' object='libaria2c_a-FtpNegotiationCommand.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-FtpNegotiationCommand.obj `if test -f 'FtpNegotiationCommand.cc'; then $(CYGPATH_W) 'FtpNegotiationCommand.cc'; else $(CYGPATH_W) '$(srcdir)/FtpNegotiationCommand.cc'; fi` + +libaria2c_a-FtpDownloadCommand.o: FtpDownloadCommand.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-FtpDownloadCommand.o -MD -MP -MF "$(DEPDIR)/libaria2c_a-FtpDownloadCommand.Tpo" -c -o libaria2c_a-FtpDownloadCommand.o `test -f 'FtpDownloadCommand.cc' || echo '$(srcdir)/'`FtpDownloadCommand.cc; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-FtpDownloadCommand.Tpo" "$(DEPDIR)/libaria2c_a-FtpDownloadCommand.Po"; else rm -f "$(DEPDIR)/libaria2c_a-FtpDownloadCommand.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='FtpDownloadCommand.cc' object='libaria2c_a-FtpDownloadCommand.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-FtpDownloadCommand.o `test -f 'FtpDownloadCommand.cc' || echo '$(srcdir)/'`FtpDownloadCommand.cc + +libaria2c_a-FtpDownloadCommand.obj: FtpDownloadCommand.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-FtpDownloadCommand.obj -MD -MP -MF "$(DEPDIR)/libaria2c_a-FtpDownloadCommand.Tpo" -c -o libaria2c_a-FtpDownloadCommand.obj `if test -f 'FtpDownloadCommand.cc'; then $(CYGPATH_W) 'FtpDownloadCommand.cc'; else $(CYGPATH_W) '$(srcdir)/FtpDownloadCommand.cc'; fi`; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-FtpDownloadCommand.Tpo" "$(DEPDIR)/libaria2c_a-FtpDownloadCommand.Po"; else rm -f "$(DEPDIR)/libaria2c_a-FtpDownloadCommand.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='FtpDownloadCommand.cc' object='libaria2c_a-FtpDownloadCommand.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-FtpDownloadCommand.obj `if test -f 'FtpDownloadCommand.cc'; then $(CYGPATH_W) 'FtpDownloadCommand.cc'; else $(CYGPATH_W) '$(srcdir)/FtpDownloadCommand.cc'; fi` + +libaria2c_a-FtpTunnelRequestCommand.o: FtpTunnelRequestCommand.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-FtpTunnelRequestCommand.o -MD -MP -MF "$(DEPDIR)/libaria2c_a-FtpTunnelRequestCommand.Tpo" -c -o libaria2c_a-FtpTunnelRequestCommand.o `test -f 'FtpTunnelRequestCommand.cc' || echo '$(srcdir)/'`FtpTunnelRequestCommand.cc; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-FtpTunnelRequestCommand.Tpo" "$(DEPDIR)/libaria2c_a-FtpTunnelRequestCommand.Po"; else rm -f "$(DEPDIR)/libaria2c_a-FtpTunnelRequestCommand.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='FtpTunnelRequestCommand.cc' object='libaria2c_a-FtpTunnelRequestCommand.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-FtpTunnelRequestCommand.o `test -f 'FtpTunnelRequestCommand.cc' || echo '$(srcdir)/'`FtpTunnelRequestCommand.cc + +libaria2c_a-FtpTunnelRequestCommand.obj: FtpTunnelRequestCommand.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-FtpTunnelRequestCommand.obj -MD -MP -MF "$(DEPDIR)/libaria2c_a-FtpTunnelRequestCommand.Tpo" -c -o libaria2c_a-FtpTunnelRequestCommand.obj `if test -f 'FtpTunnelRequestCommand.cc'; then $(CYGPATH_W) 'FtpTunnelRequestCommand.cc'; else $(CYGPATH_W) '$(srcdir)/FtpTunnelRequestCommand.cc'; fi`; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-FtpTunnelRequestCommand.Tpo" "$(DEPDIR)/libaria2c_a-FtpTunnelRequestCommand.Po"; else rm -f "$(DEPDIR)/libaria2c_a-FtpTunnelRequestCommand.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='FtpTunnelRequestCommand.cc' object='libaria2c_a-FtpTunnelRequestCommand.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-FtpTunnelRequestCommand.obj `if test -f 'FtpTunnelRequestCommand.cc'; then $(CYGPATH_W) 'FtpTunnelRequestCommand.cc'; else $(CYGPATH_W) '$(srcdir)/FtpTunnelRequestCommand.cc'; fi` + +libaria2c_a-FtpTunnelResponseCommand.o: FtpTunnelResponseCommand.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-FtpTunnelResponseCommand.o -MD -MP -MF "$(DEPDIR)/libaria2c_a-FtpTunnelResponseCommand.Tpo" -c -o libaria2c_a-FtpTunnelResponseCommand.o `test -f 'FtpTunnelResponseCommand.cc' || echo '$(srcdir)/'`FtpTunnelResponseCommand.cc; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-FtpTunnelResponseCommand.Tpo" "$(DEPDIR)/libaria2c_a-FtpTunnelResponseCommand.Po"; else rm -f "$(DEPDIR)/libaria2c_a-FtpTunnelResponseCommand.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='FtpTunnelResponseCommand.cc' object='libaria2c_a-FtpTunnelResponseCommand.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-FtpTunnelResponseCommand.o `test -f 'FtpTunnelResponseCommand.cc' || echo '$(srcdir)/'`FtpTunnelResponseCommand.cc + +libaria2c_a-FtpTunnelResponseCommand.obj: FtpTunnelResponseCommand.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-FtpTunnelResponseCommand.obj -MD -MP -MF "$(DEPDIR)/libaria2c_a-FtpTunnelResponseCommand.Tpo" -c -o libaria2c_a-FtpTunnelResponseCommand.obj `if test -f 'FtpTunnelResponseCommand.cc'; then $(CYGPATH_W) 'FtpTunnelResponseCommand.cc'; else $(CYGPATH_W) '$(srcdir)/FtpTunnelResponseCommand.cc'; fi`; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-FtpTunnelResponseCommand.Tpo" "$(DEPDIR)/libaria2c_a-FtpTunnelResponseCommand.Po"; else rm -f "$(DEPDIR)/libaria2c_a-FtpTunnelResponseCommand.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='FtpTunnelResponseCommand.cc' object='libaria2c_a-FtpTunnelResponseCommand.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-FtpTunnelResponseCommand.obj `if test -f 'FtpTunnelResponseCommand.cc'; then $(CYGPATH_W) 'FtpTunnelResponseCommand.cc'; else $(CYGPATH_W) '$(srcdir)/FtpTunnelResponseCommand.cc'; fi` + libaria2c_a-SleepCommand.o: SleepCommand.cc @am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-SleepCommand.o -MD -MP -MF "$(DEPDIR)/libaria2c_a-SleepCommand.Tpo" -c -o libaria2c_a-SleepCommand.o `test -f 'SleepCommand.cc' || echo '$(srcdir)/'`SleepCommand.cc; \ @am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-SleepCommand.Tpo" "$(DEPDIR)/libaria2c_a-SleepCommand.Po"; else rm -f "$(DEPDIR)/libaria2c_a-SleepCommand.Tpo"; exit 1; fi diff --git a/src/Request.cc b/src/Request.cc index b87cab1d..6eb4aa1e 100644 --- a/src/Request.cc +++ b/src/Request.cc @@ -22,14 +22,13 @@ #include "Request.h" #include "Util.h" -#define MAX_RETRY_COUNT 5 - -Request::Request():port(0), retryCount(0) { +Request::Request():port(0), tryCount(0) { defaultPorts["http"] = 80; #ifdef HAVE_LIBSSL // for SSL defaultPorts["https"] = 443; #endif // HAVE_LIBSSL + defaultPorts["ftp"] = 21; seg.sp = 0; seg.ep = 0; seg.ds = 0; @@ -102,15 +101,3 @@ bool Request::parseUrl(string url) { } return true; } - -void Request::resetRetryCount() { - this->retryCount = 0; -} - -void Request::addRetryCount() { - retryCount++; -} - -bool Request::noMoreRetry() { - return retryCount >= MAX_RETRY_COUNT; -} diff --git a/src/Request.h b/src/Request.h index 0d9554e6..7a7b0fce 100644 --- a/src/Request.h +++ b/src/Request.h @@ -29,6 +29,8 @@ using namespace std; +#define MAX_TRY_COUNT 5 + #define SAFE_CHARS "abcdefghijklmnopqrstuvwxyz"\ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"\ "0123456789"\ @@ -55,7 +57,7 @@ private: string dir; string file; map defaultPorts; - int retryCount; + int tryCount; bool parseUrl(string url); public: Segment seg; @@ -72,15 +74,14 @@ public: // Returns true if parsing goes successful, otherwise returns false. bool redirectUrl(string url); bool resetUrl(); - void resetRetryCount(); - void addRetryCount(); - int getRetryCount(); - bool noMoreRetry(); + void resetTryCount() { tryCount = 0; } + void addTryCount() { tryCount++; } + int getTryCount() const { return tryCount; } + bool noMoreTry() const { return tryCount >= MAX_TRY_COUNT; } string getUrl() const { return url; } string getCurrentUrl() const { return currentUrl; } string getPreviousUrl() const { return previousUrl; } - void setPreviousUrl(string url) { previousUrl = url; } string getReferer() const { return referer; } void setReferer(string url) { referer = previousUrl = url; } string getProtocol() const { return protocol; } diff --git a/src/SegmentMan.cc b/src/SegmentMan.cc index 8dca0ba0..579c2bfe 100644 --- a/src/SegmentMan.cc +++ b/src/SegmentMan.cc @@ -123,7 +123,7 @@ void SegmentMan::updateSegment(const Segment& segment) { } -bool SegmentMan::segmentFileExists() { +bool SegmentMan::segmentFileExists() const { if(!isSplittable) { return false; } @@ -154,14 +154,17 @@ void SegmentMan::load() { } } -void SegmentMan::save() { - if(!isSplittable) { +void SegmentMan::save() const { + if(!isSplittable || totalSize == 0) { return; } string segFilename = getSegmentFilePath(); logger->info(MSG_SAVING_SEGMENT_FILE, segFilename.c_str()); FILE* segFile = openSegFile(segFilename, "w"); - for(vector::iterator itr = segments.begin(); itr != segments.end(); itr++) { + if(fwrite(&totalSize, sizeof(totalSize), 1, segFile) < 1) { + throw new DlAbortEx(strerror(errno)); + } + for(vector::const_iterator itr = segments.begin(); itr != segments.end(); itr++) { if(fwrite(&*itr, sizeof(Segment), 1, segFile) < 1) { throw new DlAbortEx(strerror(errno)); } @@ -170,7 +173,7 @@ void SegmentMan::save() { logger->info(MSG_SAVED_SEGMENT_FILE); } -FILE* SegmentMan::openSegFile(string segFilename, string mode) { +FILE* SegmentMan::openSegFile(string segFilename, string mode) const { FILE* segFile = fopen(segFilename.c_str(), mode.c_str()); if(segFile == NULL) { throw new DlAbortEx(strerror(errno)); @@ -180,6 +183,9 @@ FILE* SegmentMan::openSegFile(string segFilename, string mode) { void SegmentMan::read(FILE* file) { assert(file != NULL); + if(fread(&totalSize, sizeof(totalSize), 1, file) < 1) { + throw new DlAbortEx(strerror(errno)); + } while(1) { Segment seg; if(fread(&seg, sizeof(Segment), 1, file) < 1) { @@ -193,7 +199,7 @@ void SegmentMan::read(FILE* file) { } } -void SegmentMan::remove() { +void SegmentMan::remove() const { if(!isSplittable) { return; } @@ -203,11 +209,11 @@ void SegmentMan::remove() { } } -bool SegmentMan::finished() { +bool SegmentMan::finished() const { if(!downloadStarted || segments.size() == 0) { return false; } - for(vector::iterator itr = segments.begin(); itr != segments.end(); itr++) { + for(vector::const_iterator itr = segments.begin(); itr != segments.end(); itr++) { if(!(*itr).finish) { return false; } @@ -215,15 +221,15 @@ bool SegmentMan::finished() { return true; } -void SegmentMan::removeIfFinished() { +void SegmentMan::removeIfFinished() const { if(finished()) { remove(); } } -long long int SegmentMan::getDownloadedSize() { +long long int SegmentMan::getDownloadedSize() const { long long int size = 0; - for(vector::iterator itr = segments.begin(); itr != segments.end(); itr++) { + for(vector::const_iterator itr = segments.begin(); itr != segments.end(); itr++) { size += (*itr).ds; } return size; diff --git a/src/SegmentMan.h b/src/SegmentMan.h index 1ef744e3..e971a176 100644 --- a/src/SegmentMan.h +++ b/src/SegmentMan.h @@ -37,7 +37,7 @@ using namespace std; class SegmentMan { private: void read(FILE* file); - FILE* openSegFile(string segFilename, string mode); + FILE* openSegFile(string segFilename, string mode) const; public: /** * The total number of bytes to download. @@ -80,7 +80,7 @@ public: */ string ufilename; - Logger* logger; + const Logger* logger; SegmentMan(); ~SegmentMan(); @@ -89,13 +89,13 @@ public: * Returns dir+"/"+filename. * If filename is empty, then returns dir+"/"+"inex.html"; */ - string getFilePath() { + string getFilePath() const { return (dir == "" ? "." : dir)+"/"+ (ufilename == "" ? (filename == "" ? "index.html" : filename) : ufilename); } - string getSegmentFilePath() { + string getSegmentFilePath() const { return getFilePath()+SEGMENT_FILE_EXTENSION; } @@ -129,7 +129,7 @@ public: * The file name of the segment data is filename appended by ".aria2". * If isSplittable is false, then returns simply false without any operation. */ - bool segmentFileExists(); + bool segmentFileExists() const; /** * Loads the segment data file. * If isSplittable is false, then returns without any operation. @@ -139,26 +139,26 @@ public: * Saves the segment data file. * If isSplittable is false, then returns without any operation. */ - void save(); + void save() const; /** * Removes the segment data file. * If isSplittable is false, then returns without any operation. */ - void remove(); + void remove() const; /** * Returs true when the download has finished. * If downloadStarted is false or the number of the segments of this object * holds is 0, then returns false. */ - bool finished(); + bool finished() const; /** * if finished() is true, then call remove() */ - void removeIfFinished(); + void removeIfFinished() const; /** * Returns the total number of bytes to be downloaded. */ - long long int getDownloadedSize(); + long long int getDownloadedSize() const; }; #endif // _D_SEGMENT_MAN_H_ diff --git a/src/SimpleLogger.cc b/src/SimpleLogger.cc index 9c2b28fe..476428fd 100644 --- a/src/SimpleLogger.cc +++ b/src/SimpleLogger.cc @@ -36,7 +36,8 @@ SimpleLogger::~SimpleLogger() { } } -void SimpleLogger::writeLog(int level, string msg, va_list ap, Exception* e) { +void SimpleLogger::writeLog(int level, string msg, va_list ap, Exception* e) const +{ string levelStr; switch(level) { case DEBUG: @@ -56,14 +57,14 @@ void SimpleLogger::writeLog(int level, string msg, va_list ap, Exception* e) { fflush(stdout); } -void SimpleLogger::debug(string msg, ...) { +void SimpleLogger::debug(string msg, ...) const { va_list ap; va_start(ap, msg); writeLog(DEBUG, msg, ap); va_end(ap); } -void SimpleLogger::debug(string msg, Exception* e, ...) { +void SimpleLogger::debug(string msg, Exception* e, ...) const { va_list ap; va_start(ap, e); writeLog(DEBUG, msg, ap, e); @@ -71,14 +72,14 @@ void SimpleLogger::debug(string msg, Exception* e, ...) { } -void SimpleLogger::info(string msg, ...) { +void SimpleLogger::info(string msg, ...) const { va_list ap; va_start(ap, msg); writeLog(INFO, msg, ap); va_end(ap); } -void SimpleLogger::info(string msg, Exception* e, ...) { +void SimpleLogger::info(string msg, Exception* e, ...) const { va_list ap; va_start(ap, e); writeLog(INFO, msg, ap, e); @@ -86,14 +87,14 @@ void SimpleLogger::info(string msg, Exception* e, ...) { } -void SimpleLogger::error(string msg, ...) { +void SimpleLogger::error(string msg, ...) const { va_list ap; va_start(ap, msg); writeLog(ERROR, msg, ap); va_end(ap); } -void SimpleLogger::error(string msg, Exception* e, ...) { +void SimpleLogger::error(string msg, Exception* e, ...) const { va_list ap; va_start(ap, e); writeLog(ERROR, msg, ap, e); diff --git a/src/SimpleLogger.h b/src/SimpleLogger.h index 4b3843a2..5caab527 100644 --- a/src/SimpleLogger.h +++ b/src/SimpleLogger.h @@ -30,19 +30,19 @@ private: DEBUG, INFO, ERROR}; - void writeLog(int level, string msg, va_list ap, Exception* e = NULL); + void writeLog(int level, string msg, va_list ap, Exception* e = NULL) const; FILE* file; public: SimpleLogger(string filename); SimpleLogger(FILE* logfile); ~SimpleLogger(); - void debug(string msg, ...); - void debug(string msg, Exception* ex, ...); - void info(string msg, ...); - void info(string msg, Exception* ex, ...); - void error(string msg, ...); - void error(string msg, Exception* ex, ...); + void debug(string msg, ...) const; + void debug(string msg, Exception* ex, ...) const; + void info(string msg, ...) const; + void info(string msg, Exception* ex, ...) const; + void error(string msg, ...) const; + void error(string msg, Exception* ex, ...) const; }; #endif // _D_SIMPLE_LOGGER_H_ diff --git a/src/Socket.cc b/src/Socket.cc index bf9a04f1..f4e0862d 100644 --- a/src/Socket.cc +++ b/src/Socket.cc @@ -30,6 +30,10 @@ Socket::Socket(const Socket& s) { core->use++; } +Socket::Socket(SocketCore* core) { + this->core = core; +} + Socket::~Socket() { core->use--; if(core->use == 0) { @@ -49,38 +53,54 @@ Socket& Socket::operator=(const Socket& s) { return *this; } -void Socket::establishConnection(string host, int port) { +void Socket::beginListen() const { + core->beginListen(); +} + +void Socket::getAddrInfo(pair& addrinfo) const { + core->getAddrInfo(addrinfo); +} + +Socket* Socket::acceptConnection() const { + return new Socket(core->acceptConnection()); +} + +void Socket::establishConnection(string host, int port) const { core->establishConnection(host, port); } -void Socket::setNonBlockingMode() { - core->setNonBlockingMode(); +void Socket::setBlockingMode() const { + core->setBlockingMode(); } -void Socket::closeConnection() { +void Socket::closeConnection() const { core->closeConnection(); } -bool Socket::isWritable(int timeout) { +bool Socket::isWritable(int timeout) const { return core->isWritable(timeout); } -bool Socket::isReadable(int timeout) { +bool Socket::isReadable(int timeout) const { return core->isReadable(timeout); } -void Socket::writeData(const char* data, int len, int timeout) { +void Socket::writeData(const char* data, int len, int timeout) const { core->writeData(data, len, timeout); } -void Socket::readData(char* data, int& len, int timeout) { +void Socket::writeData(string str, int timeout) const { + core->writeData(str.c_str(), str.size(), timeout); +} + +void Socket::readData(char* data, int& len, int timeout) const { core->readData(data, len, timeout); } -void Socket::peekData(char* data, int& len, int timeout) { +void Socket::peekData(char* data, int& len, int timeout) const { core->peekData(data, len, timeout); } -void Socket::initiateSecureConnection() { +void Socket::initiateSecureConnection() const { core->initiateSecureConnection(); } diff --git a/src/Socket.h b/src/Socket.h index a4624801..ea895dc5 100644 --- a/src/Socket.h +++ b/src/Socket.h @@ -32,6 +32,7 @@ class Socket { private: // socket endpoint descriptor SocketCore* core; + Socket(SocketCore* core); public: Socket(); Socket(const Socket& s); @@ -41,46 +42,51 @@ public: int getSockfd() const { return core->sockfd; } + void beginListen() const; + void getAddrInfo(pair& addrinfo) const; + Socket* acceptConnection() const; + /** * Connects to the server named host and the destination port is port. * This method make socket non-blocking mode. - * To make the socket blocking mode, call setNonBlockingMode() after + * To make the socket blocking mode, call setBlockingMode() after * the connection is established. */ - void establishConnection(string host, int port); + void establishConnection(string host, int port) const; - void setNonBlockingMode(); + void setBlockingMode() const; // Closes the connection which this socket object has - void closeConnection(); + void closeConnection() const; // examines whether the socket of this Socket object is available for writing. // returns true if the socket is available for writing, otherwise returns false. - bool isWritable(int timeout); + bool isWritable(int timeout) const; // examines whether the socket of this Socket object is available for reading. // returns true if the socket is available for reading, otherwise returns false. - bool isReadable(int timeout); + bool isReadable(int timeout) const; // writes characters into the socket. data is a pointer pointing the first // byte of the data and len is the length of the data. - void writeData(const char* data, int len, int timeout = 5); + void writeData(const char* data, int len, int timeout = 5) const; + void writeData(string str, int timeout = 5) const; // Reads up to len bytes from this socket. // data is a pointer pointing the first // byte of the data, which must be allocated before this method is called. // len is the size of the allocated memory. When this method returns // successfully, len is replaced by the size of the read data. - void readData(char* data, int& len, int timeout = 5); + void readData(char* data, int& len, int timeout = 5) const; // Reads up to len bytes from this socket, but bytes are not removed from // this socket. - void peekData(char* data, int& len, int timeout = 5); + void peekData(char* data, int& len, int timeout = 5) const; /** * Makes this socket secure. * If the system has not OpenSSL, then this method do nothing. */ - void initiateSecureConnection(); + void initiateSecureConnection() const; }; #endif // _D_SOCKET_H_ diff --git a/src/SocketCore.cc b/src/SocketCore.cc index 94b986ed..53051679 100644 --- a/src/SocketCore.cc +++ b/src/SocketCore.cc @@ -34,29 +34,92 @@ #include #include "message.h" -SocketCore::SocketCore():sockfd(-1), use(1), secure(false) +SocketCore::SocketCore():sockfd(-1) { + init(); +} + +SocketCore::SocketCore(int sockfd):sockfd(sockfd) { + init(); +} + +void SocketCore::init() { + use = 1; + secure = false; #ifdef HAVE_LIBSSL // for SSL - , sslCtx(NULL), ssl(NULL) + sslCtx = NULL; + ssl = NULL; #endif // HAVE_LIBSSL -{} +} SocketCore::~SocketCore() { closeConnection(); } +void SocketCore::beginListen() { + closeConnection(); + //sockfd = socket(AF_UNSPEC, SOCK_STREAM, PF_UNSPEC); + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if(sockfd == -1) { + throw new DlAbortEx(strerror(errno)); + } + socklen_t sockopt = 1; + if(setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &sockopt, sizeof(socklen_t)) < 0) { + close(sockfd); + sockfd = -1; + throw new DlAbortEx(strerror(errno)); + } + + struct sockaddr_in sockaddr; + memset((char*)&sockaddr, 0, sizeof(sockaddr)); + sockaddr.sin_family = AF_INET; + sockaddr.sin_addr.s_addr = INADDR_ANY; + sockaddr.sin_port = htons(0); + + if(bind(sockfd, (struct sockaddr*)&sockaddr, sizeof(sockaddr)) == -1) { + throw new DlAbortEx(strerror(errno)); + } + + if(listen(sockfd, 1) == -1) { + throw new DlAbortEx(strerror(errno)); + } +} + +SocketCore* SocketCore::acceptConnection() const { + struct sockaddr_in sockaddr; + socklen_t len = sizeof(sockaddr); + memset((char*)&sockaddr, 0, sizeof(sockaddr)); + int fd; + if((fd = accept(sockfd, (struct sockaddr*)&sockaddr, &len)) == -1) { + throw new DlAbortEx(strerror(errno)); + } + SocketCore* s = new SocketCore(fd); + return s; +} + +void SocketCore::getAddrInfo(pair& addrinfo) const { + struct sockaddr_in listenaddr; + memset((char*)&listenaddr, 0, sizeof(listenaddr)); + socklen_t len = sizeof(listenaddr); + if(getsockname(sockfd, (struct sockaddr*)&listenaddr, &len) == -1) { + throw new DlAbortEx(strerror(errno)); + } + addrinfo.first = inet_ntoa(listenaddr.sin_addr); + addrinfo.second = ntohs(listenaddr.sin_port); +} + void SocketCore::establishConnection(string host, int port) { + closeConnection(); sockfd = socket(AF_INET, SOCK_STREAM, 0); - if(sockfd >= 0) { - socklen_t sockopt = 1; - if(setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &sockopt, sizeof(int)) < 0) { - close(sockfd); - sockfd = -1; - throw new DlAbortEx(strerror(errno)); - } - } else { + if(sockfd == -1) { throw new DlAbortEx(strerror(errno)); } + socklen_t sockopt = 1; + if(setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &sockopt, sizeof(socklen_t)) < 0) { + close(sockfd); + sockfd = -1; + throw new DlAbortEx(strerror(errno)); + } struct sockaddr_in sockaddr; memset((char*)&sockaddr, 0, sizeof(sockaddr)); @@ -87,7 +150,7 @@ void SocketCore::establishConnection(string host, int port) { } } -void SocketCore::setNonBlockingMode() { +void SocketCore::setBlockingMode() const { int flags = fcntl(sockfd, F_GETFL, 0); fcntl(sockfd, F_SETFL, flags&~O_NONBLOCK); } @@ -95,7 +158,7 @@ void SocketCore::setNonBlockingMode() { void SocketCore::closeConnection() { #ifdef HAVE_LIBSSL // for SSL - if(secure) { + if(secure && ssl != NULL) { SSL_shutdown(ssl); } #endif // HAVE_LIBSSL @@ -105,7 +168,7 @@ void SocketCore::closeConnection() { } #ifdef HAVE_LIBSSL // for SSL - if(secure) { + if(secure && ssl != NULL) { SSL_free(ssl); SSL_CTX_free(sslCtx); ssl = NULL; @@ -114,7 +177,7 @@ void SocketCore::closeConnection() { #endif // HAVE_LIBSSL } -bool SocketCore::isWritable(int timeout) { +bool SocketCore::isWritable(int timeout) const { fd_set fds; FD_ZERO(&fds); FD_SET(sockfd, &fds); @@ -130,11 +193,15 @@ bool SocketCore::isWritable(int timeout) { // time out return false; } else { - throw new DlRetryEx(strerror(errno)); + if(errno == EINPROGRESS) { + return false; + } else { + throw new DlRetryEx(strerror(errno)); + } } } -bool SocketCore::isReadable(int timeout) { +bool SocketCore::isReadable(int timeout) const { fd_set fds; FD_ZERO(&fds); FD_SET(sockfd, &fds); @@ -150,11 +217,15 @@ bool SocketCore::isReadable(int timeout) { // time out return false; } else { - throw new DlRetryEx(strerror(errno)); + if(errno == EINPROGRESS) { + return false; + } else { + throw new DlRetryEx(strerror(errno)); + } } } -void SocketCore::writeData(const char* data, int len, int timeout) { +void SocketCore::writeData(const char* data, int len, int timeout) const { if(!isWritable(timeout) || !secure && send(sockfd, data, (size_t)len, 0) != len #ifdef HAVE_LIBSSL @@ -167,7 +238,7 @@ void SocketCore::writeData(const char* data, int len, int timeout) { } } -void SocketCore::readData(char* data, int& len, int timeout) { +void SocketCore::readData(char* data, int& len, int timeout) const { if(!isReadable(timeout) || !secure && (len = recv(sockfd, data, (size_t)len, 0)) < 0 #ifdef HAVE_LIBSSL @@ -180,7 +251,7 @@ void SocketCore::readData(char* data, int& len, int timeout) { } } -void SocketCore::peekData(char* data, int& len, int timeout) { +void SocketCore::peekData(char* data, int& len, int timeout) const { if(!isReadable(timeout) || !secure && (len = recv(sockfd, data, (size_t)len, MSG_PEEK)) < 0 #ifdef HAVE_LIBSSL diff --git a/src/SocketCore.h b/src/SocketCore.h index ba5fb7af..a6b7d3f8 100644 --- a/src/SocketCore.h +++ b/src/SocketCore.h @@ -23,6 +23,7 @@ #define _D_SOCKET_CORE_H_ #include +#include #include "common.h" #ifdef HAVE_LIBSSL @@ -45,50 +46,55 @@ private: SSL_CTX* sslCtx; SSL* ssl; #endif // HAVE_LIBSSL + void init(); + SocketCore(int sockfd); public: SocketCore(); ~SocketCore(); + void beginListen(); + void getAddrInfo(pair& addrinfo) const; + SocketCore* acceptConnection() const; /** * Connects to the server named host and the destination port is port. * This method make socket non-blocking mode. - * To make the socket blocking mode, call setNonBlockingMode() after + * To make the socket blocking mode, call setBlockingMode() after * the connection is established. */ void establishConnection(string host, int port); - void setNonBlockingMode(); + void setBlockingMode() const; // Closes the connection which this socket object has void closeConnection(); // examines whether the socket of this SocketCore object is available for writing. // returns true if the socket is available for writing, otherwise returns false. - bool isWritable(int timeout); + bool isWritable(int timeout) const; // examines whether the socket of this SocketCore object is available for reading. // returns true if the socket is available for reading, otherwise returns false. - bool isReadable(int timeout); + bool isReadable(int timeout) const; // writes characters into the socket. data is a pointer pointing the first // byte of the data and len is the length of the data. - void writeData(const char* data, int len, int timeout = 5); + void writeData(const char* data, int len, int timeout = 5) const; // Reads up to len bytes from this socket. // data is a pointer pointing the first // byte of the data, which must be allocated before this method is called. // len is the size of the allocated memory. When this method returns // successfully, len is replaced by the size of the read data. - void readData(char* data, int& len, int timeout = 5); + void readData(char* data, int& len, int timeout = 5) const; // Reads up to len bytes from this socket, but bytes are not removed from // this socket. - void peekData(char* data, int& len, int timeout = 5); + void peekData(char* data, int& len, int timeout = 5) const; /** * Makes this socket secure. * If the system has not OpenSSL, then this method do nothing. */ - void initiateSecureConnection(); + void initiateSecureConnection() ; }; #endif // _D_SOCKET_CORE_H_ diff --git a/src/Util.cc b/src/Util.cc index 13258e41..99b70785 100644 --- a/src/Util.cc +++ b/src/Util.cc @@ -108,3 +108,14 @@ void Util::slice(vector& result, string src, char delim) { result.push_back(trim(term)); } } + +bool Util::endsWith(string target, string part) { + if(target.size() < part.size()) { + return false; + } + if(target.compare(target.size()-part.size(), part.size(), part, 0, part.size()) == 0) { + return true; + } else { + return false; + } +} diff --git a/src/Util.h b/src/Util.h index 02f73657..ebe67910 100644 --- a/src/Util.h +++ b/src/Util.h @@ -49,6 +49,8 @@ public: static void slice(vector& result, string src, char delim); static string trim(string src); + + static bool endsWith(string target, string part); }; #endif // _D_UTIL_H_ diff --git a/src/main.cc b/src/main.cc index 718ddd76..773bdd17 100644 --- a/src/main.cc +++ b/src/main.cc @@ -147,6 +147,15 @@ int main(int argc, char* argv[]) { Option* op = new Option(); op->put("retry_wait", "5"); + // TODO warning! Delete username/password line when commit + op->put("ftp_user", "anonymous"); + op->put("ftp_passwd", "IE60USER@"); + op->put("ftp_type", "I"); + + op->put("ftp_pasv_enabled", "true"); + op->put("ftp_via_http_proxy", "tunnel"); + op->put("http_abs_uri_request_enabled", "true"); + while(1) { int optIndex = 0; int lopt; diff --git a/src/message.h b/src/message.h index 445828fa..d326953c 100644 --- a/src/message.h +++ b/src/message.h @@ -28,10 +28,12 @@ #define MSG_SEGMENT_CHANGED "CUID#%d - The segment changed. We send the request again with new Range header." #define MSG_REDIRECT "CUID#%d - Redirecting to %s" #define MSG_SENDING_HTTP_REQUEST "CUID#%d - Sending the request:\n%s" +#define MSG_SENDING_FTP_REQUEST "CUID#%d - Sending the request: %s" #define MSG_RECEIVE_RESPONSE "CUID#%d - Response received:\n%s" #define MSG_DOWNLOAD_ABORTED "CUID#%d - Download aborted." #define MSG_RESTARTING_DOWNLOAD "CUID#%d - Restarting the download." #define MSG_MAX_RETRY "CUID#%d - The retry count reached its max value. Download aborted." +#define MSG_UNREGISTER_CUID "CUID#%d - Unregistering cuid from segmentManager." #define MSG_SEGMENT_FILE_EXISTS "The segment file %s exists." #define MSG_SEGMENT_FILE_DOES_NOT_EXIST "The segment file %s does not exist." @@ -44,12 +46,16 @@ #define EX_INVALID_CHUNK_SIZE "Invalid chunk size." #define EX_TOO_LARGE_CHUNK "Too large chunk. size = %d" #define EX_INVALID_HEADER "Invalid header." +#define EX_INVALID_RESPONSE "Invalid response." #define EX_NO_HEADER "No header found." #define EX_NO_STATUS_HEADER "No status header." #define EX_PROXY_CONNECTION_FAILED "Proxy connection failed." +#define EX_CONNECTION_FAILED "Connection failed." #define EX_FILENAME_MISMATCH "The requested filename and the previously registered one are not same. %s != %s" #define EX_BAD_STATUS "The response status is not successful. status = %d" #define EX_TOO_LARGE_FILE "Too large file size. size = %d" #define EX_TRANSFER_ENCODING_NOT_SUPPORTED "Transfer encoding %s is not supported." #define EX_SSL_INIT_FAILURE "SSL initialization failed." +#define EX_SIZE_MISMATCH "Size mismatch %d != %d" + #endif // _D_MESSAGE_H_ -- cgit From e0aa767f9428eb34a7ab7c1169c1ecc55262dade Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 21 Feb 2006 12:28:42 +0000 Subject: --- src/FtpConnection.cc | 207 +++++++++++++++++++++++ src/FtpConnection.h | 65 +++++++ src/FtpDownloadCommand.cc | 38 +++++ src/FtpDownloadCommand.h | 37 ++++ src/FtpInitiateConnectionCommand.cc | 84 +++++++++ src/FtpInitiateConnectionCommand.h | 39 +++++ src/FtpNegotiationCommand.cc | 328 ++++++++++++++++++++++++++++++++++++ src/FtpNegotiationCommand.h | 87 ++++++++++ src/FtpTunnelRequestCommand.cc | 41 +++++ src/FtpTunnelRequestCommand.h | 35 ++++ src/FtpTunnelResponseCommand.cc | 49 ++++++ src/FtpTunnelResponseCommand.h | 39 +++++ src/HttpHeader.cc | 66 ++++++++ src/HttpHeader.h | 46 +++++ 14 files changed, 1161 insertions(+) create mode 100644 src/FtpConnection.cc create mode 100644 src/FtpConnection.h create mode 100644 src/FtpDownloadCommand.cc create mode 100644 src/FtpDownloadCommand.h create mode 100644 src/FtpInitiateConnectionCommand.cc create mode 100644 src/FtpInitiateConnectionCommand.h create mode 100644 src/FtpNegotiationCommand.cc create mode 100644 src/FtpNegotiationCommand.h create mode 100644 src/FtpTunnelRequestCommand.cc create mode 100644 src/FtpTunnelRequestCommand.h create mode 100644 src/FtpTunnelResponseCommand.cc create mode 100644 src/FtpTunnelResponseCommand.h create mode 100644 src/HttpHeader.cc create mode 100644 src/HttpHeader.h (limited to 'src') diff --git a/src/FtpConnection.cc b/src/FtpConnection.cc new file mode 100644 index 00000000..4c528ada --- /dev/null +++ b/src/FtpConnection.cc @@ -0,0 +1,207 @@ +/* */ +#include "FtpConnection.h" +#include "Util.h" +#include "DlAbortEx.h" +#include "message.h" + +FtpConnection::FtpConnection(int cuid, const Socket* socket, const Request* req, const Option* op, const Logger* logger):cuid(cuid), socket(socket), req(req), option(op), logger(logger) {} + +FtpConnection::~FtpConnection() {} + +void FtpConnection::sendUser() const { + string request = "USER "+option->get("ftp_user")+"\r\n"; + logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + socket->writeData(request); +} + +void FtpConnection::sendPass() const { + string request = "PASS "+option->get("ftp_passwd")+"\r\n"; + logger->info(MSG_SENDING_FTP_REQUEST, cuid, "PASS ********"); + socket->writeData(request); +} + +void FtpConnection::sendType() const { + string request = "TYPE "+option->get("ftp_type")+"\r\n"; + logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + socket->writeData(request); +} + +void FtpConnection::sendCwd() const { + string request = "CWD "+req->getDir()+"\r\n"; + logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + socket->writeData(request); +} + +void FtpConnection::sendSize() const { + string request = "SIZE "+req->getFile()+"\r\n"; + logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + socket->writeData(request); +} + +void FtpConnection::sendPasv() const { + string request = "PASV\r\n"; + logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + socket->writeData(request); +} + +Socket* FtpConnection::sendPort() const { + Socket* serverSocket = new Socket(); + try { + serverSocket->beginListen(); + + pair addrinfo; + socket->getAddrInfo(addrinfo); + int ipaddr[4]; + sscanf(addrinfo.first.c_str(), "%d.%d.%d.%d", + &ipaddr[0], &ipaddr[1], &ipaddr[2], &ipaddr[3]); + serverSocket->getAddrInfo(addrinfo); + string request = "PORT "+ + Util::itos(ipaddr[0])+","+Util::itos(ipaddr[1])+","+ + Util::itos(ipaddr[2])+","+Util::itos(ipaddr[3])+","+ + Util::itos(addrinfo.second/256)+","+Util::itos(addrinfo.second%256)+"\r\n"; + logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + socket->writeData(request); + } catch (Exception* ex) { + delete serverSocket; + throw; + } + return serverSocket; +} + +void FtpConnection::sendRest(const Segment& segment) const { + string request = "REST "+Util::llitos(segment.sp+segment.ds)+"\r\n"; + logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + socket->writeData(request); +} + +void FtpConnection::sendRetr() const { + string request = "RETR "+req->getFile()+"\r\n"; + logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + socket->writeData(request); +} + +int FtpConnection::getStatus(string response) const { + int status; + // TODO we must handle when the response is not like "%d %*s" + // In this case, we return 0 + if(sscanf(response.c_str(), "%d %*s", &status) == 1) { + return status; + } else { + return 0; + } +} + +bool FtpConnection::isEndOfResponse(int status, string response) const { + if(response.size() <= 4) { + return false; + } + // if forth character of buf is '-', then multi line response is expected. + if(response.at(3) == '-') { + // multi line response + string::size_type p; + p = response.find("\r\n"+Util::itos(status)+" "); + if(p == string::npos) { + return false; + } + } + if(Util::endsWith(response, "\r\n")) { + return true; + } else { + return false; + } +} + +bool FtpConnection::bulkReceiveResponse(pair& response) { + char buf[1024]; + while(socket->isReadable(0)) { + int size = sizeof(buf)-1; + socket->readData(buf, size); + buf[size] = '\0'; + strbuf += buf; + } + int status; + if(strbuf.size() >= 4) { + status = getStatus(strbuf); + if(status == 0) { + throw new DlAbortEx(EX_INVALID_RESPONSE); + } + } else { + return false; + } + if(isEndOfResponse(status, strbuf)) { + logger->info(MSG_RECEIVE_RESPONSE, cuid, strbuf.c_str()); + response.first = status; + response.second = strbuf; + strbuf.erase(); + return true; + } else { + // didn't receive response fully. + return false; + } +} + +int FtpConnection::receiveResponse() { + pair response; + if(bulkReceiveResponse(response)) { + return response.first; + } else { + return 0; + } +} + +int FtpConnection::receiveSizeResponse(long long int& size) { + pair response; + if(bulkReceiveResponse(response)) { + if(response.first == 213) { + sscanf(response.second.c_str(), "%*d %Ld", &size); + } + return response.first; + } else { + return 0; + } +} + +int FtpConnection::receivePasvResponse(pair& dest) { + pair response; + if(bulkReceiveResponse(response)) { + if(response.first == 227) { + // we assume the format of response is "227 Entering Passive Mode (h1,h2,h3,h4,p1,p2)." + int h1, h2, h3, h4, p1, p2; + string::size_type p = response.second.find("("); + if(p >= 4) { + sscanf(response.second.substr(response.second.find("(")).c_str(), + "(%d,%d,%d,%d,%d,%d).", + &h1, &h2, &h3, &h4, &p1, &p2); + // ip address + dest.first = Util::itos(h1)+"."+Util::itos(h2)+"."+Util::itos(h3)+"."+Util::itos(h4); + // port number + dest.second = 256*p1+p2; + } else { + throw new DlAbortEx(EX_INVALID_RESPONSE); + } + } + return response.first; + } else { + return 0; + } +} diff --git a/src/FtpConnection.h b/src/FtpConnection.h new file mode 100644 index 00000000..6dab8e68 --- /dev/null +++ b/src/FtpConnection.h @@ -0,0 +1,65 @@ +/* */ +#ifndef _D_FTP_CONNECTION_H_ +#define _D_FTP_CONNECTION_H_ + +#include "Socket.h" +#include "Option.h" +#include "Logger.h" +#include "Segment.h" +#include "Request.h" +#include + +using namespace std; + +class FtpConnection { +private: + int cuid; + const Socket* socket; + const Request* req; + const Option* option; + const Logger* logger; + + string strbuf; + + int getStatus(string response) const; + bool isEndOfResponse(int status, string response) const; + bool bulkReceiveResponse(pair& response); +public: + FtpConnection(int cuid, const Socket* socket, const Request* req, const Option* op, const Logger* logger); + ~FtpConnection(); + void sendUser() const; + void sendPass() const; + void sendType() const; + void sendCwd() const; + void sendSize() const; + void sendPasv() const; + Socket* sendPort() const; + void sendRest(const Segment& segment) const; + void sendRetr() const; + + int receiveResponse(); + int receiveSizeResponse(long long int& size); + int receivePasvResponse(pair& dest); +}; + +#endif // _D_FTP_CONNECTION_H_ diff --git a/src/FtpDownloadCommand.cc b/src/FtpDownloadCommand.cc new file mode 100644 index 00000000..3d29c84c --- /dev/null +++ b/src/FtpDownloadCommand.cc @@ -0,0 +1,38 @@ +/* */ +#include "FtpDownloadCommand.h" + +FtpDownloadCommand::FtpDownloadCommand(int cuid, Request* req, DownloadEngine* e, Socket* dataSocket, Socket* ctrlSocket): + DownloadCommand(cuid, req, e, dataSocket) +{ + this->ctrlSocket = new Socket(*ctrlSocket); +} + +FtpDownloadCommand::~FtpDownloadCommand() { + if(ctrlSocket != NULL) { + delete ctrlSocket; + } +} + +TransferEncoding* FtpDownloadCommand::getTransferEncoding(string name) { + return NULL; +} diff --git a/src/FtpDownloadCommand.h b/src/FtpDownloadCommand.h new file mode 100644 index 00000000..b2db68a8 --- /dev/null +++ b/src/FtpDownloadCommand.h @@ -0,0 +1,37 @@ +/* */ +#ifndef _D_FTP_DOWNLOAD_COMMAND_H_ +#define _D_FTP_DOWNLOAD_COMMAND_H_ + +#include "DownloadCommand.h" + +class FtpDownloadCommand : public DownloadCommand { +private: + Socket* ctrlSocket; +public: + FtpDownloadCommand(int cuid, Request* req, DownloadEngine* e, Socket* dataSocket, Socket* ctrlSocket); + ~FtpDownloadCommand(); + + TransferEncoding* getTransferEncoding(string name); +}; + +#endif // _D_FTP_DOWNLOAD_COMMAND_H_ diff --git a/src/FtpInitiateConnectionCommand.cc b/src/FtpInitiateConnectionCommand.cc new file mode 100644 index 00000000..26bec02c --- /dev/null +++ b/src/FtpInitiateConnectionCommand.cc @@ -0,0 +1,84 @@ +/* */ +#include "FtpInitiateConnectionCommand.h" +#include "FtpNegotiationCommand.h" +#include "HttpRequestCommand.h" +#include "FtpTunnelRequestCommand.h" +#include "DlAbortEx.h" +#include "message.h" + +FtpInitiateConnectionCommand::FtpInitiateConnectionCommand(int cuid, Request* req, DownloadEngine* e):AbstractCommand(cuid, req, e) {} + +FtpInitiateConnectionCommand::~FtpInitiateConnectionCommand() {} + +bool FtpInitiateConnectionCommand::executeInternal(Segment segment) { + if(!e->segmentMan->downloadStarted) { + e->segmentMan->filename = req->getFile(); + bool segFileExists = e->segmentMan->segmentFileExists(); + if(segFileExists) { + e->segmentMan->load(); + e->diskWriter->openExistingFile(e->segmentMan->getFilePath()); + e->segmentMan->downloadStarted = true; + } else { + e->diskWriter->initAndOpenFile(e->segmentMan->getFilePath()); + } + } + + socket = new Socket(); + Command* command; + if(useHttpProxy()) { + e->logger->info(MSG_CONNECTING_TO_SERVER, cuid, + e->option->get("http_proxy_host").c_str(), + e->option->getAsInt("http_proxy_port")); + socket->establishConnection(e->option->get("http_proxy_host"), + e->option->getAsInt("http_proxy_port")); + + if(useHttpProxyGet()) { + command = new HttpRequestCommand(cuid, req, e, socket); + } else if(useHttpProxyConnect()) { + command = new FtpTunnelRequestCommand(cuid, req, e, socket); + } else { + // TODO + throw new DlAbortEx("ERROR"); + } + } else { + e->logger->info(MSG_CONNECTING_TO_SERVER, cuid, req->getHost().c_str(), + req->getPort()); + socket->establishConnection(req->getHost(), req->getPort()); + command = new FtpNegotiationCommand(cuid, req, e, socket); + } + e->commands.push(command); + return true; +} + +bool FtpInitiateConnectionCommand::useHttpProxy() const { + return e->option->defined("http_proxy_enabled") && + e->option->get("http_proxy_enabled") == "true"; +} + +bool FtpInitiateConnectionCommand::useHttpProxyGet() const { + return useHttpProxy() && e->option->get("ftp_via_http_proxy") == "get"; +} + +bool FtpInitiateConnectionCommand::useHttpProxyConnect() const { + return useHttpProxy() && e->option->get("ftp_via_http_proxy") == "tunnel"; +} diff --git a/src/FtpInitiateConnectionCommand.h b/src/FtpInitiateConnectionCommand.h new file mode 100644 index 00000000..0f997bf8 --- /dev/null +++ b/src/FtpInitiateConnectionCommand.h @@ -0,0 +1,39 @@ +/* */ +#ifndef _D_FTP_INITIATE_CONNECTION_COMMAND_H_ +#define _D_FTP_INITIATE_CONNECTION_COMMAND_H_ + +#include "AbstractCommand.h" + +class FtpInitiateConnectionCommand : public AbstractCommand { +private: + bool useHttpProxy() const; + bool useHttpProxyGet() const; + bool useHttpProxyConnect() const; +protected: + bool executeInternal(Segment segment); +public: + FtpInitiateConnectionCommand(int cuid, Request* req, DownloadEngine* e); + ~FtpInitiateConnectionCommand(); +}; + +#endif // _D_FTP_INITIATE_CONNECTION_COMMAND_H_ diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc new file mode 100644 index 00000000..ee6821ec --- /dev/null +++ b/src/FtpNegotiationCommand.cc @@ -0,0 +1,328 @@ +/* */ +#include "FtpNegotiationCommand.h" +#include "FtpDownloadCommand.h" +#include "DlAbortEx.h" +#include "message.h" + +FtpNegotiationCommand::FtpNegotiationCommand(int cuid, Request* req, DownloadEngine* e, Socket* s): + AbstractCommand(cuid, req, e, s), + dataSocket(NULL), serverSocket(NULL), sequence(SEQ_RECV_GREETING) +{ + ftp = new FtpConnection(cuid, socket, req, e->option, e->logger); + setReadCheckSocket(NULL); + setWriteCheckSocket(socket); +} + +FtpNegotiationCommand::~FtpNegotiationCommand() { + if(dataSocket != NULL) { + delete dataSocket; + } + if(serverSocket != NULL) { + delete serverSocket; + } + delete ftp; +} + +bool FtpNegotiationCommand::executeInternal(Segment segment) { + while(processSequence(segment)); + if(sequence == SEQ_RETRY) { + return prepareForRetry(0); + } else if(sequence == SEQ_NEGOTIATION_COMPLETED) { + FtpDownloadCommand* command = new FtpDownloadCommand(cuid, req, e, dataSocket, socket); + e->commands.push(command); + return true; + } else { + e->commands.push(this); + return false; + } +} + +bool FtpNegotiationCommand::recvGreeting() { + socket->setBlockingMode(); + int status = ftp->receiveResponse(); + if(status == 0) { + return false; + } + if(status != 220) { + throw new DlAbortEx(EX_CONNECTION_FAILED); + } + sequence = SEQ_SEND_USER; + + setReadCheckSocket(socket); + setWriteCheckSocket(NULL); + + return true; +} + +bool FtpNegotiationCommand::sendUser() { + ftp->sendUser(); + sequence = SEQ_RECV_USER; + return false; +} + +bool FtpNegotiationCommand::recvUser() { + int status = ftp->receiveResponse(); + switch(status) { + case 0: + return false; + case 230: + sequence = SEQ_SEND_TYPE; + break; + case 331: + sequence = SEQ_SEND_PASS; + break; + default: + throw new DlAbortEx(EX_BAD_STATUS, status); + } + return true; +} + +bool FtpNegotiationCommand::sendPass() { + ftp->sendPass(); + sequence = SEQ_RECV_PASS; + return false; +} + +bool FtpNegotiationCommand::recvPass() { + int status = ftp->receiveResponse(); + if(status == 0) { + return false; + } + if(status != 230) { + throw new DlAbortEx(EX_BAD_STATUS, status); + } + sequence = SEQ_SEND_TYPE; + return true; +} + +bool FtpNegotiationCommand::sendType() { + ftp->sendType(); + sequence = SEQ_RECV_TYPE; + return false; +} + +bool FtpNegotiationCommand::recvType() { + int status = ftp->receiveResponse(); + if(status == 0) { + return false; + } + if(status != 200) { + throw new DlAbortEx(EX_BAD_STATUS, status); + } + sequence = SEQ_SEND_CWD; + return true; +} + +bool FtpNegotiationCommand::sendCwd() { + ftp->sendCwd(); + sequence = SEQ_RECV_CWD; + return false; +} + +bool FtpNegotiationCommand::recvCwd() { + int status = ftp->receiveResponse(); + if(status == 0) { + return false; + } + if(status != 250) { + throw new DlAbortEx(EX_BAD_STATUS, status); + } + sequence = SEQ_SEND_SIZE; + return true; +} + +bool FtpNegotiationCommand::sendSize() { + ftp->sendSize(); + sequence = SEQ_RECV_SIZE; + return false; +} + +bool FtpNegotiationCommand::recvSize() { + long long int size = 0; + int status = ftp->receiveSizeResponse(size); + if(status == 0) { + return false; + } + if(status != 213) { + throw new DlAbortEx(EX_BAD_STATUS, status); + } + // TODO check the value of size + if(!e->segmentMan->downloadStarted) { + e->segmentMan->downloadStarted = true; + e->segmentMan->totalSize = size; + } else if(e->segmentMan->totalSize != size) { + throw new DlAbortEx(EX_SIZE_MISMATCH, e->segmentMan->totalSize, size); + } + if(e->option->get("ftp_pasv_enabled") == "true") { + sequence = SEQ_SEND_PASV; + } else { + sequence = SEQ_SEND_PORT; + } + return true; +} + +bool FtpNegotiationCommand::sendPort() { + serverSocket = ftp->sendPort(); + sequence = SEQ_RECV_PORT; + return false; +} + +bool FtpNegotiationCommand::recvPort() { + int status = ftp->receiveResponse(); + if(status == 0) { + return false; + } + if(status != 200) { + throw new DlAbortEx(EX_BAD_STATUS, status); + } + sequence = SEQ_SEND_REST; + return true; +} + +bool FtpNegotiationCommand::sendPasv() { + ftp->sendPasv(); + sequence = SEQ_RECV_PASV; + return false; +} + +bool FtpNegotiationCommand::recvPasv() { + pair dest; + int status = ftp->receivePasvResponse(dest); + if(status == 0) { + return false; + } + if(status != 227) { + throw new DlAbortEx(EX_BAD_STATUS, status); + } + // make a data connection to the server. + dataSocket = new Socket(); + + e->logger->info(MSG_CONNECTING_TO_SERVER, cuid, + dest.first.c_str(), + dest.second); + dataSocket->establishConnection(dest.first, dest.second); + + setReadCheckSocket(NULL); + setWriteCheckSocket(dataSocket); + + sequence = SEQ_SEND_REST_PASV; + return false; +} + +bool FtpNegotiationCommand::sendRestPasv(const Segment& segment) { + dataSocket->setBlockingMode(); + setReadCheckSocket(socket); + setWriteCheckSocket(NULL); + return sendRest(segment); +} + +bool FtpNegotiationCommand::sendRest(const Segment& segment) { + ftp->sendRest(segment); + sequence = SEQ_RECV_REST; + return false; +} + +bool FtpNegotiationCommand::recvRest() { + int status = ftp->receiveResponse(); + if(status == 0) { + return false; + } + // TODO if we recieve negative response, then we set e->segmentMan->splittable = false, and continue. + if(status != 350) { + throw new DlAbortEx(EX_BAD_STATUS, status); + } + sequence = SEQ_SEND_RETR; + return true; +} + +bool FtpNegotiationCommand::sendRetr() { + ftp->sendRetr(); + sequence = SEQ_RECV_RETR; + return false; +} + +bool FtpNegotiationCommand::recvRetr() { + int status = ftp->receiveResponse(); + if(status == 0) { + return false; + } + if(status != 150) { + throw new DlAbortEx(EX_BAD_STATUS, status); + } + if(e->option->get("ftp_pasv_enabled") != "true") { + assert(serverSocket); + dataSocket = serverSocket->acceptConnection(); + } + sequence = SEQ_NEGOTIATION_COMPLETED; + + return false; +} + +bool FtpNegotiationCommand::processSequence(const Segment& segment) { + bool doNextSequence = true; + switch(sequence) { + case SEQ_RECV_GREETING: + return recvGreeting(); + case SEQ_SEND_USER: + return sendUser(); + case SEQ_RECV_USER: + return recvUser(); + case SEQ_SEND_PASS: + return sendPass(); + case SEQ_RECV_PASS: + return recvPass(); + case SEQ_SEND_TYPE: + return sendType(); + case SEQ_RECV_TYPE: + return recvType(); + case SEQ_SEND_CWD: + return sendCwd(); + case SEQ_RECV_CWD: + return recvCwd(); + case SEQ_SEND_SIZE: + return sendSize(); + case SEQ_RECV_SIZE: + return recvSize(); + case SEQ_SEND_PORT: + return sendPort(); + case SEQ_RECV_PORT: + return recvPort(); + case SEQ_SEND_PASV: + return sendPasv(); + case SEQ_RECV_PASV: + return recvPasv(); + case SEQ_SEND_REST_PASV: + return sendRestPasv(segment); + case SEQ_SEND_REST: + return sendRest(segment); + case SEQ_RECV_REST: + return recvRest(); + case SEQ_SEND_RETR: + return sendRetr(); + case SEQ_RECV_RETR: + return recvRetr(); + default: + abort(); + } + return doNextSequence; +} diff --git a/src/FtpNegotiationCommand.h b/src/FtpNegotiationCommand.h new file mode 100644 index 00000000..29e2c0e6 --- /dev/null +++ b/src/FtpNegotiationCommand.h @@ -0,0 +1,87 @@ +/* */ +#ifndef _D_FTP_NEGOTIATION_COMMAND_H_ +#define _D_FTP_NEGOTIATION_COMMAND_H_ + +#include "AbstractCommand.h" +#include "FtpConnection.h" + +class FtpNegotiationCommand : public AbstractCommand { +private: + enum Seq { + SEQ_RECV_GREETING, + SEQ_SEND_USER, + SEQ_RECV_USER, + SEQ_SEND_PASS, + SEQ_RECV_PASS, + SEQ_SEND_TYPE, + SEQ_RECV_TYPE, + SEQ_SEND_CWD, + SEQ_RECV_CWD, + SEQ_SEND_SIZE, + SEQ_RECV_SIZE, + SEQ_SEND_PORT, + SEQ_RECV_PORT, + SEQ_SEND_PASV, + SEQ_RECV_PASV, + SEQ_SEND_REST_PASV, + SEQ_SEND_REST, + SEQ_RECV_REST, + SEQ_SEND_RETR, + SEQ_RECV_RETR, + SEQ_NEGOTIATION_COMPLETED, + SEQ_RETRY + }; + bool recvGreeting(); + bool sendUser(); + bool recvUser(); + bool sendPass(); + bool recvPass(); + bool sendType(); + bool recvType(); + bool sendCwd(); + bool recvCwd(); + bool sendSize(); + bool recvSize(); + bool sendPort(); + bool recvPort(); + bool sendPasv(); + bool recvPasv(); + bool sendRest(const Segment& segment); + bool sendRestPasv(const Segment& segment); + bool recvRest(); + bool sendRetr(); + bool recvRetr(); + bool processSequence(const Segment& segment); + + Socket* dataSocket; + Socket* serverSocket; + int sequence; + FtpConnection* ftp; +protected: + bool executeInternal(Segment segment); +public: + FtpNegotiationCommand(int cuid, Request* req, DownloadEngine* e, Socket* s); + ~FtpNegotiationCommand(); +}; + +#endif // _D_FTP_NEGOTIATION_COMMAND_H_ diff --git a/src/FtpTunnelRequestCommand.cc b/src/FtpTunnelRequestCommand.cc new file mode 100644 index 00000000..f103f7ce --- /dev/null +++ b/src/FtpTunnelRequestCommand.cc @@ -0,0 +1,41 @@ +/* */ +#include "FtpTunnelRequestCommand.h" +#include "FtpTunnelResponseCommand.h" +#include "HttpConnection.h" + +FtpTunnelRequestCommand::FtpTunnelRequestCommand(int cuid, Request* req, DownloadEngine* e, Socket* s):AbstractCommand(cuid, req, e, s) { + setReadCheckSocket(NULL); + setWriteCheckSocket(NULL); +} + +FtpTunnelRequestCommand::~FtpTunnelRequestCommand() {} + +bool FtpTunnelRequestCommand::executeInternal(Segment segment) { + socket->setBlockingMode(); + HttpConnection httpConnection(cuid, socket, req, e->option, e->logger); + httpConnection.sendProxyRequest(); + + FtpTunnelResponseCommand* command = new FtpTunnelResponseCommand(cuid, req, e, socket); + e->commands.push(command); + return true; +} diff --git a/src/FtpTunnelRequestCommand.h b/src/FtpTunnelRequestCommand.h new file mode 100644 index 00000000..817f1c9d --- /dev/null +++ b/src/FtpTunnelRequestCommand.h @@ -0,0 +1,35 @@ +/* */ +#ifndef _D_FTP_TUNNEL_REQUEST_COMMAND_H_ +#define _D_FTP_TUNNEL_REQUEST_COMMAND_H_ + +#include "AbstractCommand.h" + +class FtpTunnelRequestCommand : public AbstractCommand { +protected: + bool executeInternal(Segment segment); +public: + FtpTunnelRequestCommand(int cuid, Request* req, DownloadEngine* e, Socket* s); + ~FtpTunnelRequestCommand(); +}; + +#endif // _D_FTP_TUNNEL_REQUEST_COMMAND_H_ diff --git a/src/FtpTunnelResponseCommand.cc b/src/FtpTunnelResponseCommand.cc new file mode 100644 index 00000000..7009310c --- /dev/null +++ b/src/FtpTunnelResponseCommand.cc @@ -0,0 +1,49 @@ +/* */ +#include "FtpTunnelResponseCommand.h" +#include "FtpNegotiationCommand.h" +#include "DlRetryEx.h" +#include "message.h" + +FtpTunnelResponseCommand::FtpTunnelResponseCommand(int cuid, Request* req, DownloadEngine* e, Socket* s):AbstractCommand(cuid, req, e, s) { + http = new HttpConnection(cuid, socket, req, e->option, e->logger); +} + +FtpTunnelResponseCommand::~FtpTunnelResponseCommand() { + delete http; +} + +bool FtpTunnelResponseCommand::executeInternal(Segment segment) { + HttpHeader headers; + int status = http->receiveResponse(headers); + if(status == 0) { + // we didn't receive all of headers yet. + e->commands.push(this); + return false; + } + if(status != 200) { + throw new DlRetryEx(EX_PROXY_CONNECTION_FAILED); + } + FtpNegotiationCommand* command = new FtpNegotiationCommand(cuid, req, e, socket); + e->commands.push(command); + return true; +} diff --git a/src/FtpTunnelResponseCommand.h b/src/FtpTunnelResponseCommand.h new file mode 100644 index 00000000..eeba75ac --- /dev/null +++ b/src/FtpTunnelResponseCommand.h @@ -0,0 +1,39 @@ +/* */ +#ifndef _D_FTP_TUNNEL_RESPONSE_COMMAND_H_ +#define _D_FTP_TUNNEL_RESPONSE_COMMAND_H_ + +#include "AbstractCommand.h" +#include "HttpConnection.h" + +class FtpTunnelResponseCommand : public AbstractCommand { +private: + HttpConnection* http; +protected: + bool executeInternal(Segment segment); +public: + FtpTunnelResponseCommand(int cuid, Request* req, DownloadEngine* e, Socket* s); + ~FtpTunnelResponseCommand(); +}; + +#endif // _D_FTP_TUNNEL_RESPONSE_COMMAND_H_ + diff --git a/src/HttpHeader.cc b/src/HttpHeader.cc new file mode 100644 index 00000000..eb80cf5b --- /dev/null +++ b/src/HttpHeader.cc @@ -0,0 +1,66 @@ +/* */ +#include "HttpHeader.h" + +void HttpHeader::put(const string& name, const string& value) { + multimap::value_type vt(name, value); + table.insert(vt); +} + +bool HttpHeader::defined(const string& name) const { + return table.count(name) == 1; +} + +string HttpHeader::getFirst(const string& name) const { + multimap::const_iterator itr = table.find(name); + if(itr == table.end()) { + return ""; + } else { + return (*itr).second; + } +} + +vector HttpHeader::get(const string& name) const { + vector v; + for(multimap::const_iterator itr = table.find(name); itr != table.end(); itr++) { + v.push_back((*itr).second); + } + return v; +} + +int HttpHeader::getFirstAsInt(const string& name) const { + multimap::const_iterator itr = table.find(name); + if(itr == table.end()) { + return 0; + } else { + return (int)strtol((*itr).second.c_str(), NULL, 10); + } +} + +long long int HttpHeader::getFirstAsLLInt(const string& name) const { + multimap::const_iterator itr = table.find(name); + if(itr == table.end()) { + return 0; + } else { + return strtoll((*itr).second.c_str(), NULL, 10); + } +} diff --git a/src/HttpHeader.h b/src/HttpHeader.h new file mode 100644 index 00000000..3bcaa479 --- /dev/null +++ b/src/HttpHeader.h @@ -0,0 +1,46 @@ +/* */ +#ifndef _D_HTTP_HEADER_H_ +#define _D_HTTP_HEADER_H_ + +#include +#include +#include + +using namespace std; + +class HttpHeader { +private: + multimap table; +public: + HttpHeader() {} + ~HttpHeader() {} + + void put(const string& name, const string& value); + bool defined(const string& name) const; + string getFirst(const string& name) const; + vector get(const string& name) const; + int getFirstAsInt(const string& name) const; + long long int getFirstAsLLInt(const string& name) const; +}; + +#endif // _D_HTTP_HEADER_H_ -- cgit From 32e8f2b011d1e1db9f724108894d9e2cd01d9f2a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 21 Feb 2006 14:00:58 +0000 Subject: --- README | 2 +- src/AbstractCommand.cc | 11 ++-- src/DownloadEngine.cc | 3 +- src/FtpConnection.cc | 18 ++++-- src/FtpInitiateConnectionCommand.cc | 16 ++--- src/FtpNegotiationCommand.cc | 30 +++++---- src/HttpConnection.cc | 22 ++++--- src/HttpInitiateConnectionCommand.cc | 12 ++-- src/HttpResponseCommand.cc | 5 +- src/Request.h | 4 +- src/main.cc | 120 ++++++++++++++++++++++++++--------- src/message.h | 2 +- src/prefs.h | 79 +++++++++++++++++++++++ 13 files changed, 237 insertions(+), 87 deletions(-) create mode 100644 src/prefs.h (limited to 'src') diff --git a/README b/README index 785c7da0..ed44c9d6 100644 --- a/README +++ b/README @@ -7,7 +7,7 @@ You must use this program at your own risk. 2. About aria2 -------------- -aria2 has segmented downloading engine in its core. By segmented downloding, +aria2 has segmented downloading engine in its core. By segmented downloading, it can download files very much faster than ordinary browsers. aria2 is in very early development stage. Currently it has following features: diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index 89683a70..a16bf6a6 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -27,8 +27,7 @@ #include "Util.h" #include "message.h" #include "SleepCommand.h" - -#define TIMEOUT_SEC 60 +#include "prefs.h" AbstractCommand::AbstractCommand(int cuid, Request* req, DownloadEngine* e, Socket* s): Command(cuid), req(req), e(e), checkSocketIsReadable(false), checkSocketIsWritable(false) { @@ -63,7 +62,7 @@ bool AbstractCommand::isTimeoutDetected() { return false; } else { long long int elapsed = Util::difftv(now, checkPoint); - if(elapsed >= TIMEOUT_SEC*1000000) { + if(elapsed >= e->option->getAsInt(PREF_TIMEOUT)*1000000) { return true; } else { return false; @@ -103,11 +102,11 @@ bool AbstractCommand::execute() { delete(err); //req->resetUrl(); req->addTryCount(); - if(req->noMoreTry()) { - e->logger->error(MSG_MAX_RETRY, cuid); + if(req->getTryCount() >= e->option->getAsInt(PREF_MAX_TRY)) { + e->logger->error(MSG_MAX_TRY, cuid, req->getTryCount()); return true; } else { - return prepareForRetry(e->option->getAsInt("retry_wait")); + return prepareForRetry(e->option->getAsInt(PREF_RETRY_WAIT)); } } } diff --git a/src/DownloadEngine.cc b/src/DownloadEngine.cc index 337e221c..80b98141 100644 --- a/src/DownloadEngine.cc +++ b/src/DownloadEngine.cc @@ -20,12 +20,11 @@ */ /* copyright --> */ #include "DownloadEngine.h" +#include "Util.h" #include #include #include #include -#include "DlAbortEx.h" -#include "Util.h" using namespace std; diff --git a/src/FtpConnection.cc b/src/FtpConnection.cc index 4c528ada..4614304e 100644 --- a/src/FtpConnection.cc +++ b/src/FtpConnection.cc @@ -22,26 +22,34 @@ #include "FtpConnection.h" #include "Util.h" #include "DlAbortEx.h" +#include "DlRetryEx.h" #include "message.h" +#include "prefs.h" FtpConnection::FtpConnection(int cuid, const Socket* socket, const Request* req, const Option* op, const Logger* logger):cuid(cuid), socket(socket), req(req), option(op), logger(logger) {} FtpConnection::~FtpConnection() {} void FtpConnection::sendUser() const { - string request = "USER "+option->get("ftp_user")+"\r\n"; + string request = "USER "+option->get(PREF_FTP_USER)+"\r\n"; logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); socket->writeData(request); } void FtpConnection::sendPass() const { - string request = "PASS "+option->get("ftp_passwd")+"\r\n"; + string request = "PASS "+option->get(PREF_FTP_PASSWD)+"\r\n"; logger->info(MSG_SENDING_FTP_REQUEST, cuid, "PASS ********"); socket->writeData(request); } void FtpConnection::sendType() const { - string request = "TYPE "+option->get("ftp_type")+"\r\n"; + string type; + if(option->get(PREF_FTP_TYPE) == V_ASCII) { + type = "A"; + } else { + type = "I"; + } + string request = "TYPE "+type+"\r\n"; logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); socket->writeData(request); } @@ -143,7 +151,7 @@ bool FtpConnection::bulkReceiveResponse(pair& response) { if(strbuf.size() >= 4) { status = getStatus(strbuf); if(status == 0) { - throw new DlAbortEx(EX_INVALID_RESPONSE); + throw new DlRetryEx(EX_INVALID_RESPONSE); } } else { return false; @@ -197,7 +205,7 @@ int FtpConnection::receivePasvResponse(pair& dest) { // port number dest.second = 256*p1+p2; } else { - throw new DlAbortEx(EX_INVALID_RESPONSE); + throw new DlRetryEx(EX_INVALID_RESPONSE); } } return response.first; diff --git a/src/FtpInitiateConnectionCommand.cc b/src/FtpInitiateConnectionCommand.cc index 26bec02c..03af4232 100644 --- a/src/FtpInitiateConnectionCommand.cc +++ b/src/FtpInitiateConnectionCommand.cc @@ -25,6 +25,7 @@ #include "FtpTunnelRequestCommand.h" #include "DlAbortEx.h" #include "message.h" +#include "prefs.h" FtpInitiateConnectionCommand::FtpInitiateConnectionCommand(int cuid, Request* req, DownloadEngine* e):AbstractCommand(cuid, req, e) {} @@ -47,10 +48,10 @@ bool FtpInitiateConnectionCommand::executeInternal(Segment segment) { Command* command; if(useHttpProxy()) { e->logger->info(MSG_CONNECTING_TO_SERVER, cuid, - e->option->get("http_proxy_host").c_str(), - e->option->getAsInt("http_proxy_port")); - socket->establishConnection(e->option->get("http_proxy_host"), - e->option->getAsInt("http_proxy_port")); + e->option->get(PREF_HTTP_PROXY_HOST).c_str(), + e->option->getAsInt(PREF_HTTP_PROXY_PORT)); + socket->establishConnection(e->option->get(PREF_HTTP_PROXY_HOST), + e->option->getAsInt(PREF_HTTP_PROXY_PORT)); if(useHttpProxyGet()) { command = new HttpRequestCommand(cuid, req, e, socket); @@ -71,14 +72,13 @@ bool FtpInitiateConnectionCommand::executeInternal(Segment segment) { } bool FtpInitiateConnectionCommand::useHttpProxy() const { - return e->option->defined("http_proxy_enabled") && - e->option->get("http_proxy_enabled") == "true"; + return e->option->get(PREF_HTTP_PROXY_ENABLED) == V_TRUE; } bool FtpInitiateConnectionCommand::useHttpProxyGet() const { - return useHttpProxy() && e->option->get("ftp_via_http_proxy") == "get"; + return useHttpProxy() && e->option->get(PREF_FTP_VIA_HTTP_PROXY) == V_GET; } bool FtpInitiateConnectionCommand::useHttpProxyConnect() const { - return useHttpProxy() && e->option->get("ftp_via_http_proxy") == "tunnel"; + return useHttpProxy() && e->option->get(PREF_FTP_VIA_HTTP_PROXY) == V_TUNNEL; } diff --git a/src/FtpNegotiationCommand.cc b/src/FtpNegotiationCommand.cc index ee6821ec..2f5bb342 100644 --- a/src/FtpNegotiationCommand.cc +++ b/src/FtpNegotiationCommand.cc @@ -22,7 +22,9 @@ #include "FtpNegotiationCommand.h" #include "FtpDownloadCommand.h" #include "DlAbortEx.h" +#include "DlRetryEx.h" #include "message.h" +#include "prefs.h" FtpNegotiationCommand::FtpNegotiationCommand(int cuid, Request* req, DownloadEngine* e, Socket* s): AbstractCommand(cuid, req, e, s), @@ -64,7 +66,7 @@ bool FtpNegotiationCommand::recvGreeting() { return false; } if(status != 220) { - throw new DlAbortEx(EX_CONNECTION_FAILED); + throw new DlRetryEx(EX_CONNECTION_FAILED); } sequence = SEQ_SEND_USER; @@ -92,7 +94,7 @@ bool FtpNegotiationCommand::recvUser() { sequence = SEQ_SEND_PASS; break; default: - throw new DlAbortEx(EX_BAD_STATUS, status); + throw new DlRetryEx(EX_BAD_STATUS, status); } return true; } @@ -109,7 +111,7 @@ bool FtpNegotiationCommand::recvPass() { return false; } if(status != 230) { - throw new DlAbortEx(EX_BAD_STATUS, status); + throw new DlRetryEx(EX_BAD_STATUS, status); } sequence = SEQ_SEND_TYPE; return true; @@ -127,7 +129,7 @@ bool FtpNegotiationCommand::recvType() { return false; } if(status != 200) { - throw new DlAbortEx(EX_BAD_STATUS, status); + throw new DlRetryEx(EX_BAD_STATUS, status); } sequence = SEQ_SEND_CWD; return true; @@ -145,7 +147,7 @@ bool FtpNegotiationCommand::recvCwd() { return false; } if(status != 250) { - throw new DlAbortEx(EX_BAD_STATUS, status); + throw new DlRetryEx(EX_BAD_STATUS, status); } sequence = SEQ_SEND_SIZE; return true; @@ -164,16 +166,18 @@ bool FtpNegotiationCommand::recvSize() { return false; } if(status != 213) { - throw new DlAbortEx(EX_BAD_STATUS, status); + throw new DlRetryEx(EX_BAD_STATUS, status); + } + if(size == LONG_LONG_MAX || size < 0) { + throw new DlAbortEx(EX_TOO_LARGE_FILE, size); } - // TODO check the value of size if(!e->segmentMan->downloadStarted) { e->segmentMan->downloadStarted = true; e->segmentMan->totalSize = size; } else if(e->segmentMan->totalSize != size) { throw new DlAbortEx(EX_SIZE_MISMATCH, e->segmentMan->totalSize, size); } - if(e->option->get("ftp_pasv_enabled") == "true") { + if(e->option->get(PREF_FTP_PASV_ENABLED) == V_TRUE) { sequence = SEQ_SEND_PASV; } else { sequence = SEQ_SEND_PORT; @@ -193,7 +197,7 @@ bool FtpNegotiationCommand::recvPort() { return false; } if(status != 200) { - throw new DlAbortEx(EX_BAD_STATUS, status); + throw new DlRetryEx(EX_BAD_STATUS, status); } sequence = SEQ_SEND_REST; return true; @@ -212,7 +216,7 @@ bool FtpNegotiationCommand::recvPasv() { return false; } if(status != 227) { - throw new DlAbortEx(EX_BAD_STATUS, status); + throw new DlRetryEx(EX_BAD_STATUS, status); } // make a data connection to the server. dataSocket = new Socket(); @@ -249,7 +253,7 @@ bool FtpNegotiationCommand::recvRest() { } // TODO if we recieve negative response, then we set e->segmentMan->splittable = false, and continue. if(status != 350) { - throw new DlAbortEx(EX_BAD_STATUS, status); + throw new DlRetryEx(EX_BAD_STATUS, status); } sequence = SEQ_SEND_RETR; return true; @@ -267,9 +271,9 @@ bool FtpNegotiationCommand::recvRetr() { return false; } if(status != 150) { - throw new DlAbortEx(EX_BAD_STATUS, status); + throw new DlRetryEx(EX_BAD_STATUS, status); } - if(e->option->get("ftp_pasv_enabled") != "true") { + if(e->option->get(PREF_FTP_PASV_ENABLED) != V_TRUE) { assert(serverSocket); dataSocket = serverSocket->acceptConnection(); } diff --git a/src/HttpConnection.cc b/src/HttpConnection.cc index 9f373d33..535e745f 100644 --- a/src/HttpConnection.cc +++ b/src/HttpConnection.cc @@ -20,11 +20,11 @@ */ /* copyright --> */ #include "HttpConnection.h" -#include "DlAbortEx.h" #include "DlRetryEx.h" #include "Util.h" #include "Base64.h" #include "message.h" +#include "prefs.h" HttpConnection::HttpConnection(int cuid, const Socket* socket, const Request* req, const Option* op, const Logger* logger): cuid(cuid), socket(socket), req(req), option(op), logger(logger) {} @@ -36,12 +36,14 @@ void HttpConnection::sendRequest(const Segment& segment) const { } void HttpConnection::sendProxyRequest() const { - string request = string("CONNECT ")+req->getHost()+":"+Util::llitos(req->getPort())+ + string request = + string("CONNECT ")+req->getHost()+":"+Util::llitos(req->getPort())+ string(" HTTP/1.1\r\n")+ "Host: "+getHost(req->getHost(), req->getPort())+"\r\n"; if(useProxyAuth()) { request += "Proxy-Authorization: Basic "+ - Base64::encode(option->get("http_proxy_user")+":"+option->get("http_proxy_passwd"))+"\r\n"; + Base64::encode(option->get(PREF_HTTP_PROXY_USER)+":"+ + option->get(PREF_HTTP_PROXY_PORT))+"\r\n"; } request += "\r\n"; logger->info(MSG_SENDING_HTTP_REQUEST, cuid, request.c_str()); @@ -65,11 +67,13 @@ string HttpConnection::createRequest(const Segment& segment) const { "Pragma: no-cache\r\n"+ "Cache-Control: no-cache\r\n"; if(segment.sp+segment.ds > 0) { - request += "Range: bytes="+Util::llitos(segment.sp+segment.ds)+"-"+Util::llitos(segment.ep)+"\r\n"; + request += "Range: bytes="+ + Util::llitos(segment.sp+segment.ds)+"-"+Util::llitos(segment.ep)+"\r\n"; } - if(option->get("http_auth_scheme") == "BASIC") { + if(option->get(PREF_HTTP_AUTH_SCHEME) == V_BASIC) { request += "Authorization: Basic "+ - Base64::encode(option->get("http_user")+":"+option->get("http_passwd"))+"\r\n"; + Base64::encode(option->get(PREF_HTTP_USER)+":"+ + option->get(PREF_HTTP_PASSWD))+"\r\n"; } if(req->getPreviousUrl().size()) { request += "Referer: "+req->getPreviousUrl()+"\r\n"; @@ -135,11 +139,9 @@ int HttpConnection::receiveResponse(HttpHeader& headers) { } bool HttpConnection::useProxy() const { - return option->defined("http_proxy_enabled") && - option->get("http_proxy_enabled") == "true"; + return option->get(PREF_HTTP_PROXY_ENABLED) == V_TRUE; } bool HttpConnection::useProxyAuth() const { - return option->defined("http_proxy_auth_enabled") && - option->get("http_proxy_auth_enabled") == "true"; + return option->get(PREF_HTTP_PROXY_AUTH_ENABLED) == V_TRUE; } diff --git a/src/HttpInitiateConnectionCommand.cc b/src/HttpInitiateConnectionCommand.cc index e94c39c9..8b53aab8 100644 --- a/src/HttpInitiateConnectionCommand.cc +++ b/src/HttpInitiateConnectionCommand.cc @@ -24,6 +24,7 @@ #include "HttpProxyRequestCommand.h" #include "Util.h" #include "message.h" +#include "prefs.h" HttpInitiateConnectionCommand::HttpInitiateConnectionCommand(int cuid, Request* req, DownloadEngine* e):AbstractCommand(cuid, req, e) {} @@ -35,10 +36,10 @@ bool HttpInitiateConnectionCommand::executeInternal(Segment segment) { Command* command; if(useProxy()) { e->logger->info(MSG_CONNECTING_TO_SERVER, cuid, - e->option->get("http_proxy_host").c_str(), - e->option->getAsInt("http_proxy_port")); - socket->establishConnection(e->option->get("http_proxy_host"), - e->option->getAsInt("http_proxy_port")); + e->option->get(PREF_HTTP_PROXY_HOST).c_str(), + e->option->getAsInt(PREF_HTTP_PROXY_PORT)); + socket->establishConnection(e->option->get(PREF_HTTP_PROXY_HOST), + e->option->getAsInt(PREF_HTTP_PROXY_PORT)); command = new HttpProxyRequestCommand(cuid, req, e, socket); } else { @@ -52,6 +53,5 @@ bool HttpInitiateConnectionCommand::executeInternal(Segment segment) { } bool HttpInitiateConnectionCommand::useProxy() { - return e->option->defined("http_proxy_enabled") && - e->option->get("http_proxy_enabled") == "true"; + return e->option->get(PREF_HTTP_PROXY_ENABLED) == V_TRUE; } diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index 1fd273bb..f5788be7 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -21,6 +21,7 @@ /* copyright --> */ #include "HttpResponseCommand.h" #include "DlAbortEx.h" +#include "DlRetryEx.h" #include "HttpDownloadCommand.h" #include "HttpInitiateConnectionCommand.h" #include "message.h" @@ -77,7 +78,7 @@ void HttpResponseCommand::checkResponse(int status, const Segment& segment) { if(!(status < 400 && status >= 300 || (segment.sp+segment.ds == 0 && status == 200) || (segment.sp+segment.ds > 0 && status == 206))) { - throw new DlAbortEx(EX_BAD_STATUS, status); + throw new DlRetryEx(EX_BAD_STATUS, status); } } @@ -90,7 +91,7 @@ bool HttpResponseCommand::handleRedirect(string url, const HttpHeader& headers) bool HttpResponseCommand::handleDefaultEncoding(const HttpHeader& headers) { long long int size = headers.getFirstAsLLInt("Content-Length"); - if(size == LONG_LONG_MAX || size == LONG_LONG_MIN || size < 0) { + if(size == LONG_LONG_MAX || size < 0) { throw new DlAbortEx(EX_TOO_LARGE_FILE, size); } e->segmentMan->isSplittable = !(size == 0); diff --git a/src/Request.h b/src/Request.h index 7a7b0fce..8d8676ef 100644 --- a/src/Request.h +++ b/src/Request.h @@ -29,8 +29,6 @@ using namespace std; -#define MAX_TRY_COUNT 5 - #define SAFE_CHARS "abcdefghijklmnopqrstuvwxyz"\ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"\ "0123456789"\ @@ -77,7 +75,7 @@ public: void resetTryCount() { tryCount = 0; } void addTryCount() { tryCount++; } int getTryCount() const { return tryCount; } - bool noMoreTry() const { return tryCount >= MAX_TRY_COUNT; } + //bool noMoreTry() const { return tryCount >= PREF_MAX_TRY; } string getUrl() const { return url; } string getCurrentUrl() const { return currentUrl; } diff --git a/src/main.cc b/src/main.cc index 773bdd17..63c46a95 100644 --- a/src/main.cc +++ b/src/main.cc @@ -27,6 +27,7 @@ #include "DefaultDiskWriter.h" #include "Util.h" #include "InitiateConnectionCommandFactory.h" +#include "prefs.h" #include #include #include @@ -96,7 +97,9 @@ void showVersion() { } void showUsage() { + cout << endl; cout << "Usage: " << PACKAGE_NAME << " [options] URL ..." << endl; + cout << endl; cout << "Options:" << endl; cout << " -d, --dir=DIR The directory to store downloaded file." << endl; cout << " -o, --out=FILE The file name for downloaded file." << endl; @@ -106,31 +109,47 @@ void showUsage() { cout << " -s, --split=N Download a file using s connections. s must be" << endl; cout << " between 1 and 5. If this option is specified the" << endl; cout << " first URL is used, and the other URLs are ignored." << endl; + cout << " --retry-wait=SEC Set amount of time in second between requests" << endl; + cout << " for errors. Specify a value between 0 and 60." << endl; + cout << " -t, --timeout=SEC Set timeout in second." << endl; cout << " --http-proxy=HOST:PORT Use HTTP proxy server. This affects to all" << endl; cout << " URLs." << endl; cout << " --http-user=USER Set HTTP user. This affects to all URLs." << endl; cout << " --http-passwd=PASSWD Set HTTP password. This affects to all URLs." << endl; cout << " --http-proxy-user=USER Set HTTP proxy user. This affects to all URLs" << endl; cout << " --http-proxy-passwd=PASSWD Set HTTP proxy password. This affects to all URLs." << endl; - cout << " --http-auth-scheme=SCHEME Set HTTP authentication scheme. Currently, BASIC" << endl; + cout << " --http-auth-scheme=SCHEME Set HTTP authentication scheme. Currently, basic" << endl; cout << " is the only supported scheme. You MUST specify" << endl; cout << " this option in order to use HTTP authentication" << endl; cout << " as well as --http-proxy option." << endl; - cout << " --referer Set Referer. This affects to all URLs." << endl; - cout << " --retry-wait Set amount of time in second between requests" << endl; - cout << " for errors. Specify a value between 0 and 60." << endl; + cout << " --referer=REFERER Set Referer. This affects to all URLs." << endl; + cout << " --ftp-user=USER Set FTP user. This affects to all URLs." << endl; + cout << " Default: anonymous" << endl; + cout << " --ftp-passwd=PASSWD Set FTP password. This affects to all URLs." << endl; + cout << " Default: ARIA2USER@" << endl; + cout << " --ftp-type=TYPE Set FTP transfer type. TYPE is either 'binary'" << endl; + cout << " or 'ascii'." << endl; + cout << " Default: binary" << endl; + cout << " -p, --ftp-pasv Use passive mode in FTP." << endl; + cout << " --ftp-via-http-proxy=WAY Use HTTP proxy in FTP. WAY is either 'get' or" << endl; + cout << " 'tunnel'." << endl; cout << " -v, --version Print the version number and exit." << endl; cout << " -h, --help Print this message and exit." << endl; + cout << endl; cout << "URL:" << endl; cout << " You can specify multiple URLs. All URLs must point to the same file" << endl; - cout << " or a download fails." << endl; + cout << " or downloading fails." << endl; + cout << endl; cout << "Examples:" << endl; cout << " Download a file by 1 connection:" << endl; cout << " aria2c http://AAA.BBB.CCC/file.zip" << endl; cout << " Download a file by 2 connections:" << endl; cout << " aria2c -s 2 http://AAA.BBB.CCC/file.zip" << endl; - cout << " Download a file by 2 connections, each connects to a different server." << endl; + cout << " Download a file by 2 connections, each connects to a different server:" << endl; cout << " aria2c http://AAA.BBB.CCC/file.zip http://DDD.EEE.FFF/GGG/file.zip" << endl; + cout << " You can mix up different protocols:" << endl; + cout << " aria2c http://AAA.BBB.CCC/file.zip ftp://DDD.EEE.FFF/GGG/file.zip" << endl; + cout << endl; cout << "Reports bugs to " << endl; } @@ -145,16 +164,13 @@ int main(int argc, char* argv[]) { int c; Option* op = new Option(); - op->put("retry_wait", "5"); - - // TODO warning! Delete username/password line when commit - op->put("ftp_user", "anonymous"); - op->put("ftp_passwd", "IE60USER@"); - op->put("ftp_type", "I"); - - op->put("ftp_pasv_enabled", "true"); - op->put("ftp_via_http_proxy", "tunnel"); - op->put("http_abs_uri_request_enabled", "true"); + op->put(PREF_RETRY_WAIT, "5"); + op->put(PREF_TIMEOUT, "60"); + op->put(PREF_FTP_USER, "anonymous"); + op->put(PREF_FTP_PASSWD, "ARIA2USER@"); + op->put(PREF_FTP_TYPE, V_BINARY); + op->put(PREF_FTP_PASV_ENABLED, V_TRUE); + op->put(PREF_FTP_VIA_HTTP_PROXY, V_TUNNEL); while(1) { int optIndex = 0; @@ -165,6 +181,7 @@ int main(int argc, char* argv[]) { { "out", required_argument, NULL, 'o' }, { "log", required_argument, NULL, 'l' }, { "split", required_argument, NULL, 's' }, + { "timeout", required_argument, NULL, 't' }, { "http-proxy", required_argument, &lopt, 1 }, { "http-user", required_argument, &lopt, 2 }, { "http-passwd", required_argument, &lopt, 3 }, @@ -173,11 +190,16 @@ int main(int argc, char* argv[]) { { "http-auth-scheme", required_argument, &lopt, 6 }, { "referer", required_argument, &lopt, 7 }, { "retry-wait", required_argument, &lopt, 8 }, + { "ftp-user", required_argument, &lopt, 9 }, + { "ftp-passwd", required_argument, &lopt, 10 }, + { "ftp-type", required_argument, &lopt, 11 }, + { "ftp-pasv", no_argument, NULL, 'p' }, + { "ftp-via-http-proxy", required_argument, &lopt, 12 }, { "version", no_argument, NULL, 'v' }, { "help", no_argument, NULL, 'h' }, { 0, 0, 0, 0 } }; - c = getopt_long(argc, argv, "Dd:o:l:s:vh", longOpts, &optIndex); + c = getopt_long(argc, argv, "Dd:o:l:s:pt:vh", longOpts, &optIndex); if(c == -1) { break; } @@ -194,29 +216,29 @@ int main(int argc, char* argv[]) { showUsage(); exit(1); } - op->put("http_proxy_host", proxy.first); - op->put("http_proxy_port", Util::itos(port)); - op->put("http_proxy_enabled", "true"); + op->put(PREF_HTTP_PROXY_HOST, proxy.first); + op->put(PREF_HTTP_PROXY_PORT, Util::itos(port)); + op->put(PREF_HTTP_PROXY_ENABLED, V_TRUE); break; } case 2: - op->put("http_user", optarg); + op->put(PREF_HTTP_USER, optarg); break; case 3: - op->put("http_passwd", optarg); + op->put(PREF_HTTP_PASSWD, optarg); break; case 4: - op->put("http_proxy_user", optarg); - op->put("http_proxy_auth_enabled", "true"); + op->put(PREF_HTTP_PROXY_USER, optarg); + op->put(PREF_HTTP_PROXY_AUTH_ENABLED, V_TRUE); break; case 5: - op->put("http_proxy_passwd", optarg); + op->put(PREF_HTTP_PROXY_PASSWD, optarg); break; case 6: - if(string("BASIC") == optarg) { - op->put("http_auth_scheme", "BASIC"); + if(string(V_BASIC) == optarg) { + op->put(PREF_HTTP_AUTH_SCHEME, V_BASIC); } else { - cerr << "Currently, supported authentication scheme is BASIC." << endl; + cerr << "Currently, supported authentication scheme is basic." << endl; } break; case 7: @@ -229,9 +251,33 @@ int main(int argc, char* argv[]) { showUsage(); exit(1); } - op->put("retry-wait", Util::itos(wait)); + op->put(PREF_RETRY_WAIT, Util::itos(wait)); break; } + case 9: + op->put(PREF_FTP_USER, optarg); + break; + case 10: + op->put(PREF_FTP_PASSWD, optarg); + break; + case 11: + if(string(optarg) == V_BINARY || string(optarg) == V_ASCII) { + op->put(PREF_FTP_TYPE, optarg); + } else { + cerr << "ftp-type must be either 'binary' or 'ascii'." << endl; + showUsage(); + exit(1); + } + break; + case 12: + if(string(optarg) == V_GET || string(optarg) == V_TUNNEL) { + op->put(PREF_FTP_VIA_HTTP_PROXY, optarg); + } else { + cerr << "ftp-via-http-proxy must be either 'get' or 'tunnel'." << endl; + showUsage(); + exit(1); + } + break; } break; } @@ -253,12 +299,26 @@ int main(int argc, char* argv[]) { break; case 's': split = (int)strtol(optarg, NULL, 10); - if(!(1 < split && split < 5)) { + if(!(1 <= split && split <= 5)) { cerr << "split must be between 1 and 5." << endl; showUsage(); exit(1); } break; + case 't': { + int timeout = (int)strtol(optarg, NULL, 10); + if(1 <= timeout && timeout <= 600) { + op->put(PREF_TIMEOUT, Util::itos(timeout)); + } else { + cerr << "timeout must be between 1 and 600" << endl; + showUsage(); + exit(1); + } + break; + } + case 'p': + op->put(PREF_FTP_PASV_ENABLED, V_TRUE); + break; case 'v': showVersion(); exit(0); diff --git a/src/message.h b/src/message.h index d326953c..7323c58e 100644 --- a/src/message.h +++ b/src/message.h @@ -32,7 +32,7 @@ #define MSG_RECEIVE_RESPONSE "CUID#%d - Response received:\n%s" #define MSG_DOWNLOAD_ABORTED "CUID#%d - Download aborted." #define MSG_RESTARTING_DOWNLOAD "CUID#%d - Restarting the download." -#define MSG_MAX_RETRY "CUID#%d - The retry count reached its max value. Download aborted." +#define MSG_MAX_TRY "CUID#%d - %d times attempted, but no success. Download aborted." #define MSG_UNREGISTER_CUID "CUID#%d - Unregistering cuid from segmentManager." #define MSG_SEGMENT_FILE_EXISTS "The segment file %s exists." diff --git a/src/prefs.h b/src/prefs.h new file mode 100644 index 00000000..847990da --- /dev/null +++ b/src/prefs.h @@ -0,0 +1,79 @@ +/* */ +#ifndef _D_PREFS_H_ +#define _D_PREFS_H_ + +/** + * Constants + */ +#define V_TRUE "true" +//#define V_FALSE "false" + +/** + * General preferences + */ +// values: 1*digit +#define PREF_RETRY_WAIT "retry_wait" +// values: 1*digit +#define PREF_TIMEOUT "timeout" +// values: 1*digit +#define PREF_MAX_TRY "max_try" + +/** + * FTP related preferences + */ +#define PREF_FTP_USER "ftp_user" +#define PREF_FTP_PASSWD "ftp_passwd" +// values: binary | ascii +#define PREF_FTP_TYPE "ftp_type" +# define V_BINARY "binary" +# define V_ASCII "ascii" +// values: get | tunnel +#define PREF_FTP_VIA_HTTP_PROXY "ftp_via_http_proxy" +# define V_GET "get" +# define V_TUNNEL "tunnel" +// values: true | false +#define PREF_FTP_PASV_ENABLED "ftp_pasv_enabled" + +/** + * HTTP related preferences + */ +#define PREF_HTTP_USER "http_user" +#define PREF_HTTP_PASSWD "http_passwd" +// values: basic +#define PREF_HTTP_AUTH_SCHEME "http_auth_scheme" +# define V_BASIC "basic" + +/** + * HTTP proxy related preferences + */ +#define PREF_HTTP_PROXY_USER "http_proxy_user" +#define PREF_HTTP_PROXY_PASSWD "http_proxy_passwd" +#define PREF_HTTP_PROXY_HOST "http_proxy_host" +#define PREF_HTTP_PROXY_PORT "http_proxy_port" +// values: true | false +#define PREF_HTTP_PROXY_ENABLED "http_proxy_enabled" +// values: true | false +#define PREF_HTTP_PROXY_AUTH_ENABLED "http_proxy_auth_enabled" + + +#endif // _D_PREFS_H_ -- cgit From 28fc3405dddd3998e3cb66242ef40b8d6fdad129 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Tue, 21 Feb 2006 15:01:05 +0000 Subject: --- ChangeLog | 48 +++++++++++++++++++++++++++++++++--------------- README | 6 ++++-- TODO | 1 - src/FtpConnection.cc | 18 +++++++++--------- src/HttpConnection.cc | 4 ++-- src/SimpleLogger.cc | 10 ++++++++-- src/Util.cc | 18 ++++++++++++++++++ src/Util.h | 2 ++ src/message.h | 3 +-- test/UtilTest.cc | 10 ++++++++++ 10 files changed, 87 insertions(+), 33 deletions(-) (limited to 'src') diff --git a/ChangeLog b/ChangeLog index d7f788b0..04756041 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,23 +1,41 @@ +2006-02-21 Tatsuhiro Tsujikawa + + * FtpInitiateConnectionCommand.{h,cc}: + * FtpNegotiationCommand.{h,cc}: + * FtpDownloadCommand.{h,cc}: + * FtpConnection.{h,cc}: Added FTP support + * SimpleLogger.cc: Log message now includes time information. + * main.cc: The value of --http-auth-scheme option is chagned from + 'BASIC' to 'basic' + * main.cc: Added --timeout command-line option. + * prefs.h: option string constants are now defined in prefs.h + 2006-02-19 Tatsuhiro Tsujikawa - * Fixed timeout bug in AbstractCommand - * Added totalSize entry to .aria2 file. No compatibility for - version 0.1.0's. + * AbstractCommand.cc: Fixed timeout bug in AbstractCommand + * SegmentMan.cc: Added totalSize entry to .aria2 file. No compatibility + with version 0.1.0's one. 2006-02-18 Tatsuhiro Tsujikawa - * Added --enable-ssl option to configure script. - * Make Request-URI an absolute path. Some servers cannot permit - absoluteURI as Request-URI. - * Added Referer support. - * Added referer command-line option. - * Added rety-wait command-line option. - * Fixed formating bug in Exception::setMsg() - * Added HTTPS support. - * Added SocketCore. Socket is now handle class for SocketCore. - * Fixed bug in ChunkedEncoding: expanding buffer size is wrong - * Fixed bug in DownloadCommand: In Chunked Encoding, it wrongly - adds to Segment.ds buff length from the socket. + * configure.in: Added --enable-ssl option to configure script. + * HttpConnection.cc: Make Request-URI an absolute path. Some servers + cannot permit absoluteURI as Request-URI. + * HttpConnection.cc: Added Referer support. + * main.cc: Added referer command-line option. + * main.cc: Added rety-wait command-line option. + * Exception.h: Fixed formating bug in Exception::setMsg() + * SocketCore.{h,cc}: + * Socket.{h, cc}: + * Request.cc: + * InitiateConnectionCommandFactory.cc: + * HttpRequestCommand.cc: Added HTTPS support. + * SocketCore.{h,cc}: Added SocketCore. Socket becomes a handle class + for SocketCore. + * ChunkedEncoding.cc: Fixed bug in ChunkedEncoding: expanding buffer + size is wrong + * DownloadCommand.cc: Fixed bug in DownloadCommand: In Chunked + Encoding, it wrongly adds to Segment.ds buff length from the socket. 2006-02-17 Tatsuhiro Tsujikawa diff --git a/README b/README index ed44c9d6..ba64745b 100644 --- a/README +++ b/README @@ -1,4 +1,4 @@ -aria2 - a simple utility for downloading files faster. +aria2 - a simple utility for downloading files. 1. Disclaimer ------------- @@ -11,10 +11,12 @@ aria2 has segmented downloading engine in its core. By segmented downloading, it can download files very much faster than ordinary browsers. aria2 is in very early development stage. Currently it has following features: -* HTTP GET support +* HTTP/HTTPS GET support * HTTP Proxy support * HTTP BASIC authentication support * HTTP Proxy authentication support +* FTP support(active, passive mode) +* FTP through HTTP proxy(GET command or tunneling) * Segmented download * Cookie support(currently aria2 ignores "expires") * It can run as a daemon process. diff --git a/TODO b/TODO index 96cda6a7..90d5a5f9 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,5 @@ * Add HTTP POST support * Add expires handling for Cookie -* Add FTP support * Add SSL server cert verification * Add SSL client cert support * Better HTTP status handling diff --git a/src/FtpConnection.cc b/src/FtpConnection.cc index 4614304e..1d478f1b 100644 --- a/src/FtpConnection.cc +++ b/src/FtpConnection.cc @@ -32,13 +32,13 @@ FtpConnection::~FtpConnection() {} void FtpConnection::sendUser() const { string request = "USER "+option->get(PREF_FTP_USER)+"\r\n"; - logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request); } void FtpConnection::sendPass() const { string request = "PASS "+option->get(PREF_FTP_PASSWD)+"\r\n"; - logger->info(MSG_SENDING_FTP_REQUEST, cuid, "PASS ********"); + logger->info(MSG_SENDING_REQUEST, cuid, "PASS ********"); socket->writeData(request); } @@ -50,25 +50,25 @@ void FtpConnection::sendType() const { type = "I"; } string request = "TYPE "+type+"\r\n"; - logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request); } void FtpConnection::sendCwd() const { string request = "CWD "+req->getDir()+"\r\n"; - logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request); } void FtpConnection::sendSize() const { string request = "SIZE "+req->getFile()+"\r\n"; - logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request); } void FtpConnection::sendPasv() const { string request = "PASV\r\n"; - logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request); } @@ -87,7 +87,7 @@ Socket* FtpConnection::sendPort() const { Util::itos(ipaddr[0])+","+Util::itos(ipaddr[1])+","+ Util::itos(ipaddr[2])+","+Util::itos(ipaddr[3])+","+ Util::itos(addrinfo.second/256)+","+Util::itos(addrinfo.second%256)+"\r\n"; - logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request); } catch (Exception* ex) { delete serverSocket; @@ -98,13 +98,13 @@ Socket* FtpConnection::sendPort() const { void FtpConnection::sendRest(const Segment& segment) const { string request = "REST "+Util::llitos(segment.sp+segment.ds)+"\r\n"; - logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request); } void FtpConnection::sendRetr() const { string request = "RETR "+req->getFile()+"\r\n"; - logger->info(MSG_SENDING_FTP_REQUEST, cuid, request.c_str()); + logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request); } diff --git a/src/HttpConnection.cc b/src/HttpConnection.cc index 535e745f..38949bd0 100644 --- a/src/HttpConnection.cc +++ b/src/HttpConnection.cc @@ -31,7 +31,7 @@ HttpConnection::HttpConnection(int cuid, const Socket* socket, const Request* re void HttpConnection::sendRequest(const Segment& segment) const { string request = createRequest(segment); - logger->info(MSG_SENDING_HTTP_REQUEST, cuid, request.c_str()); + logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request.c_str(), request.size()); } @@ -46,7 +46,7 @@ void HttpConnection::sendProxyRequest() const { option->get(PREF_HTTP_PROXY_PORT))+"\r\n"; } request += "\r\n"; - logger->info(MSG_SENDING_HTTP_REQUEST, cuid, request.c_str()); + logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request.c_str(), request.size()); } diff --git a/src/SimpleLogger.cc b/src/SimpleLogger.cc index 476428fd..efb95374 100644 --- a/src/SimpleLogger.cc +++ b/src/SimpleLogger.cc @@ -20,6 +20,8 @@ */ /* copyright --> */ #include "SimpleLogger.h" +#include "Util.h" +#include #include SimpleLogger::SimpleLogger(string filename) { @@ -50,9 +52,13 @@ void SimpleLogger::writeLog(int level, string msg, va_list ap, Exception* e) con default: levelStr = "INFO"; } - vfprintf(file, string(levelStr+" - "+msg+"\n").c_str(), ap); + time_t now = time(NULL); + char datestr[26]; + ctime_r(&now, datestr); + datestr[strlen(datestr)-1] = '\0'; + vfprintf(file, string(string(datestr)+" - "+levelStr+" - "+Util::replace(msg, "\r", "")+"\n").c_str(), ap); if(e != NULL) { - fprintf(file, string(levelStr+" - exception: "+e->getMsg()+"\n").c_str()); + fprintf(file, string(string(datestr)+" - "+levelStr+" - exception: "+Util::replace(e->getMsg(), "\r", "")+"\n").c_str()); } fflush(stdout); } diff --git a/src/Util.cc b/src/Util.cc index 99b70785..72e487e4 100644 --- a/src/Util.cc +++ b/src/Util.cc @@ -119,3 +119,21 @@ bool Util::endsWith(string target, string part) { return false; } } + +string Util::replace(string target, string oldstr, string newstr) { + if(target == "" || oldstr == "" ) { + return target; + } + string result; + string::size_type p = 0; + string::size_type np = target.find(oldstr); + while(np != string::npos) { + result += target.substr(p, np-p)+newstr; + p = np+oldstr.size(); + np = target.find(oldstr, p); + } + result += target.substr(p); + + return result; +} + diff --git a/src/Util.h b/src/Util.h index ebe67910..0377234b 100644 --- a/src/Util.h +++ b/src/Util.h @@ -51,6 +51,8 @@ public: static string trim(string src); static bool endsWith(string target, string part); + + static string replace(string target, string oldstr, string newstr); }; #endif // _D_UTIL_H_ diff --git a/src/message.h b/src/message.h index 7323c58e..0a75b239 100644 --- a/src/message.h +++ b/src/message.h @@ -27,8 +27,7 @@ #define MSG_CONNECTING_TO_SERVER "CUID#%d - Connecting to %s:%d" #define MSG_SEGMENT_CHANGED "CUID#%d - The segment changed. We send the request again with new Range header." #define MSG_REDIRECT "CUID#%d - Redirecting to %s" -#define MSG_SENDING_HTTP_REQUEST "CUID#%d - Sending the request:\n%s" -#define MSG_SENDING_FTP_REQUEST "CUID#%d - Sending the request: %s" +#define MSG_SENDING_REQUEST "CUID#%d - Requesting:\n%s" #define MSG_RECEIVE_RESPONSE "CUID#%d - Response received:\n%s" #define MSG_DOWNLOAD_ABORTED "CUID#%d - Download aborted." #define MSG_RESTARTING_DOWNLOAD "CUID#%d - Restarting the download." diff --git a/test/UtilTest.cc b/test/UtilTest.cc index 0100adbb..e6ab1a60 100644 --- a/test/UtilTest.cc +++ b/test/UtilTest.cc @@ -11,6 +11,7 @@ class UtilTest:public CppUnit::TestFixture { CPPUNIT_TEST(testSplit); CPPUNIT_TEST(testSlice); CPPUNIT_TEST(testEndsWith); + CPPUNIT_TEST(testReplace); CPPUNIT_TEST_SUITE_END(); private: @@ -22,6 +23,7 @@ public: void testSplit(); void testSlice(); void testEndsWith(); + void testReplace(); }; @@ -111,3 +113,11 @@ void UtilTest::testEndsWith() { part = "g"; CPPUNIT_ASSERT(!Util::endsWith(target, part)); } + +void UtilTest::testReplace() { + CPPUNIT_ASSERT_EQUAL(string("abc\n"), Util::replace("abc\r\n", "\r", "")); + CPPUNIT_ASSERT_EQUAL(string("abc"), Util::replace("abc\r\n", "\r\n", "")); + CPPUNIT_ASSERT_EQUAL(string(""), Util::replace("", "\r\n", "")); + CPPUNIT_ASSERT_EQUAL(string("abc"), Util::replace("abc", "", "a")); + CPPUNIT_ASSERT_EQUAL(string("xbc"), Util::replace("abc", "a", "x")); +} -- cgit From ebdb4b86cff8247b1a275748c60173581e034f46 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 22 Feb 2006 11:18:47 +0000 Subject: --- ChangeLog | 2 + TODO | 1 + src/AbstractCommand.cc | 3 +- src/DownloadEngine.h | 4 +- src/FtpConnection.cc | 8 +++- src/HttpResponseCommand.cc | 7 +-- src/Option.cc | 9 ++++ src/Option.h | 1 + src/Request.cc | 4 +- src/SegmentMan.cc | 3 +- src/SegmentMan.h | 2 + src/main.cc | 110 ++++++++++++++++++++++++++++++--------------- src/prefs.h | 4 +- 13 files changed, 110 insertions(+), 48 deletions(-) (limited to 'src') diff --git a/ChangeLog b/ChangeLog index 04756041..c7cbe795 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,6 +8,8 @@ * main.cc: The value of --http-auth-scheme option is chagned from 'BASIC' to 'basic' * main.cc: Added --timeout command-line option. + * main.cc: Added --min-segment-size command-line option. + * main.cc: Added --max-retries command-line option. * prefs.h: option string constants are now defined in prefs.h 2006-02-19 Tatsuhiro Tsujikawa diff --git a/TODO b/TODO index 90d5a5f9..2435e4b0 100644 --- a/TODO +++ b/TODO @@ -4,3 +4,4 @@ * Add SSL client cert support * Better HTTP status handling * Download files listed in a specifed file. +* check MD5 checksum \ No newline at end of file diff --git a/src/AbstractCommand.cc b/src/AbstractCommand.cc index a16bf6a6..6fe1d2ae 100644 --- a/src/AbstractCommand.cc +++ b/src/AbstractCommand.cc @@ -102,7 +102,8 @@ bool AbstractCommand::execute() { delete(err); //req->resetUrl(); req->addTryCount(); - if(req->getTryCount() >= e->option->getAsInt(PREF_MAX_TRY)) { + if(e->option->getAsInt(PREF_MAX_TRIES) != 0 && + req->getTryCount() >= e->option->getAsInt(PREF_MAX_TRIES)) { e->logger->error(MSG_MAX_TRY, cuid, req->getTryCount()); return true; } else { diff --git a/src/DownloadEngine.h b/src/DownloadEngine.h index f8eaa97d..b3d2a70b 100644 --- a/src/DownloadEngine.h +++ b/src/DownloadEngine.h @@ -47,8 +47,8 @@ public: queue commands; SegmentMan* segmentMan; DiskWriter* diskWriter; - Logger* logger; - Option* option; + const Logger* logger; + const Option* option; DownloadEngine(); ~DownloadEngine(); diff --git a/src/FtpConnection.cc b/src/FtpConnection.cc index 1d478f1b..54844e74 100644 --- a/src/FtpConnection.cc +++ b/src/FtpConnection.cc @@ -110,8 +110,12 @@ void FtpConnection::sendRetr() const { int FtpConnection::getStatus(string response) const { int status; - // TODO we must handle when the response is not like "%d %*s" - // In this case, we return 0 + // When the response is not like "%d %*s", + // we return 0. + if(response.find_first_not_of("0123456789") != 3 + || response.find(" ") != 3) { + return 0; + } if(sscanf(response.c_str(), "%d %*s", &status) == 1) { return status; } else { diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index f5788be7..4f808a8b 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -65,10 +65,13 @@ bool HttpResponseCommand::executeInternal(Segment seg) { return handleDefaultEncoding(headers); } } else { - // TODO we must check headers["size"] == e->segmentMan->totalSize here if(req->getFile() != e->segmentMan->filename) { throw new DlAbortEx(EX_FILENAME_MISMATCH, req->getFile().c_str(), e->segmentMan->filename.c_str()); } + long long int size = headers.getFirstAsLLInt("Content-Length"); + if(e->segmentMan->totalSize != size) { + throw new DlAbortEx(EX_SIZE_MISMATCH, e->segmentMan->totalSize, size); + } createHttpDownloadCommand(); return true; } @@ -99,9 +102,7 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpHeader& headers) { bool segFileExists = e->segmentMan->segmentFileExists(); e->segmentMan->downloadStarted = true; if(segFileExists) { - e->diskWriter->openExistingFile(e->segmentMan->getFilePath()); - // we must check headers["size"] == e->segmentMan->totalSize here if(e->segmentMan->totalSize != size) { return new DlAbortEx(EX_SIZE_MISMATCH, e->segmentMan->totalSize, size); } diff --git a/src/Option.cc b/src/Option.cc index 96576b83..35e4a35c 100644 --- a/src/Option.cc +++ b/src/Option.cc @@ -50,3 +50,12 @@ int Option::getAsInt(const string& name) const { return (int)strtol((*itr).second.c_str(), NULL, 10); } } + +long long int Option::getAsLLInt(const string& name) const { + map::const_iterator itr = table.find(name); + if(itr == table.end()) { + return 0; + } else { + return (int)strtoll((*itr).second.c_str(), NULL, 10); + } +} diff --git a/src/Option.h b/src/Option.h index fa1e0c67..7cacef50 100644 --- a/src/Option.h +++ b/src/Option.h @@ -38,6 +38,7 @@ public: bool defined(const string& name) const; string get(const string& name) const; int getAsInt(const string& name) const; + long long int getAsLLInt(const string& name) const; }; #endif // _D_OPTION_H_ diff --git a/src/Request.cc b/src/Request.cc index 6eb4aa1e..9d183cd4 100644 --- a/src/Request.cc +++ b/src/Request.cc @@ -79,14 +79,12 @@ bool Request::parseUrl(string url) { Util::split(hostAndPort, url.substr(hp, hep-hp), ':'); host = hostAndPort.first; if(hostAndPort.second != "") { - // TODO rewrite this using strtoul function. If strtoul fails, - // return false. port = (int)strtol(hostAndPort.second.c_str(), NULL, 10); if(!(0 < port && port <= 65535)) { return false; } } else { - // If port is not specified, then we leave it 0. + // If port is not specified, then we set it to default port of its protocol.. port = defPort; } string::size_type direp = url.find_last_of("/"); diff --git a/src/SegmentMan.cc b/src/SegmentMan.cc index 579c2bfe..0089575a 100644 --- a/src/SegmentMan.cc +++ b/src/SegmentMan.cc @@ -28,6 +28,7 @@ #include "Util.h" #include "File.h" #include "message.h" +#include "prefs.h" SegmentMan::SegmentMan():totalSize(0),isSplittable(true),downloadStarted(false),dir(".") {} @@ -85,7 +86,7 @@ bool SegmentMan::getSegment(Segment& seg, int cuid) { if(s.finish) { continue; } - if(s.ep-(s.sp+s.ds) > 524288) { + if(s.ep-(s.sp+s.ds) > option->getAsLLInt(PREF_MIN_SEGMENT_SIZE)) { long long int nep = (s.ep-(s.sp+s.ds))/2+(s.sp+s.ds); //nseg = { cuid, nep+1, s.ep, 0, false }; seg.cuid = cuid; diff --git a/src/SegmentMan.h b/src/SegmentMan.h index e971a176..075453ff 100644 --- a/src/SegmentMan.h +++ b/src/SegmentMan.h @@ -26,6 +26,7 @@ #include "common.h" #include "Logger.h" #include "Segment.h" +#include "Option.h" using namespace std; @@ -81,6 +82,7 @@ public: string ufilename; const Logger* logger; + const Option* option; SegmentMan(); ~SegmentMan(); diff --git a/src/main.cc b/src/main.cc index 63c46a95..a352a674 100644 --- a/src/main.cc +++ b/src/main.cc @@ -101,40 +101,45 @@ void showUsage() { cout << "Usage: " << PACKAGE_NAME << " [options] URL ..." << endl; cout << endl; cout << "Options:" << endl; - cout << " -d, --dir=DIR The directory to store downloaded file." << endl; - cout << " -o, --out=FILE The file name for downloaded file." << endl; - cout << " -l, --log=LOG The file path to store log. If '-' is specified," << endl; - cout << " log is written to stdout." << endl; - cout << " -D, --daemon Run as daemon." << endl; - cout << " -s, --split=N Download a file using s connections. s must be" << endl; - cout << " between 1 and 5. If this option is specified the" << endl; - cout << " first URL is used, and the other URLs are ignored." << endl; - cout << " --retry-wait=SEC Set amount of time in second between requests" << endl; - cout << " for errors. Specify a value between 0 and 60." << endl; - cout << " -t, --timeout=SEC Set timeout in second." << endl; - cout << " --http-proxy=HOST:PORT Use HTTP proxy server. This affects to all" << endl; - cout << " URLs." << endl; - cout << " --http-user=USER Set HTTP user. This affects to all URLs." << endl; - cout << " --http-passwd=PASSWD Set HTTP password. This affects to all URLs." << endl; - cout << " --http-proxy-user=USER Set HTTP proxy user. This affects to all URLs" << endl; - cout << " --http-proxy-passwd=PASSWD Set HTTP proxy password. This affects to all URLs." << endl; - cout << " --http-auth-scheme=SCHEME Set HTTP authentication scheme. Currently, basic" << endl; - cout << " is the only supported scheme. You MUST specify" << endl; - cout << " this option in order to use HTTP authentication" << endl; - cout << " as well as --http-proxy option." << endl; - cout << " --referer=REFERER Set Referer. This affects to all URLs." << endl; - cout << " --ftp-user=USER Set FTP user. This affects to all URLs." << endl; - cout << " Default: anonymous" << endl; - cout << " --ftp-passwd=PASSWD Set FTP password. This affects to all URLs." << endl; - cout << " Default: ARIA2USER@" << endl; - cout << " --ftp-type=TYPE Set FTP transfer type. TYPE is either 'binary'" << endl; - cout << " or 'ascii'." << endl; - cout << " Default: binary" << endl; - cout << " -p, --ftp-pasv Use passive mode in FTP." << endl; - cout << " --ftp-via-http-proxy=WAY Use HTTP proxy in FTP. WAY is either 'get' or" << endl; - cout << " 'tunnel'." << endl; - cout << " -v, --version Print the version number and exit." << endl; - cout << " -h, --help Print this message and exit." << endl; + cout << " -d, --dir=DIR The directory to store downloaded file." << endl; + cout << " -o, --out=FILE The file name for downloaded file." << endl; + cout << " -l, --log=LOG The file path to store log. If '-' is specified," << endl; + cout << " log is written to stdout." << endl; + cout << " -D, --daemon Run as daemon." << endl; + cout << " -s, --split=N Download a file using s connections. s must be" << endl; + cout << " between 1 and 5. If this option is specified the" << endl; + cout << " first URL is used, and the other URLs are ignored." << endl; + cout << " --retry-wait=SEC Set amount of time in second between requests" << endl; + cout << " for errors. Specify a value between 0 and 60." << endl; + cout << " Default: 5" << endl; + cout << " -t, --timeout=SEC Set timeout in second. Default: 60" << endl; + cout << " -m, --max-tries=N Set number of tries. 0 means unlimited." << endl; + cout << " Default: 5" << endl; + cout << " --min-segment-size=SIZE[K|M] Set minimum segment size. You can append" << endl; + cout << " K or M(1K = 1024, 1M = 1024K)." << endl; + cout << " --http-proxy=HOST:PORT Use HTTP proxy server. This affects to all" << endl; + cout << " URLs." << endl; + cout << " --http-user=USER Set HTTP user. This affects to all URLs." << endl; + cout << " --http-passwd=PASSWD Set HTTP password. This affects to all URLs." << endl; + cout << " --http-proxy-user=USER Set HTTP proxy user. This affects to all URLs" << endl; + cout << " --http-proxy-passwd=PASSWD Set HTTP proxy password. This affects to all URLs." << endl; + cout << " --http-auth-scheme=SCHEME Set HTTP authentication scheme. Currently, basic" << endl; + cout << " is the only supported scheme. You MUST specify" << endl; + cout << " this option in order to use HTTP authentication" << endl; + cout << " as well as --http-proxy option." << endl; + cout << " --referer=REFERER Set Referer. This affects to all URLs." << endl; + cout << " --ftp-user=USER Set FTP user. This affects to all URLs." << endl; + cout << " Default: anonymous" << endl; + cout << " --ftp-passwd=PASSWD Set FTP password. This affects to all URLs." << endl; + cout << " Default: ARIA2USER@" << endl; + cout << " --ftp-type=TYPE Set FTP transfer type. TYPE is either 'binary'" << endl; + cout << " or 'ascii'." << endl; + cout << " Default: binary" << endl; + cout << " -p, --ftp-pasv Use passive mode in FTP." << endl; + cout << " --ftp-via-http-proxy=WAY Use HTTP proxy in FTP. WAY is either 'get' or" << endl; + cout << " 'tunnel'." << endl; + cout << " -v, --version Print the version number and exit." << endl; + cout << " -h, --help Print this message and exit." << endl; cout << endl; cout << "URL:" << endl; cout << " You can specify multiple URLs. All URLs must point to the same file" << endl; @@ -166,6 +171,8 @@ int main(int argc, char* argv[]) { Option* op = new Option(); op->put(PREF_RETRY_WAIT, "5"); op->put(PREF_TIMEOUT, "60"); + op->put(PREF_MIN_SEGMENT_SIZE, "1048576");// 1M + op->put(PREF_MAX_TRIES, "5"); op->put(PREF_FTP_USER, "anonymous"); op->put(PREF_FTP_PASSWD, "ARIA2USER@"); op->put(PREF_FTP_TYPE, V_BINARY); @@ -182,6 +189,7 @@ int main(int argc, char* argv[]) { { "log", required_argument, NULL, 'l' }, { "split", required_argument, NULL, 's' }, { "timeout", required_argument, NULL, 't' }, + { "max-retries", required_argument, NULL, 'm' }, { "http-proxy", required_argument, &lopt, 1 }, { "http-user", required_argument, &lopt, 2 }, { "http-passwd", required_argument, &lopt, 3 }, @@ -195,11 +203,12 @@ int main(int argc, char* argv[]) { { "ftp-type", required_argument, &lopt, 11 }, { "ftp-pasv", no_argument, NULL, 'p' }, { "ftp-via-http-proxy", required_argument, &lopt, 12 }, + { "min-segment-size", required_argument, &lopt, 13 }, { "version", no_argument, NULL, 'v' }, { "help", no_argument, NULL, 'h' }, { 0, 0, 0, 0 } }; - c = getopt_long(argc, argv, "Dd:o:l:s:pt:vh", longOpts, &optIndex); + c = getopt_long(argc, argv, "Dd:o:l:s:pt:m:vh", longOpts, &optIndex); if(c == -1) { break; } @@ -278,6 +287,26 @@ int main(int argc, char* argv[]) { exit(1); } break; + case 13: { + string::size_type p = string(optarg).find_first_of("KM"); + int mult = 1; + if(p != string::npos) { + if(optarg[p] == 'K') { + mult = 1024; + } else if(optarg[p] == 'M') { + mult = 1024*1024; + } + optarg[p] = '\0'; + } + long long int size = strtoll(optarg, NULL, 10)*mult; + if(size <= 0) { + cerr << "min-segment-size invalid" << endl; + showUsage(); + exit(1); + } + op->put(PREF_MIN_SEGMENT_SIZE, Util::llitos(size)); + break; + } } break; } @@ -316,6 +345,16 @@ int main(int argc, char* argv[]) { } break; } + case 'm': { + int retries = (int)strtol(optarg, NULL, 10); + if(retries < 0) { + cerr << "max-retires invalid" << endl; + showUsage(); + exit(1); + } + op->put(PREF_MAX_TRIES, Util::itos(retries)); + break; + } case 'p': op->put(PREF_FTP_PASV_ENABLED, V_TRUE); break; @@ -363,6 +402,7 @@ int main(int argc, char* argv[]) { e->segmentMan->dir = dir; e->segmentMan->ufilename = ufilename; e->segmentMan->logger = logger; + e->segmentMan->option = op; vector requests; if(split > 0) { for(int i = 1; i <= split; i++) { diff --git a/src/prefs.h b/src/prefs.h index 847990da..bce670e4 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -36,7 +36,9 @@ // values: 1*digit #define PREF_TIMEOUT "timeout" // values: 1*digit -#define PREF_MAX_TRY "max_try" +#define PREF_MAX_TRIES "max_try" +// values: 1*digit +#define PREF_MIN_SEGMENT_SIZE "min_segment_size" /** * FTP related preferences -- cgit From 27d392d5f2c0cbddeacd53995b2831ddc7f5d08a Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 22 Feb 2006 12:16:10 +0000 Subject: --- ChangeLog | 6 ++++++ TODO | 4 +++- src/DlAbortEx.h | 4 ++-- src/DlRetryEx.h | 2 +- src/HttpResponseCommand.cc | 8 +------- src/main.cc | 18 ++++++------------ src/message.h | 4 ++-- 7 files changed, 21 insertions(+), 25 deletions(-) (limited to 'src') diff --git a/ChangeLog b/ChangeLog index c7cbe795..af586269 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,9 @@ +2006-02-22 Tatsuhiro Tsujikawa + + * main.cc: -s option now affects all URLs in command-line arguemtns. + * HttpResponseCommand.cc: Fixed bug that segment file is not loaded. + * message.h: Change file size related %d to %lld. + 2006-02-21 Tatsuhiro Tsujikawa * FtpInitiateConnectionCommand.{h,cc}: diff --git a/TODO b/TODO index 2435e4b0..e8c1e273 100644 --- a/TODO +++ b/TODO @@ -4,4 +4,6 @@ * Add SSL client cert support * Better HTTP status handling * Download files listed in a specifed file. -* check MD5 checksum \ No newline at end of file +* check MD5 checksum +* Apply "split longest remianing time first" algorithm to SegmentMan. +* split algorithm must be separate class. \ No newline at end of file diff --git a/src/DlAbortEx.h b/src/DlAbortEx.h index 151e2e60..4d7fe7e6 100644 --- a/src/DlAbortEx.h +++ b/src/DlAbortEx.h @@ -26,10 +26,10 @@ class DlAbortEx:public Exception { public: DlAbortEx():Exception() {} - DlAbortEx(string msg, ...):Exception() { + DlAbortEx(const char* msg, ...):Exception() { va_list ap; va_start(ap, msg); - setMsg(msg, ap); + setMsg(string(msg), ap); va_end(ap); } }; diff --git a/src/DlRetryEx.h b/src/DlRetryEx.h index 29bf6f70..0ba5f50b 100644 --- a/src/DlRetryEx.h +++ b/src/DlRetryEx.h @@ -26,7 +26,7 @@ class DlRetryEx:public Exception { public: DlRetryEx():Exception() {} - DlRetryEx(string msg, ...):Exception() { + DlRetryEx(const char* msg, ...):Exception() { va_list ap; va_start(ap, msg); setMsg(msg, ap); diff --git a/src/HttpResponseCommand.cc b/src/HttpResponseCommand.cc index 4f808a8b..30e65db1 100644 --- a/src/HttpResponseCommand.cc +++ b/src/HttpResponseCommand.cc @@ -68,10 +68,6 @@ bool HttpResponseCommand::executeInternal(Segment seg) { if(req->getFile() != e->segmentMan->filename) { throw new DlAbortEx(EX_FILENAME_MISMATCH, req->getFile().c_str(), e->segmentMan->filename.c_str()); } - long long int size = headers.getFirstAsLLInt("Content-Length"); - if(e->segmentMan->totalSize != size) { - throw new DlAbortEx(EX_SIZE_MISMATCH, e->segmentMan->totalSize, size); - } createHttpDownloadCommand(); return true; } @@ -102,10 +98,8 @@ bool HttpResponseCommand::handleDefaultEncoding(const HttpHeader& headers) { bool segFileExists = e->segmentMan->segmentFileExists(); e->segmentMan->downloadStarted = true; if(segFileExists) { + e->segmentMan->load(); e->diskWriter->openExistingFile(e->segmentMan->getFilePath()); - if(e->segmentMan->totalSize != size) { - return new DlAbortEx(EX_SIZE_MISMATCH, e->segmentMan->totalSize, size); - } // send request again to the server with Range header return prepareForRetry(0); } else { diff --git a/src/main.cc b/src/main.cc index a352a674..1f08d4ba 100644 --- a/src/main.cc +++ b/src/main.cc @@ -107,8 +107,8 @@ void showUsage() { cout << " log is written to stdout." << endl; cout << " -D, --daemon Run as daemon." << endl; cout << " -s, --split=N Download a file using s connections. s must be" << endl; - cout << " between 1 and 5. If this option is specified the" << endl; - cout << " first URL is used, and the other URLs are ignored." << endl; + cout << " between 1 and 5. This option affects all URLs." << endl; + cout << " Thus, aria2 connects to each URL with N connections." << endl; cout << " --retry-wait=SEC Set amount of time in second between requests" << endl; cout << " for errors. Specify a value between 0 and 60." << endl; cout << " Default: 5" << endl; @@ -163,7 +163,7 @@ int main(int argc, char* argv[]) { string logfile; string dir; string ufilename; - int split = 0; + int split = 1; bool daemonMode = false; string referer; @@ -393,7 +393,6 @@ int main(int argc, char* argv[]) { } else { logger = new SimpleLogger("/dev/null"); } - e = new DownloadEngine(); e->logger = logger; e->option = op; @@ -404,16 +403,11 @@ int main(int argc, char* argv[]) { e->segmentMan->logger = logger; e->segmentMan->option = op; vector requests; - if(split > 0) { - for(int i = 1; i <= split; i++) { - addCommand(i, argv[optind], referer, requests); - } - } else { - for(int i = 1; optind < argc; i++) { - addCommand(i, argv[optind++], referer, requests); + for(int i = 1; optind+i-1 < argc; i++) { + for(int s = 1; s <= split; s++) { + addCommand(split*(i-1)+s, argv[optind+i-1], referer, requests); } } - struct sigaction sigact; sigact.sa_handler = handler; sigact.sa_flags = 0; diff --git a/src/message.h b/src/message.h index 0a75b239..ca8514f3 100644 --- a/src/message.h +++ b/src/message.h @@ -52,9 +52,9 @@ #define EX_CONNECTION_FAILED "Connection failed." #define EX_FILENAME_MISMATCH "The requested filename and the previously registered one are not same. %s != %s" #define EX_BAD_STATUS "The response status is not successful. status = %d" -#define EX_TOO_LARGE_FILE "Too large file size. size = %d" +#define EX_TOO_LARGE_FILE "Too large file size. size = %lld" #define EX_TRANSFER_ENCODING_NOT_SUPPORTED "Transfer encoding %s is not supported." #define EX_SSL_INIT_FAILURE "SSL initialization failed." -#define EX_SIZE_MISMATCH "Size mismatch %d != %d" +#define EX_SIZE_MISMATCH "Size mismatch %lld != %lld" #endif // _D_MESSAGE_H_ -- cgit From 2462a3b06cd54b0b0534dc7c133b382e498266f2 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 22 Feb 2006 14:30:47 +0000 Subject: --- ChangeLog | 9 +++++++++ TODO | 3 +-- src/DownloadCommand.cc | 19 ++++++++++++++++- src/DownloadCommand.h | 3 +++ src/FtpConnection.cc | 2 +- src/Makefile.am | 3 +++ src/Makefile.in | 55 ++++++++++++++++++++++++++++++++++++++++++++++++-- src/Segment.h | 7 +++++++ src/SegmentMan.cc | 31 ++-------------------------- src/SegmentMan.h | 4 +++- src/Util.h | 2 +- src/main.cc | 9 +++++++-- 12 files changed, 108 insertions(+), 39 deletions(-) (limited to 'src') diff --git a/ChangeLog b/ChangeLog index af586269..7678a435 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,14 @@ 2006-02-22 Tatsuhiro Tsujikawa + * SplitSlowestSegmentSplitter.{h,cc}: This class provies algorithm + that splits slowest segment of SegmentMan::commands vector. + This is the default split algorithm of aria2. + * SplitFirstSegmentSplitter.{h,cc}: This class provides algorithm + that splits first segment of SegmentMan::commands vector. + * SegmentSplitter.{h,cc}: Added. This class provides split algorithm. + * DownloadCommand.{h,cc}: Added downloading speed calculation. + * Segment.h: + * SegmentMan.cc: Added speed field to Segment.h * main.cc: -s option now affects all URLs in command-line arguemtns. * HttpResponseCommand.cc: Fixed bug that segment file is not loaded. * message.h: Change file size related %d to %lld. diff --git a/TODO b/TODO index e8c1e273..e05708c1 100644 --- a/TODO +++ b/TODO @@ -5,5 +5,4 @@ * Better HTTP status handling * Download files listed in a specifed file. * check MD5 checksum -* Apply "split longest remianing time first" algorithm to SegmentMan. -* split algorithm must be separate class. \ No newline at end of file +* Add the feature which adds or removes URLs on-the-fly. \ No newline at end of file diff --git a/src/DownloadCommand.cc b/src/DownloadCommand.cc index 0cc15661..b771cbe5 100644 --- a/src/DownloadCommand.cc +++ b/src/DownloadCommand.cc @@ -27,7 +27,10 @@ #include "InitiateConnectionCommandFactory.h" #include "message.h" -DownloadCommand::DownloadCommand(int cuid, Request* req, DownloadEngine* e, Socket* s):AbstractCommand(cuid, req, e, s) {} +DownloadCommand::DownloadCommand(int cuid, Request* req, DownloadEngine* e, Socket* s):AbstractCommand(cuid, req, e, s), lastSize(0) { + sw.tv_sec = 0; + sw.tv_usec = 0; +} DownloadCommand::~DownloadCommand() {} @@ -50,6 +53,20 @@ bool DownloadCommand::executeInternal(Segment seg) { e->diskWriter->writeData(buf, bufSize, seg.sp+seg.ds); seg.ds += bufSize; } + // calculate downloading speed + struct timeval now; + gettimeofday(&now, NULL); + if(sw.tv_sec == 0 && sw.tv_usec == 0) { + sw = now; + lastSize = seg.ds; + } else { + long long int diff = Util::difftv(now, sw); + if(diff >= 1000000) { + seg.speed = (int)((seg.ds-lastSize)/(diff/1000000.0)); + sw = now; + lastSize = seg.ds; + } + } if(te != NULL && te->finished() || te == NULL && seg.ds >= seg.ep-seg.sp+1 diff --git a/src/DownloadCommand.h b/src/DownloadCommand.h index af44901c..3dd374e7 100644 --- a/src/DownloadCommand.h +++ b/src/DownloadCommand.h @@ -29,6 +29,9 @@ using namespace std; class DownloadCommand : public AbstractCommand { +private: + struct timeval sw; + long long int lastSize; protected: bool executeInternal(Segment segment); diff --git a/src/FtpConnection.cc b/src/FtpConnection.cc index 54844e74..9e7b9d44 100644 --- a/src/FtpConnection.cc +++ b/src/FtpConnection.cc @@ -113,7 +113,7 @@ int FtpConnection::getStatus(string response) const { // When the response is not like "%d %*s", // we return 0. if(response.find_first_not_of("0123456789") != 3 - || response.find(" ") != 3) { + || !(response.find(" ") == 3 || response.find("-") == 3)) { return 0; } if(sscanf(response.c_str(), "%d %*s", &status) == 1) { diff --git a/src/Makefile.am b/src/Makefile.am index 4ae30cca..796cdd82 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -24,6 +24,9 @@ SRCS = Socket.cc Socket.h\ DownloadEngine.cc DownloadEngine.h\ Segment.h\ SegmentMan.cc SegmentMan.h\ + SegmentSplitter.cc SegmentSplitter.h\ + SplitFirstSegmentSplitter.cc SplitFirstSegmentSplitter.h\ + SplitSlowestSegmentSplitter.cc SplitSlowestSegmentSplitter.h\ Util.cc Util.h\ Request.cc Request.h\ common.h\ diff --git a/src/Makefile.in b/src/Makefile.in index 760a3bce..51f4454a 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -71,8 +71,11 @@ am__objects_1 = libaria2c_a-Socket.$(OBJEXT) \ libaria2c_a-FtpTunnelResponseCommand.$(OBJEXT) \ libaria2c_a-SleepCommand.$(OBJEXT) \ libaria2c_a-DownloadEngine.$(OBJEXT) \ - libaria2c_a-SegmentMan.$(OBJEXT) libaria2c_a-Util.$(OBJEXT) \ - libaria2c_a-Request.$(OBJEXT) \ + libaria2c_a-SegmentMan.$(OBJEXT) \ + libaria2c_a-SegmentSplitter.$(OBJEXT) \ + libaria2c_a-SplitFirstSegmentSplitter.$(OBJEXT) \ + libaria2c_a-SplitSlowestSegmentSplitter.$(OBJEXT) \ + libaria2c_a-Util.$(OBJEXT) libaria2c_a-Request.$(OBJEXT) \ libaria2c_a-SimpleLogger.$(OBJEXT) \ libaria2c_a-ChunkedEncoding.$(OBJEXT) \ libaria2c_a-DefaultDiskWriter.$(OBJEXT) \ @@ -211,6 +214,9 @@ SRCS = Socket.cc Socket.h\ DownloadEngine.cc DownloadEngine.h\ Segment.h\ SegmentMan.cc SegmentMan.h\ + SegmentSplitter.cc SegmentSplitter.h\ + SplitFirstSegmentSplitter.cc SplitFirstSegmentSplitter.h\ + SplitSlowestSegmentSplitter.cc SplitSlowestSegmentSplitter.h\ Util.cc Util.h\ Request.cc Request.h\ common.h\ @@ -337,10 +343,13 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-Option.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-Request.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-SegmentMan.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-SegmentSplitter.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-SimpleLogger.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-SleepCommand.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-Socket.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-SocketCore.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-SplitFirstSegmentSplitter.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-SplitSlowestSegmentSplitter.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libaria2c_a-Util.Po@am__quote@ .cc.o: @@ -665,6 +674,48 @@ libaria2c_a-SegmentMan.obj: SegmentMan.cc @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-SegmentMan.obj `if test -f 'SegmentMan.cc'; then $(CYGPATH_W) 'SegmentMan.cc'; else $(CYGPATH_W) '$(srcdir)/SegmentMan.cc'; fi` +libaria2c_a-SegmentSplitter.o: SegmentSplitter.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-SegmentSplitter.o -MD -MP -MF "$(DEPDIR)/libaria2c_a-SegmentSplitter.Tpo" -c -o libaria2c_a-SegmentSplitter.o `test -f 'SegmentSplitter.cc' || echo '$(srcdir)/'`SegmentSplitter.cc; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-SegmentSplitter.Tpo" "$(DEPDIR)/libaria2c_a-SegmentSplitter.Po"; else rm -f "$(DEPDIR)/libaria2c_a-SegmentSplitter.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='SegmentSplitter.cc' object='libaria2c_a-SegmentSplitter.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-SegmentSplitter.o `test -f 'SegmentSplitter.cc' || echo '$(srcdir)/'`SegmentSplitter.cc + +libaria2c_a-SegmentSplitter.obj: SegmentSplitter.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-SegmentSplitter.obj -MD -MP -MF "$(DEPDIR)/libaria2c_a-SegmentSplitter.Tpo" -c -o libaria2c_a-SegmentSplitter.obj `if test -f 'SegmentSplitter.cc'; then $(CYGPATH_W) 'SegmentSplitter.cc'; else $(CYGPATH_W) '$(srcdir)/SegmentSplitter.cc'; fi`; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-SegmentSplitter.Tpo" "$(DEPDIR)/libaria2c_a-SegmentSplitter.Po"; else rm -f "$(DEPDIR)/libaria2c_a-SegmentSplitter.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='SegmentSplitter.cc' object='libaria2c_a-SegmentSplitter.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-SegmentSplitter.obj `if test -f 'SegmentSplitter.cc'; then $(CYGPATH_W) 'SegmentSplitter.cc'; else $(CYGPATH_W) '$(srcdir)/SegmentSplitter.cc'; fi` + +libaria2c_a-SplitFirstSegmentSplitter.o: SplitFirstSegmentSplitter.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-SplitFirstSegmentSplitter.o -MD -MP -MF "$(DEPDIR)/libaria2c_a-SplitFirstSegmentSplitter.Tpo" -c -o libaria2c_a-SplitFirstSegmentSplitter.o `test -f 'SplitFirstSegmentSplitter.cc' || echo '$(srcdir)/'`SplitFirstSegmentSplitter.cc; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-SplitFirstSegmentSplitter.Tpo" "$(DEPDIR)/libaria2c_a-SplitFirstSegmentSplitter.Po"; else rm -f "$(DEPDIR)/libaria2c_a-SplitFirstSegmentSplitter.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='SplitFirstSegmentSplitter.cc' object='libaria2c_a-SplitFirstSegmentSplitter.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-SplitFirstSegmentSplitter.o `test -f 'SplitFirstSegmentSplitter.cc' || echo '$(srcdir)/'`SplitFirstSegmentSplitter.cc + +libaria2c_a-SplitFirstSegmentSplitter.obj: SplitFirstSegmentSplitter.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-SplitFirstSegmentSplitter.obj -MD -MP -MF "$(DEPDIR)/libaria2c_a-SplitFirstSegmentSplitter.Tpo" -c -o libaria2c_a-SplitFirstSegmentSplitter.obj `if test -f 'SplitFirstSegmentSplitter.cc'; then $(CYGPATH_W) 'SplitFirstSegmentSplitter.cc'; else $(CYGPATH_W) '$(srcdir)/SplitFirstSegmentSplitter.cc'; fi`; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-SplitFirstSegmentSplitter.Tpo" "$(DEPDIR)/libaria2c_a-SplitFirstSegmentSplitter.Po"; else rm -f "$(DEPDIR)/libaria2c_a-SplitFirstSegmentSplitter.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='SplitFirstSegmentSplitter.cc' object='libaria2c_a-SplitFirstSegmentSplitter.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-SplitFirstSegmentSplitter.obj `if test -f 'SplitFirstSegmentSplitter.cc'; then $(CYGPATH_W) 'SplitFirstSegmentSplitter.cc'; else $(CYGPATH_W) '$(srcdir)/SplitFirstSegmentSplitter.cc'; fi` + +libaria2c_a-SplitSlowestSegmentSplitter.o: SplitSlowestSegmentSplitter.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-SplitSlowestSegmentSplitter.o -MD -MP -MF "$(DEPDIR)/libaria2c_a-SplitSlowestSegmentSplitter.Tpo" -c -o libaria2c_a-SplitSlowestSegmentSplitter.o `test -f 'SplitSlowestSegmentSplitter.cc' || echo '$(srcdir)/'`SplitSlowestSegmentSplitter.cc; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-SplitSlowestSegmentSplitter.Tpo" "$(DEPDIR)/libaria2c_a-SplitSlowestSegmentSplitter.Po"; else rm -f "$(DEPDIR)/libaria2c_a-SplitSlowestSegmentSplitter.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='SplitSlowestSegmentSplitter.cc' object='libaria2c_a-SplitSlowestSegmentSplitter.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-SplitSlowestSegmentSplitter.o `test -f 'SplitSlowestSegmentSplitter.cc' || echo '$(srcdir)/'`SplitSlowestSegmentSplitter.cc + +libaria2c_a-SplitSlowestSegmentSplitter.obj: SplitSlowestSegmentSplitter.cc +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-SplitSlowestSegmentSplitter.obj -MD -MP -MF "$(DEPDIR)/libaria2c_a-SplitSlowestSegmentSplitter.Tpo" -c -o libaria2c_a-SplitSlowestSegmentSplitter.obj `if test -f 'SplitSlowestSegmentSplitter.cc'; then $(CYGPATH_W) 'SplitSlowestSegmentSplitter.cc'; else $(CYGPATH_W) '$(srcdir)/SplitSlowestSegmentSplitter.cc'; fi`; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-SplitSlowestSegmentSplitter.Tpo" "$(DEPDIR)/libaria2c_a-SplitSlowestSegmentSplitter.Po"; else rm -f "$(DEPDIR)/libaria2c_a-SplitSlowestSegmentSplitter.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='SplitSlowestSegmentSplitter.cc' object='libaria2c_a-SplitSlowestSegmentSplitter.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -c -o libaria2c_a-SplitSlowestSegmentSplitter.obj `if test -f 'SplitSlowestSegmentSplitter.cc'; then $(CYGPATH_W) 'SplitSlowestSegmentSplitter.cc'; else $(CYGPATH_W) '$(srcdir)/SplitSlowestSegmentSplitter.cc'; fi` + libaria2c_a-Util.o: Util.cc @am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libaria2c_a_CXXFLAGS) $(CXXFLAGS) -MT libaria2c_a-Util.o -MD -MP -MF "$(DEPDIR)/libaria2c_a-Util.Tpo" -c -o libaria2c_a-Util.o `test -f 'Util.cc' || echo '$(srcdir)/'`Util.cc; \ @am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/libaria2c_a-Util.Tpo" "$(DEPDIR)/libaria2c_a-Util.Po"; else rm -f "$(DEPDIR)/libaria2c_a-Util.Tpo"; exit 1; fi diff --git a/src/Segment.h b/src/Segment.h index c6c2b36c..035282ae 100644 --- a/src/Segment.h +++ b/src/Segment.h @@ -22,6 +22,10 @@ #ifndef _D_SEGMENT_H_ #define _D_SEGMENT_H_ +#include + +using namespace std; + /** * Segment represents a download segment. * sp, ep is a offset from a begining of a file. @@ -36,9 +40,12 @@ typedef struct { long long int sp; long long int ep; long long int ds; + int speed; bool finish; } Segment; +typedef vector Segments; + #define SEGMENT_EQUAL(X, Y) (X.cuid == Y.cuid && X.sp == Y.sp && X.ep == Y.ep && X.ds == Y.ds && X.finish == Y.finish ? true : false) #endif // _D_SEGMENT_H_ diff --git a/src/SegmentMan.cc b/src/SegmentMan.cc index 0089575a..8046bf7b 100644 --- a/src/SegmentMan.cc +++ b/src/SegmentMan.cc @@ -52,6 +52,7 @@ bool SegmentMan::getSegment(Segment& seg, int cuid) { seg.sp = 0; seg.ep = totalSize == 0 ? 0 : totalSize-1; seg.ds = 0; + seg.speed = 0; seg.finish = false; segments.push_back(seg); return true; @@ -81,35 +82,7 @@ bool SegmentMan::getSegment(Segment& seg, int cuid) { return true; } } - for(vector::iterator itr = segments.begin(); itr != segments.end(); itr++) { - Segment& s = *itr; - if(s.finish) { - continue; - } - if(s.ep-(s.sp+s.ds) > option->getAsLLInt(PREF_MIN_SEGMENT_SIZE)) { - long long int nep = (s.ep-(s.sp+s.ds))/2+(s.sp+s.ds); - //nseg = { cuid, nep+1, s.ep, 0, false }; - seg.cuid = cuid; - seg.sp = nep+1; - seg.ep = s.ep; - seg.ds = 0; - seg.finish = false; - s.ep = nep; - logger->debug("return new segment { " - "sp = "+Util::llitos(seg.sp)+", "+ - "ep = "+Util::llitos(seg.ep)+", " - "ds = "+Util::llitos(seg.ds)+" } to "+ - "cuid "+Util::llitos(cuid)); - logger->debug("update segment { " - "sp = "+Util::llitos(s.sp)+", "+ - "ep = "+Util::llitos(s.ep)+", " - "ds = "+Util::llitos(s.ds)+" } of "+ - "cuid "+Util::llitos(s.cuid)); - segments.push_back(seg); - return true; - } - } - return false; + return splitter->splitSegment(seg, cuid, segments); } void SegmentMan::updateSegment(const Segment& segment) { diff --git a/src/SegmentMan.h b/src/SegmentMan.h index 075453ff..970a4fd0 100644 --- a/src/SegmentMan.h +++ b/src/SegmentMan.h @@ -27,6 +27,7 @@ #include "Logger.h" #include "Segment.h" #include "Option.h" +#include "SegmentSplitter.h" using namespace std; @@ -64,7 +65,7 @@ public: /** * Holds segments. */ - vector segments; + Segments segments; /** * Respresents the file name of the downloaded file. * If the URL does not contain file name part(http://www.rednoah.com/, for @@ -83,6 +84,7 @@ public: const Logger* logger; const Option* option; + SegmentSplitter* splitter; SegmentMan(); ~SegmentMan(); diff --git a/src/Util.h b/src/Util.h index 0377234b..98bb116e 100644 --- a/src/Util.h +++ b/src/Util.h @@ -37,7 +37,7 @@ public: static string llitos(long long int value, bool comma = false); static string itos(int value, bool comma = false); /** - * Computes difference in milli second between tv1 and tv2, + * Computes difference in micro-seconds between tv1 and tv2, * assuming tv1 is newer than tv2. * If tv1 is older than tv2, then this method returns 0. */ diff --git a/src/main.cc b/src/main.cc index 1f08d4ba..4abad821 100644 --- a/src/main.cc +++ b/src/main.cc @@ -22,6 +22,7 @@ #include "HttpInitiateConnectionCommand.h" #include "DownloadEngine.h" #include "SegmentMan.h" +#include "SplitSlowestSegmentSplitter.h" #include "SimpleLogger.h" #include "common.h" #include "DefaultDiskWriter.h" @@ -106,7 +107,7 @@ void showUsage() { cout << " -l, --log=LOG The file path to store log. If '-' is specified," << endl; cout << " log is written to stdout." << endl; cout << " -D, --daemon Run as daemon." << endl; - cout << " -s, --split=N Download a file using s connections. s must be" << endl; + cout << " -s, --split=N Download a file using N connections. N must be" << endl; cout << " between 1 and 5. This option affects all URLs." << endl; cout << " Thus, aria2 connects to each URL with N connections." << endl; cout << " --retry-wait=SEC Set amount of time in second between requests" << endl; @@ -138,6 +139,7 @@ void showUsage() { cout << " -p, --ftp-pasv Use passive mode in FTP." << endl; cout << " --ftp-via-http-proxy=WAY Use HTTP proxy in FTP. WAY is either 'get' or" << endl; cout << " 'tunnel'." << endl; + cout << " Default: tunnel" << endl; cout << " -v, --version Print the version number and exit." << endl; cout << " -h, --help Print this message and exit." << endl; cout << endl; @@ -176,7 +178,6 @@ int main(int argc, char* argv[]) { op->put(PREF_FTP_USER, "anonymous"); op->put(PREF_FTP_PASSWD, "ARIA2USER@"); op->put(PREF_FTP_TYPE, V_BINARY); - op->put(PREF_FTP_PASV_ENABLED, V_TRUE); op->put(PREF_FTP_VIA_HTTP_PROXY, V_TUNNEL); while(1) { @@ -393,6 +394,9 @@ int main(int argc, char* argv[]) { } else { logger = new SimpleLogger("/dev/null"); } + SegmentSplitter* splitter = new SplitSlowestSegmentSplitter(); + splitter->setMinSegmentSize(op->getAsLLInt(PREF_MIN_SEGMENT_SIZE)); + splitter->logger = logger; e = new DownloadEngine(); e->logger = logger; e->option = op; @@ -402,6 +406,7 @@ int main(int argc, char* argv[]) { e->segmentMan->ufilename = ufilename; e->segmentMan->logger = logger; e->segmentMan->option = op; + e->segmentMan->splitter = splitter; vector requests; for(int i = 1; optind+i-1 < argc; i++) { for(int s = 1; s <= split; s++) { -- cgit From 5a624f5a3dba9a55bc8659338f694c5ea9059940 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 22 Feb 2006 14:31:33 +0000 Subject: new --- src/SegmentSplitter.cc | 46 +++++++++++++++++++++++++++++++++++ src/SegmentSplitter.h | 42 ++++++++++++++++++++++++++++++++ src/SplitFirstSegmentSplitter.cc | 37 ++++++++++++++++++++++++++++ src/SplitFirstSegmentSplitter.h | 34 ++++++++++++++++++++++++++ src/SplitSlowestSegmentSplitter.cc | 50 ++++++++++++++++++++++++++++++++++++++ src/SplitSlowestSegmentSplitter.h | 35 ++++++++++++++++++++++++++ 6 files changed, 244 insertions(+) create mode 100644 src/SegmentSplitter.cc create mode 100644 src/SegmentSplitter.h create mode 100644 src/SplitFirstSegmentSplitter.cc create mode 100644 src/SplitFirstSegmentSplitter.h create mode 100644 src/SplitSlowestSegmentSplitter.cc create mode 100644 src/SplitSlowestSegmentSplitter.h (limited to 'src') diff --git a/src/SegmentSplitter.cc b/src/SegmentSplitter.cc new file mode 100644 index 00000000..ef144d1f --- /dev/null +++ b/src/SegmentSplitter.cc @@ -0,0 +1,46 @@ +/* */ +#include "SegmentSplitter.h" +#include "Util.h" + +void SegmentSplitter::split(Segment& seg, int cuid, Segment& s) const { + long long int nep = (s.ep-(s.sp+s.ds))/2+(s.sp+s.ds); + seg.cuid = cuid; + seg.sp = nep+1; + seg.ep = s.ep; + seg.ds = 0; + seg.speed = s.speed; + seg.finish = false; + s.ep = nep; + logger->debug("return new segment { " + "sp = "+Util::llitos(seg.sp)+", "+ + "ep = "+Util::llitos(seg.ep)+", "+ + "ds = "+Util::llitos(seg.ds)+", "+ + "speed = "+Util::itos(seg.speed)+" } to "+ + "cuid "+Util::llitos(cuid)); + logger->debug("update segment { " + "sp = "+Util::llitos(s.sp)+", "+ + "ep = "+Util::llitos(s.ep)+", "+ + "ds = "+Util::llitos(s.ds)+", "+ + "speed = "+Util::itos(s.speed)+" } of "+ + "cuid "+Util::llitos(s.cuid)); +} diff --git a/src/SegmentSplitter.h b/src/SegmentSplitter.h new file mode 100644 index 00000000..887250ae --- /dev/null +++ b/src/SegmentSplitter.h @@ -0,0 +1,42 @@ +/* */ +#ifndef _D_SEGMENT_SPLITTER_H_ +#define _D_SEGMENT_SPLITTER_H_ + +#include "Segment.h" +#include "Logger.h" + +class SegmentSplitter { +protected: + long long int minSegmentSize; + + void split(Segment& seg, int cuid, Segment& s) const; +public: + const Logger* logger; + + virtual ~SegmentSplitter() {} + virtual bool splitSegment(Segment& newSegment, int cuid, Segments& segments) = 0; + void setMinSegmentSize(long long int size) { minSegmentSize = size; } + long long int getMinSegmentSize() const { return minSegmentSize; } +}; + +#endif // _D_SEGMENT_SPLITTER_H_ diff --git a/src/SplitFirstSegmentSplitter.cc b/src/SplitFirstSegmentSplitter.cc new file mode 100644 index 00000000..72a3f2d1 --- /dev/null +++ b/src/SplitFirstSegmentSplitter.cc @@ -0,0 +1,37 @@ +/* */ +#include "SplitFirstSegmentSplitter.h" + +bool SplitFirstSegmentSplitter::splitSegment(Segment& seg, int cuid, Segments& segments) { + for(vector::iterator itr = segments.begin(); itr != segments.end(); itr++) { + Segment& s = *itr; + if(s.finish) { + continue; + } + if(s.ep-(s.sp+s.ds) > minSegmentSize) { + split(seg, cuid, s); + segments.push_back(seg); + return true; + } + } + return false; +} diff --git a/src/SplitFirstSegmentSplitter.h b/src/SplitFirstSegmentSplitter.h new file mode 100644 index 00000000..73f76abd --- /dev/null +++ b/src/SplitFirstSegmentSplitter.h @@ -0,0 +1,34 @@ +/* */ +#ifndef _D_SPLIT_FIRST_SEGMENT_SPLITTER_H_ +#define _D_SPLIT_FIRST_SEGMENT_SPLITTER_H_ + +#include "SegmentSplitter.h" + +class SplitFirstSegmentSplitter : public SegmentSplitter { +public: + SplitFirstSegmentSplitter() {} + ~SplitFirstSegmentSplitter() {} + bool splitSegment(Segment& newSegment, int cuid, Segments& segments); +}; + +#endif // _D_SPLIT_FIRST_SEGMENT_SPLITTER_H_ diff --git a/src/SplitSlowestSegmentSplitter.cc b/src/SplitSlowestSegmentSplitter.cc new file mode 100644 index 00000000..47c571bf --- /dev/null +++ b/src/SplitSlowestSegmentSplitter.cc @@ -0,0 +1,50 @@ +/* */ +#include "SplitSlowestSegmentSplitter.h" + +bool SplitSlowestSegmentSplitter::splitSegment(Segment& seg, int cuid, Segments& segments) { + vector::iterator slowest = segments.end();; + for(vector::iterator itr = segments.begin(); itr != segments.end(); itr++) { + Segment& s = *itr; + if(s.finish) { + continue; + } + if(s.ep-(s.sp+s.ds) <= minSegmentSize) { + continue; + } + if(slowest == segments.end()) { + slowest = itr; + } else { + Segment sl = *slowest; + if((sl.ep-(sl.sp+sl.ds))/(sl.speed+1) < (s.ep-(s.sp+s.ds))/(s.speed+1)) { + slowest = itr; + } + } + } + if(slowest == segments.end()) { + return false; + } else { + split(seg, cuid, *slowest); + segments.push_back(seg); + return true; + } +} diff --git a/src/SplitSlowestSegmentSplitter.h b/src/SplitSlowestSegmentSplitter.h new file mode 100644 index 00000000..90ff3753 --- /dev/null +++ b/src/SplitSlowestSegmentSplitter.h @@ -0,0 +1,35 @@ +/* */ +#ifndef _D_SPLIT_SLOWEST_SEGMENT_SPLITTER_H_ +#define _D_SPLIT_SLOWEST_SEGMENT_SPLITTER_H_ + +#include "SegmentSplitter.h" + +class SplitSlowestSegmentSplitter : public SegmentSplitter { +public: + SplitSlowestSegmentSplitter() {} + ~SplitSlowestSegmentSplitter() {} + + bool splitSegment(Segment& segment, int cuid, Segments& segments); +}; + +#endif // _D_SPLIT_SLOWEST_SEGMENT_SPLITTER_H_ -- cgit From cd5f591c417ec4924181af6c7f3c439b20f614b1 Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Wed, 22 Feb 2006 15:40:04 +0000 Subject: --- ChangeLog | 11 ++++++++++- src/HttpConnection.cc | 22 ++++++++++++++++++---- src/HttpConnection.h | 2 ++ src/HttpInitiateConnectionCommand.cc | 19 +++++++++++++++++-- src/HttpInitiateConnectionCommand.h | 2 ++ src/main.cc | 16 +++++++++++++++- src/prefs.h | 2 ++ 7 files changed, 66 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/ChangeLog b/ChangeLog index 7678a435..3c2db43e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,14 @@ -2006-02-22 Tatsuhiro Tsujikawa +2006-02-23 Tatsuhiro Tsujikawa + + * Release 0.2.0 + * main.cc: + * HttpInitiateConnectionCommand.{h,cc}: + * prefs.h: + * HttpConnection.{h,cc}: added --http-proxy-method option. + We can now use GET command in http proxy. +2006-02-22 Tatsuhiro Tsujikawa + * SplitSlowestSegmentSplitter.{h,cc}: This class provies algorithm that splits slowest segment of SegmentMan::commands vector. This is the default split algorithm of aria2. diff --git a/src/HttpConnection.cc b/src/HttpConnection.cc index 38949bd0..815975d2 100644 --- a/src/HttpConnection.cc +++ b/src/HttpConnection.cc @@ -41,22 +41,26 @@ void HttpConnection::sendProxyRequest() const { string(" HTTP/1.1\r\n")+ "Host: "+getHost(req->getHost(), req->getPort())+"\r\n"; if(useProxyAuth()) { - request += "Proxy-Authorization: Basic "+ - Base64::encode(option->get(PREF_HTTP_PROXY_USER)+":"+ - option->get(PREF_HTTP_PROXY_PORT))+"\r\n"; + request += getProxyAuthString(); } request += "\r\n"; logger->info(MSG_SENDING_REQUEST, cuid, request.c_str()); socket->writeData(request.c_str(), request.size()); } +string HttpConnection::getProxyAuthString() const { + return "Proxy-Authorization: Basic "+ + Base64::encode(option->get(PREF_HTTP_PROXY_USER)+":"+ + option->get(PREF_HTTP_PROXY_PORT))+"\r\n"; +} + string HttpConnection::getHost(const string& host, int port) const { return host+(port == 80 || port == 443 ? "" : ":"+Util::llitos(port)); } string HttpConnection::createRequest(const Segment& segment) const { string request = string("GET ")+ - (req->getProtocol() == "ftp" ? + (req->getProtocol() == "ftp" || useProxy() && useProxyGet() ? req->getCurrentUrl() : ((req->getDir() == "/" ? "/" : req->getDir()+"/")+req->getFile()))+ string(" HTTP/1.1\r\n")+ @@ -70,6 +74,9 @@ string HttpConnection::createRequest(const Segment& segment) const { request += "Range: bytes="+ Util::llitos(segment.sp+segment.ds)+"-"+Util::llitos(segment.ep)+"\r\n"; } + if(useProxy() && useProxyAuth() && useProxyGet()) { + request += getProxyAuthString(); + } if(option->get(PREF_HTTP_AUTH_SCHEME) == V_BASIC) { request += "Authorization: Basic "+ Base64::encode(option->get(PREF_HTTP_USER)+":"+ @@ -96,6 +103,9 @@ int HttpConnection::receiveResponse(HttpHeader& headers) { while(socket->isReadable(0)) { int size = sizeof(buf)-1; socket->peekData(buf, size); + if(size == 0) { + throw new DlRetryEx(EX_INVALID_RESPONSE); + } buf[size] = '\0'; int hlenTemp = header.size(); header += buf; @@ -145,3 +155,7 @@ bool HttpConnection::useProxy() const { bool HttpConnection::useProxyAuth() const { return option->get(PREF_HTTP_PROXY_AUTH_ENABLED) == V_TRUE; } + +bool HttpConnection::useProxyGet() const { + return option->get(PREF_HTTP_PROXY_METHOD) == V_GET; +} diff --git a/src/HttpConnection.h b/src/HttpConnection.h index a5a5eceb..f37ed892 100644 --- a/src/HttpConnection.h +++ b/src/HttpConnection.h @@ -41,6 +41,8 @@ private: string createRequest(const Segment& segment) const; bool useProxy() const; bool useProxyAuth() const; + bool useProxyGet() const; + string getProxyAuthString() const; int cuid; const Socket* socket; const Request* req; diff --git a/src/HttpInitiateConnectionCommand.cc b/src/HttpInitiateConnectionCommand.cc index 8b53aab8..f3cdc655 100644 --- a/src/HttpInitiateConnectionCommand.cc +++ b/src/HttpInitiateConnectionCommand.cc @@ -23,6 +23,7 @@ #include "HttpRequestCommand.h" #include "HttpProxyRequestCommand.h" #include "Util.h" +#include "DlAbortEx.h" #include "message.h" #include "prefs.h" @@ -40,8 +41,14 @@ bool HttpInitiateConnectionCommand::executeInternal(Segment segment) { e->option->getAsInt(PREF_HTTP_PROXY_PORT)); socket->establishConnection(e->option->get(PREF_HTTP_PROXY_HOST), e->option->getAsInt(PREF_HTTP_PROXY_PORT)); - command = new HttpProxyRequestCommand(cuid, req, e, socket); - + if(useProxyTunnel()) { + command = new HttpProxyRequestCommand(cuid, req, e, socket); + } else if(useProxyGet()) { + command = new HttpRequestCommand(cuid, req, e, socket); + } else { + // TODO + throw new DlAbortEx("ERROR"); + } } else { e->logger->info(MSG_CONNECTING_TO_SERVER, cuid, req->getHost().c_str(), req->getPort()); @@ -55,3 +62,11 @@ bool HttpInitiateConnectionCommand::executeInternal(Segment segment) { bool HttpInitiateConnectionCommand::useProxy() { return e->option->get(PREF_HTTP_PROXY_ENABLED) == V_TRUE; } + +bool HttpInitiateConnectionCommand::useProxyGet() { + return e->option->get(PREF_HTTP_PROXY_METHOD) == V_GET; +} + +bool HttpInitiateConnectionCommand::useProxyTunnel() { + return e->option->get(PREF_HTTP_PROXY_METHOD) == V_TUNNEL; +} diff --git a/src/HttpInitiateConnectionCommand.h b/src/HttpInitiateConnectionCommand.h index 19cc9bd3..e605ec1b 100644 --- a/src/HttpInitiateConnectionCommand.h +++ b/src/HttpInitiateConnectionCommand.h @@ -27,6 +27,8 @@ class HttpInitiateConnectionCommand : public AbstractCommand { private: bool useProxy(); + bool useProxyGet(); + bool useProxyTunnel(); protected: /** * Connect to the server. diff --git a/src/main.cc b/src/main.cc index 4abad821..b01853b9 100644 --- a/src/main.cc +++ b/src/main.cc @@ -124,6 +124,9 @@ void showUsage() { cout << " --http-passwd=PASSWD Set HTTP password. This affects to all URLs." << endl; cout << " --http-proxy-user=USER Set HTTP proxy user. This affects to all URLs" << endl; cout << " --http-proxy-passwd=PASSWD Set HTTP proxy password. This affects to all URLs." << endl; + cout << " --http-proxy-method=METHOD Set the method to use in proxy request." << endl; + cout << " METHOD is either 'get' or 'tunnel'." << endl; + cout << " Default: tunnel" << endl; cout << " --http-auth-scheme=SCHEME Set HTTP authentication scheme. Currently, basic" << endl; cout << " is the only supported scheme. You MUST specify" << endl; cout << " this option in order to use HTTP authentication" << endl; @@ -137,7 +140,7 @@ void showUsage() { cout << " or 'ascii'." << endl; cout << " Default: binary" << endl; cout << " -p, --ftp-pasv Use passive mode in FTP." << endl; - cout << " --ftp-via-http-proxy=WAY Use HTTP proxy in FTP. WAY is either 'get' or" << endl; + cout << " --ftp-via-http-proxy=METHOD Use HTTP proxy in FTP. METHOD is either 'get' or" << endl; cout << " 'tunnel'." << endl; cout << " Default: tunnel" << endl; cout << " -v, --version Print the version number and exit." << endl; @@ -175,6 +178,7 @@ int main(int argc, char* argv[]) { op->put(PREF_TIMEOUT, "60"); op->put(PREF_MIN_SEGMENT_SIZE, "1048576");// 1M op->put(PREF_MAX_TRIES, "5"); + op->put(PREF_HTTP_PROXY_METHOD, V_TUNNEL); op->put(PREF_FTP_USER, "anonymous"); op->put(PREF_FTP_PASSWD, "ARIA2USER@"); op->put(PREF_FTP_TYPE, V_BINARY); @@ -205,6 +209,7 @@ int main(int argc, char* argv[]) { { "ftp-pasv", no_argument, NULL, 'p' }, { "ftp-via-http-proxy", required_argument, &lopt, 12 }, { "min-segment-size", required_argument, &lopt, 13 }, + { "http-proxy-method", required_argument, &lopt, 14 }, { "version", no_argument, NULL, 'v' }, { "help", no_argument, NULL, 'h' }, { 0, 0, 0, 0 } @@ -308,6 +313,15 @@ int main(int argc, char* argv[]) { op->put(PREF_MIN_SEGMENT_SIZE, Util::llitos(size)); break; } + case 14: + if(string(optarg) == V_GET || string(optarg) == V_TUNNEL) { + op->put(PREF_HTTP_PROXY_METHOD, optarg); + } else { + cerr << "http-proxy-method must be either 'get' or 'tunnel'." << endl; + showUsage(); + exit(1); + } + break; } break; } diff --git a/src/prefs.h b/src/prefs.h index bce670e4..080ee961 100644 --- a/src/prefs.h +++ b/src/prefs.h @@ -72,6 +72,8 @@ #define PREF_HTTP_PROXY_PASSWD "http_proxy_passwd" #define PREF_HTTP_PROXY_HOST "http_proxy_host" #define PREF_HTTP_PROXY_PORT "http_proxy_port" +// values: get | tunnel +#define PREF_HTTP_PROXY_METHOD "http_proxy_method" // values: true | false #define PREF_HTTP_PROXY_ENABLED "http_proxy_enabled" // values: true | false -- cgit