diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/Cookie.cc | 71 | ||||
| -rw-r--r-- | src/Cookie.h | 24 | ||||
| -rw-r--r-- | src/CookieStorage.cc | 456 | ||||
| -rw-r--r-- | src/CookieStorage.h | 219 | ||||
| -rw-r--r-- | src/HttpRequest.cc | 12 | ||||
| -rw-r--r-- | src/NsCookieParser.cc | 47 | ||||
| -rw-r--r-- | src/NsCookieParser.h | 4 | ||||
| -rw-r--r-- | src/Sqlite3CookieParser.cc | 36 | ||||
| -rw-r--r-- | src/Sqlite3CookieParser.h | 3 | ||||
| -rw-r--r-- | src/cookie_helper.cc | 48 | ||||
| -rw-r--r-- | src/cookie_helper.h | 6 |
11 files changed, 510 insertions, 416 deletions
diff --git a/src/Cookie.cc b/src/Cookie.cc index 0628bc27..2c8b881e 100644 --- a/src/Cookie.cc +++ b/src/Cookie.cc @@ -43,23 +43,23 @@ namespace aria2 { Cookie::Cookie -(const std::string& name, - const std::string& value, +(std::string name, + std::string value, time_t expiryTime, bool persistent, - const std::string& domain, + std::string domain, bool hostOnly, - const std::string& path, + std::string path, bool secure, bool httpOnly, time_t creationTime): - name_(name), - value_(value), + name_(std::move(name)), + value_(std::move(value)), expiryTime_(expiryTime), persistent_(persistent), - domain_(domain), + domain_(std::move(domain)), hostOnly_(hostOnly), - path_(path), + path_(std::move(path)), secure_(secure), httpOnly_(httpOnly), creationTime_(creationTime), @@ -74,40 +74,6 @@ Cookie::Cookie(): creationTime_(0), lastAccessTime_(0) {} -Cookie::Cookie(const Cookie& c) - : name_(c.name_), - value_(c.value_), - expiryTime_(c.expiryTime_), - persistent_(c.persistent_), - domain_(c.domain_), - hostOnly_(c.hostOnly_), - path_(c.path_), - secure_(c.secure_), - httpOnly_(c.httpOnly_), - creationTime_(c.creationTime_), - lastAccessTime_(c.lastAccessTime_) -{} - -Cookie::~Cookie() {} - -Cookie& Cookie::operator=(const Cookie& c) -{ - if(this != &c) { - name_ = c.name_; - value_ = c.value_; - expiryTime_ = c.expiryTime_; - persistent_ = c.persistent_; - domain_ = c.domain_; - hostOnly_ = c.hostOnly_; - path_ = c.path_; - secure_ = c.secure_; - httpOnly_ = c.httpOnly_; - creationTime_ = c.creationTime_; - lastAccessTime_ = c.lastAccessTime_; - } - return *this; -} - std::string Cookie::toString() const { std::string s = name_; @@ -138,6 +104,11 @@ bool Cookie::operator==(const Cookie& cookie) const name_ == cookie.name_; } +bool Cookie::operator!=(const Cookie& cookie) const +{ + return !(*this == cookie); +} + bool Cookie::isExpired(time_t base) const { return persistent_ && base > expiryTime_; @@ -174,24 +145,24 @@ std::string Cookie::toNsCookieFormat() const return ss.str(); } -void Cookie::setName(const std::string& name) +void Cookie::setName(std::string name) { - name_ = name; + name_ = std::move(name); } -void Cookie::setValue(const std::string& value) +void Cookie::setValue(std::string value) { - value_ = value; + value_ = std::move(value); } -void Cookie::setDomain(const std::string& domain) +void Cookie::setDomain(std::string domain) { - domain_ = domain; + domain_ = std::move(domain); } -void Cookie::setPath(const std::string& path) +void Cookie::setPath(std::string path) { - path_ = path; + path_ = std::move(path); } } // namespace aria2 diff --git a/src/Cookie.h b/src/Cookie.h index e571225d..d97ddaeb 100644 --- a/src/Cookie.h +++ b/src/Cookie.h @@ -63,23 +63,17 @@ public: Cookie(); Cookie - (const std::string& name, - const std::string& value, + (std::string name, + std::string value, time_t expiryTime, bool persistent, - const std::string& domain, + std::string domain, bool hostOnly, - const std::string& path, + std::string path, bool secure, bool httpOnly, time_t creationTime); - Cookie(const Cookie& c); - - ~Cookie(); - - Cookie& operator=(const Cookie& c); - std::string toString() const; bool match @@ -88,6 +82,8 @@ public: bool operator==(const Cookie& cookie) const; + bool operator!=(const Cookie& cookie) const; + bool isExpired(time_t base) const; const std::string& getName() const @@ -95,7 +91,7 @@ public: return name_; } - void setName(const std::string& name); + void setName(std::string name); template<typename InputIterator> void setName(InputIterator first, InputIterator last) @@ -108,7 +104,7 @@ public: return value_; } - void setValue(const std::string& value); + void setValue(std::string value); template<typename InputIterator> void setValue(InputIterator first, InputIterator last) @@ -141,7 +137,7 @@ public: return domain_; } - void setDomain(const std::string& domain); + void setDomain(std::string domain); template<typename InputIterator> void setDomain(InputIterator first, InputIterator last) @@ -164,7 +160,7 @@ public: return path_; } - void setPath(const std::string& path); + void setPath(std::string path); template<typename InputIterator> void setPath(InputIterator first, InputIterator last) diff --git a/src/CookieStorage.cc b/src/CookieStorage.cc index e9353091..c21c4e6d 100644 --- a/src/CookieStorage.cc +++ b/src/CookieStorage.cc @@ -2,7 +2,7 @@ /* * aria2 - The high speed download utility * - * Copyright (C) 2006 Tatsuhiro Tsujikawa + * Copyright (C) 2013 Tatsuhiro Tsujikawa * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -56,131 +56,196 @@ namespace aria2 { -CookieStorage::DomainEntry::DomainEntry(const std::string& domain) - : key_(util::isNumericHost(domain)?domain:cookie::reverseDomainLevel(domain)), - lastAccessTime_(0) +DomainNode::DomainNode(std::string label, DomainNode* parent) + : label_{std::move(label)}, parent_{parent}, lastAccessTime_{0}, + lruAccessTime_{0}, inLru_{false} {} -CookieStorage::DomainEntry::DomainEntry -(const DomainEntry& c) - : key_(c.key_), - lastAccessTime_(c.lastAccessTime_), - cookies_(c.cookies_) -{} - -CookieStorage::DomainEntry::~DomainEntry() {} - -CookieStorage::DomainEntry& CookieStorage::DomainEntry::operator= -(const DomainEntry& c) -{ - if(this != &c) { - key_ = c.key_; - lastAccessTime_ = c.lastAccessTime_; - cookies_ = c.cookies_; - } - return *this; -} - -void CookieStorage::DomainEntry::swap(CookieStorage::DomainEntry& c) -{ - using std::swap; - swap(key_, c.key_); - swap(lastAccessTime_, c.lastAccessTime_); - swap(cookies_, c.cookies_); -} - -void swap(CookieStorage::DomainEntry& a, CookieStorage::DomainEntry& b) -{ - a.swap(b); -} - -void CookieStorage::DomainEntry::findCookie -(std::vector<Cookie>& out, +void DomainNode::findCookie +(std::vector<const Cookie*>& out, const std::string& requestHost, const std::string& requestPath, time_t now, bool secure) { - for(std::deque<Cookie>::iterator i = cookies_.begin(), - eoi = cookies_.end(); i != eoi; ++i) { - if((*i).match(requestHost, requestPath, now, secure)) { - (*i).setLastAccessTime(now); - out.push_back(*i); + if(cookies_) { + for(auto& c : *cookies_) { + if(c->match(requestHost, requestPath, now, secure)) { + c->setLastAccessTime(now); + out.push_back(c.get()); + } } } } -bool CookieStorage::DomainEntry::addCookie(const Cookie& cookie, time_t now) +bool DomainNode::addCookie(std::unique_ptr<Cookie> cookie, time_t now) { using namespace std::placeholders; setLastAccessTime(now); - std::deque<Cookie>::iterator i = - std::find(cookies_.begin(), cookies_.end(), cookie); - if(i == cookies_.end()) { - if(cookie.isExpired(now)) { + if(!cookies_) { + if(cookie->isExpired(now)) { return false; } else { - if(cookies_.size() >= CookieStorage::MAX_COOKIE_PER_DOMAIN) { - cookies_.erase - (std::remove_if(cookies_.begin(), cookies_.end(), + cookies_ = make_unique<std::deque<std::unique_ptr<Cookie>>>(); + cookies_->push_back(std::move(cookie)); + return true; + } + } + + auto i = std::find_if(std::begin(*cookies_), std::end(*cookies_), + [&](const std::unique_ptr<Cookie>& c) + { + return *c == *cookie; + }); + if(i == std::end(*cookies_)) { + if(cookie->isExpired(now)) { + return false; + } else { + if(cookies_->size() >= CookieStorage::MAX_COOKIE_PER_DOMAIN) { + cookies_->erase + (std::remove_if(std::begin(*cookies_), std::end(*cookies_), std::bind(&Cookie::isExpired, _1, now)), - cookies_.end()); - if(cookies_.size() >= CookieStorage::MAX_COOKIE_PER_DOMAIN) { - std::deque<Cookie>::iterator m = std::min_element - (cookies_.begin(), cookies_.end(), LeastRecentAccess<Cookie>()); - *m = cookie; + std::end(*cookies_)); + if(cookies_->size() >= CookieStorage::MAX_COOKIE_PER_DOMAIN) { + auto m = std::min_element(std::begin(*cookies_), std::end(*cookies_), + [](const std::unique_ptr<Cookie>& lhs, + const std::unique_ptr<Cookie>& rhs) + { + return lhs->getLastAccessTime() < + rhs->getLastAccessTime(); + }); + *m = std::move(cookie); } else { - cookies_.push_back(cookie); + cookies_->push_back(std::move(cookie)); } } else { - cookies_.push_back(cookie); + cookies_->push_back(std::move(cookie)); } return true; } - } else if(cookie.isExpired(now)) { - cookies_.erase(i); + } else if(cookie->isExpired(now)) { + cookies_->erase(i); return false; } else { - *i = cookie; + *i = std::move(cookie); return true; } } -bool CookieStorage::DomainEntry::contains(const Cookie& cookie) const +bool DomainNode::contains(const Cookie& cookie) const { - return std::find(cookies_.begin(), cookies_.end(), cookie) != cookies_.end(); + if(cookies_) { + for(auto& i : *cookies_) { + if(*i == cookie) { + return true; + } + } + } + return false; } -bool CookieStorage::DomainEntry::writeCookie(BufferedFile& fp) const +bool DomainNode::writeCookie(BufferedFile& fp) const { - for(std::deque<Cookie>::const_iterator i = cookies_.begin(), - eoi = cookies_.end(); i != eoi; ++i) { - std::string data = (*i).toNsCookieFormat(); - data += "\n"; - if(fp.write(data.data(), data.size()) != data.size()) { - return false; + if(cookies_) { + for(const auto& c : *cookies_) { + std::string data = c->toNsCookieFormat(); + data += "\n"; + if(fp.write(data.data(), data.size()) != data.size()) { + return false; + } } } return true; } -size_t CookieStorage::DomainEntry::countCookie() const +size_t DomainNode::countCookie() const +{ + if(cookies_) { + return cookies_->size(); + } else { + return 0; + } +} + +void DomainNode::clearCookie() +{ + cookies_->clear(); +} + +void DomainNode::setLastAccessTime(time_t lastAccessTime) { - return cookies_.size(); + lastAccessTime_ = lastAccessTime; } -bool CookieStorage::DomainEntry::operator==(const DomainEntry& de) const +time_t DomainNode::getLastAccessTime() const { - return key_ == de.key_; + return lastAccessTime_; } -bool CookieStorage::DomainEntry::operator<(const DomainEntry& de) const +void DomainNode::setLruAccessTime(time_t t) { - return key_ < de.key_; + lruAccessTime_ = t; } -CookieStorage::CookieStorage() {} +time_t DomainNode::getLruAccessTime() const +{ + return lruAccessTime_; +} -CookieStorage::~CookieStorage() {} +bool DomainNode::empty() const +{ + return !cookies_ || cookies_->empty(); +} + +bool DomainNode::hasNext() const +{ + return !next_.empty(); +} + +DomainNode* DomainNode::getParent() const +{ + return parent_; +} + +void DomainNode::removeNode(DomainNode* node) +{ + next_.erase(node->getLabel()); +} + +DomainNode* DomainNode::findNext(const std::string& label) const +{ + auto i = next_.find(label); + if(i == std::end(next_)) { + return nullptr; + } else { + return (*i).second.get(); + } +} + +DomainNode* DomainNode::addNext(std::string label, + std::unique_ptr<DomainNode> node) +{ + auto& res = next_[std::move(label)] = std::move(node); + return res.get(); +} + +const std::string& DomainNode::getLabel() const +{ + return label_; +} + +bool DomainNode::getInLru() const +{ + return inLru_; +} + +void DomainNode::setInLru(bool f) +{ + inLru_ = f; +} + +CookieStorage::CookieStorage() + : rootNode_{make_unique<DomainNode>("", nullptr)} +{} namespace { // See CookieStorageTest::testDomainIsFull() in CookieStorageTest.cc @@ -189,29 +254,86 @@ const size_t DOMAIN_EVICTION_TRIGGER = 2000; const double DOMAIN_EVICTION_RATE = 0.1; } // namespace -bool CookieStorage::store(const Cookie& cookie, time_t now) +namespace { +std::vector<std::string> splitDomainLabel(const std::string& domain) { - if(domains_.size() >= DOMAIN_EVICTION_TRIGGER) { - std::vector<std::shared_ptr<DomainEntry> > evictions(domains_.begin(), - domains_.end()); - std::sort(evictions.begin(), evictions.end(), - LeastRecentAccess<DomainEntry>()); - size_t delnum = (size_t)(evictions.size()*DOMAIN_EVICTION_RATE); - domains_.clear(); - domains_.insert(evictions.begin()+delnum, evictions.end()); - } - std::shared_ptr<DomainEntry> v(new DomainEntry(cookie.getDomain())); - DomainEntrySet::iterator i = domains_.lower_bound(v); - bool added = false; - if(i != domains_.end() && *(*i) == *v) { - added = (*i)->addCookie(cookie, now); + auto labels = std::vector<std::string>{}; + if(util::isNumericHost(domain)) { + labels.push_back(domain); } else { - added = v->addCookie(cookie, now); - if(added) { - domains_.insert(i, v); + util::split(std::begin(domain), std::end(domain), + std::back_inserter(labels), '.'); + } + return labels; +} +} // namespace + + +size_t CookieStorage::getLruTrackerSize() const +{ + return lruTracker_.size(); +} + +void CookieStorage::evictNode(size_t delnum) +{ + for(; delnum > 0 && !lruTracker_.empty(); --delnum) { + auto node = (*lruTracker_.begin()).second; + lruTracker_.erase(lruTracker_.begin()); + node->setInLru(false); + node->clearCookie(); + while(node->empty() && !node->hasNext()) { + auto parent = node->getParent(); + parent->removeNode(node); + if(!parent->empty() || parent->hasNext() || + parent == rootNode_.get()) { + break; + } + node = parent; + if(node->getInLru()) { + lruTracker_.erase({node->getLruAccessTime(), node}); + node->setInLru(false); + } } } - return added; +} + +const DomainNode* CookieStorage::getRootNode() const +{ + return rootNode_.get(); +} + +bool CookieStorage::store(std::unique_ptr<Cookie> cookie, time_t now) +{ + if(lruTracker_.size() >= DOMAIN_EVICTION_TRIGGER) { + auto delnum = size_t(lruTracker_.size()*DOMAIN_EVICTION_RATE); + evictNode(delnum); + } + auto labels = splitDomainLabel(cookie->getDomain()); + auto node = rootNode_.get(); + for(auto i = labels.rbegin(), eoi = labels.rend(); i != eoi; ++i) { + auto nextNode = node->findNext(*i); + if(nextNode) { + node = nextNode; + } else { + node = node->addNext(*i, make_unique<DomainNode>(*i, node)); + } + } + bool ok = node->addCookie(std::move(cookie), now); + if(ok) { + updateLru(node, now); + } + return ok; +} + +void CookieStorage::updateLru(DomainNode* node, time_t now) +{ + if(node->getInLru()) { + lruTracker_.erase({node->getLruAccessTime(), node}); + } else { + node->setInLru(true); + } + node->setLruAccessTime(now); + lruTracker_.insert({node->getLruAccessTime(), node}); } bool CookieStorage::parseAndStore @@ -220,20 +342,17 @@ bool CookieStorage::parseAndStore const std::string& defaultPath, time_t now) { - Cookie cookie; - if(cookie::parse(cookie, setCookieString, requestHost, defaultPath, now)) { - return store(cookie, now); - } else { - return false; - } + auto cookie = cookie::parse(setCookieString, requestHost, defaultPath, now); + return cookie && store(std::move(cookie), now); } +namespace { struct CookiePathDivider { - Cookie cookie_; + const Cookie* cookie_; int pathDepth_; - CookiePathDivider(const Cookie& cookie):cookie_(cookie), pathDepth_(0) + CookiePathDivider(const Cookie* cookie):cookie_(cookie), pathDepth_(0) { - const std::string& path = cookie_.getPath(); + const std::string& path = cookie_->getPath(); if(!path.empty()) { for(size_t i = 1, len = path.size(); i < len; ++i) { if(path[i] == '/' && path[i-1] != '/') { @@ -246,16 +365,17 @@ struct CookiePathDivider { } } }; +} // namespace namespace { class CookiePathDividerConverter { public: - CookiePathDivider operator()(const Cookie& cookie) const + CookiePathDivider operator()(const Cookie* cookie) const { return CookiePathDivider(cookie); } - Cookie operator()(const CookiePathDivider& cookiePathDivider) const + const Cookie* operator()(const CookiePathDivider& cookiePathDivider) const { return cookiePathDivider.cookie_; } @@ -280,80 +400,69 @@ public: // creation-times. return lhs.pathDepth_ > rhs.pathDepth_ || (!(rhs.pathDepth_ > lhs.pathDepth_) && - lhs.cookie_.getCreationTime() < rhs.cookie_.getCreationTime()); + lhs.cookie_->getCreationTime() < rhs.cookie_->getCreationTime()); } }; } // namespace -void CookieStorage::searchCookieByDomainSuffix -(std::vector<Cookie>& out, - const std::string& domain, - const std::string& requestHost, - const std::string& requestPath, - time_t now, bool secure) +namespace { +DomainNode* findNode(const std::string& domain, DomainNode* node) { - std::shared_ptr<DomainEntry> v(new DomainEntry(domain)); - DomainEntrySet::iterator i = domains_.lower_bound(v); - if(i != domains_.end() && *(*i) == *v) { - (*i)->setLastAccessTime(now); - (*i)->findCookie(out, requestHost, requestPath, now, secure); + auto labels = splitDomainLabel(domain); + for(auto i = labels.rbegin(), eoi = labels.rend(); i != eoi && node; ++i) { + node = node->findNext(*i); } + return node; } +} // namespace bool CookieStorage::contains(const Cookie& cookie) const { - std::shared_ptr<DomainEntry> v(new DomainEntry(cookie.getDomain())); - DomainEntrySet::iterator i = domains_.find(v); - if(i != domains_.end()) { - return (*i)->contains(cookie); - } else { - return false; - } + auto node = findNode(cookie.getDomain(), rootNode_.get()); + return node && node->contains(cookie); } -std::vector<Cookie> CookieStorage::criteriaFind +std::vector<const Cookie*> CookieStorage::criteriaFind (const std::string& requestHost, const std::string& requestPath, time_t now, bool secure) { - std::vector<Cookie> res; + auto res = std::vector<const Cookie*>{}; if(requestPath.empty()) { return res; } - if(util::isNumericHost(requestHost)) { - searchCookieByDomainSuffix - (res, requestHost, requestHost, requestPath, now, secure); - } else { - std::vector<Scip> levels; - util::splitIter(requestHost.begin(), requestHost.end(), - std::back_inserter(levels), '.'); - std::string domain; - for(std::vector<Scip>::const_reverse_iterator i = - levels.rbegin(), eoi = levels.rend(); - i != eoi; ++i, domain.insert(domain.begin(), '.')) { - domain.insert(domain.begin(), (*i).first, (*i).second); - searchCookieByDomainSuffix - (res, domain, requestHost, requestPath, now, secure); + auto labels = splitDomainLabel(requestHost); + auto node = rootNode_.get(); + for(auto i = labels.rbegin(), eoi = labels.rend(); i != eoi; ++i) { + auto nextNode = node->findNext(*i); + if(nextNode) { + nextNode->setLastAccessTime(now); + if(nextNode->getInLru()) { + updateLru(nextNode, now); + } + nextNode->findCookie(res, requestHost, requestPath, now, secure); + node = nextNode; + } else { + break; } } - std::vector<CookiePathDivider> divs; - std::transform(res.begin(), res.end(), std::back_inserter(divs), - CookiePathDividerConverter()); - std::sort(divs.begin(), divs.end(), OrderByPathDepthDesc()); - std::transform(divs.begin(), divs.end(), res.begin(), - CookiePathDividerConverter()); + auto divs = std::vector<CookiePathDivider>{}; + std::transform(std::begin(res), std::end(res), std::back_inserter(divs), + CookiePathDividerConverter{}); + std::sort(std::begin(divs), std::end(divs), OrderByPathDepthDesc{}); + std::transform(std::begin(divs), std::end(divs), std::begin(res), + CookiePathDividerConverter{}); return res; } size_t CookieStorage::size() const { - size_t numCookie = 0; - for(DomainEntrySet::iterator i = domains_.begin(), eoi = domains_.end(); - i != eoi; ++i) { - numCookie += (*i)->countCookie(); + size_t n = 0; + for(auto& p : lruTracker_) { + n += p.second->countCookie(); } - return numCookie; + return n; } bool CookieStorage::load(const std::string& filename, time_t now) @@ -361,7 +470,7 @@ bool CookieStorage::load(const std::string& filename, time_t now) char header[16]; // "SQLite format 3" plus \0 size_t headlen; { - BufferedFile fp(filename.c_str(), BufferedFile::READ); + BufferedFile fp{filename.c_str(), BufferedFile::READ}; if(!fp) { A2_LOG_ERROR(fmt("Failed to open cookie file %s", filename.c_str())); return false; @@ -371,25 +480,28 @@ bool CookieStorage::load(const std::string& filename, time_t now) try { if(headlen == 16 && memcmp(header, "SQLite format 3\0", 16) == 0) { #ifdef HAVE_SQLITE3 - std::vector<Cookie> cookies; try { - Sqlite3MozCookieParser(filename).parse(cookies); + auto cookies = Sqlite3MozCookieParser(filename).parse(); + storeCookies(std::make_move_iterator(std::begin(cookies)), + std::make_move_iterator(std::end(cookies)), now); } catch(RecoverableException& e) { A2_LOG_INFO_EX(EX_EXCEPTION_CAUGHT, e); A2_LOG_INFO("This does not look like Firefox3 cookie file." " Retrying, assuming it is Chromium cookie file."); // Try chrome cookie format - Sqlite3ChromiumCookieParser(filename).parse(cookies); + auto cookies = Sqlite3ChromiumCookieParser(filename).parse(); + storeCookies(std::make_move_iterator(std::begin(cookies)), + std::make_move_iterator(std::end(cookies)), now); } - storeCookies(cookies.begin(), cookies.end(), now); #else // !HAVE_SQLITE3 throw DL_ABORT_EX ("Cannot read SQLite3 database because SQLite3 support is disabled by" " configuration."); #endif // !HAVE_SQLITE3 } else { - std::vector<Cookie> cookies = NsCookieParser().parse(filename, now); - storeCookies(cookies.begin(), cookies.end(), now); + auto cookies = NsCookieParser().parse(filename, now); + storeCookies(std::make_move_iterator(std::begin(cookies)), + std::make_move_iterator(std::end(cookies)), now); } return true; } catch(RecoverableException& e) { @@ -400,17 +512,16 @@ bool CookieStorage::load(const std::string& filename, time_t now) bool CookieStorage::saveNsFormat(const std::string& filename) { - std::string tempfilename = filename; + auto tempfilename = filename; tempfilename += "__temp"; { - BufferedFile fp(tempfilename.c_str(), BufferedFile::WRITE); + BufferedFile fp{tempfilename.c_str(), BufferedFile::WRITE}; if(!fp) { A2_LOG_ERROR(fmt("Cannot create cookie file %s", filename.c_str())); return false; } - for(DomainEntrySet::iterator i = domains_.begin(), eoi = domains_.end(); - i != eoi; ++i) { - if(!(*i)->writeCookie(fp)) { + for(auto& p : lruTracker_) { + if(!p.second->writeCookie(fp)) { A2_LOG_ERROR(fmt("Failed to save cookies to %s", filename.c_str())); return false; } @@ -431,14 +542,3 @@ bool CookieStorage::saveNsFormat(const std::string& filename) } } // namespace aria2 - -namespace std { -template<> -void swap<aria2::CookieStorage::DomainEntry> -(aria2::CookieStorage::DomainEntry& a, - aria2::CookieStorage::DomainEntry& b) -{ - a.swap(b); -} -} // namespace std - diff --git a/src/CookieStorage.h b/src/CookieStorage.h index 530e0d62..de3eaebc 100644 --- a/src/CookieStorage.h +++ b/src/CookieStorage.h @@ -2,7 +2,7 @@ /* * aria2 - The high speed download utility * - * Copyright (C) 2006 Tatsuhiro Tsujikawa + * Copyright (C) 2013 Tatsuhiro Tsujikawa * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -42,6 +42,7 @@ #include <vector> #include <set> #include <algorithm> +#include <unordered_map> #include "a2time.h" #include "Cookie.h" @@ -51,89 +52,97 @@ namespace aria2 { class BufferedFile; -class CookieStorage { +// This object represents one domain label. +class DomainNode { public: + DomainNode(std::string label, DomainNode* parent); + // Stores the matching cookies in |out|. The |now| is used to update + // the last access time of this node. + void findCookie + (std::vector<const Cookie*>& out, + const std::string& requestHost, + const std::string& requestPath, + time_t now, bool secure); + // Returns the number of cookies this node has. + size_t countCookie() const; + // Add |cookie| using update time |now|. Returns true if the + // function succeeds. + bool addCookie(std::unique_ptr<Cookie> cookie, time_t now); + // Sets the last access time of this node. + void setLastAccessTime(time_t lastAccessTime); + // Returns the last access time of this node. + time_t getLastAccessTime() const; + // Sets the time |t| as a time used as key in LRU tracker. + void setLruAccessTime(time_t t); + time_t getLruAccessTime() const; + + bool writeCookie(BufferedFile& fp) const; + // Returns true if this node contains the |cookie|. + bool contains(const Cookie& cookie) const; + // Returns true if this node contains no cookie. + bool empty() const; + // Returns true if this node has any next nodes. + bool hasNext() const; + // Returns the parent node. If this is the root node, returns + // nullptr. + DomainNode* getParent() const; + // Removes the child node |node|. Nothing happens if |node| is not a + // child of this node. + void removeNode(DomainNode* node); + // Returns the child node having label |label. Returns nullptr if + // there is no such node. + DomainNode* findNext(const std::string& label) const; + // Add the |node| as a child using label |label and returns the raw + // pointer of |node|. + DomainNode* addNext(std::string label, std::unique_ptr<DomainNode> node); + // Returns the |label|. + const std::string& getLabel() const; + // Deletes all cookies this node has. + void clearCookie(); + // Returns value set by setInLru(). This is typically used to know + // this node is tracked by LRU tracker or not. + bool getInLru() const; + void setInLru(bool f); - static const size_t MAX_COOKIE_PER_DOMAIN = 50; - - class DomainEntry { - private: - // This is reversed domain level string. - // e.g. net.sourceforge.aria2 - // e.g. 192.168.0.1 - std::string key_; - - time_t lastAccessTime_; - - std::deque<Cookie> cookies_; - public: - DomainEntry(const std::string& domain); - DomainEntry(const DomainEntry& c); - ~DomainEntry(); - - void swap(DomainEntry& c); - - DomainEntry& operator=(const DomainEntry& c); - - const std::string& getKey() const - { - return key_; - } - - void findCookie - (std::vector<Cookie>& out, - const std::string& requestHost, - const std::string& requestPath, - time_t now, bool secure); - - size_t countCookie() const; - - bool addCookie(const Cookie& cookie, time_t now); - - void setLastAccessTime(time_t lastAccessTime) - { - lastAccessTime_ = lastAccessTime; - } - - time_t getLastAccessTime() const - { - return lastAccessTime_; - } - - bool writeCookie(BufferedFile& fp) const; - - bool contains(const Cookie& cookie) const; - - template<typename OutputIterator> - OutputIterator dumpCookie(OutputIterator out) const - { - return std::copy(cookies_.begin(), cookies_.end(), out); + template<typename OutputIterator> + OutputIterator dumpCookie(OutputIterator out) const + { + if(cookies_) { + for(auto& c : *cookies_) { + out++ = c.get(); + } } + return out; + } +private: + std::string label_; + DomainNode* parent_; + time_t lastAccessTime_; + time_t lruAccessTime_; + bool inLru_; + std::unique_ptr<std::deque<std::unique_ptr<Cookie>>> cookies_; + // domain label string to DomainNode + // e.g. net, sourceforge + // For numerical addresses, this is address itself. + // e.g. 192.168.0.1 + std::unordered_map<std::string, std::unique_ptr<DomainNode>> next_; +}; - bool operator==(const DomainEntry& de) const; - bool operator<(const DomainEntry& de) const; - }; +class CookieStorage { +public: + static const size_t MAX_COOKIE_PER_DOMAIN = 50; private: - typedef std::set<std::shared_ptr<DomainEntry>, - DerefLess<std::shared_ptr<DomainEntry> > > DomainEntrySet; - DomainEntrySet domains_; + // typedef std::set<std::shared_ptr<DomainEntry>, + // DerefLess<std::shared_ptr<DomainEntry> > > DomainEntrySet; + // DomainEntrySet domains_; - template<typename InputIterator> - void storeCookies(InputIterator first, InputIterator last, time_t now) - { - for(; first != last; ++first) { - store(*first, now); - } - } public: CookieStorage(); - ~CookieStorage(); - // Returns true if cookie is stored or updated existing cookie. // Returns false if cookie is expired. now is used as last access // time. - bool store(const Cookie& cookie, time_t now); + bool store(std::unique_ptr<Cookie> cookie, time_t now); // Returns true if cookie is stored or updated existing cookie. // Otherwise, returns false. now is used as creation time and last @@ -146,9 +155,10 @@ public: // Finds cookies matched with given criteria and returns them. // Matched cookies' lastAccess_ property is updated. - std::vector<Cookie> criteriaFind(const std::string& requestHost, - const std::string& requestPath, - time_t now, bool secure); + std::vector<const Cookie*> + criteriaFind(const std::string& requestHost, + const std::string& requestPath, + time_t now, bool secure); // Loads Cookies from file denoted by filename. If compiled with // libsqlite3, this method automatically detects the specified file @@ -170,36 +180,51 @@ public: // satisfies. bool contains(const Cookie& cookie) const; - // Searches Cookie using given domain, requestHost, requestPath, - // current time and secure flag. The found Cookies are stored in - // out. - void searchCookieByDomainSuffix - (std::vector<Cookie>& out, - const std::string& domain, - const std::string& requestHost, - const std::string& requestPath, - time_t now, bool secure); - template<typename OutputIterator> OutputIterator dumpCookie(OutputIterator out) const { - for(DomainEntrySet::iterator i = domains_.begin(), eoi = domains_.end(); - i != eoi; ++i) { - out = (*i)->dumpCookie(out); + for(auto& i : lruTracker_) { + out = i.second->dumpCookie(out); } return out; } -}; -void swap(CookieStorage::DomainEntry& a, CookieStorage::DomainEntry& b); + // Force eviction of delnum nodes. Exposed for unittest + void evictNode(size_t delnum); + // Returns size of LRU tracker. Exposed for unittest + size_t getLruTrackerSize() const; + // Returns root node. Exposed for unittest + const DomainNode* getRootNode() const; +private: + template<typename InputIterator> + void storeCookies(InputIterator first, InputIterator last, time_t now) + { + for(; first != last; ++first) { + store(*first, now); + } + } + + void updateLru(DomainNode* node, time_t now); + + // rootNode_ is a root node of tree structure of reversed domain + // labels. rootNode_ always contans no cookie. It has the child + // nodes of the top level domain label (e.g., net, com and org). And + // those top level domain nodes have 2nd doman label (e.g., + // sourcforge, github), and so on. The numeric host name are always + // stored as a child node of rootNode_. So the domain name of a + // paricular node is constructed as follows. First traverse the + // target node from root node. The concatenation of the visited + // node's label in the reverse order, delimited by ".", is the + std::unique_ptr<DomainNode> rootNode_; + // This object tracks the node which has cookies or it once had. The + // order is sorted by the least recent updated node first. This + // object does not track the node which has not contan cookie. For + // example, adding cookies in aria2.sourceforge.net, and no node + // labeled "sourceforge" is present, only node labelded "aria2" is + // tracked and node labeled "sourceforge" and "net" are not. + std::set<std::pair<time_t, DomainNode*>> lruTracker_; +}; } // namespace aria2 -namespace std { -template<> -void swap<aria2::CookieStorage::DomainEntry> -(aria2::CookieStorage::DomainEntry& a, - aria2::CookieStorage::DomainEntry& b); -} // namespace std - #endif // D_COOKIE_STORAGE_H diff --git a/src/HttpRequest.cc b/src/HttpRequest.cc index 3e7d1a4b..33469813 100644 --- a/src/HttpRequest.cc +++ b/src/HttpRequest.cc @@ -234,13 +234,11 @@ std::string HttpRequest::createRequest() std::string cookiesValue; std::string path = getDir(); path += getFile(); - std::vector<Cookie> cookies = - cookieStorage_->criteriaFind(getHost(), path, - Time().getTime(), - getProtocol() == "https"); - for(std::vector<Cookie>::const_iterator itr = cookies.begin(), - eoi = cookies.end(); itr != eoi; ++itr) { - cookiesValue += (*itr).toString(); + auto cookies = cookieStorage_->criteriaFind(getHost(), path, + Time().getTime(), + getProtocol() == "https"); + for(auto c : cookies) { + cookiesValue += c->toString(); cookiesValue += ";"; } if(!cookiesValue.empty()) { diff --git a/src/NsCookieParser.cc b/src/NsCookieParser.cc index 61672d18..e107a295 100644 --- a/src/NsCookieParser.cc +++ b/src/NsCookieParser.cc @@ -54,58 +54,59 @@ NsCookieParser::NsCookieParser() {} NsCookieParser::~NsCookieParser() {} namespace { -bool parseNsCookie -(Cookie& cookie, const std::string& cookieStr, time_t creationTime) +std::unique_ptr<Cookie> parseNsCookie +(const std::string& cookieStr, time_t creationTime) { std::vector<Scip> vs; util::splitIter(cookieStr.begin(), cookieStr.end(), std::back_inserter(vs), '\t', true); if(vs.size() < 6) { - return false; + return std::unique_ptr<Cookie>{}; } vs[0].first = util::lstripIter(vs[0].first, vs[0].second, '.'); if(vs[5].first == vs[5].second || vs[0].first == vs[0].second || !cookie::goodPath(vs[2].first, vs[2].second)) { - return false; + return std::unique_ptr<Cookie>{}; } int64_t expiryTime; if(!util::parseLLIntNoThrow(expiryTime, std::string(vs[4].first, vs[4].second))) { - return false; + return std::unique_ptr<Cookie>{}; } if(std::numeric_limits<time_t>::max() < expiryTime) { expiryTime = std::numeric_limits<time_t>::max(); } else if(std::numeric_limits<time_t>::min() > expiryTime) { expiryTime = std::numeric_limits<time_t>::min(); } - cookie.setName(vs[5].first, vs[5].second); + auto cookie = make_unique<Cookie>(); + cookie->setName(vs[5].first, vs[5].second); if(vs.size() >= 7) { - cookie.setValue(vs[6].first, vs[6].second); + cookie->setValue(vs[6].first, vs[6].second); } else { - cookie.setValue(A2STR::NIL.begin(), A2STR::NIL.end()); + cookie->setValue(""); } - cookie.setExpiryTime(expiryTime == 0? + cookie->setExpiryTime(expiryTime == 0? std::numeric_limits<time_t>::max():expiryTime); - // aria2 treats expiryTime == 0 means session cookie. - cookie.setPersistent(expiryTime != 0); - cookie.setDomain(vs[0].first, vs[0].second); - cookie.setHostOnly(util::isNumericHost(cookie.getDomain()) || + // aria2 treats expiryTime == 0 means session cookie-> + cookie->setPersistent(expiryTime != 0); + cookie->setDomain(vs[0].first, vs[0].second); + cookie->setHostOnly(util::isNumericHost(cookie->getDomain()) || !util::streq(vs[1].first, vs[1].second, "TRUE")); - cookie.setPath(vs[2].first, vs[2].second); - cookie.setSecure(util::streq(vs[3].first, vs[3].second, "TRUE")); - cookie.setCreationTime(creationTime); - return true; + cookie->setPath(vs[2].first, vs[2].second); + cookie->setSecure(util::streq(vs[3].first, vs[3].second, "TRUE")); + cookie->setCreationTime(creationTime); + return cookie; } } // namespace -std::vector<Cookie> NsCookieParser::parse +std::vector<std::unique_ptr<Cookie>> NsCookieParser::parse (const std::string& filename, time_t creationTime) { - BufferedFile fp(filename.c_str(), BufferedFile::READ); + BufferedFile fp{filename.c_str(), BufferedFile::READ}; if(!fp) { throw DL_ABORT_EX(fmt("Failed to open file %s", filename.c_str())); } - std::vector<Cookie> cookies; + std::vector<std::unique_ptr<Cookie>> cookies; while(1) { std::string line = fp.getLine(); if(line.empty()) { @@ -120,9 +121,9 @@ std::vector<Cookie> NsCookieParser::parse if(line[0] == '#') { continue; } - Cookie c; - if(parseNsCookie(c, line, creationTime)) { - cookies.push_back(c); + auto c = parseNsCookie(line, creationTime); + if(c) { + cookies.push_back(std::move(c)); } } return cookies; diff --git a/src/NsCookieParser.h b/src/NsCookieParser.h index 5515692f..19668ed2 100644 --- a/src/NsCookieParser.h +++ b/src/NsCookieParser.h @@ -39,6 +39,7 @@ #include <string> #include <vector> +#include <memory> namespace aria2 { @@ -50,7 +51,8 @@ public: ~NsCookieParser(); - std::vector<Cookie> parse(const std::string& filename, time_t creationTime); + std::vector<std::unique_ptr<Cookie>> parse + (const std::string& filename, time_t creationTime); }; } // namespace aria2 diff --git a/src/Sqlite3CookieParser.cc b/src/Sqlite3CookieParser.cc index ed89aef7..08c2b836 100644 --- a/src/Sqlite3CookieParser.cc +++ b/src/Sqlite3CookieParser.cc @@ -102,8 +102,8 @@ int cookieRowMapper(void* data, int columns, char** values, char** names) if(columns != 7 || !values[0] || !values[1] || !values[4]) { return 0; } - std::vector<Cookie>& cookies = - *reinterpret_cast<std::vector<Cookie>*>(data); + std::vector<std::unique_ptr<Cookie>>& cookies = + *reinterpret_cast<std::vector<std::unique_ptr<Cookie>>*>(data); size_t val0len = strlen(values[0]); std::string cookieDomain (util::lstripIter(&values[0][0], &values[0][val0len], '.'), @@ -122,29 +122,29 @@ int cookieRowMapper(void* data, int columns, char** values, char** names) if(!values[6] || !parseTime(lastAccessTime, values[6])) { return 0; } - Cookie c(cookieName, - toString(values[5]), // value - expiryTime, - true, // persistent - cookieDomain, - util::isNumericHost(cookieDomain) || - (values[0] && values[0][0] != '.'), // hostOnly - cookiePath, - values[2] && strcmp(values[2], "1") == 0, //secure - false, - lastAccessTime // creation time. Set this later. - ); - cookies.push_back(c); + cookies.push_back(make_unique<Cookie> + (std::move(cookieName), + toString(values[5]), // value + expiryTime, + true, // persistent + std::move(cookieDomain), + util::isNumericHost(cookieDomain) || + (values[0] && values[0][0] != '.'), // hostOnly + std::move(cookiePath), + values[2] && strcmp(values[2], "1") == 0, //secure + false, + lastAccessTime // creation time. Set this later. + )); return 0; } } // namespace -void Sqlite3CookieParser::parse(std::vector<Cookie>& cookies) +std::vector<std::unique_ptr<Cookie>> Sqlite3CookieParser::parse() { if(!db_) { throw DL_ABORT_EX(fmt("SQLite3 database is not opened.")); } - std::vector<Cookie> tcookies; + auto tcookies = std::vector<std::unique_ptr<Cookie>>{}; char* sqlite3ErrMsg = 0; int ret = sqlite3_exec(db_, getQuery(), cookieRowMapper, &tcookies, &sqlite3ErrMsg); @@ -157,7 +157,7 @@ void Sqlite3CookieParser::parse(std::vector<Cookie>& cookies) throw DL_ABORT_EX (fmt("Failed to read SQLite3 database: %s", errMsg.c_str())); } - cookies.swap(tcookies); + return tcookies; } } // namespace aria2 diff --git a/src/Sqlite3CookieParser.h b/src/Sqlite3CookieParser.h index 24b590df..b83a2175 100644 --- a/src/Sqlite3CookieParser.h +++ b/src/Sqlite3CookieParser.h @@ -39,6 +39,7 @@ #include <string> #include <vector> +#include <memory> #include <sqlite3.h> @@ -55,7 +56,7 @@ public: // Loads cookies from sqlite3 database and stores them in cookies. // When loading is successful, cookies stored in cookies initially // are removed. Otherwise, the content of cookies is unchanged. - void parse(std::vector<Cookie>& cookies); + std::vector<std::unique_ptr<Cookie>> parse(); protected: // Returns SQL select statement to get 1 record of cookie. The sql // must return 6 columns in the following order: host, path, diff --git a/src/cookie_helper.cc b/src/cookie_helper.cc index bec1dba0..58a4c69c 100644 --- a/src/cookie_helper.cc +++ b/src/cookie_helper.cc @@ -216,9 +216,8 @@ bool parseDate return time != -1; } -bool parse -(Cookie& cookie, - const std::string& cookieStr, +std::unique_ptr<Cookie> parse +(const std::string& cookieStr, const std::string& requestHost, const std::string& defaultPath, time_t creationTime) @@ -231,13 +230,13 @@ bool parse std::string::const_iterator eq = cookieStr.begin(); for(; eq != nvEnd && *eq != '='; ++eq); if(eq == nvEnd) { - return false; + return std::unique_ptr<Cookie>{}; } std::pair<std::string::const_iterator, std::string::const_iterator> p = util::stripIter(cookieStr.begin(), eq); if(p.first == p.second) { - return false; + return std::unique_ptr<Cookie>{}; } Scip cookieName(p.first, p.second); p = util::stripIter(eq+1, nvEnd); @@ -276,18 +275,18 @@ bool parse if(parseDate(expiryTime, attrp.first, attrp.second)) { foundExpires = true; } else { - return false; + return std::unique_ptr<Cookie>{}; } } else if(util::strieq(p.first, p.second, "max-age")) { if(attrp.first == attrp.second || (!in(static_cast<unsigned char>(*attrp.first), 0x30u, 0x39u) && *attrp.first != '-')) { - return false; + return std::unique_ptr<Cookie>{}; } for(std::string::const_iterator s = attrp.first+1, eos = attrp.second; s != eos; ++s) { if(!in(static_cast<unsigned char>(*s), 0x30u, 0x39u)) { - return false; + return std::unique_ptr<Cookie>{}; } } int64_t delta; @@ -306,17 +305,17 @@ bool parse } } } else { - return false; + return std::unique_ptr<Cookie>{}; } } else if(util::strieq(p.first, p.second, "domain")) { if(attrp.first == attrp.second) { - return false; + return std::unique_ptr<Cookie>{}; } std::string::const_iterator noDot = attrp.first; std::string::const_iterator end = attrp.second; for(; noDot != end && *noDot == '.'; ++noDot); if(noDot == end) { - return false; + return std::unique_ptr<Cookie>{}; } cookieDomain.assign(noDot, end); } else if(util::strieq(p.first, p.second, "path")) { @@ -349,26 +348,27 @@ bool parse } else if(domainMatch(canonicalizedHost, cookieDomain)) { hostOnly = util::isNumericHost(canonicalizedHost); } else { - return false; + return std::unique_ptr<Cookie>{}; } if(cookiePath.empty()) { cookiePath = defaultPath; } - cookie.setName(cookieName.first, cookieName.second); - cookie.setValue(cookieValue.first, cookieValue.second); - cookie.setExpiryTime(expiryTime); - cookie.setPersistent(persistent); - cookie.setDomain(cookieDomain); - cookie.setHostOnly(hostOnly); - cookie.setPath(cookiePath); - cookie.setSecure(secure); - cookie.setHttpOnly(httpOnly); - cookie.setCreationTime(creationTime); - cookie.setLastAccessTime(creationTime); + auto cookie = make_unique<Cookie>(); + cookie->setName(cookieName.first, cookieName.second); + cookie->setValue(cookieValue.first, cookieValue.second); + cookie->setExpiryTime(expiryTime); + cookie->setPersistent(persistent); + cookie->setDomain(std::move(cookieDomain)); + cookie->setHostOnly(hostOnly); + cookie->setPath(std::move(cookiePath)); + cookie->setSecure(secure); + cookie->setHttpOnly(httpOnly); + cookie->setCreationTime(creationTime); + cookie->setLastAccessTime(creationTime); - return true; + return cookie; } bool goodPath diff --git a/src/cookie_helper.h b/src/cookie_helper.h index 7e481f53..79264b09 100644 --- a/src/cookie_helper.h +++ b/src/cookie_helper.h @@ -38,6 +38,7 @@ #include "common.h" #include <string> +#include <memory> #include "a2time.h" @@ -52,9 +53,8 @@ bool parseDate std::string::const_iterator first, std::string::const_iterator last); -bool parse -(Cookie& cookie, - const std::string& cookieStr, +std::unique_ptr<Cookie> parse +(const std::string& cookieStr, const std::string& requestHost, const std::string& defaultPath, time_t creationTime); |