Disard cache when checking checksum

This will slow down checksum checking but does not thrash cache.

10 files changed, 57 insertions, 2 deletions

diff --git a/src/AbstractDiskWriter.cc b/src/AbstractDiskWriter.cc
index e3653233..0944194c 100644
--- a/src/AbstractDiskWriter.cc
+++ b/src/AbstractDiskWriter.cc
@@ -38,6 +38,7 @@
 #ifdef HAVE_MMAP
 #  include <sys/mman.h>
 #endif // HAVE_MMAP
+#include <fcntl.h>
 
 #include <cerrno>
 #include <cstring>
@@ -547,4 +548,11 @@ void AbstractDiskWriter::enableMmap()
   enableMmap_ = true;
 }
 
+void AbstractDiskWriter::dropCache(int64_t len, int64_t offset)
+{
+#ifdef HAVE_POSIX_FADVISE
+  posix_fadvise(fd_, offset, len, POSIX_FADV_DONTNEED);
+#endif // HAVE_POSIX_FADVISE
+}
+
 } // namespace aria2
diff --git a/src/AbstractDiskWriter.h b/src/AbstractDiskWriter.h
index 64e8ade0..f469ab4f 100644
--- a/src/AbstractDiskWriter.h
+++ b/src/AbstractDiskWriter.h
@@ -96,6 +96,8 @@ public:
   virtual void disableReadOnly() CXX11_OVERRIDE;
 
   virtual void enableMmap() CXX11_OVERRIDE;
+
+  virtual void dropCache(int64_t len, int64_t offset) CXX11_OVERRIDE;
 };
 
 } // namespace aria2
diff --git a/src/AbstractSingleDiskAdaptor.cc b/src/AbstractSingleDiskAdaptor.cc
index c3a2ba0c..974af822 100644
--- a/src/AbstractSingleDiskAdaptor.cc
+++ b/src/AbstractSingleDiskAdaptor.cc
@@ -83,6 +83,18 @@ ssize_t AbstractSingleDiskAdaptor::readData
   return diskWriter_->readData(data, len, offset);
 }
 
+ssize_t AbstractSingleDiskAdaptor::readDataDropCache
+(unsigned char* data, size_t len, int64_t offset)
+{
+  auto rv = readData(data, len, offset);
+
+  if(rv > 0) {
+    diskWriter_->dropCache(len, offset);
+  }
+
+  return rv;
+}
+
 void AbstractSingleDiskAdaptor::writeCache(const WrDiskCacheEntry* entry)
 {
   // Write cached data in 4KiB aligned offset. This reduces disk
diff --git a/src/AbstractSingleDiskAdaptor.h b/src/AbstractSingleDiskAdaptor.h
index 209bf571..ef279df1 100644
--- a/src/AbstractSingleDiskAdaptor.h
+++ b/src/AbstractSingleDiskAdaptor.h
@@ -66,6 +66,10 @@ public:
   virtual ssize_t readData(unsigned char* data, size_t len, int64_t offset)
     CXX11_OVERRIDE;
 
+  virtual ssize_t readDataDropCache(unsigned char* data, size_t len,
+                                    int64_t offset)
+    CXX11_OVERRIDE;
+
   virtual void writeCache(const WrDiskCacheEntry* entry) CXX11_OVERRIDE;
 
   virtual bool fileExists() CXX11_OVERRIDE;
diff --git a/src/DiskAdaptor.h b/src/DiskAdaptor.h
index bb070707..b4c55bec 100644
--- a/src/DiskAdaptor.h
+++ b/src/DiskAdaptor.h
@@ -107,6 +107,10 @@ public:
   // successfully changed.
   virtual size_t utime(const Time& actime, const Time& modtime) = 0;
 
+  // Just like readData(), but drop cache after read.
+  virtual ssize_t readDataDropCache(unsigned char* data, size_t len,
+                                    int64_t offset) = 0;
+
   // Writes cached data to the underlying disk.
   virtual void writeCache(const WrDiskCacheEntry* entry) = 0;
 
diff --git a/src/DiskWriter.h b/src/DiskWriter.h
index 88715e96..94512a95 100644
--- a/src/DiskWriter.h
+++ b/src/DiskWriter.h
@@ -82,6 +82,9 @@ public:
 
   // Enables mmap.
   virtual void enableMmap() {}
+
+  // Drops cache in range [offset, offset + len)
+  virtual void dropCache(int64_t len, int64_t offset) {}
 };
 
 } // namespace aria2
diff --git a/src/IteratableChecksumValidator.cc b/src/IteratableChecksumValidator.cc
index c07daafb..ecf36cb4 100644
--- a/src/IteratableChecksumValidator.cc
+++ b/src/IteratableChecksumValidator.cc
@@ -64,7 +64,7 @@ void IteratableChecksumValidator::validateChunk()
   // Don't guard with !finished() to allow zero-length file to be
   // verified.
   unsigned char buf[4096];
-  size_t length = pieceStorage_->getDiskAdaptor()->readData
+  size_t length = pieceStorage_->getDiskAdaptor()->readDataDropCache
     (buf, sizeof(buf), currentOffset_);
   ctx_->update(buf, length);
   currentOffset_ += length;
diff --git a/src/IteratableChunkChecksumValidator.cc b/src/IteratableChunkChecksumValidator.cc
index a2fbce41..515f62a7 100644
--- a/src/IteratableChunkChecksumValidator.cc
+++ b/src/IteratableChunkChecksumValidator.cc
@@ -125,7 +125,7 @@ std::string IteratableChunkChecksumValidator::digest(int64_t offset, size_t leng
   ctx_->reset();
   int64_t max = offset+length;
   while(offset < max) {
-    size_t r = pieceStorage_->getDiskAdaptor()->readData
+    size_t r = pieceStorage_->getDiskAdaptor()->readDataDropCache
       (buf, std::min(static_cast<int64_t>(sizeof(buf)), max-offset), offset);
     if(r == 0) {
       throw DL_ABORT_EX
diff --git a/src/MultiDiskAdaptor.cc b/src/MultiDiskAdaptor.cc
index 4a2b0ec4..fbc6127f 100644
--- a/src/MultiDiskAdaptor.cc
+++ b/src/MultiDiskAdaptor.cc
@@ -367,6 +367,18 @@ void MultiDiskAdaptor::writeData(const unsigned char* data, size_t len,
 ssize_t MultiDiskAdaptor::readData
 (unsigned char* data, size_t len, int64_t offset)
 {
+  return readData(data, len, offset, false);
+}
+
+ssize_t MultiDiskAdaptor::readDataDropCache
+(unsigned char* data, size_t len, int64_t offset)
+{
+  return readData(data, len, offset, true);
+}
+
+ssize_t MultiDiskAdaptor::readData
+(unsigned char* data, size_t len, int64_t offset, bool dropCache)
+{
   auto first = findFirstDiskWriterEntry(diskWriterEntries_, offset);
   ssize_t rem = len;
   ssize_t totalReadLength = 0;
@@ -380,6 +392,9 @@ ssize_t MultiDiskAdaptor::readData
     totalReadLength +=
       (*i)->getDiskWriter()->readData(data+(len-rem), readLength, fileOffset);
     rem -= readLength;
+    if(dropCache) {
+      (*i)->getDiskWriter()->dropCache(readLength, fileOffset);
+    }
     fileOffset = 0;
     if(rem == 0) {
       break;
diff --git a/src/MultiDiskAdaptor.h b/src/MultiDiskAdaptor.h
index 0b540803..2d98588f 100644
--- a/src/MultiDiskAdaptor.h
+++ b/src/MultiDiskAdaptor.h
@@ -123,6 +123,9 @@ private:
 
   void openIfNot(DiskWriterEntry* entry, void (DiskWriterEntry::*f)());
 
+  ssize_t readData(unsigned char* data, size_t len, int64_t offset,
+                   bool dropCache);
+
   static const int DEFAULT_MAX_OPEN_FILES = 100;
 
 public:
@@ -143,6 +146,10 @@ public:
   virtual ssize_t readData(unsigned char* data, size_t len, int64_t offset)
     CXX11_OVERRIDE;
 
+  virtual ssize_t readDataDropCache(unsigned char* data, size_t len,
+                                    int64_t offset)
+    CXX11_OVERRIDE;
+
   virtual void writeCache(const WrDiskCacheEntry* entry) CXX11_OVERRIDE;
 
   virtual bool fileExists() CXX11_OVERRIDE;