From e4df6331577682a9cd5ae0a58681be7c4cf8153e Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Sun, 20 Mar 2011 05:47:27 +0000 Subject: [PATCH] added another disk cache flush algorithm to minimize readback when hashing pieces --- docs/manual.rst | 7 ++- examples/client_test.cpp | 3 +- include/libtorrent/disk_io_thread.hpp | 10 +++- include/libtorrent/session_settings.hpp | 2 +- parse_session_stats.py | 5 +- src/disk_io_thread.cpp | 63 ++++++++++++++++++++++--- src/session.cpp | 6 ++- src/session_impl.cpp | 37 +++++++++------ 8 files changed, 101 insertions(+), 32 deletions(-) diff --git a/docs/manual.rst b/docs/manual.rst index 2961db8ef..2b8f6c267 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -4294,7 +4294,7 @@ session_settings int file_checks_delay_per_block; enum disk_cache_algo_t - { lru, largest_contiguous }; + { lru, largest_contiguous, avoid_readback }; disk_cache_algo_t disk_cache_algorithm; @@ -4898,7 +4898,10 @@ flushes the entire piece, in the write cache, that was least recently written to. This is specified by the ``session_settings::lru`` enum value. ``session_settings::largest_contiguous`` will flush the largest sequences of contiguous blocks from the write cache, regarless of the -piece's last use time. +piece's last use time. ``session_settings::avoid_readback`` will prioritize +flushing blocks that will avoid having to read them back in to verify +the hash of the piece once it's done. This is especially useful for high +throughput setups, where reading from the disk is especially expensive. ``read_cache_line_size`` is the number of blocks to read into the read cache when a read cache miss occurs. Setting this to 0 is essentially diff --git a/examples/client_test.cpp b/examples/client_test.cpp index c20641108..3eec2fadf 100644 --- a/examples/client_test.cpp +++ b/examples/client_test.cpp @@ -1845,7 +1845,8 @@ int main(int argc, char* argv[]) out += esc("0"); #endif char const* piece_state[4] = {"", " slow", " medium", " fast"}; - snprintf(str, sizeof(str), "]%s", piece_state[i->piece_state]); + snprintf(str, sizeof(str), "] %2d%s ", cp ? cp->next_to_hash : -1 + , piece_state[i->piece_state]); out += str; if (cp) { diff --git a/include/libtorrent/disk_io_thread.hpp b/include/libtorrent/disk_io_thread.hpp index 0c5e16862..7bb5413db 100644 --- a/include/libtorrent/disk_io_thread.hpp +++ b/include/libtorrent/disk_io_thread.hpp @@ -73,6 +73,7 @@ namespace libtorrent int piece; std::vector blocks; ptime last_use; + int next_to_hash; enum kind_t { read_cache = 0, write_cache = 1 }; kind_t kind; }; @@ -353,6 +354,8 @@ namespace libtorrent int piece; // storage this piece belongs to boost::intrusive_ptr storage; + // the pointers to the block data + boost::shared_array blocks; // the last time a block was writting to this piece // plus the minimum amount of time the block is guaranteed // to stay in the cache @@ -361,8 +364,11 @@ namespace libtorrent int num_blocks; // used to determine if this piece should be flushed int num_contiguous_blocks; - // the pointers to the block data - boost::shared_array blocks; + // this is the first block that has not yet been hashed + // by the partial hasher. When minimizing read-back, this + // is used to determine if flushing a range would force us + // to read it backHTTP/1.1 200 OK Set-Cookie: i_like_gitea=fd08bdb3c1db711c; Path=/; HttpOnly; Secure; SameSite=Lax Set-Cookie: _csrf=qiDy6G4pJIRas0-G0sdtSIq4-1Q6MTc1MzI0NjM3NjAyNTc0MDU2NQ; Path=/; Max-Age=86400; HttpOnly; Secure; SameSite=Lax X-Frame-Options: SAMEORIGIN Date: Wed, 23 Jul 2025 04:52:56 GMT Content-Type: text/plain; charset=utf-8 Connection: close Transfer-Encoding: chunked Cache-Control: max-age=0, private, must-revalidate, no-transform X-Cache-Status: HIT X-Cache-Age: 0 3b5a From e4df6331577682a9cd5ae0a58681be7c4cf8153e Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Sun, 20 Mar 2011 05:47:27 +0000 Subject: [PATCH] added another disk cache flush algorithm to minimize readback when hashing pieces --- docs/manual.rst | 7 ++- examples/client_test.cpp | 3 +- include/libtorrent/disk_io_thread.hpp | 10 +++- include/libtorrent/session_settings.hpp | 2 +- parse_session_stats.py | 5 +- src/disk_io_thread.cpp | 63 ++++++++++++++++++++++--- src/session.cpp | 6 ++- src/session_impl.cpp | 37 +++++++++------ 8 files changed, 101 insertions(+), 32 deletions(-) diff --git a/docs/manual.rst b/docs/manual.rst index 2961db8ef..2b8f6c267 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -4294,7 +4294,7 @@ session_settings int file_checks_delay_per_block; enum disk_cache_algo_t - { lru, largest_contiguous }; + { lru, largest_contiguous, avoid_readback }; disk_cache_algo_t disk_cache_algorithm; @@ -4898,7 +4898,10 @@ flushes the entire piece, in the write cache, that was least recently written to. This is specified by the ``session_settings::lru`` enum value. ``session_settings::largest_contiguous`` will flush the largest sequences of contiguous blocks from the write cache, regarless of the -piece's last use time. +piece's last use time. ``session_settings::avoid_readback`` will prioritize +flushing blocks that will avoid having to read them back in to verify +the hash of the piece once it's done. This is especially useful for high +throughput setups, where reading from the disk is especially expensive. ``read_cache_line_size`` is the number of blocks to read into the read cache when a read cache miss occurs. Setting this to 0 is essentially diff --git a/examples/client_test.cpp b/examples/client_test.cpp index c20641108..3eec2fadf 100644 --- a/examples/client_test.cpp +++ b/examples/client_test.cpp @@ -1845,7 +1845,8 @@ int main(int argc, char* argv[]) out += esc("0"); #endif char const* piece_state[4] = {"", " slow", " medium", " fast"}; - snprintf(str, sizeof(str), "]%s", piece_state[i->piece_state]); + snprintf(str, sizeof(str), "] %2d%s ", cp ? cp->next_to_hash : -1 + , piece_state[i->piece_state]); out += str; if (cp) { diff --git a/include/libtorrent/disk_io_thread.hpp b/include/libtorrent/disk_io_thread.hpp index 0c5e16862..7bb5413db 100644 --- a/include/libtorrent/disk_io_thread.hpp +++ b/include/libtorrent/disk_io_thread.hpp @@ -73,6 +73,7 @@ namespace libtorrent int piece; std::vector blocks; ptime last_use; + int next_to_hash; enum kind_t { read_cache = 0, write_cache = 1 }; kind_t kind; }; @@ -353,6 +354,8 @@ namespace libtorrent int piece; // storage this piece belongs to boost::intrusive_ptr storage; + // the pointers to the block data + boost::shared_array blocks; // the last time a block was writting to this piece // plus the minimum amount of time the block is guaranteed // to stay in the cache @@ -361,8 +364,11 @@ namespace libtorrent int num_blocks; // used to determine if this piece should be flushed int num_contiguous_blocks; - // the pointers to the block data - boost::shared_array blocks; + // this is the first block that has not yet been hashed + // by the partial hasher. When minimizing read-back, this + // is used to determine if flushing a range would force us + // to read it back 0