diff --git a/docs/manual.rst b/docs/manual.rst index a3e1630ba..c1cf4cf23 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -1699,6 +1699,8 @@ requested. The entry in the vector (``partial_piece_info``) looks like this:: std::bitset finished_blocks; address peer[max_blocks_per_piece]; int num_downloads[max_blocks_per_piece]; + enum state_t { none, slow. medium, fast }; + state_t piece_state; }; ``piece_index`` is the index of the piece in question. ``blocks_in_piece`` is the @@ -1718,6 +1720,13 @@ or not. And the ``num_downloads`` array says how many times that block has been When a piece fails a hash verification, single blocks may be re-downloaded to see if the hash test may pass then. +``piece_state`` is set to either ``fast``, ``medium``, ``slow`` or ``none``. It tells which +download rate category the peers downloading this piece falls into. ``none`` means that no +peer is currently downloading any part of the piece. Peers prefer picking pieces from +the same category as themselves. The reason for this is to keep the number of partially +downloaded pieces down. Pieces set to ``none`` can be converted into any of ``fast``, +``medium`` or ``slow`` as soon as a peer want to download from it. + get_peer_info() --------------- diff --git a/examples/client_test.cpp b/examples/client_test.cpp index 5375d4eef..891392bb4 100644 --- a/examples/client_test.cpp +++ b/examples/client_test.cpp @@ -995,7 +995,8 @@ int main(int ac, char* av[]) else out << "-"; #endif } - out << "]\n"; + char* piece_state[4] = {"", "slow", "medium", "fast"}; + out << "] " << piece_state[i->piece_state] << "\n"; } out << "___________________________________\n"; diff --git a/include/libtorrent/peer_connection.hpp b/include/libtorrent/peer_connection.hpp index 42164656e..d513b955d 100755 --- a/include/libtorrent/peer_connection.hpp +++ b/include/libtorrent/peer_connection.hpp @@ -136,6 +136,9 @@ namespace libtorrent policy::peer* peer_info_struct() const { return m_peer_info; } + enum peer_speed_t { slow, medium, fast }; + peer_speed_t peer_speed(); + #ifndef TORRENT_DISABLE_EXTENSIONS void add_extension(boost::shared_ptr); #endif @@ -650,6 +653,10 @@ namespace libtorrent // and hasn't been added to a torrent yet. policy::peer* m_peer_info; + // this is a measurement of how fast the peer + // it allows some variance without changing + // back and forth between states + peer_speed_t m_speed; #ifndef NDEBUG public: bool m_in_constructor; diff --git a/include/libtorrent/piece_picker.hpp b/include/libtorrent/piece_picker.hpp index b4009ee54..bedddc1c0 100755 --- a/include/libtorrent/piece_picker.hpp +++ b/include/libtorrent/piece_picker.hpp @@ -100,8 +100,18 @@ namespace libtorrent int num_downloads; }; + // the peers that are downloading this piece + // are considered fast peers or slow peers. + // none is set if the blocks were downloaded + // in a previous session + enum piece_state_t + { none, slow, medium, fast }; + struct downloading_piece { + piece_state_t state; + + // the index of the piece int index; // each bit represents a block in the piece // set to one if the block has been requested @@ -173,7 +183,7 @@ namespace libtorrent void pick_pieces(const std::vector& pieces , std::vector& interesting_blocks , int num_pieces, bool prefer_whole_pieces - , tcp::endpoint peer) const; + , tcp::endpoint peer, piece_state_t speed) const; // returns true if any client is currently downloading this // piece-block, or if it's queued for downloading by some client @@ -182,7 +192,8 @@ namespace libtorrent bool is_finished(piece_block block) const; // marks this piece-block as queued for downloading - void mark_as_downloading(piece_block block, tcp::endpoint const& peer); + void mark_as_downloading(piece_block block, tcp::endpoint const& peer + , piece_state_t s); void mark_as_finished(piece_block block, tcp::endpoint const& peer); // if a piece had a hash-failure, it must be restored and @@ -320,15 +331,13 @@ namespace libtorrent void add(int index); void move(int vec_index, int elem_index); -// void remove(int vec_index, int elem_index); int add_interesting_blocks(const std::vector& piece_list , const std::vector& pieces , std::vector& interesting_blocks , std::vector& backup_blocks , int num_blocks, bool prefer_whole_pieces - , tcp::endpoint peer) const; - + , tcp::endpoint peer, piece_state_t speed) const; // this vector contains all pieces we don't have. // in the first entry (index 0) is a vector of all pieces diff --git a/include/libtorrent/torrent_handle.hpp b/include/libtorrent/torrent_handle.hpp index f75171f57..86bccd94a 100755 --- a/include/libtorrent/torrent_handle.hpp +++ b/include/libtorrent/torrent_handle.hpp @@ -211,6 +211,8 @@ namespace libtorrent std::bitset finished_blocks; tcp::endpoint peer[max_blocks_per_piece]; int num_downloads[max_blocks_per_piece]; + enum state_t { none, slow, medium, fast }; + state_t piece_state; }; struct TORRENT_EXPORT torrent_handle diff --git a/src/peer_connection.cpp b/src/peer_connection.cpp index 2987fc5dc..b72b57759 100755 --- a/src/peer_connection.cpp +++ b/src/peer_connection.cpp @@ -124,6 +124,7 @@ namespace libtorrent , m_upload_limit(resource_request::inf) , m_download_limit(resource_request::inf) , m_peer_info(peerinfo) + , m_speed(slow) #ifndef NDEBUG , m_in_constructor(true) #endif @@ -189,6 +190,7 @@ namespace libtorrent , m_upload_limit(resource_request::inf) , m_download_limit(resource_request::inf) , m_peer_info(peerinfo) + , m_speed(slow) #ifndef NDEBUG , m_in_constructor(true) #endif @@ -1328,7 +1330,14 @@ namespace libtorrent assert(block.block_index < t->torrent_file().piece_size(block.piece_index)); assert(!t->picker().is_downloading(block)); - t->picker().mark_as_downloading(block, m_remote); + piece_picker::piece_state_t state; + peer_speed_t speed = peer_speed(); + if (speed == fast) state = piece_picker::fast; + else if (speed == medium) state = piece_picker::medium; + else if (speed == slow) state = piece_picker::slow; + + t->picker().mark_as_downloading(block, m_remote, state); + m_request_queue.push_back(block); } @@ -2411,6 +2420,25 @@ namespace libtorrent return false; } + peer_connection::peer_speed_t peer_connection::peer_speed() + { + shared_ptr t = m_torrent.lock(); + assert(t); + + int download_rate = statistics().download_payload_rate(); + int torrent_download_rate = t->statistics().download_payload_rate(); + + if (download_rate > 512 && download_rate > torrent_download_rate / 16) + m_speed = fast; + else if (download_rate > 4096 && download_rate > torrent_download_rate / 64) + m_speed = medium; + else if (download_rate < torrent_download_rate / 15 && m_speed == fast) + m_speed = medium; + else if (download_rate < torrent_download_rate / 63 && m_speed == medium) + m_speed = slow; + + return m_speed; + } void peer_connection::keep_alive() { diff --git a/src/piece_picker.cpp b/src/piece_picker.cpp index fcb016314..bb5bf131a 100755 --- a/src/piece_picker.cpp +++ b/src/piece_picker.cpp @@ -205,6 +205,24 @@ namespace libtorrent for (int i = m_sequenced_download_threshold * 2 + 1; i < int(m_piece_info.size()); ++i) assert(m_piece_info[i].empty()); + for (std::vector::const_iterator i = m_downloads.begin() + , end(m_downloads.end()); i != end; ++i) + { + bool blocks_requested = false; + int num_blocks = blocks_in_piece(i->index); + for (int k = 0; k < num_blocks; ++k) + { + if (i->finished_blocks[k]) continue; + if (i->requested_blocks[k]) + { + blocks_requested = true; + break; + } + } + assert(blocks_requested == (i->state != none)); + } + + int num_filtered = 0; int num_have_filtered = 0; for (std::vector::const_iterator i = m_piece_map.begin(); @@ -971,28 +989,40 @@ namespace libtorrent } // ============ end deprecation ============== - + + // pieces describes which pieces the peer we're requesting from + // has. + // interesting_blocks is an out parameter, and will be filled + // with (up to) num_blocks of interesting blocks that the peer has. + // prefer_whole_pieces can be set if this peer should download + // whole pieces rather than trying to download blocks from the + // same piece as other peers. + // the endpoint is the address of the peer we're picking pieces + // from. This is used when downloading whole pieces, to only + // pick from the same piece the same peer is downloading from. + // state is supposed to be set to fast if the peer is downloading + // relatively fast, by some notion. Slow peers will prefer not + // to pick blocks from the same pieces as fast peers, and vice + // versa. Downloading pieces are marked as being fast, medium + // or slow once they're started. void piece_picker::pick_pieces(const std::vector& pieces , std::vector& interesting_blocks , int num_blocks, bool prefer_whole_pieces - , tcp::endpoint peer) const + , tcp::endpoint peer, piece_state_t speed) const { TORRENT_PIECE_PICKER_INVARIANT_CHECK; assert(num_blocks > 0); assert(pieces.size() == m_piece_map.size()); assert(m_files_checked_called); - // free refers to pieces that are free to download, no one else - // is downloading them. - // partial is pieces that are partially being downloaded, and - // parts of them may be free for download as well, the - // partially downloaded pieces will be prioritized assert(m_piece_info.begin() != m_piece_info.end()); - // +1 is to ignore pieces that no peer has. The bucket with index 0 contains - // pieces that 0 other peers has. - std::vector >::const_iterator free - = m_piece_info.begin() + 1; + // this will be filled with blocks that we should not request + // unless we can't find num_blocks among the other ones. + // blocks that belong to pieces with a mismatching speed + // category for instance, or if we prefer whole pieces, + // blocks belonging to a piece that others have + // downloaded to std::vector backup_blocks; // this loop will loop from pieces with priority 1 and up @@ -1004,29 +1034,29 @@ namespace libtorrent // fast peers) the partial pieces will not be prioritized, but actually // ignored as long as possible. - while (free != m_piece_info.end()) + // +1 is to ignore pieces that no peer has. The bucket with index 0 contains + // pieces that 0 other peers have. bucket will point to a bucket with + // pieces with the same priority. It will be iterated in priority + // order (high priority/rare pices first). The content of each + // bucket is randomized + for (std::vector >::const_iterator bucket + = m_piece_info.begin() + 1; bucket != m_piece_info.end(); + ++bucket) { - num_blocks = add_interesting_blocks(*free, pieces + if (bucket->empty()) continue; + num_blocks = add_interesting_blocks(*bucket, pieces , interesting_blocks, backup_blocks, num_blocks - , prefer_whole_pieces, peer); + , prefer_whole_pieces, peer, speed); assert(num_blocks >= 0); if (num_blocks == 0) return; - ++free; } -// TODO: what's up with this? - if (!prefer_whole_pieces) return; assert(num_blocks > 0); -#ifdef TORRENT_VERBOSE_LOGGING -// std::ofstream f("piece_picker.log", std::ios_base::app); -// f << "backup_blocks: " << backup_blocks.size() << "\n" -// << "used: " << std::min(num_blocks, (int)backup_blocks.size()) << "\n----\n"; -#endif - - interesting_blocks.insert(interesting_blocks.end() - , backup_blocks.begin(), backup_blocks.begin() - + (std::min)(num_blocks, (int)backup_blocks.size())); + if (!backup_blocks.empty()) + interesting_blocks.insert(interesting_blocks.end() + , backup_blocks.begin(), backup_blocks.begin() + + (std::min)(num_blocks, (int)backup_blocks.size())); } namespace @@ -1053,7 +1083,7 @@ namespace libtorrent , std::vector& interesting_blocks , std::vector& backup_blocks , int num_blocks, bool prefer_whole_pieces - , tcp::endpoint peer) const + , tcp::endpoint peer, piece_state_t speed) const { for (std::vector::const_iterator i = piece_list.begin(); i != piece_list.end(); ++i) @@ -1083,7 +1113,7 @@ namespace libtorrent if (prefer_whole_pieces && !exclusively_requested_from(*p, num_blocks_in_piece, peer)) { - if ((int)backup_blocks.size() >= num_blocks) continue; + if (int(backup_blocks.size()) >= num_blocks) continue; for (int j = 0; j < num_blocks_in_piece; ++j) { if (p->finished_blocks[j] == 1) continue; @@ -1096,9 +1126,19 @@ namespace libtorrent for (int j = 0; j < num_blocks_in_piece; ++j) { + // ignore completed blocks if (p->finished_blocks[j] == 1) continue; + // ignore blocks requested from this peer already if (p->requested_blocks[j] == 1 && p->info[j].peer == peer) continue; + // if the piece is fast and the peer is slow, or vice versa, + // add the block as a backup + if (p->state != none && p->state != speed) + { + if (int(backup_blocks.size()) >= num_blocks) continue; + backup_blocks.push_back(piece_block(*i, j)); + continue; + } // this block is interesting (we don't have it // yet). But it may already have been requested // from another peer. We have to add it anyway @@ -1113,6 +1153,8 @@ namespace libtorrent { // we have found a block that's free to download num_blocks--; + // if we prefer whole pieces, continue picking from this + // piece even though we have num_blocks if (prefer_whole_pieces) continue; assert(num_blocks >= 0); if (num_blocks == 0) return num_blocks; @@ -1193,7 +1235,8 @@ namespace libtorrent } - void piece_picker::mark_as_downloading(piece_block block, const tcp::endpoint& peer) + void piece_picker::mark_as_downloading(piece_block block + , const tcp::endpoint& peer, piece_state_t state) { TORRENT_PIECE_PICKER_INVARIANT_CHECK; @@ -1210,6 +1253,7 @@ namespace libtorrent move(prio, p.index); downloading_piece dp; + dp.state = state; dp.index = block.piece_index; dp.requested_blocks[block.block_index] = 1; dp.info[block.block_index].peer = peer; @@ -1223,6 +1267,7 @@ namespace libtorrent assert(i->requested_blocks[block.block_index] == 0); i->info[block.block_index].peer = peer; i->requested_blocks[block.block_index] = 1; + if (i->state == none) i->state = state; } } @@ -1245,6 +1290,7 @@ namespace libtorrent else assert(p.priority(m_sequenced_download_threshold) == 0); downloading_piece dp; + dp.state = none; dp.index = block.piece_index; dp.requested_blocks[block.block_index] = 1; dp.finished_blocks[block.block_index] = 1; @@ -1259,6 +1305,27 @@ namespace libtorrent i->info[block.block_index].peer = peer; i->requested_blocks[block.block_index] = 1; i->finished_blocks[block.block_index] = 1; + + // TODO: maintain requested and finished counters so that + // we don't have to count every time + bool blocks_requested = false; + int num_blocks = blocks_in_piece(i->index); + for (int k = 0; k < num_blocks; ++k) + { + if (i->finished_blocks[k]) continue; + if (i->requested_blocks[k]) + { + blocks_requested = true; + break; + } + } + + if (!blocks_requested) + { + // there are no blocks requested in this piece. + // remove the fast/slow state from it + i->state = none; + } } } /* @@ -1373,6 +1440,27 @@ namespace libtorrent assert(std::find_if(m_downloads.begin(), m_downloads.end() , has_index(block.piece_index)) == m_downloads.end()); } + + // TODO: maintain requested and finished counters so that + // we don't have to count every time + bool blocks_requested = false; + int num_blocks = blocks_in_piece(i->index); + for (int k = 0; k < num_blocks; ++k) + { + if (i->finished_blocks[k]) continue; + if (i->requested_blocks[k]) + { + blocks_requested = true; + break; + } + } + + if (!blocks_requested) + { + // there are no blocks requested in this piece. + // remove the fast/slow state from it + i->state = none; + } } int piece_picker::unverified_blocks() const diff --git a/src/policy.cpp b/src/policy.cpp index fff6778ef..8ec873e4f 100755 --- a/src/policy.cpp +++ b/src/policy.cpp @@ -200,6 +200,12 @@ namespace libtorrent // than we requested. assert(c.remote() == c.get_socket()->remote_endpoint()); + piece_picker::piece_state_t state; + peer_connection::peer_speed_t speed = c.peer_speed(); + if (speed == peer_connection::fast) state = piece_picker::fast; + else if (speed == peer_connection::medium) state = piece_picker::medium; + else if (speed == peer_connection::slow) state = piece_picker::slow; + // picks the interesting pieces from this peer // the integer is the number of pieces that // should be guaranteed to be available for download @@ -209,7 +215,7 @@ namespace libtorrent // for this peer. If we're downloading one piece in 20 seconds // then use this mode. p.pick_pieces(c.get_bitfield(), interesting_pieces - , num_requests, prefer_whole_pieces, c.remote()); + , num_requests, prefer_whole_pieces, c.remote(), state); // this vector is filled with the interesting pieces // that some other peer is currently downloading diff --git a/src/torrent_handle.cpp b/src/torrent_handle.cpp index eccc4971e..c7584f87e 100755 --- a/src/torrent_handle.cpp +++ b/src/torrent_handle.cpp @@ -764,6 +764,7 @@ namespace libtorrent = q.begin(); i != q.end(); ++i) { partial_piece_info pi; + pi.piece_state = (partial_piece_info::state_t)i->state; pi.finished_blocks = i->finished_blocks; pi.requested_blocks = i->requested_blocks; for (int j = 0; j < partial_piece_info::max_blocks_per_piece; ++j) diff --git a/test/test_piece_picker.cpp b/test/test_piece_picker.cpp index f24038ee4..8c4ab1eb7 100644 --- a/test/test_piece_picker.cpp +++ b/test/test_piece_picker.cpp @@ -88,21 +88,21 @@ int test_main() std::vector picked; picked.clear(); - p.pick_pieces(peer1, picked, 1, false, tcp::endpoint()); + p.pick_pieces(peer1, picked, 1, false, tcp::endpoint(), piece_picker::fast); TEST_CHECK(picked.size() == 1); TEST_CHECK(picked.front().piece_index == 2); // now pick a piece from peer2. The block is supposed to be // from piece 3, since it is the rarest piece that peer has. picked.clear(); - p.pick_pieces(peer2, picked, 1, false, tcp::endpoint()); + p.pick_pieces(peer2, picked, 1, false, tcp::endpoint(), piece_picker::fast); TEST_CHECK(picked.size() == 1); TEST_CHECK(picked.front().piece_index == 3); // same thing for peer3. picked.clear(); - p.pick_pieces(peer3, picked, 1, false, tcp::endpoint()); + p.pick_pieces(peer3, picked, 1, false, tcp::endpoint(), piece_picker::fast); TEST_CHECK(picked.size() == 1); TEST_CHECK(picked.front().piece_index == 5); @@ -112,7 +112,7 @@ int test_main() p.inc_refcount(1); picked.clear(); - p.pick_pieces(peer3, picked, 1, false, tcp::endpoint()); + p.pick_pieces(peer3, picked, 1, false, tcp::endpoint(), piece_picker::fast); TEST_CHECK(picked.size() == 1); TEST_CHECK(picked.front().piece_index == 1); // and the block picked should not be 0 or 2 @@ -136,9 +136,12 @@ int test_main() // we have block 0 and 2 already, so we can't mark // them as begin downloaded. - p.mark_as_downloading(piece_block(1, 1), tcp::endpoint(address::from_string("1.1.1.1"), 0)); - p.mark_as_downloading(piece_block(1, 3), tcp::endpoint(address::from_string("1.1.1.1"), 0)); - p.mark_as_downloading(piece_block(2, 0), tcp::endpoint(address::from_string("1.1.1.1"), 0)); + p.mark_as_downloading(piece_block(1, 1), tcp::endpoint( + address::from_string("1.1.1.1"), 0), piece_picker::fast); + p.mark_as_downloading(piece_block(1, 3), tcp::endpoint( + address::from_string("1.1.1.1"), 0), piece_picker::fast); + p.mark_as_downloading(piece_block(2, 0), tcp::endpoint( + address::from_string("1.1.1.1"), 0), piece_picker::fast); std::vector const& downloads = p.get_download_queue(); TEST_CHECK(downloads.size() == 2); @@ -166,7 +169,7 @@ int test_main() TEST_CHECK(!p.is_downloading(piece_block(2, 1))); picked.clear(); - p.pick_pieces(peer1, picked, 1, false, tcp::endpoint()); + p.pick_pieces(peer1, picked, 1, false, tcp::endpoint(), piece_picker::fast); TEST_CHECK(picked.size() == 2); piece_block expected3[] = { piece_block(2, 0), piece_block(2, 1) }; @@ -179,7 +182,7 @@ int test_main() // partially selected) picked.clear(); - p.pick_pieces(peer1, picked, 1, true, tcp::endpoint()); + p.pick_pieces(peer1, picked, 1, true, tcp::endpoint(), piece_picker::fast); // it will pick 4 blocks, since we said we // wanted whole pieces. @@ -197,7 +200,7 @@ int test_main() // to make sure it can still fall back on partial pieces picked.clear(); - p.pick_pieces(peer1, picked, 100, true, tcp::endpoint()); + p.pick_pieces(peer1, picked, 100, true, tcp::endpoint(), piece_picker::fast); TEST_CHECK(picked.size() == 12); @@ -218,7 +221,8 @@ int test_main() // to make sure it can still fall back on partial pieces picked.clear(); - p.pick_pieces(peer1, picked, 100, true, tcp::endpoint(address::from_string("1.1.1.1"), 0)); + p.pick_pieces(peer1, picked, 100, true + , tcp::endpoint(address::from_string("1.1.1.1"), 0), piece_picker::fast); TEST_CHECK(picked.size() == 11);