From b41cdd6469b9d392ceded785c5ad0df6bec5511d Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Mon, 8 Dec 2008 06:36:22 +0000 Subject: [PATCH] initial super seeding support --- ChangeLog | 2 +- docs/manual.rst | 15 +++++ include/libtorrent/peer_connection.hpp | 14 +++- include/libtorrent/torrent.hpp | 10 +++ include/libtorrent/torrent_handle.hpp | 3 + src/bt_peer_connection.cpp | 27 +++++++- src/peer_connection.cpp | 93 ++++++++++++++++++++++++++ src/torrent.cpp | 73 ++++++++++++++++++++ src/torrent_handle.cpp | 12 ++++ test/setup_transfer.cpp | 3 +- test/setup_transfer.hpp | 2 +- test/test_swarm.cpp | 37 +++++----- 12 files changed, 267 insertions(+), 24 deletions(-) diff --git a/ChangeLog b/ChangeLog index c6f126554..1546aca4d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ - + * added super seeding * added add_piece() function to inject data from external sources * add_tracker() function added to torrent_handle * if there is no working tracker, current_tracker is the diff --git a/docs/manual.rst b/docs/manual.rst index 4af647c60..84058de91 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -1717,6 +1717,9 @@ Its declaration looks like this:: void rename_file(int index, boost::filesystem::wpath) const; storage_interface* get_storage_impl() const; + bool super_seeding() const; + void super_seeding(bool on) const; + enum flags_t { overwrite_existing = 1 }; void add_piece(int piece, char const* data, int flags = 0) const; @@ -1871,6 +1874,18 @@ get_storage_impl() Returns the storage implementation for this torrent. This depends on the storage contructor function that was passed to ``session::add_torrent``. +super_seeding() +--------------- + + :: + + bool super_seeding() const; + void super_seeding(bool on) const; + +Enables or disabled super seeding/initial seeding for this torrent. The torrent +needs to be a seed for this to take effect. The overload that returns a bool +tells you of super seeding is enabled or not. + add_piece() ----------- diff --git a/include/libtorrent/peer_connection.hpp b/include/libtorrent/peer_connection.hpp index 9371a7d5f..bb55b2eaf 100644 --- a/include/libtorrent/peer_connection.hpp +++ b/include/libtorrent/peer_connection.hpp @@ -210,6 +210,11 @@ namespace libtorrent // this adds an announcement in the announcement queue // it will let the peer know that we have the given piece void announce_piece(int index); + + // this will tell the peer to announce the given piece + // and only allow it to request that piece + void superseed_piece(int index); + int superseed_piece() const { return m_superseed_piece; } // tells if this connection has data it want to send // and has enough upload bandwidth quota left to send it. @@ -758,7 +763,14 @@ namespace libtorrent // so that it can be removed from the queue // once the connection completes int m_connection_ticket; - + + // if this is -1, superseeding is not active. If it is >= 0 + // this is the piece that is available to this peer. Only + // this piece can be downloaded from us by this peer. + // This will remain the current piece for this peer until + // another peer sends us a have message for this piece + int m_superseed_piece; + // bytes downloaded since last second // timer timeout; used for determining // approx download rate diff --git a/include/libtorrent/torrent.hpp b/include/libtorrent/torrent.hpp index aac562aaf..4409fa267 100644 --- a/include/libtorrent/torrent.hpp +++ b/include/libtorrent/torrent.hpp @@ -422,6 +422,12 @@ namespace libtorrent // -------------------------------------------- // PIECE MANAGEMENT + bool super_seeding() const + { return m_super_seeding; } + + void super_seeding(bool on); + int get_piece_to_super_seed(bitfield const&); + // returns true if we have downloaded the given piece bool have_piece(int index) const { @@ -967,6 +973,10 @@ namespace libtorrent // has been initialized with files_checked(). bool m_connections_initialized:1; + // if this is true, we're currently super seeding this + // torrent. + bool m_super_seeding:1; + // is set to true every time there is an incoming // connection to this torrent bool m_has_incoming:1; diff --git a/include/libtorrent/torrent_handle.hpp b/include/libtorrent/torrent_handle.hpp index 2a21894b4..128931033 100644 --- a/include/libtorrent/torrent_handle.hpp +++ b/include/libtorrent/torrent_handle.hpp @@ -479,6 +479,9 @@ namespace libtorrent void rename_file(int index, fs::path const& new_name) const; void rename_file(int index, fs::wpath const& new_name) const; + bool super_seeding() const; + void super_seeding(bool on) const; + sha1_hash info_hash() const; bool operator==(const torrent_handle& h) const diff --git a/src/bt_peer_connection.cpp b/src/bt_peer_connection.cpp index 25d02392e..d150840d2 100644 --- a/src/bt_peer_connection.cpp +++ b/src/bt_peer_connection.cpp @@ -987,7 +987,7 @@ namespace libtorrent r.piece = detail::read_int32(ptr); r.start = detail::read_int32(ptr); r.length = detail::read_int32(ptr); - + incoming_request(r); } @@ -1452,7 +1452,22 @@ namespace libtorrent // in this case, have_all or have_none should be sent instead TORRENT_ASSERT(!m_supports_fast || !t->is_seed() || t->num_have() != 0); - if (m_supports_fast && t->is_seed()) + if (t->super_seeding()) + { + if (m_supports_fast) write_have_none(); + + // if we are super seeding, pretend to not have any piece + // and don't send a bitfield +#ifdef TORRENT_DEBUG + m_sent_bitfield = true; +#endif + + // bootstrap superseeding by sending one have message + superseed_piece(t->get_piece_to_super_seed( + get_bitfield())); + return; + } + else if (m_supports_fast && t->is_seed()) { write_have_all(); send_allowed_set(); @@ -1477,6 +1492,7 @@ namespace libtorrent } int num_pieces = t->torrent_file().num_pieces(); + int lazy_pieces[50]; int num_lazy_pieces = 0; int lazy_piece = 0; @@ -1553,6 +1569,7 @@ namespace libtorrent << " ==> HAVE [ piece: " << lazy_pieces[i] << "]\n"; #endif } + // TODO: if we're finished, send upload_only message } if (m_supports_fast) @@ -1588,7 +1605,11 @@ namespace libtorrent handshake["reqq"] = m_ses.settings().max_allowed_in_request_queue; boost::shared_ptr t = associated_torrent().lock(); TORRENT_ASSERT(t); - if (t->is_finished()) handshake["upload_only"] = 1; + + // if we're using lazy bitfields or if we're super seeding, don't say + // we're upload only, since it might make peers disconnect + if (t->is_finished() && !t->super_seeding() && !m_ses.settings().lazy_bitfields) + handshake["upload_only"] = 1; tcp::endpoint ep = m_ses.get_ipv6_interface(); if (!is_any(ep.address())) diff --git a/src/peer_connection.cpp b/src/peer_connection.cpp index 2d030369f..79ac4dc2c 100644 --- a/src/peer_connection.cpp +++ b/src/peer_connection.cpp @@ -106,6 +106,7 @@ namespace libtorrent , m_peer_info(peerinfo) , m_speed(slow) , m_connection_ticket(-1) + , m_superseed_piece(-1) , m_remote_bytes_dled(0) , m_remote_dl_rate(0) , m_outstanding_writing_bytes(0) @@ -214,6 +215,7 @@ namespace libtorrent , m_peer_info(peerinfo) , m_speed(slow) , m_connection_ticket(-1) + , m_superseed_piece(-1) , m_remote_bytes_dled(0) , m_remote_dl_rate(0) , m_outstanding_writing_bytes(0) @@ -395,6 +397,15 @@ namespace libtorrent boost::shared_ptr t = m_torrent.lock(); TORRENT_ASSERT(t); + if (t->super_seeding()) + { +#ifdef TORRENT_VERBOSE_LOGGING + (*m_logger) << time_now_string() + << " *** SKIPPING ALLOWED SET BECAUSE OF SUPER SEEDING\n"; +#endif + return; + } + int num_allowed_pieces = m_ses.settings().allowed_fast_set_size; int num_pieces = t->torrent_file().num_pieces(); @@ -1157,6 +1168,13 @@ namespace libtorrent } t->get_policy().not_interested(*this); + + if (t->super_seeding() && m_superseed_piece != -1) + { + // assume the peer has the piece we're superseeding to it + // and give it another one + if (!m_have_piece[m_superseed_piece]) incoming_have(m_superseed_piece); + } } // ----------------------------- @@ -1216,6 +1234,20 @@ namespace libtorrent return; } + if (t->super_seeding()) + { + // if we're superseeding and the peer just told + // us that it completed the piece we're superseeding + // to it, change the superseeding piece for this peer + // if the peer optimizes out redundant have messages + // this will be handled when the peer sends not-interested + // instead. + if (m_superseed_piece == index) + { + superseed_piece(t->get_piece_to_super_seed(m_have_piece)); + } + } + if (m_have_piece[index]) { #if defined TORRENT_VERBOSE_LOGGING || defined TORRENT_ERROR_LOGGING @@ -1408,6 +1440,31 @@ namespace libtorrent boost::shared_ptr t = m_torrent.lock(); TORRENT_ASSERT(t); + if (m_superseed_piece != -1 + && r.piece != m_superseed_piece) + { + ++m_num_invalid_requests; +#ifdef TORRENT_VERBOSE_LOGGING + (*m_logger) << time_now_string() + << " <== INVALID_SUPER_SEED_REQUEST [ " + "piece: " << r.piece << " | " + "s: " << r.start << " | " + "l: " << r.length << " | " + "i: " << m_peer_interested << " | " + "t: " << (int)t->torrent_file().piece_size(r.piece) << " | " + "n: " << t->torrent_file().num_pieces() << " | " + "h: " << t->have_piece(r.piece) << " | " + "ss: " << m_superseed_piece << " ]\n"; +#endif + + if (t->alerts().should_post()) + { + t->alerts().post_alert(invalid_request_alert( + t->get_handle(), m_remote, m_peer_id, r)); + } + return; + } + // if we haven't received a bitfield, it was // probably omitted, which is the same as 'have_none' if (!m_bitfield_received) incoming_have_none(); @@ -2755,6 +2812,42 @@ namespace libtorrent m_packet_size = packet_size; } + void peer_connection::superseed_piece(int index) + { + if (index == -1) + { + if (m_superseed_piece == -1) return; + m_superseed_piece = -1; + +#ifdef TORRENT_VERBOSE_LOGGING + (*m_logger) << time_now_string() + << " *** ending super seed mode\n"; +#endif + boost::shared_ptr t = m_torrent.lock(); + assert(t); + + for (int i = 0; i < m_have_piece.size(); ++i) + { + if (m_have_piece[i] || !t->have_piece(i)) continue; +#ifdef TORRENT_VERBOSE_LOGGING + (*m_logger) << " ==> HAVE [ piece: " << i << "] (ending super seed)\n"; +#endif + write_have(i); + } + + return; + } + + assert(!has_piece(index)); + +#ifdef TORRENT_VERBOSE_LOGGING + (*m_logger) << time_now_string() + << " ==> HAVE [ piece: " << index << "] (super seed)\n"; +#endif + write_have(index); + m_superseed_piece = index; + } + void peer_connection::second_tick(float tick_interval) { ptime now(time_now()); diff --git a/src/torrent.cpp b/src/torrent.cpp index 10793cf50..b455a7d28 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -184,6 +184,7 @@ namespace libtorrent , m_sequential_download(false) , m_got_tracker_response(false) , m_connections_initialized(true) + , m_super_seeding(false) , m_has_incoming(false) , m_files_checked(false) , m_announcing(false) @@ -258,6 +259,7 @@ namespace libtorrent , m_sequential_download(false) , m_got_tracker_response(false) , m_connections_initialized(false) + , m_super_seeding(false) , m_has_incoming(false) , m_files_checked(false) , m_announcing(false) @@ -1741,6 +1743,74 @@ namespace libtorrent m_host_resolver.cancel(); } + void torrent::super_seeding(bool on) + { + if (on == m_super_seeding) return; + + // don't turn on super seeding if we're not a seed + TORRENT_ASSERT(!on || is_seed() || !m_files_checked); + if (on && !is_seed() && m_files_checked) return; + m_super_seeding = on; + + if (m_super_seeding) return; + + // disable super seeding for all peers + for (peer_iterator i = begin(); i != end(); ++i) + { + (*i)->superseed_piece(-1); + } + } + + int torrent::get_piece_to_super_seed(bitfield const& bits) + { + // return a piece with low availability that is not in + // the bitfield and that is not currently being super + // seeded by any peer + TORRENT_ASSERT(m_super_seeding); + + // do a linear search from the first piece + int min_availability = 9999; + std::vector avail_vec; + for (int i = 0; i < m_torrent_file->num_pieces(); ++i) + { + if (bits[i]) continue; + + int availability = 0; + for (const_peer_iterator j = begin(); j != end(); ++j) + { + if ((*j)->superseed_piece() == i) + { + // avoid superseeding the same piece to more than one + // peer if we can avoid it. Do this by artificially + // increase the availability + availability = 999; + break; + } + if ((*j)->has_piece(i)) ++availability; + } + if (availability > min_availability) continue; + if (availability == min_availability) + { + avail_vec.push_back(i); + continue; + } + TORRENT_ASSERT(availability < min_availability); + min_availability = availability; + avail_vec.clear(); + avail_vec.push_back(i); + } + + if (min_availability > 1) + { + // if the minimum availability is 2 or more, + // we shouldn't be super seeding any more + super_seeding(false); + return -1; + } + + return avail_vec[rand() % avail_vec.size()]; + } + void torrent::on_files_deleted(int ret, disk_io_job const& j) { session_impl::mutex_t::scoped_lock l(m_ses.m_mutex); @@ -3581,6 +3651,9 @@ namespace libtorrent if (!is_seed()) { + // turn off super seeding if we're not a seed + if (m_super_seeding) m_super_seeding = false; + // if we just finished checking and we're not a seed, we are // likely to be unpaused if (m_ses.m_auto_manage_time_scaler > 1) diff --git a/src/torrent_handle.cpp b/src/torrent_handle.cpp index d15698600..09e5ec2ff 100644 --- a/src/torrent_handle.cpp +++ b/src/torrent_handle.cpp @@ -632,6 +632,18 @@ namespace libtorrent TORRENT_FORWARD(scrape_tracker()); } + bool torrent_handle::super_seeding() const + { + INVARIANT_CHECK; + TORRENT_FORWARD_RETURN(super_seeding(), false); + } + + void torrent_handle::super_seeding(bool on) const + { + INVARIANT_CHECK; + TORRENT_FORWARD(super_seeding(on)); + } + void torrent_handle::set_ratio(float ratio) const { INVARIANT_CHECK; diff --git a/test/setup_transfer.cpp b/test/setup_transfer.cpp index 8fe17ab1e..bbf20e098 100644 --- a/test/setup_transfer.cpp +++ b/test/setup_transfer.cpp @@ -241,7 +241,7 @@ boost::tuple setup_transfer(session* ses1, session* ses2, session* ses3 , bool clear_files, bool use_metadata_transfer, bool connect_peers , std::string suffix, int piece_size - , boost::intrusive_ptr* torrent) + , boost::intrusive_ptr* torrent, bool super_seeding) { using namespace boost::filesystem; @@ -286,6 +286,7 @@ setup_transfer(session* ses1, session* ses2, session* ses3 // use the same files sha1_hash info_hash = t->info_hash(); torrent_handle tor1 = ses1->add_torrent(clone_ptr(t), "./tmp1" + suffix); + tor1.super_seeding(super_seeding); TEST_CHECK(!ses1->get_torrents().empty()); torrent_handle tor2; torrent_handle tor3; diff --git a/test/setup_transfer.hpp b/test/setup_transfer.hpp index 8c9a4a476..c7472ef5d 100644 --- a/test/setup_transfer.hpp +++ b/test/setup_transfer.hpp @@ -50,7 +50,7 @@ boost::tuple* torrent = 0); + , boost::intrusive_ptr* torrent = 0, bool super_seeding = false); void start_web_server(int port, bool ssl = false); void stop_web_server(int port); diff --git a/test/test_swarm.cpp b/test/test_swarm.cpp index 6475da217..4a08b90f0 100644 --- a/test/test_swarm.cpp +++ b/test/test_swarm.cpp @@ -44,10 +44,15 @@ POSSIBILITY OF SUCH DAMAGE. using boost::filesystem::remove_all; using boost::filesystem::exists; -void test_swarm() +void test_swarm(bool super_seeding = false) { using namespace libtorrent; + // in case the previous run was terminated + try { remove_all("./tmp1_swarm"); } catch (std::exception&) {} + try { remove_all("./tmp2_swarm"); } catch (std::exception&) {} + try { remove_all("./tmp3_swarm"); } catch (std::exception&) {} + session ses1(fingerprint("LT", 0, 1, 0, 0), std::make_pair(48000, 49000)); ses1.set_alert_mask(alert::all_categories & ~alert::progress_notification); session ses2(fingerprint("LT", 0, 1, 0, 0), std::make_pair(49000, 50000)); @@ -87,7 +92,8 @@ void test_swarm() torrent_handle tor3; // test using piece sizes smaller than 16kB - boost::tie(tor1, tor2, tor3) = setup_transfer(&ses1, &ses2, &ses3, true, false, true, "_swarm", 8 * 1024); + boost::tie(tor1, tor2, tor3) = setup_transfer(&ses1, &ses2, &ses3, true + , false, true, "_swarm", 8 * 1024, 0, super_seeding); float sum_dl_rate2 = 0.f; float sum_dl_rate3 = 0.f; @@ -171,7 +177,7 @@ void test_swarm() // about 2 seconds ptime start = time_now(); alert const* ret; - while (ret = ses1.wait_for_alert(seconds(2))) + while ((ret = ses1.wait_for_alert(seconds(2)))) { a = ses1.pop_alert(); std::cerr << ret->message() << std::endl; @@ -179,21 +185,7 @@ void test_swarm() } TEST_CHECK(time_now() - start < seconds(3)); TEST_CHECK(time_now() - start >= seconds(2)); -} -int test_main() -{ - using namespace libtorrent; - using namespace boost::filesystem; - - // in case the previous run was terminated - try { remove_all("./tmp1_swarm"); } catch (std::exception&) {} - try { remove_all("./tmp2_swarm"); } catch (std::exception&) {} - try { remove_all("./tmp3_swarm"); } catch (std::exception&) {} - - test_swarm(); - - test_sleep(2000); TEST_CHECK(!exists("./tmp1_swarm/temporary")); TEST_CHECK(!exists("./tmp2_swarm/temporary")); TEST_CHECK(!exists("./tmp3_swarm/temporary")); @@ -201,7 +193,18 @@ int test_main() remove_all("./tmp1_swarm"); remove_all("./tmp2_swarm"); remove_all("./tmp3_swarm"); +} +int test_main() +{ + using namespace libtorrent; + using namespace boost::filesystem; + + test_swarm(); + + // with super seeding + test_swarm(true); + return 0; }