From 456c9448a576a296d851f4f043e6363cf789c62f Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Sun, 28 Mar 2021 08:29:42 +0200 Subject: [PATCH] add new torrent_file_with_hashes() which includes piece layers (#6083) for creating .torrent files --- ChangeLog | 2 ++ include/libtorrent/torrent.hpp | 4 ++- include/libtorrent/torrent_handle.hpp | 35 +++++++++++++++---- include/libtorrent/torrent_info.hpp | 1 + simulation/test_transfer.cpp | 13 ++++++- src/torrent.cpp | 29 ++++++++++++++-- src/torrent_handle.cpp | 8 ++++- src/torrent_info.cpp | 7 ++++ test/test_torrent_info.cpp | 49 ++++++++++++++++++++++++++- test/test_utils.cpp | 13 +++++++ test/test_utils.hpp | 3 ++ 11 files changed, 151 insertions(+), 13 deletions(-) diff --git a/ChangeLog b/ChangeLog index d7c5c4342..85462e5e5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ * 2.0.3 released + * add new torrent_file_with_hashes() which includes piece layers for + creating .torrent files * add file_prio_alert, posted when file priorities are updated * fix issue where set_piece_hashes() would not propagate file errors * add missing python binding for event_t diff --git a/include/libtorrent/torrent.hpp b/include/libtorrent/torrent.hpp index 8e98e1faa..cf2cc810e 100644 --- a/include/libtorrent/torrent.hpp +++ b/include/libtorrent/torrent.hpp @@ -1066,7 +1066,9 @@ namespace libtorrent { void hashes_rejected(hash_request const& req); void verify_block_hashes(piece_index_t index); - std::shared_ptr get_torrent_copy(); + std::shared_ptr get_torrent_file() const; + + std::shared_ptr get_torrent_copy_with_hashes() const; std::vector> get_piece_layers() const; diff --git a/include/libtorrent/torrent_handle.hpp b/include/libtorrent/torrent_handle.hpp index be656f6a8..af8210d69 100644 --- a/include/libtorrent/torrent_handle.hpp +++ b/include/libtorrent/torrent_handle.hpp @@ -843,17 +843,40 @@ namespace aux { , std::string const& private_key , std::string const& dh_params); - // Returns a pointer to the torrent_info object associated with this - // torrent. The torrent_info object may be a copy of the internal object. - // If the torrent doesn't have metadata, the pointer will not be - // initialized (i.e. a nullptr). The torrent may be in a state - // without metadata only if it was started without a .torrent file, e.g. - // by being added by magnet link. + // torrent_file() returns a pointer to the torrent_info object + // associated with this torrent. The torrent_info object may be a copy + // of the internal object. If the torrent doesn't have metadata, the + // pointer will not be initialized (i.e. a nullptr). The torrent may be + // in a state without metadata only if it was started without a .torrent + // file, e.g. by being added by magnet link. + // // Note that the torrent_info object returned here may be a different // instance than the one added to the session, with different attributes // like piece layers, dht nodes and trackers. A torrent_info object does // not round-trip cleanly when added to a session. + // + // This means if you want to create a .torrent file by passing the + // torrent_info object into create_torrent, you need to use + // torrent_file_with_hashes() instead. + // + // torrent_file_with_hashes() returns a *copy* of the internal + // torrent_info and piece layer hashes (if it's a v2 torrent). The piece + // layers will only be included if they are available. If this torrent + // was added from a .torrent file with piece layers or if it's seeding, + // the piece layers are available. This function is more expensive than + // torrent_file() since it needs to make copies of this information. + // + // When constructing a create_torrent object from a torrent_info that's + // in a session, you need to use this function. + // + // Note that a torrent added from a magnet link may not have the full + // merkle trees for all files, and hence not have the complete piece + // layers. In that state, you cannot create a .torrent file even from + // the torrent_info returned from torrent_file_with_hashes(). Once the + // torrent completes downloading all files, becoming a seed, you can + // make a .torrent file from it. std::shared_ptr torrent_file() const; + std::shared_ptr torrent_file_with_hashes() const; // returns the piece layers for all files in the torrent. If this is a // v1 torrent (and doesn't have any piece layers) it returns an empty diff --git a/include/libtorrent/torrent_info.hpp b/include/libtorrent/torrent_info.hpp index 5d372675c..3d63d6aef 100644 --- a/include/libtorrent/torrent_info.hpp +++ b/include/libtorrent/torrent_info.hpp @@ -482,6 +482,7 @@ TORRENT_VERSION_NAMESPACE_3 // internal bool v2_piece_hashes_verified() const { return bool(m_flags & v2_has_piece_hashes); } + void set_piece_layers(aux::vector, file_index_t> pl); // returns the piece size of file with ``index``. This will be the same as piece_length(), // except for the last piece, which may be shorter. diff --git a/simulation/test_transfer.cpp b/simulation/test_transfer.cpp index dc344044b..f8258041c 100644 --- a/simulation/test_transfer.cpp +++ b/simulation/test_transfer.cpp @@ -46,6 +46,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "simulator/utils.hpp" #include "setup_swarm.hpp" #include "utils.hpp" +#include "test_utils.hpp" #include "setup_transfer.hpp" // for addr() using namespace sim; @@ -139,6 +140,7 @@ void run_test( params.save_path = save_path(1); ses[1]->async_add_torrent(params); + auto torrent = params.ti; params.save_path = save_path(0); if (flags & tx::magnet_download) @@ -148,9 +150,18 @@ void run_test( } ses[0]->async_add_torrent(params); - sim::timer t(sim, lt::seconds(60), [&](boost::system::error_code const&) { + auto h = ses[0]->get_torrents(); + auto ti = h[0].torrent_file_with_hashes(); + + if (ti->v2()) + TEST_EQUAL(ti->v2_piece_hashes_verified(), true); + + auto downloaded = serialize(*ti); + auto added = serialize(*torrent); + TEST_CHECK(downloaded == added); + test(ses); // shut down diff --git a/src/torrent.cpp b/src/torrent.cpp index a92426662..02f0592ba 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -1890,7 +1890,7 @@ bool is_downloading_state(int const st) // complete and just look at those if (!t->is_seed()) continue; - res.match(t->get_torrent_copy(), t->save_path()); + res.match(t->get_torrent_file(), t->save_path()); } for (auto const& c : m_torrent_file->collections()) { @@ -1903,7 +1903,7 @@ bool is_downloading_state(int const st) // complete and just look at those if (!t->is_seed()) continue; - res.match(t->get_torrent_copy(), t->save_path()); + res.match(t->get_torrent_file(), t->save_path()); } } @@ -6723,12 +6723,35 @@ namespace { m_hash_picker->verify_block_hashes(index); } - std::shared_ptr torrent::get_torrent_copy() + std::shared_ptr torrent::get_torrent_file() const { if (!m_torrent_file->is_valid()) return {}; return m_torrent_file; } + std::shared_ptr torrent::get_torrent_copy_with_hashes() const + { + if (!m_torrent_file->is_valid()) return {}; + auto ret = std::make_shared(*m_torrent_file); + + if (ret->v2()) + { + aux::vector, file_index_t> v2_hashes; + for (auto const& tree : m_merkle_trees) + { + auto const& layer = tree.get_piece_layer(); + std::vector out_layer; + out_layer.reserve(layer.size() * sha256_hash::size()); + for (auto const& h : layer) + out_layer.insert(out_layer.end(), h.data(), h.data() + sha256_hash::size()); + v2_hashes.emplace_back(std::move(out_layer)); + } + ret->set_piece_layers(std::move(v2_hashes)); + } + + return ret; + } + std::vector> torrent::get_piece_layers() const { std::vector> ret; diff --git a/src/torrent_handle.cpp b/src/torrent_handle.cpp index f8f12453a..ba3fc2600 100644 --- a/src/torrent_handle.cpp +++ b/src/torrent_handle.cpp @@ -716,7 +716,13 @@ namespace libtorrent { std::shared_ptr torrent_handle::torrent_file() const { return sync_call_ret>( - std::shared_ptr(), &torrent::get_torrent_copy); + std::shared_ptr(), &torrent::get_torrent_file); + } + + std::shared_ptr torrent_handle::torrent_file_with_hashes() const + { + return sync_call_ret>( + std::shared_ptr(), &torrent::get_torrent_copy_with_hashes); } std::vector> torrent_handle::piece_layers() const diff --git a/src/torrent_info.cpp b/src/torrent_info.cpp index b33c373fe..af586cc2a 100644 --- a/src/torrent_info.cpp +++ b/src/torrent_info.cpp @@ -1055,6 +1055,13 @@ namespace { torrent_info::~torrent_info() = default; + // internal + void torrent_info::set_piece_layers(aux::vector, file_index_t> pl) + { + m_piece_layers = pl; + m_flags |= v2_has_piece_hashes; + } + sha1_hash torrent_info::hash_for_piece(piece_index_t const index) const { return sha1_hash(hash_for_piece_ptr(index)); } diff --git a/test/test_torrent_info.cpp b/test/test_torrent_info.cpp index d3a463fbf..00bf709fd 100644 --- a/test/test_torrent_info.cpp +++ b/test/test_torrent_info.cpp @@ -1005,7 +1005,7 @@ TORRENT_TEST(parse_torrents) TORRENT_TEST(parse_invalid_torrents) { - std::string root_dir = parent_path(current_working_directory()); + std::string const root_dir = parent_path(current_working_directory()); for (auto const& e : test_error_torrents) { error_code ec; @@ -1245,3 +1245,50 @@ TORRENT_TEST(copy_ptr) a->val = 5; TEST_EQUAL(b->val, 4); } + +TORRENT_TEST(torrent_info_with_hashes_roundtrip) +{ + std::string const root_dir = parent_path(current_working_directory()); + std::string const filename = combine_path(combine_path(root_dir, "test_torrents"), "v2_only.torrent"); + + error_code ec; + std::vector data; + TEST_CHECK(load_file(filename, data, ec) == 0); + + auto ti = std::make_shared(data, ec, from_span); + TEST_CHECK(!ec); + if (ec) std::printf(" loading(\"%s\") -> failed %s\n", filename.c_str() + , ec.message().c_str()); + + TEST_CHECK(ti->v2()); + TEST_CHECK(!ti->v1()); + TEST_EQUAL(ti->v2_piece_hashes_verified(), true); + + add_torrent_params atp; + atp.ti = ti; + atp.save_path = "."; + + session ses; + torrent_handle h = ses.add_torrent(atp); + + TEST_CHECK(ti->v2()); + TEST_CHECK(!ti->v1()); + + { + auto ti2 = h.torrent_file(); + TEST_CHECK(ti2->v2()); + TEST_CHECK(!ti2->v1()); + TEST_EQUAL(ti2->v2_piece_hashes_verified(), false); + } + + ti = h.torrent_file_with_hashes(); + + TEST_CHECK(ti->v2()); + TEST_CHECK(!ti->v1()); + TEST_EQUAL(ti->v2_piece_hashes_verified(), true); + + std::vector out_buffer = serialize(*ti); + + TEST_EQUAL(out_buffer, data); +} + diff --git a/test/test_utils.cpp b/test/test_utils.cpp index 1eb9eacce..ada2e80e9 100644 --- a/test/test_utils.cpp +++ b/test/test_utils.cpp @@ -33,6 +33,8 @@ POSSIBILITY OF SUCH DAMAGE. #include "test_utils.hpp" #include "libtorrent/time.hpp" +#include "libtorrent/torrent_info.hpp" +#include "libtorrent/create_torrent.hpp" #include "libtorrent/aux_/merkle.hpp" #ifdef _WIN32 @@ -107,3 +109,14 @@ bool exists(std::string const& f) lt::error_code ec; return lt::exists(f, ec); } + +std::vector serialize(lt::torrent_info const& ti) +{ + lt::create_torrent ct(ti); + ct.set_creation_date(0); + entry e = ct.generate(); + std::vector out_buffer; + bencode(std::back_inserter(out_buffer), e); + return out_buffer; +} + diff --git a/test/test_utils.hpp b/test/test_utils.hpp index 54650d213..cdd1bd3c3 100644 --- a/test/test_utils.hpp +++ b/test/test_utils.hpp @@ -38,6 +38,7 @@ POSSIBILITY OF SUCH DAMAGE. #include "test.hpp" #include "libtorrent/time.hpp" #include "libtorrent/download_priority.hpp" +#include "libtorrent/fwd.hpp" #include "libtorrent/sha1_hash.hpp" #include "libtorrent/aux_/vector.hpp" @@ -60,6 +61,8 @@ constexpr inline lt::file_index_t operator "" _file(unsigned long long const p) constexpr inline lt::piece_index_t operator "" _piece(unsigned long long const p) { return lt::piece_index_t(static_cast(p)); } +EXPORT std::vector serialize(lt::torrent_info const& ti); + EXPORT lt::aux::vector build_tree(int const size); #ifdef _WIN32