add new torrent_file_with_hashes() which includes piece layers (#6083)

for creating .torrent files

ChangeLog

@@ -1,5 +1,7 @@
 * 2.0.3 released
 
+	* add new torrent_file_with_hashes() which includes piece layers for
+	  creating .torrent files
 	* add file_prio_alert, posted when file priorities are updated
 	* fix issue where set_piece_hashes() would not propagate file errors
 	* add missing python binding for event_t

include/libtorrent/torrent.hpp

@@ -1066,7 +1066,9 @@ namespace libtorrent {
 		void hashes_rejected(hash_request const& req);
 		void verify_block_hashes(piece_index_t index);
 
-		std::shared_ptr<const torrent_info> get_torrent_copy();
+		std::shared_ptr<const torrent_info> get_torrent_file() const;
+
+		std::shared_ptr<torrent_info> get_torrent_copy_with_hashes() const;
 
 		std::vector<std::vector<sha256_hash>> get_piece_layers() const;
 

include/libtorrent/torrent_handle.hpp

@@ -843,17 +843,40 @@ namespace aux {
 			, std::string const& private_key
 			, std::string const& dh_params);
 
-		// Returns a pointer to the torrent_info object associated with this
-		// torrent. The torrent_info object may be a copy of the internal object.
-		// If the torrent doesn't have metadata, the pointer will not be
-		// initialized (i.e. a nullptr). The torrent may be in a state
-		// without metadata only if it was started without a .torrent file, e.g.
-		// by being added by magnet link.
+		// torrent_file() returns a pointer to the torrent_info object
+		// associated with this torrent. The torrent_info object may be a copy
+		// of the internal object. If the torrent doesn't have metadata, the
+		// pointer will not be initialized (i.e. a nullptr). The torrent may be
+		// in a state without metadata only if it was started without a .torrent
+		// file, e.g. by being added by magnet link.
+		//
 		// Note that the torrent_info object returned here may be a different
 		// instance than the one added to the session, with different attributes
 		// like piece layers, dht nodes and trackers. A torrent_info object does
 		// not round-trip cleanly when added to a session.
+		//
+		// This means if you want to create a .torrent file by passing the
+		// torrent_info object into create_torrent, you need to use
+		// torrent_file_with_hashes() instead.
+		//
+		// torrent_file_with_hashes() returns a *copy* of the internal
+		// torrent_info and piece layer hashes (if it's a v2 torrent). The piece
+		// layers will only be included if they are available. If this torrent
+		// was added from a .torrent file with piece layers or if it's seeding,
+		// the piece layers are available. This function is more expensive than
+		// torrent_file() since it needs to make copies of this information.
+		//
+		// When constructing a create_torrent object from a torrent_info that's
+		// in a session, you need to use this function.
+		//
+		// Note that a torrent added from a magnet link may not have the full
+		// merkle trees for all files, and hence not have the complete piece
+		// layers. In that state, you cannot create a .torrent file even from
+		// the torrent_info returned from torrent_file_with_hashes(). Once the
+		// torrent completes downloading all files, becoming a seed, you can
+		// make a .torrent file from it.
 		std::shared_ptr<const torrent_info> torrent_file() const;
+		std::shared_ptr<torrent_info> torrent_file_with_hashes() const;
 
 		// returns the piece layers for all files in the torrent. If this is a
 		// v1 torrent (and doesn't have any piece layers) it returns an empty

include/libtorrent/torrent_info.hpp

@@ -482,6 +482,7 @@ TORRENT_VERSION_NAMESPACE_3
 
 		// internal
 		bool v2_piece_hashes_verified() const { return bool(m_flags & v2_has_piece_hashes); }
+		void set_piece_layers(aux::vector<aux::vector<char>, file_index_t> pl);
 
 		// returns the piece size of file with ``index``. This will be the same as piece_length(),
 		// except for the last piece, which may be shorter.

simulation/test_transfer.cpp

@@ -46,6 +46,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "simulator/utils.hpp"
 #include "setup_swarm.hpp"
 #include "utils.hpp"
+#include "test_utils.hpp"
 #include "setup_transfer.hpp" // for addr()
 
 using namespace sim;
@@ -139,6 +140,7 @@ void run_test(
 
 	params.save_path = save_path(1);
 	ses[1]->async_add_torrent(params);
+	auto torrent = params.ti;
 
 	params.save_path = save_path(0);
 	if (flags & tx::magnet_download)
@@ -148,9 +150,18 @@ void run_test(
 	}
 	ses[0]->async_add_torrent(params);
 
-
 	sim::timer t(sim, lt::seconds(60), [&](boost::system::error_code const&)
 	{
+		auto h = ses[0]->get_torrents();
+		auto ti = h[0].torrent_file_with_hashes();
+
+		if (ti->v2())
+			TEST_EQUAL(ti->v2_piece_hashes_verified(), true);
+
+		auto downloaded = serialize(*ti);
+		auto added = serialize(*torrent);
+		TEST_CHECK(downloaded == added);
+
 		test(ses);
 
 		// shut down

src/torrent.cpp

@@ -1890,7 +1890,7 @@ bool is_downloading_state(int const st)
 				// complete and just look at those
 				if (!t->is_seed()) continue;
 
-				res.match(t->get_torrent_copy(), t->save_path());
+				res.match(t->get_torrent_file(), t->save_path());
 			}
 			for (auto const& c : m_torrent_file->collections())
 			{
@@ -1903,7 +1903,7 @@ bool is_downloading_state(int const st)
 					// complete and just look at those
 					if (!t->is_seed()) continue;
 
-					res.match(t->get_torrent_copy(), t->save_path());
+					res.match(t->get_torrent_file(), t->save_path());
 				}
 			}
 
@@ -6723,12 +6723,35 @@ namespace {
 		m_hash_picker->verify_block_hashes(index);
 	}
 
-	std::shared_ptr<const torrent_info> torrent::get_torrent_copy()
+	std::shared_ptr<const torrent_info> torrent::get_torrent_file() const
 	{
 		if (!m_torrent_file->is_valid()) return {};
 		return m_torrent_file;
 	}
 
+	std::shared_ptr<torrent_info> torrent::get_torrent_copy_with_hashes() const
+	{
+		if (!m_torrent_file->is_valid()) return {};
+		auto ret = std::make_shared<torrent_info>(*m_torrent_file);
+
+		if (ret->v2())
+		{
+			aux::vector<aux::vector<char>, file_index_t> v2_hashes;
+			for (auto const& tree : m_merkle_trees)
+			{
+				auto const& layer = tree.get_piece_layer();
+				std::vector<char> out_layer;
+				out_layer.reserve(layer.size() * sha256_hash::size());
+				for (auto const& h : layer)
+					out_layer.insert(out_layer.end(), h.data(), h.data() + sha256_hash::size());
+				v2_hashes.emplace_back(std::move(out_layer));
+			}
+			ret->set_piece_layers(std::move(v2_hashes));
+		}
+
+		return ret;
+	}
+
 	std::vector<std::vector<sha256_hash>> torrent::get_piece_layers() const
 	{
 		std::vector<std::vector<sha256_hash>> ret;

src/torrent_handle.cpp

@@ -716,7 +716,13 @@ namespace libtorrent {
 	std::shared_ptr<const torrent_info> torrent_handle::torrent_file() const
 	{
 		return sync_call_ret<std::shared_ptr<const torrent_info>>(
-			std::shared_ptr<const torrent_info>(), &torrent::get_torrent_copy);
+			std::shared_ptr<const torrent_info>(), &torrent::get_torrent_file);
+	}
+
+	std::shared_ptr<torrent_info> torrent_handle::torrent_file_with_hashes() const
+	{
+		return sync_call_ret<std::shared_ptr<torrent_info>>(
+			std::shared_ptr<torrent_info>(), &torrent::get_torrent_copy_with_hashes);
 	}
 
 	std::vector<std::vector<sha256_hash>> torrent_handle::piece_layers() const

src/torrent_info.cpp

@@ -1055,6 +1055,13 @@ namespace {
 
 	torrent_info::~torrent_info() = default;
 
+	// internal
+	void torrent_info::set_piece_layers(aux::vector<aux::vector<char>, file_index_t> pl)
+	{
+		m_piece_layers = pl;
+		m_flags |= v2_has_piece_hashes;
+	}
+
 	sha1_hash torrent_info::hash_for_piece(piece_index_t const index) const
 	{ return sha1_hash(hash_for_piece_ptr(index)); }
 

test/test_torrent_info.cpp

@@ -1005,7 +1005,7 @@ TORRENT_TEST(parse_torrents)
 
 TORRENT_TEST(parse_invalid_torrents)
 {
-	std::string root_dir = parent_path(current_working_directory());
+	std::string const root_dir = parent_path(current_working_directory());
 	for (auto const& e : test_error_torrents)
 	{
 		error_code ec;
@@ -1245,3 +1245,50 @@ TORRENT_TEST(copy_ptr)
 	a->val = 5;
 	TEST_EQUAL(b->val, 4);
 }
+
+TORRENT_TEST(torrent_info_with_hashes_roundtrip)
+{
+	std::string const root_dir = parent_path(current_working_directory());
+	std::string const filename = combine_path(combine_path(root_dir, "test_torrents"), "v2_only.torrent");
+
+	error_code ec;
+	std::vector<char> data;
+	TEST_CHECK(load_file(filename, data, ec) == 0);
+
+	auto ti = std::make_shared<torrent_info>(data, ec, from_span);
+	TEST_CHECK(!ec);
+	if (ec) std::printf(" loading(\"%s\") -> failed %s\n", filename.c_str()
+		, ec.message().c_str());
+
+	TEST_CHECK(ti->v2());
+	TEST_CHECK(!ti->v1());
+	TEST_EQUAL(ti->v2_piece_hashes_verified(), true);
+
+	add_torrent_params atp;
+	atp.ti = ti;
+	atp.save_path = ".";
+
+	session ses;
+	torrent_handle h = ses.add_torrent(atp);
+
+	TEST_CHECK(ti->v2());
+	TEST_CHECK(!ti->v1());
+
+	{
+		auto ti2 = h.torrent_file();
+		TEST_CHECK(ti2->v2());
+		TEST_CHECK(!ti2->v1());
+		TEST_EQUAL(ti2->v2_piece_hashes_verified(), false);
+	}
+
+	ti = h.torrent_file_with_hashes();
+
+	TEST_CHECK(ti->v2());
+	TEST_CHECK(!ti->v1());
+	TEST_EQUAL(ti->v2_piece_hashes_verified(), true);
+
+	std::vector<char> out_buffer = serialize(*ti);
+
+	TEST_EQUAL(out_buffer, data);
+}
+

test/test_utils.cpp

@@ -33,6 +33,8 @@ POSSIBILITY OF SUCH DAMAGE.
 
 #include "test_utils.hpp"
 #include "libtorrent/time.hpp"
+#include "libtorrent/torrent_info.hpp"
+#include "libtorrent/create_torrent.hpp"
 #include "libtorrent/aux_/merkle.hpp"
 
 #ifdef _WIN32
@@ -107,3 +109,14 @@ bool exists(std::string const& f)
 	lt::error_code ec;
 	return lt::exists(f, ec);
 }
+
+std::vector<char> serialize(lt::torrent_info const& ti)
+{
+	lt::create_torrent ct(ti);
+	ct.set_creation_date(0);
+	entry e = ct.generate();
+	std::vector<char> out_buffer;
+	bencode(std::back_inserter(out_buffer), e);
+	return out_buffer;
+}
+

test/test_utils.hpp

@@ -38,6 +38,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include "test.hpp"
 #include "libtorrent/time.hpp"
 #include "libtorrent/download_priority.hpp"
+#include "libtorrent/fwd.hpp"
 
 #include "libtorrent/sha1_hash.hpp"
 #include "libtorrent/aux_/vector.hpp"
@@ -60,6 +61,8 @@ constexpr inline lt::file_index_t operator "" _file(unsigned long long const p)
 constexpr inline lt::piece_index_t operator "" _piece(unsigned long long const p)
 { return lt::piece_index_t(static_cast<int>(p)); }
 
+EXPORT std::vector<char> serialize(lt::torrent_info const& ti);
+
 EXPORT lt::aux::vector<lt::sha256_hash> build_tree(int const size);
 
 #ifdef _WIN32