From 8ba970018c6e908146ab50e93632fd55e8601b76 Mon Sep 17 00:00:00 2001 From: Arvid Norberg Date: Fri, 19 Oct 2012 01:28:47 +0000 Subject: [PATCH] make bottled http requests response size limit configurable and bump default to 2 MiB --- ChangeLog | 1 + bindings/python/src/session_settings.cpp | 2 ++ docs/manual.rst | 6 ++++++ include/libtorrent/http_connection.hpp | 12 +++++++++++- include/libtorrent/session_settings.hpp | 4 ++++ src/http_connection.cpp | 13 +++++++------ src/http_tracker_connection.cpp | 2 +- src/session.cpp | 7 +++++++ src/session_impl.cpp | 2 ++ src/torrent.cpp | 7 +++++-- src/upnp.cpp | 6 +++--- test/test_http_connection.cpp | 2 +- 12 files changed, 50 insertions(+), 14 deletions(-) diff --git a/ChangeLog b/ChangeLog index 241bb3755..0770d95df 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,7 @@ * fix uTP edge case where udp socket buffer fills up * fix nagle implementation in uTP + * raised the limit for bottled http downloads to 2 MiB * add support for magnet links and URLs in python example client * fixed typo in python bindings' add_torrent_params * introduce a way to add built-in plugins from python diff --git a/bindings/python/src/session_settings.cpp b/bindings/python/src/session_settings.cpp index b3494353f..315886d48 100644 --- a/bindings/python/src/session_settings.cpp +++ b/bindings/python/src/session_settings.cpp @@ -172,6 +172,8 @@ void bind_session_settings() .def_readwrite("enable_incoming_utp", &session_settings::enable_incoming_utp) .def_readwrite("ssl_listen", &session_settings::ssl_listen) .def_readwrite("tracker_backoff", &session_settings::tracker_backoff) + .def_readwrite("ban_web_seeds", &session_settings::ban_web_seeds) + .def_readwrite("max_http_recv_buffer_size", &session_settings::max_http_recv_buffer_size) ; enum_("proxy_type") diff --git a/docs/manual.rst b/docs/manual.rst index e93da86b6..7c1df0c95 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -4601,6 +4601,7 @@ session_settings int tracker_backoff; bool ban_web_seeds; + int max_http_recv_buffer_size; }; ``version`` is automatically set to the libtorrent version you're using @@ -5495,6 +5496,11 @@ trackers. ``ban_web_seeds`` enables banning web seeds. By default, web seeds that send corrupt data are banned. +``max_http_recv_buffer_size`` specifies the max number of bytes to receive into +RAM buffers when downloading stuff over HTTP. Specifically when specifying a +URL to a .torrent file when adding a torrent or when announcing to an HTTP +tracker. The default is 2 MiB. + pe_settings =========== diff --git a/include/libtorrent/http_connection.hpp b/include/libtorrent/http_connection.hpp index 91794a4a6..5b54dfcdd 100644 --- a/include/libtorrent/http_connection.hpp +++ b/include/libtorrent/http_connection.hpp @@ -63,6 +63,8 @@ namespace libtorrent struct http_connection; class connection_queue; + +const int default_max_bottled_buffer_size = 2*1024*1024; typedef boost::function http_handler; @@ -73,10 +75,13 @@ typedef boost::function&)> http_ // when bottled, the last two arguments to the handler // will always be 0 -struct TORRENT_EXTRA_EXPORT http_connection : boost::enable_shared_from_this, boost::noncopyable +struct TORRENT_EXTRA_EXPORT http_connection + : boost::enable_shared_from_this + , boost::noncopyable { http_connection(io_service& ios, connection_queue& cc , http_handler const& handler, bool bottled = true + , int max_bottled_buffer_size = default_max_bottled_buffer_size , http_connect_handler const& ch = http_connect_handler() , http_filter_handler const& fh = http_filter_handler() #ifdef TORRENT_USE_OPENSSL @@ -152,11 +157,16 @@ private: time_duration m_completion_timeout; ptime m_last_receive; ptime m_start_time; + // bottled means that the handler is called once, when // everything is received (and buffered in memory). // non bottled means that once the headers have been // received, data is streamed to the handler bool m_bottled; + + // maximum size of bottled buffer + int m_max_bottled_buffer_size; + // set to true the first time the handler is called bool m_called; std::string m_hostname; diff --git a/include/libtorrent/session_settings.hpp b/include/libtorrent/session_settings.hpp index f93b26f0b..38f2a86a0 100644 --- a/include/libtorrent/session_settings.hpp +++ b/include/libtorrent/session_settings.hpp @@ -933,6 +933,10 @@ namespace libtorrent // when true, web seeds sending bad data will be banned bool ban_web_seeds; + + // http_connection maximum receive buffer size + // limits torrent file size for URL torrents + int max_http_recv_buffer_size; }; #ifndef TORRENT_DISABLE_DHT diff --git a/src/http_connection.cpp b/src/http_connection.cpp index adebd757f..ab1e39953 100644 --- a/src/http_connection.cpp +++ b/src/http_connection.cpp @@ -49,10 +49,10 @@ POSSIBILITY OF SUCH DAMAGE. namespace libtorrent { -enum { max_bottled_buffer = 1024 * 1024 }; - http_connection::http_connection(io_service& ios, connection_queue& cc - , http_handler const& handler, bool bottled + , http_handler const& handler + , bool bottled + , int max_bottled_buffer_size , http_connect_handler const& ch , http_filter_handler const& fh #ifdef TORRENT_USE_OPENSSL @@ -72,6 +72,7 @@ http_connection::http_connection(io_service& ios, connection_queue& cc , m_last_receive(time_now()) , m_start_time(time_now()) , m_bottled(bottled) + , m_max_bottled_buffer_size(max_bottled_buffer_size) , m_called(false) #ifdef TORRENT_USE_OPENSSL , m_ssl_ctx(ssl_ctx) @@ -625,7 +626,7 @@ void http_connection::callback(error_code e, char const* data, int size) if ((encoding == "gzip" || encoding == "x-gzip") && size > 0 && data) { std::string error; - if (inflate_gzip(data, size, buf, max_bottled_buffer, error)) + if (inflate_gzip(data, size, buf, m_max_bottled_buffer_size, error)) { if (m_handler) m_handler(errors::http_failed_decompress, m_parser, data, size, *this); close(); @@ -837,8 +838,8 @@ void http_connection::on_read(error_code const& e } if (int(m_recvbuffer.size()) == m_read_pos) - m_recvbuffer.resize((std::min)(m_read_pos + 2048, int(max_bottled_buffer))); - if (m_read_pos == max_bottled_buffer) + m_recvbuffer.resize((std::min)(m_read_pos + 2048, m_max_bottled_buffer_size)); + if (m_read_pos == m_max_bottled_buffer_size) { callback(asio::error::eof); close(); diff --git a/src/http_tracker_connection.cpp b/src/http_tracker_connection.cpp index 4cf572d22..d3660ef34 100644 --- a/src/http_tracker_connection.cpp +++ b/src/http_tracker_connection.cpp @@ -214,7 +214,7 @@ namespace libtorrent m_tracker_connection.reset(new http_connection(m_ios, m_cc , boost::bind(&http_tracker_connection::on_response, self(), _1, _2, _3, _4) - , true + , true, settings.max_http_recv_buffer_size , boost::bind(&http_tracker_connection::on_connect, self(), _1) , boost::bind(&http_tracker_connection::on_filter, self(), _1, _2) #ifdef TORRENT_USE_OPENSSL diff --git a/src/session.cpp b/src/session.cpp index 6766fc2ab..47f8c05e0 100644 --- a/src/session.cpp +++ b/src/session.cpp @@ -174,6 +174,9 @@ namespace libtorrent // disallow the buffer size to grow for the uTP socket set.utp_dynamic_sock_buf = false; + + // max 'bottled' http receive buffer/url torrent size + set.max_http_recv_buffer_size = 1024 * 1024; return set; } @@ -292,6 +295,9 @@ namespace libtorrent // allow the buffer size to grow for the uTP socket set.utp_dynamic_sock_buf = true; + // max 'bottled' http receive buffer/url torrent size + set.max_http_recv_buffer_size = 6 * 1024 * 1024; + return set; } @@ -1283,6 +1289,7 @@ namespace libtorrent , ssl_listen(4433) , tracker_backoff(250) , ban_web_seeds(true) + , max_http_recv_buffer_size(2*1024*1024) {} session_settings::~session_settings() {} diff --git a/src/session_impl.cpp b/src/session_impl.cpp index b44780b53..9615940c1 100644 --- a/src/session_impl.cpp +++ b/src/session_impl.cpp @@ -445,6 +445,8 @@ namespace aux { TORRENT_SETTING(boolean, lock_files) TORRENT_SETTING(integer, ssl_listen) TORRENT_SETTING(integer, tracker_backoff) + TORRENT_SETTING(boolean, ban_web_seeds) + TORRENT_SETTING(integer, max_http_recv_buffer_size) }; #undef TORRENT_SETTING diff --git a/src/torrent.cpp b/src/torrent.cpp index b13694f0e..be3aa2176 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -908,9 +908,12 @@ namespace libtorrent TORRENT_ASSERT(!m_url.empty()); TORRENT_ASSERT(!m_torrent_file->is_valid()); boost::shared_ptr conn( - new http_connection(m_ses.m_io_service, m_ses.m_half_open + new http_connection(m_ses.m_io_service, m_ses.m_half_open , boost::bind(&torrent::on_torrent_download, shared_from_this() - , _1, _2, _3, _4))); + , _1, _2, _3, _4) + , true //bottled + , m_ses.settings().max_http_recv_buffer_size //bottled buffer size + )); conn->get(m_url, seconds(30), 0, 0, 5, m_ses.m_settings.user_agent); set_state(torrent_status::downloading_metadata); } diff --git a/src/upnp.cpp b/src/upnp.cpp index c9f3da9a6..4999e4739 100644 --- a/src/upnp.cpp +++ b/src/upnp.cpp @@ -699,7 +699,7 @@ void upnp::update_map(rootdevice& d, int i, mutex::scoped_lock& l) if (d.upnp_connection) d.upnp_connection->close(); d.upnp_connection.reset(new http_connection(m_io_service , m_cc, boost::bind(&upnp::on_upnp_map_response, self(), _1, _2 - , boost::ref(d), i, _5), true + , boost::ref(d), i, _5), true, default_max_bottled_buffer_size , boost::bind(&upnp::create_port_mapping, self(), _1, boost::ref(d), i))); d.upnp_connection->start(d.hostname, to_string(d.port).elems @@ -710,7 +710,7 @@ void upnp::update_map(rootdevice& d, int i, mutex::scoped_lock& l) if (d.upnp_connection) d.upnp_connection->close(); d.upnp_connection.reset(new http_connection(m_io_service , m_cc, boost::bind(&upnp::on_upnp_unmap_response, self(), _1, _2 - , boost::ref(d), i, _5), true + , boost::ref(d), i, _5), true, default_max_bottled_buffer_size , boost::bind(&upnp::delete_port_mapping, self(), boost::ref(d), i))); d.upnp_connection->start(d.hostname, to_string(d.port).elems , seconds(10), 1); @@ -958,7 +958,7 @@ void upnp::on_upnp_xml(error_code const& e d.upnp_connection.reset(new http_connection(m_io_service , m_cc, boost::bind(&upnp::on_upnp_get_ip_address_response, self(), _1, _2 - , boost::ref(d), _5), true + , boost::ref(d), _5), true, default_max_bottled_buffer_size , boost::bind(&upnp::get_ip_address, self(), boost::ref(d)))); d.upnp_connection->start(d.hostname, to_string(d.port).elems , seconds(10), 1); diff --git a/test/test_http_connection.cpp b/test/test_http_connection.cpp index f49e044e7..c7d01f1b6 100644 --- a/test/test_http_connection.cpp +++ b/test/test_http_connection.cpp @@ -116,7 +116,7 @@ void run_test(std::string const& url, int size, int status, int connected << " error: " << (ec?ec->message():"no error") << std::endl; boost::shared_ptr h(new http_connection(ios, cq - , &::http_handler, true, &::http_connect_handler)); + , &::http_handler, true, 1024*1024, &::http_connect_handler)); h->get(url, seconds(1), 0, &ps); ios.reset(); error_code e;