added support for chunked encoding for web seeds

ChangeLog

@@ -1,3 +1,4 @@
+	* support chunked encoding for web seeds (only for BEP 19, web seeds)
 	* optimized session startup time
 	* support SSL for web seeds, through all proxies
 	* support extending web seeds with custom authorization and extra headers

include/libtorrent/http_parser.hpp

@@ -89,6 +89,29 @@ namespace libtorrent
 		std::pair<size_type, size_type> content_range() const
 		{ return std::make_pair(m_range_start, m_range_end); }
 
+		// returns true if this response is using chunked encoding.
+		// in this case the body is split up into chunks. You need
+		// to call parse_chunk_header() for each chunk, starting with
+		// the start of the body.
+		bool chunked_encoding() const { return m_chunked_encoding; }
+
+		// returns false if the buffer doesn't contain a complete
+		// chunk header. In this case, call the function again with
+		// a bigger buffer once more bytes have been received.
+		// chunk_size is filled in with the number of bytes in the
+		// chunk that follows. 0 means the response terminated. In
+		// this case there might be additional headers in the parser
+		// object.
+		// header_size is filled in with the number of bytes the header
+		// itself was. Skip this number of bytes to get to the actual
+		// chunk data.
+		// if the function returns false, the chunk size and header
+		// size may still have been modified, but their values are
+		// undefined
+		bool parse_chunk_header(buffer::const_interval buf
+			, size_type* chunk_size, int* header_size);
+
+		// reset the whole state and start over
 		void reset();
 
 		std::map<std::string, std::string> const& headers() const { return m_header; }
@@ -111,6 +134,7 @@ namespace libtorrent
 		buffer::const_interval m_recv_buffer;
 		int m_body_start_pos;
 
+		bool m_chunked_encoding;
 		bool m_finished;
 	};
 

include/libtorrent/peer_connection.hpp

@@ -621,7 +621,7 @@ namespace libtorrent
 		bool allocate_disk_receive_buffer(int disk_buffer_size);
 		char* release_disk_receive_buffer();
 		bool has_disk_receive_buffer() const { return m_disk_recv_buffer; }
-		void cut_receive_buffer(int size, int packet_size);
+		void cut_receive_buffer(int size, int packet_size, int offset = 0);
 		void reset_recv_buffer(int packet_size);
 		void set_soft_packet_size(int size) { m_soft_packet_size = size; }
 

include/libtorrent/web_peer_connection.hpp

@@ -133,6 +133,18 @@ namespace libtorrent
 
 		// the position in the current block
 		int m_block_pos;
+
+		// this is the offset inside the current receive
+		// buffer where the next chunk header will be.
+		// this is updated for each chunk header that's
+		// parsed. It does not necessarily point to a valid
+		// offset in the receive buffer, if we haven't received
+		// it yet. This offset never includes the HTTP header
+		int m_chunk_pos;
+
+		// this is the number of bytes we've already received
+		// from the next chunk header we're waiting for
+		int m_partial_chunk_header;
 	};
 }
 

src/http_parser.cpp

@@ -69,6 +69,7 @@ namespace libtorrent
 		, m_state(read_status)
 		, m_recv_buffer(0, 0)
 		, m_body_start_pos(0)
+		, m_chunked_encoding(false)
 		, m_finished(false)
 	{}
 
@@ -214,6 +215,10 @@ namespace libtorrent
 					// the http range is inclusive
 					m_content_length = m_range_end - m_range_start + 1;
 				}
+				else if (name == "transfer-encoding")
+				{
+					m_chunked_encoding = string_begins_no_case("chunked", value.c_str());
+				}
 
 				TORRENT_ASSERT(m_recv_pos <= (int)recv_buffer.left());
 				newline = std::find(pos, recv_buffer.end, '\n');
@@ -241,6 +246,89 @@ namespace libtorrent
 		return ret;
 	}
 	
+	bool http_parser::parse_chunk_header(buffer::const_interval buf
+		, size_type* chunk_size, int* header_size)
+	{
+		char const* pos = buf.begin;
+
+		// ignore one optional new-line. This is since each chunk
+		// is terminated by a newline. we're likely to see one
+		// before the actual header.
+
+		if (pos[0] == '\r' && pos[1] == '\n') pos += 2;
+		else if (pos[0] == '\n') pos += 1;
+
+		char const* newline = std::find(pos, buf.end, '\n');
+		if (newline == buf.end) return false;
+		++newline;
+
+		// the chunk header is a single line, a hex length of the
+		// chunk followed by an optional semi-colon with a comment
+		// in case the length is 0, the stream is terminated and
+		// there are extra tail headers, which is terminated by an
+		// empty line
+
+		// first, read the chunk length
+		*chunk_size = strtoll(pos, 0, 16);
+		if (*chunk_size != 0)
+		{
+			*header_size = newline - buf.begin;
+			// the newline alone is two bytes
+			TORRENT_ASSERT(newline - buf.begin > 2);
+			return true;
+		}
+
+		// this is the terminator of the stream. Also read headers
+		std::map<std::string, std::string> tail_headers;
+		pos = newline;
+		newline = std::find(pos, buf.end, '\n');
+
+		std::string line;
+		while (newline != buf.end)
+		{
+			// if the LF character is preceeded by a CR
+			// charachter, don't copy it into the line string.
+			char const* line_end = newline;
+			if (pos != line_end && *(line_end - 1) == '\r') --line_end;
+			line.assign(pos, line_end);
+			++newline;
+			pos = newline;
+
+			std::string::size_type separator = line.find(':');
+			if (separator == std::string::npos)
+			{
+				// this means we got a blank line,
+				// the header is finished and the body
+				// starts.
+				*header_size = newline - buf.begin;
+
+				// the newline alone is two bytes
+				TORRENT_ASSERT(newline - buf.begin > 2);
+
+				// we were successfull in parsing the headers.
+				// add them to the headers in the parser
+				for (std::map<std::string, std::string>::const_iterator i = tail_headers.begin();
+					i != tail_headers.end(); ++i)
+					m_header[i->first] = i->second;
+
+				return true;
+			}
+
+			std::string name = line.substr(0, separator);
+			std::transform(name.begin(), name.end(), name.begin(), &to_lower);
+			++separator;
+			// skip whitespace
+			while (separator < line.size()
+				&& (line[separator] == ' ' || line[separator] == '\t'))
+				++separator;
+			std::string value = line.substr(separator, std::string::npos);
+			tail_headers.insert(std::make_pair(name, value));
+
+			newline = std::find(pos, buf.end, '\n');
+		}
+		return false;
+	}
+
 	buffer::const_interval http_parser::get_body() const
 	{
 		TORRENT_ASSERT(m_state == read_body);

src/peer_connection.cpp

@@ -3635,17 +3635,18 @@ namespace libtorrent
 	
 	// size = the packet size to remove from the receive buffer
 	// packet_size = the next packet size to receive in the buffer
-	void peer_connection::cut_receive_buffer(int size, int packet_size)
+	void peer_connection::cut_receive_buffer(int size, int packet_size, int offset)
 	{
 		INVARIANT_CHECK;
 
 		TORRENT_ASSERT(packet_size > 0);
 		TORRENT_ASSERT(int(m_recv_buffer.size()) >= size);
 		TORRENT_ASSERT(int(m_recv_buffer.size()) >= m_recv_pos);
-		TORRENT_ASSERT(m_recv_pos >= size);
+		TORRENT_ASSERT(m_recv_pos >= size + offset);
+		TORRENT_ASSERT(offset >= 0);
 
 		if (size > 0)		
-			std::memmove(&m_recv_buffer[0], &m_recv_buffer[0] + size, m_recv_pos - size);
+			std::memmove(&m_recv_buffer[0] + offset, &m_recv_buffer[0] + offset + size, m_recv_pos - size - offset);
 
 		m_recv_pos -= size;
 

src/torrent.cpp

@@ -6848,8 +6848,8 @@ namespace libtorrent
 		TORRENT_ASSERT(b > 0);
 		m_total_failed_bytes += b;
 		m_ses.add_failed_bytes(b);
-		TORRENT_ASSERT(m_total_redundant_bytes + m_total_failed_bytes
+//		TORRENT_ASSERT(m_total_redundant_bytes + m_total_failed_bytes
-			<= m_stat.total_payload_download());
+//			<= m_stat.total_payload_download());
 	}
 
 	int torrent::num_seeds() const

src/web_peer_connection.cpp

@@ -68,6 +68,8 @@ namespace libtorrent
 		, m_url(url)
 		, m_range_pos(0)
 		, m_block_pos(0)
+		, m_chunk_pos(0)
+		, m_partial_chunk_header(0)
 	{
 		INVARIANT_CHECK;
 
@@ -89,7 +91,7 @@ namespace libtorrent
 		request_large_blocks(true);
 
 #ifdef TORRENT_VERBOSE_LOGGING
-		(*m_logger) << "*** web_peer_connection " << url << "\n";
+		(*m_logger) << time_now_string() << " *** web_peer_connection " << url << "\n";
 #endif
 	}
 
@@ -237,7 +239,7 @@ namespace libtorrent
 		}
 
 #ifdef TORRENT_VERBOSE_LOGGING
-		(*m_logger) << request << "\n";
+		(*m_logger) << time_now_string() << " " << request << "\n";
 #endif
 
 		send_buffer(request.c_str(), request.size(), message_type_request);
@@ -263,12 +265,22 @@ namespace libtorrent
 	{
 		INVARIANT_CHECK;
 
+#ifdef TORRENT_DEBUG
+		size_type dl_target = m_statistics.last_payload_downloaded()
+			+ m_statistics.last_protocol_downloaded() + bytes_transferred;
+#endif
+
 		if (error)
 		{
 			m_statistics.received_bytes(0, bytes_transferred);
 #ifdef TORRENT_VERBOSE_LOGGING
-			(*m_logger) << "*** web_peer_connection error: "
+			(*m_logger) << time_now_string() << " *** web_peer_connection error: "
 				<< error.message() << "\n";
+#endif
+#ifdef TORRENT_DEBUG
+			TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+				+ m_statistics.last_protocol_downloaded()
+				== dl_target);
 #endif
 			return;
 		}
@@ -278,6 +290,12 @@ namespace libtorrent
 
 		for (;;)
 		{
+#ifdef TORRENT_DEBUG
+			TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+				+ m_statistics.last_protocol_downloaded() + bytes_transferred
+				== dl_target);
+#endif
+
 			buffer::const_interval recv_buffer = receive_buffer();
 
 			int payload;
@@ -288,6 +306,7 @@ namespace libtorrent
 				bool error = false;
 				boost::tie(payload, protocol) = m_parser.incoming(recv_buffer, error);
 				m_statistics.received_bytes(0, protocol);
+				TORRENT_ASSERT(bytes_transferred >= protocol);
 				bytes_transferred -= protocol;
 
 				if (error)
@@ -297,6 +316,11 @@ namespace libtorrent
 					(*m_logger) << "*** " << std::string(recv_buffer.begin, recv_buffer.end) << "\n";
 #endif
 					disconnect(errors::http_parse_error, 2);
+#ifdef TORRENT_DEBUG
+					TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+						+ m_statistics.last_protocol_downloaded()
+						== dl_target);
+#endif
 					return;
 				}
 
@@ -309,6 +333,11 @@ namespace libtorrent
 				{
 					TORRENT_ASSERT(payload == 0);
 					TORRENT_ASSERT(bytes_transferred == 0);
+#ifdef TORRENT_DEBUG
+					TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+						+ m_statistics.last_protocol_downloaded() + bytes_transferred
+						== dl_target);
+#endif
 					break;
 				}
 
@@ -316,6 +345,11 @@ namespace libtorrent
 				{
 					TORRENT_ASSERT(payload == 0);
 					TORRENT_ASSERT(bytes_transferred == 0);
+#ifdef TORRENT_DEBUG
+					TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+						+ m_statistics.last_protocol_downloaded() + bytes_transferred
+						== dl_target);
+#endif
 					break;
 				}
 
@@ -327,7 +361,7 @@ namespace libtorrent
 			if (!header_finished)
 			{
 #ifdef TORRENT_VERBOSE_LOGGING
-				(*m_logger) << "*** STATUS: " << m_parser.status_code()
+				(*m_logger) << time_now_string() << " *** STATUS: " << m_parser.status_code()
 					<< " " << m_parser.message() << "\n";
 				std::map<std::string, std::string> const& headers = m_parser.headers();
 				for (std::map<std::string, std::string>::const_iterator i = headers.begin()
@@ -350,6 +384,11 @@ namespace libtorrent
 					}
 					m_statistics.received_bytes(0, bytes_transferred);
 					disconnect(errors::http_error, 1);
+#ifdef TORRENT_DEBUG
+					TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+						+ m_statistics.last_protocol_downloaded()
+						== dl_target);
+#endif
 					return;
 				}
 				if (is_redirect(m_parser.status_code()))
@@ -364,6 +403,11 @@ namespace libtorrent
 						// we should not try this server again.
 						t->remove_web_seed(this);
 						disconnect(errors::missing_location, 2);
+#ifdef TORRENT_DEBUG
+						TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+							+ m_statistics.last_protocol_downloaded()
+							== dl_target);
+#endif
 						return;
 					}
 					
@@ -389,13 +433,23 @@ namespace libtorrent
 						{
 							t->remove_web_seed(this);
 							disconnect(errors::invalid_redirection, 2);
+#ifdef TORRENT_DEBUG
+							TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+								+ m_statistics.last_protocol_downloaded()
+								== dl_target);
+#endif
 							return;
 						}
 						location.resize(i);
 					}
-					t->add_web_seed(location, web_seed_entry::url_seed);
+					t->add_web_seed(location, web_seed_entry::url_seed, m_external_auth, m_extra_headers);
 					t->remove_web_seed(this);
 					disconnect(errors::redirecting, 2);
+#ifdef TORRENT_DEBUG
+					TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+						+ m_statistics.last_protocol_downloaded()
+						== dl_target);
+#endif
 					return;
 				}
 
@@ -417,7 +471,15 @@ namespace libtorrent
 			recv_buffer.begin += m_body_start;
 
 			// we only received the header, no data
-			if (recv_buffer.left() == 0) break;
+			if (recv_buffer.left() == 0)
+			{
+#ifdef TORRENT_DEBUG
+				TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+					+ m_statistics.last_protocol_downloaded()
+					== dl_target);
+#endif
+				break;
+			}
 
 			size_type range_start;
 			size_type range_end;
@@ -430,6 +492,11 @@ namespace libtorrent
 					// we should not try this server again.
 					t->remove_web_seed(this);
 					disconnect(errors::invalid_range);
+#ifdef TORRENT_DEBUG
+					TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+						+ m_statistics.last_protocol_downloaded()
+						== dl_target);
+#endif
 					return;
 				}
 				// the http range is inclusive
@@ -445,6 +512,11 @@ namespace libtorrent
 					// we should not try this server again.
 					t->remove_web_seed(this);
 					disconnect(errors::no_content_length, 2);
+#ifdef TORRENT_DEBUG
+					TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+						+ m_statistics.last_protocol_downloaded()
+						== dl_target);
+#endif
 					return;
 				}
 			}
@@ -453,9 +525,67 @@ namespace libtorrent
 			{
 				m_statistics.received_bytes(0, bytes_transferred);
 				disconnect(errors::http_error, 2);
+#ifdef TORRENT_DEBUG
+				TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+					+ m_statistics.last_protocol_downloaded()
+					== dl_target);
+#endif
 				return;
 			}
 
+			// =========================
+			// === CHUNKED ENCODING  ===
+			// =========================
+			while (m_parser.chunked_encoding()
+				&& m_chunk_pos >= 0
+				&& m_chunk_pos < recv_buffer.left())
+			{
+				int header_size = 0;
+				size_type chunk_size = 0;
+				buffer::const_interval chunk_start = recv_buffer;
+				chunk_start.begin += m_chunk_pos;
+				TORRENT_ASSERT(chunk_start.begin[0] == '\r' || is_hex(chunk_start.begin, 1));
+				bool ret = m_parser.parse_chunk_header(chunk_start, &chunk_size, &header_size);
+				if (!ret)
+				{
+					TORRENT_ASSERT(bytes_transferred >= chunk_start.left() - m_partial_chunk_header);
+					bytes_transferred -= chunk_start.left() - m_partial_chunk_header;
+					m_statistics.received_bytes(0, chunk_start.left() - m_partial_chunk_header);
+					m_partial_chunk_header = chunk_start.left();
+					if (bytes_transferred == 0)
+					{
+						return;
+					}
+					break;
+				}
+				else
+				{
+#ifdef TORRENT_VERBOSE_LOGGING
+					(*m_logger) << time_now_string() << " *** parsed chunk: " << chunk_size
+						<< " header_size: " << header_size << "\n";
+#endif
+					TORRENT_ASSERT(bytes_transferred >= header_size - m_partial_chunk_header);
+					bytes_transferred -= header_size - m_partial_chunk_header;
+					m_statistics.received_bytes(0, header_size - m_partial_chunk_header);
+					m_partial_chunk_header = 0;
+					TORRENT_ASSERT(chunk_size != 0 || chunk_start.left() <= header_size || chunk_start.begin[header_size] == 'H');
+					// cut out the chunk header from the receive buffer
+					cut_receive_buffer(header_size, t->block_size() + 1024, m_chunk_pos);
+					recv_buffer = receive_buffer();
+					recv_buffer.begin += m_body_start;
+					m_chunk_pos += chunk_size;
+					if (chunk_size == 0)
+					{
+#ifdef TORRENT_DEBUG
+						chunk_start = recv_buffer;
+						chunk_start.begin += m_chunk_pos;
+						TORRENT_ASSERT(chunk_start.left() == 0 || chunk_start.begin[0] == 'H');
+#endif
+						m_chunk_pos = -1;
+					}
+				}
+			}
+
 			int left_in_response = range_end - range_start - m_range_pos;
 			int payload_transferred = (std::min)(left_in_response, int(bytes_transferred));
 
@@ -466,11 +596,12 @@ namespace libtorrent
 			TORRENT_ASSERT(m_block_pos >= 0);
 
 #ifdef TORRENT_VERBOSE_LOGGING
-			(*m_logger) << "*** payload_transferred: " << payload_transferred
+			(*m_logger) << time_now_string() << " *** payload_transferred: " << payload_transferred
 				<< " [" << front_request.piece << ":" << front_request.start
 				<< " = " << front_request.length <<  "]\n";
 #endif
 			m_statistics.received_bytes(payload_transferred, 0);
+			TORRENT_ASSERT(bytes_transferred >= payload_transferred);
 			bytes_transferred -= payload_transferred;
 			m_range_pos += payload_transferred;
 			m_block_pos += payload_transferred;
@@ -577,6 +708,11 @@ namespace libtorrent
 				TORRENT_ASSERT(receive_buffer().begin + m_body_start == recv_buffer.begin);
 				TORRENT_ASSERT(m_received_body <= range_end - range_start);
 				cut_receive_buffer(r.length + m_body_start, t->block_size() + 1024);
+				if (m_chunk_pos > 0)
+				{
+					TORRENT_ASSERT(m_chunk_pos >= r.length);
+					m_chunk_pos -= r.length;
+				}
 				m_body_start = 0;
 				recv_buffer = receive_buffer();
 			}
@@ -608,19 +744,38 @@ namespace libtorrent
 			TORRENT_ASSERT(m_received_body <= range_end - range_start);
 			if (m_received_body == range_end - range_start)
 			{
-				cut_receive_buffer(recv_buffer.begin - receive_buffer().begin
+				int size_to_cut = recv_buffer.begin - receive_buffer().begin;
-					, t->block_size() + 1024);
+				cut_receive_buffer(size_to_cut, t->block_size() + 1024);
+				if (m_chunk_pos > 0)
+				{
+					TORRENT_ASSERT(m_chunk_pos >= size_to_cut);
+					m_chunk_pos -= size_to_cut;
+				}
 				recv_buffer = receive_buffer();
 				m_file_requests.pop_front();
 				m_parser.reset();
 				m_body_start = 0;
 				m_received_body = 0;
+				m_chunk_pos = 0;
+				m_partial_chunk_header = 0;
 				continue;
 			}
-			if (bytes_transferred == 0) break;
+			if (bytes_transferred == 0)
+			{
+#ifdef TORRENT_DEBUG
+				TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+					+ m_statistics.last_protocol_downloaded()
+					== dl_target);
+#endif
+				break;
+			}
 			TORRENT_ASSERT(payload_transferred > 0);
 		}
 		TORRENT_ASSERT(bytes_transferred == 0);
+#ifdef TORRENT_DEBUG
+		TORRENT_ASSERT(m_statistics.last_payload_downloaded()
+			+ m_statistics.last_protocol_downloaded() == dl_target);
+#endif
 	}
 
 	void web_peer_connection::get_specific_peer_info(peer_info& p) const

test/setup_transfer.cpp

@@ -511,9 +511,9 @@ void stop_web_server()
 	}
 }
 
-void web_server_thread(int* port, bool ssl);
+void web_server_thread(int* port, bool ssl, bool chunked);
 
-int start_web_server(bool ssl)
+int start_web_server(bool ssl, bool chunked_encoding)
 {
 	stop_web_server();
 
@@ -537,7 +537,8 @@ int start_web_server(bool ssl)
 
 	int port = 0;
 
-	web_server.reset(new libtorrent::thread(boost::bind(&web_server_thread, &port, ssl)));
+	web_server.reset(new libtorrent::thread(boost::bind(
+		&web_server_thread, &port, ssl, chunked_encoding)));
 
 	{
 		libtorrent::mutex::scoped_lock l(web_lock);
@@ -553,16 +554,22 @@ int start_web_server(bool ssl)
 }
 
 void send_response(socket_type& s, error_code& ec
-	, int code, char const* status_message, char const* extra_header
+	, int code, char const* status_message, char const** extra_header
 	, int len)
 {
-	char msg[400];
+	char msg[600];
 	int pkt_len = snprintf(msg, sizeof(msg), "HTTP/1.0 %d %s\r\n"
 		"content-length: %d\r\n"
 		"%s"
+		"%s"
+		"%s"
+		"%s"
 		"\r\n"
 		, code, status_message, len
-		, extra_header ? extra_header : "");
+		, extra_header[0]
+		, extra_header[1]
+		, extra_header[2]
+		, extra_header[3]);
 //	fprintf(stderr, ">> %s\n", msg);
 	write(s, boost::asio::buffer(msg, pkt_len), boost::asio::transfer_all(), ec);
 }
@@ -583,7 +590,42 @@ void on_accept(error_code const& ec)
 	}
 }
 
-void web_server_thread(int* port, bool ssl)
+void send_content(socket_type& s, char const* file, int size, bool chunked)
+{
+	error_code ec;
+	if (chunked)
+	{
+		int chunk_size = 13;
+		char head[20];
+		std::vector<boost::asio::const_buffer> bufs(3);
+		bufs[2] = asio::const_buffer("\r\n", 2);
+		while (chunk_size > 0)
+		{
+			chunk_size = std::min(chunk_size, size);
+			int len = snprintf(head, sizeof(head), "%x\r\n", chunk_size);
+			bufs[0] = asio::const_buffer(head, len);
+			if (chunk_size == 0)
+			{
+				// terminate
+				bufs.erase(bufs.begin()+1);
+			}
+			else
+			{
+				bufs[1] = asio::const_buffer(file, chunk_size);
+			}
+			write(s, bufs, boost::asio::transfer_all(), ec);
+			size -= chunk_size;
+			file += chunk_size;
+			chunk_size *= 2;
+		}
+	}
+	else
+	{
+		write(s, boost::asio::buffer(file, size), boost::asio::transfer_all(), ec);
+	}
+}
+
+void web_server_thread(int* port, bool ssl, bool chunked)
 {
 	io_service ios;
 	socket_acceptor acceptor(ios);
@@ -699,6 +741,8 @@ void web_server_thread(int* port, bool ssl)
 
 			p.incoming(buffer::const_interval(buf + offset, buf + len), error);
 
+			char const* extra_header[4] = {"","","",""};
+
 			TEST_CHECK(error == false);
 			if (error)
 			{
@@ -766,19 +810,22 @@ void web_server_thread(int* port, bool ssl)
 
 			if (path == "/redirect")
 			{
-				send_response(s, ec, 301, "Moved Permanently", "Location: /test_file\r\n", 0);
+				extra_header[0] = "Location: /test_file\r\n";
+				send_response(s, ec, 301, "Moved Permanently", extra_header, 0);
 				break;
 			}
 
 			if (path == "/infinite_redirect")
 			{
-				send_response(s, ec, 301, "Moved Permanently", "Location: /infinite_redirect\r\n", 0);
+				extra_header[0] = "Location: /infinite_redirext\r\n";
+				send_response(s, ec, 301, "Moved Permanently", extra_header, 0);
 				break;
 			}
 
 			if (path == "/relative/redirect")
 			{
-				send_response(s, ec, 301, "Moved Permanently", "Location: ../test_file\r\n", 0);
+				extra_header[0] = "Location: ../test_file\r\n";
+				send_response(s, ec, 301, "Moved Permanently", extra_header, 0);
 				break;
 			}
 
@@ -792,7 +839,7 @@ void web_server_thread(int* port, bool ssl)
 				std::vector<char> buf;
 				bencode(std::back_inserter(buf), announce);
 			
-				send_response(s, ec, 200, "OK", 0, buf.size());
+				send_response(s, ec, 200, "OK", extra_header, buf.size());
 				write(s, boost::asio::buffer(&buf[0], buf.size()), boost::asio::transfer_all(), ec);
 			}
 
@@ -834,10 +881,10 @@ void web_server_thread(int* port, bool ssl)
 				error_code ec;
 				if (res == -1 || file_buf.empty())
 				{
-					send_response(s, ec, 404, "Not Found", 0, 0);
+					send_response(s, ec, 404, "Not Found", extra_header, 0);
 					continue;
 				}
-				send_response(s, ec, 200, "OK", 0, size);
+				send_response(s, ec, 200, "OK", extra_header, size);
 //				fprintf(stderr, "sending %d bytes of payload [%d, %d)\n"
 //					, size, int(off), int(off + size));
 				write(s, boost::asio::buffer(&file_buf[0] + off, size)
@@ -856,24 +903,27 @@ void web_server_thread(int* port, bool ssl)
 			int res = load_file(path, file_buf);
 			if (res == -1)
 			{
-				send_response(s, ec, 404, "Not Found", 0, 0);
+				send_response(s, ec, 404, "Not Found", extra_header, 0);
 				continue;
 			}
 
 			if (res != 0)
 			{
 				// this means the file was either too big or couldn't be read
-				send_response(s, ec, 503, "Internal Error", 0, 0);
+				send_response(s, ec, 503, "Internal Error", extra_header, 0);
 				continue;
 			}
 
 			// serve file
 
-			char const* extra_header = 0;
-
 			if (extension(path) == ".gz")
 			{
-				extra_header = "Content-Encoding: gzip\r\n";
+				extra_header[0] = "Content-Encoding: gzip\r\n";
+			}
+
+			if (chunked)
+			{
+				extra_header[2] = "Transfer-Encoding: chunked\r\n";
 			}
 
 			if (!p.header("range").empty())
@@ -881,14 +931,13 @@ void web_server_thread(int* port, bool ssl)
 				std::string range = p.header("range");
 				int start, end;
 				sscanf(range.c_str(), "bytes=%d-%d", &start, &end);
-				char eh[200];
+				char eh[400];
-				snprintf(eh, sizeof(eh), "%sContent-Range: bytes %d-%d\r\n"
+				snprintf(eh, sizeof(eh), "Content-Range: bytes %d-%d\r\n", start, end);
-						, extra_header ? extra_header : "", start, end);
+				extra_header[1] = eh;
-				send_response(s, ec, 206, "Partial", eh, end - start + 1);
+				send_response(s, ec, 206, "Partial", extra_header, end - start + 1);
 				if (!file_buf.empty())
 				{
-					write(s, boost::asio::buffer(&file_buf[0] + start, end - start + 1)
+					send_content(s, &file_buf[0] + start, end - start + 1, chunked);
-						, boost::asio::transfer_all(), ec);
 				}
 //				fprintf(stderr, "send %d bytes of payload\n", end - start + 1);
 			}
@@ -896,7 +945,7 @@ void web_server_thread(int* port, bool ssl)
 			{
 				send_response(s, ec, 200, "OK", extra_header, file_buf.size());
 				if (!file_buf.empty())
-					write(s, boost::asio::buffer(&file_buf[0], file_buf.size()), boost::asio::transfer_all(), ec);
+					send_content(s, &file_buf[0], file_buf.size(), chunked);
 			}
 //			fprintf(stderr, "%d bytes left in receive buffer. offset: %d\n", len - offset, offset);
 			memmove(buf, buf + offset, len - offset);

test/setup_transfer.hpp

@@ -62,7 +62,7 @@ setup_transfer(libtorrent::session* ses1, libtorrent::session* ses2
 	, boost::intrusive_ptr<libtorrent::torrent_info>* torrent = 0, bool super_seeding = false
 	, libtorrent::add_torrent_params const* p = 0);
 
-int start_web_server(bool ssl = false);
+int start_web_server(bool ssl = false, bool chunked = false);
 void stop_web_server();
 void start_proxy(int port, int type);
 void stop_proxy(int port);

test/test_primitives.cpp

@@ -815,7 +815,6 @@ int test_main()
 	std::cerr << unescape_string(escape_string(test_string, strlen(test_string)), ec) << std::endl;
 
 	// verify_encoding
-
 	test = "\b?filename=4";
 	TEST_CHECK(!verify_encoding(test));
 #ifdef TORRENT_WINDOWS
@@ -827,8 +826,8 @@ int test_main()
 	test = "filename=4";
 	TEST_CHECK(verify_encoding(test));
 	TEST_CHECK(test == "filename=4");
-	// HTTP request parser
 
+	// HTTP request parser
 	http_parser parser;
 	boost::tuple<int, int, bool> received;
 
@@ -939,8 +938,38 @@ int test_main()
 	TEST_CHECK(received == make_tuple(5, int(strlen(web_seed_response) - 5), false));
 	TEST_CHECK(parser.content_range() == (std::pair<size_type, size_type>(0, 4)));
 	TEST_CHECK(parser.content_length() == 5);
-	// test xml parser
 
+	{
+		// test chunked encoding parser
+		char const chunk_header1[] = "f;this is a comment\r\n";
+		size_type chunk_size;
+		int header_size;
+		bool ret = parser.parse_chunk_header(buffer::const_interval(chunk_header1, chunk_header1 + 10)
+			, &chunk_size, &header_size);
+		TEST_EQUAL(ret, false);
+		ret = parser.parse_chunk_header(buffer::const_interval(chunk_header1, chunk_header1 + sizeof(chunk_header1))
+			, &chunk_size, &header_size);
+		TEST_EQUAL(ret, true);
+		TEST_EQUAL(chunk_size, 15);
+		TEST_EQUAL(header_size, sizeof(chunk_header1) - 1);
+
+		char const chunk_header2[] =
+			"0;this is a comment\r\n"
+			"test1: foo\r\n"
+			"test2: bar\r\n"
+			"\r\n";
+
+		ret = parser.parse_chunk_header(buffer::const_interval(chunk_header2, chunk_header2 + sizeof(chunk_header2))
+			, &chunk_size, &header_size);
+		TEST_EQUAL(ret, true);
+		TEST_EQUAL(chunk_size, 0);
+		TEST_EQUAL(header_size, sizeof(chunk_header2) - 1);
+
+		TEST_EQUAL(parser.headers().find("test1")->second, "foo");
+		TEST_EQUAL(parser.headers().find("test2")->second, "bar");
+	}
+
+	// test xml parser
 	char xml1[] = "<a>foo<b/>bar</a>";
 	std::string out1;
 

test/test_web_seed.cpp

@@ -48,7 +48,7 @@ using namespace libtorrent;
 
 // proxy: 0=none, 1=socks4, 2=socks5, 3=socks5_pw 4=http 5=http_pw
 void test_transfer(boost::intrusive_ptr<torrent_info> torrent_file
-	, int proxy, int port, char const* protocol, bool url_seed)
+	, int proxy, int port, char const* protocol, bool url_seed, bool chunked_encoding)
 {
 	using namespace libtorrent;
 
@@ -63,8 +63,8 @@ void test_transfer(boost::intrusive_ptr<torrent_info> torrent_file
 
 	char const* test_name[] = {"no", "SOCKS4", "SOCKS5", "SOCKS5 password", "HTTP", "HTTP password"};
 
-	fprintf(stderr, "\n\n  ==== TESTING %s proxy ==== %s ==== %s ===\n\n\n"
+	fprintf(stderr, "\n\n  ==== TESTING === proxy: %s ==== protocol: %s ==== seed: %s === transfer-encoding: %s\n\n\n"
-		, test_name[proxy], protocol, url_seed ? "URL seed" : "HTTP seed");
+		, test_name[proxy], protocol, url_seed ? "URL seed" : "HTTP seed", chunked_encoding ? "chunked": "none");
 	
 	if (proxy)
 	{
@@ -187,7 +187,7 @@ sha1_hash file_hash(std::string const& name)
 }
 
 // test_url_seed determines whether to use url-seed or http-seed
-int run_suite(char const* protocol, bool test_url_seed)
+int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding)
 {
 	using namespace libtorrent;
 
@@ -221,7 +221,7 @@ int run_suite(char const* protocol, bool test_url_seed)
 		fs.add_file("seed", sizeof(random_data));
 	}
 
-	int port = start_web_server(strcmp(protocol, "https") == 0);
+	int port = start_web_server(strcmp(protocol, "https") == 0, chunked_encoding);
 
 	libtorrent::create_torrent t(fs, 16, 0, libtorrent::create_torrent::calculate_file_hashes);
 	char tmp[512];
@@ -262,12 +262,12 @@ int run_suite(char const* protocol, bool test_url_seed)
 	}
 
 	for (int i = 0; i < 6; ++i)
-		test_transfer(torrent_file, i, port, protocol, test_url_seed);
+		test_transfer(torrent_file, i, port, protocol, test_url_seed, chunked_encoding);
 	
 	if (test_url_seed)
 	{
 		torrent_file->rename_file(0, "./tmp2_web_seed/test_torrent_dir/renamed_test1");
-		test_transfer(torrent_file, 0, port, protocol, test_url_seed);
+		test_transfer(torrent_file, 0, port, protocol, test_url_seed, chunked_encoding);
 	}
 
 	stop_web_server();
@@ -280,10 +280,16 @@ int test_main()
 	int ret = 0;
 	for (int i = 0; i < 2; ++i)
 	{
+		// we only support chunked encoding for
+		// URL seeds (not HTTP seeds).
+		// that's why the variable limit on this loop
+		for (int j = 0; j < (i==0?2:1); ++j)
+		{
 #ifdef TORRENT_USE_OPENSSL
-		run_suite("https", i);
+			run_suite("https", i, j);
 #endif
-		run_suite("http", i);
+			run_suite("http", i, j);
+		}
 	}
 	return ret;
 }