diff --git a/ChangeLog b/ChangeLog index fa2fd39a7..4ce21e1c7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,4 @@ + * support chunked encoding for web seeds (only for BEP 19, web seeds) * optimized session startup time * support SSL for web seeds, through all proxies * support extending web seeds with custom authorization and extra headers diff --git a/include/libtorrent/http_parser.hpp b/include/libtorrent/http_parser.hpp index 7b639d940..6ee18d3d5 100644 --- a/include/libtorrent/http_parser.hpp +++ b/include/libtorrent/http_parser.hpp @@ -89,6 +89,29 @@ namespace libtorrent std::pair content_range() const { return std::make_pair(m_range_start, m_range_end); } + // returns true if this response is using chunked encoding. + // in this case the body is split up into chunks. You need + // to call parse_chunk_header() for each chunk, starting with + // the start of the body. + bool chunked_encoding() const { return m_chunked_encoding; } + + // returns false if the buffer doesn't contain a complete + // chunk header. In this case, call the function again with + // a bigger buffer once more bytes have been received. + // chunk_size is filled in with the number of bytes in the + // chunk that follows. 0 means the response terminated. In + // this case there might be additional headers in the parser + // object. + // header_size is filled in with the number of bytes the header + // itself was. Skip this number of bytes to get to the actual + // chunk data. + // if the function returns false, the chunk size and header + // size may still have been modified, but their values are + // undefined + bool parse_chunk_header(buffer::const_interval buf + , size_type* chunk_size, int* header_size); + + // reset the whole state and start over void reset(); std::map const& headers() const { return m_header; } @@ -111,6 +134,7 @@ namespace libtorrent buffer::const_interval m_recv_buffer; int m_body_start_pos; + bool m_chunked_encoding; bool m_finished; }; diff --git a/include/libtorrent/peer_connection.hpp b/include/libtorrent/peer_connection.hpp index b07326a43..f92a8bec7 100644 --- a/include/libtorrent/peer_connection.hpp +++ b/include/libtorrent/peer_connection.hpp @@ -621,7 +621,7 @@ namespace libtorrent bool allocate_disk_receive_buffer(int disk_buffer_size); char* release_disk_receive_buffer(); bool has_disk_receive_buffer() const { return m_disk_recv_buffer; } - void cut_receive_buffer(int size, int packet_size); + void cut_receive_buffer(int size, int packet_size, int offset = 0); void reset_recv_buffer(int packet_size); void set_soft_packet_size(int size) { m_soft_packet_size = size; } diff --git a/include/libtorrent/web_peer_connection.hpp b/include/libtorrent/web_peer_connection.hpp index 162003b66..e32ea24f4 100644 --- a/include/libtorrent/web_peer_connection.hpp +++ b/include/libtorrent/web_peer_connection.hpp @@ -133,6 +133,18 @@ namespace libtorrent // the position in the current block int m_block_pos; + + // this is the offset inside the current receive + // buffer where the next chunk header will be. + // this is updated for each chunk header that's + // parsed. It does not necessarily point to a valid + // offset in the receive buffer, if we haven't received + // it yet. This offset never includes the HTTP header + int m_chunk_pos; + + // this is the number of bytes we've already received + // from the next chunk header we're waiting for + int m_partial_chunk_header; }; } diff --git a/src/http_parser.cpp b/src/http_parser.cpp index a6b7fd331..46e2214a0 100644 --- a/src/http_parser.cpp +++ b/src/http_parser.cpp @@ -69,6 +69,7 @@ namespace libtorrent , m_state(read_status) , m_recv_buffer(0, 0) , m_body_start_pos(0) + , m_chunked_encoding(false) , m_finished(false) {} @@ -214,6 +215,10 @@ namespace libtorrent // the http range is inclusive m_content_length = m_range_end - m_range_start + 1; } + else if (name == "transfer-encoding") + { + m_chunked_encoding = string_begins_no_case("chunked", value.c_str()); + } TORRENT_ASSERT(m_recv_pos <= (int)recv_buffer.left()); newline = std::find(pos, recv_buffer.end, '\n'); @@ -241,6 +246,89 @@ namespace libtorrent return ret; } + bool http_parser::parse_chunk_header(buffer::const_interval buf + , size_type* chunk_size, int* header_size) + { + char const* pos = buf.begin; + + // ignore one optional new-line. This is since each chunk + // is terminated by a newline. we're likely to see one + // before the actual header. + + if (pos[0] == '\r' && pos[1] == '\n') pos += 2; + else if (pos[0] == '\n') pos += 1; + + char const* newline = std::find(pos, buf.end, '\n'); + if (newline == buf.end) return false; + ++newline; + + // the chunk header is a single line, a hex length of the + // chunk followed by an optional semi-colon with a comment + // in case the length is 0, the stream is terminated and + // there are extra tail headers, which is terminated by an + // empty line + + // first, read the chunk length + *chunk_size = strtoll(pos, 0, 16); + if (*chunk_size != 0) + { + *header_size = newline - buf.begin; + // the newline alone is two bytes + TORRENT_ASSERT(newline - buf.begin > 2); + return true; + } + + // this is the terminator of the stream. Also read headers + std::map tail_headers; + pos = newline; + newline = std::find(pos, buf.end, '\n'); + + std::string line; + while (newline != buf.end) + { + // if the LF character is preceeded by a CR + // charachter, don't copy it into the line string. + char const* line_end = newline; + if (pos != line_end && *(line_end - 1) == '\r') --line_end; + line.assign(pos, line_end); + ++newline; + pos = newline; + + std::string::size_type separator = line.find(':'); + if (separator == std::string::npos) + { + // this means we got a blank line, + // the header is finished and the body + // starts. + *header_size = newline - buf.begin; + + // the newline alone is two bytes + TORRENT_ASSERT(newline - buf.begin > 2); + + // we were successfull in parsing the headers. + // add them to the headers in the parser + for (std::map::const_iterator i = tail_headers.begin(); + i != tail_headers.end(); ++i) + m_header[i->first] = i->second; + + return true; + } + + std::string name = line.substr(0, separator); + std::transform(name.begin(), name.end(), name.begin(), &to_lower); + ++separator; + // skip whitespace + while (separator < line.size() + && (line[separator] == ' ' || line[separator] == '\t')) + ++separator; + std::string value = line.substr(separator, std::string::npos); + tail_headers.insert(std::make_pair(name, value)); + + newline = std::find(pos, buf.end, '\n'); + } + return false; + } + buffer::const_interval http_parser::get_body() const { TORRENT_ASSERT(m_state == read_body); diff --git a/src/peer_connection.cpp b/src/peer_connection.cpp index 5b7b55acf..232467a0f 100644 --- a/src/peer_connection.cpp +++ b/src/peer_connection.cpp @@ -3635,17 +3635,18 @@ namespace libtorrent // size = the packet size to remove from the receive buffer // packet_size = the next packet size to receive in the buffer - void peer_connection::cut_receive_buffer(int size, int packet_size) + void peer_connection::cut_receive_buffer(int size, int packet_size, int offset) { INVARIANT_CHECK; TORRENT_ASSERT(packet_size > 0); TORRENT_ASSERT(int(m_recv_buffer.size()) >= size); TORRENT_ASSERT(int(m_recv_buffer.size()) >= m_recv_pos); - TORRENT_ASSERT(m_recv_pos >= size); + TORRENT_ASSERT(m_recv_pos >= size + offset); + TORRENT_ASSERT(offset >= 0); if (size > 0) - std::memmove(&m_recv_buffer[0], &m_recv_buffer[0] + size, m_recv_pos - size); + std::memmove(&m_recv_buffer[0] + offset, &m_recv_buffer[0] + offset + size, m_recv_pos - size - offset); m_recv_pos -= size; diff --git a/src/torrent.cpp b/src/torrent.cpp index 00fb2dfa9..ec4d7045c 100644 --- a/src/torrent.cpp +++ b/src/torrent.cpp @@ -6848,8 +6848,8 @@ namespace libtorrent TORRENT_ASSERT(b > 0); m_total_failed_bytes += b; m_ses.add_failed_bytes(b); - TORRENT_ASSERT(m_total_redundant_bytes + m_total_failed_bytes - <= m_stat.total_payload_download()); +// TORRENT_ASSERT(m_total_redundant_bytes + m_total_failed_bytes +// <= m_stat.total_payload_download()); } int torrent::num_seeds() const diff --git a/src/web_peer_connection.cpp b/src/web_peer_connection.cpp index 51afe9255..b71e8bf9f 100644 --- a/src/web_peer_connection.cpp +++ b/src/web_peer_connection.cpp @@ -68,6 +68,8 @@ namespace libtorrent , m_url(url) , m_range_pos(0) , m_block_pos(0) + , m_chunk_pos(0) + , m_partial_chunk_header(0) { INVARIANT_CHECK; @@ -89,7 +91,7 @@ namespace libtorrent request_large_blocks(true); #ifdef TORRENT_VERBOSE_LOGGING - (*m_logger) << "*** web_peer_connection " << url << "\n"; + (*m_logger) << time_now_string() << " *** web_peer_connection " << url << "\n"; #endif } @@ -237,7 +239,7 @@ namespace libtorrent } #ifdef TORRENT_VERBOSE_LOGGING - (*m_logger) << request << "\n"; + (*m_logger) << time_now_string() << " " << request << "\n"; #endif send_buffer(request.c_str(), request.size(), message_type_request); @@ -263,12 +265,22 @@ namespace libtorrent { INVARIANT_CHECK; +#ifdef TORRENT_DEBUG + size_type dl_target = m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + bytes_transferred; +#endif + if (error) { m_statistics.received_bytes(0, bytes_transferred); #ifdef TORRENT_VERBOSE_LOGGING - (*m_logger) << "*** web_peer_connection error: " + (*m_logger) << time_now_string() << " *** web_peer_connection error: " << error.message() << "\n"; +#endif +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + == dl_target); #endif return; } @@ -278,6 +290,12 @@ namespace libtorrent for (;;) { +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + bytes_transferred + == dl_target); +#endif + buffer::const_interval recv_buffer = receive_buffer(); int payload; @@ -288,6 +306,7 @@ namespace libtorrent bool error = false; boost::tie(payload, protocol) = m_parser.incoming(recv_buffer, error); m_statistics.received_bytes(0, protocol); + TORRENT_ASSERT(bytes_transferred >= protocol); bytes_transferred -= protocol; if (error) @@ -297,6 +316,11 @@ namespace libtorrent (*m_logger) << "*** " << std::string(recv_buffer.begin, recv_buffer.end) << "\n"; #endif disconnect(errors::http_parse_error, 2); +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + == dl_target); +#endif return; } @@ -309,6 +333,11 @@ namespace libtorrent { TORRENT_ASSERT(payload == 0); TORRENT_ASSERT(bytes_transferred == 0); +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + bytes_transferred + == dl_target); +#endif break; } @@ -316,6 +345,11 @@ namespace libtorrent { TORRENT_ASSERT(payload == 0); TORRENT_ASSERT(bytes_transferred == 0); +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + bytes_transferred + == dl_target); +#endif break; } @@ -327,7 +361,7 @@ namespace libtorrent if (!header_finished) { #ifdef TORRENT_VERBOSE_LOGGING - (*m_logger) << "*** STATUS: " << m_parser.status_code() + (*m_logger) << time_now_string() << " *** STATUS: " << m_parser.status_code() << " " << m_parser.message() << "\n"; std::map const& headers = m_parser.headers(); for (std::map::const_iterator i = headers.begin() @@ -350,6 +384,11 @@ namespace libtorrent } m_statistics.received_bytes(0, bytes_transferred); disconnect(errors::http_error, 1); +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + == dl_target); +#endif return; } if (is_redirect(m_parser.status_code())) @@ -364,6 +403,11 @@ namespace libtorrent // we should not try this server again. t->remove_web_seed(this); disconnect(errors::missing_location, 2); +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + == dl_target); +#endif return; } @@ -389,13 +433,23 @@ namespace libtorrent { t->remove_web_seed(this); disconnect(errors::invalid_redirection, 2); +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + == dl_target); +#endif return; } location.resize(i); } - t->add_web_seed(location, web_seed_entry::url_seed); + t->add_web_seed(location, web_seed_entry::url_seed, m_external_auth, m_extra_headers); t->remove_web_seed(this); disconnect(errors::redirecting, 2); +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + == dl_target); +#endif return; } @@ -417,7 +471,15 @@ namespace libtorrent recv_buffer.begin += m_body_start; // we only received the header, no data - if (recv_buffer.left() == 0) break; + if (recv_buffer.left() == 0) + { +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + == dl_target); +#endif + break; + } size_type range_start; size_type range_end; @@ -430,6 +492,11 @@ namespace libtorrent // we should not try this server again. t->remove_web_seed(this); disconnect(errors::invalid_range); +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + == dl_target); +#endif return; } // the http range is inclusive @@ -445,6 +512,11 @@ namespace libtorrent // we should not try this server again. t->remove_web_seed(this); disconnect(errors::no_content_length, 2); +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + == dl_target); +#endif return; } } @@ -453,9 +525,67 @@ namespace libtorrent { m_statistics.received_bytes(0, bytes_transferred); disconnect(errors::http_error, 2); +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + == dl_target); +#endif return; } + // ========================= + // === CHUNKED ENCODING === + // ========================= + while (m_parser.chunked_encoding() + && m_chunk_pos >= 0 + && m_chunk_pos < recv_buffer.left()) + { + int header_size = 0; + size_type chunk_size = 0; + buffer::const_interval chunk_start = recv_buffer; + chunk_start.begin += m_chunk_pos; + TORRENT_ASSERT(chunk_start.begin[0] == '\r' || is_hex(chunk_start.begin, 1)); + bool ret = m_parser.parse_chunk_header(chunk_start, &chunk_size, &header_size); + if (!ret) + { + TORRENT_ASSERT(bytes_transferred >= chunk_start.left() - m_partial_chunk_header); + bytes_transferred -= chunk_start.left() - m_partial_chunk_header; + m_statistics.received_bytes(0, chunk_start.left() - m_partial_chunk_header); + m_partial_chunk_header = chunk_start.left(); + if (bytes_transferred == 0) + { + return; + } + break; + } + else + { +#ifdef TORRENT_VERBOSE_LOGGING + (*m_logger) << time_now_string() << " *** parsed chunk: " << chunk_size + << " header_size: " << header_size << "\n"; +#endif + TORRENT_ASSERT(bytes_transferred >= header_size - m_partial_chunk_header); + bytes_transferred -= header_size - m_partial_chunk_header; + m_statistics.received_bytes(0, header_size - m_partial_chunk_header); + m_partial_chunk_header = 0; + TORRENT_ASSERT(chunk_size != 0 || chunk_start.left() <= header_size || chunk_start.begin[header_size] == 'H'); + // cut out the chunk header from the receive buffer + cut_receive_buffer(header_size, t->block_size() + 1024, m_chunk_pos); + recv_buffer = receive_buffer(); + recv_buffer.begin += m_body_start; + m_chunk_pos += chunk_size; + if (chunk_size == 0) + { +#ifdef TORRENT_DEBUG + chunk_start = recv_buffer; + chunk_start.begin += m_chunk_pos; + TORRENT_ASSERT(chunk_start.left() == 0 || chunk_start.begin[0] == 'H'); +#endif + m_chunk_pos = -1; + } + } + } + int left_in_response = range_end - range_start - m_range_pos; int payload_transferred = (std::min)(left_in_response, int(bytes_transferred)); @@ -466,11 +596,12 @@ namespace libtorrent TORRENT_ASSERT(m_block_pos >= 0); #ifdef TORRENT_VERBOSE_LOGGING - (*m_logger) << "*** payload_transferred: " << payload_transferred + (*m_logger) << time_now_string() << " *** payload_transferred: " << payload_transferred << " [" << front_request.piece << ":" << front_request.start << " = " << front_request.length << "]\n"; #endif m_statistics.received_bytes(payload_transferred, 0); + TORRENT_ASSERT(bytes_transferred >= payload_transferred); bytes_transferred -= payload_transferred; m_range_pos += payload_transferred; m_block_pos += payload_transferred; @@ -577,6 +708,11 @@ namespace libtorrent TORRENT_ASSERT(receive_buffer().begin + m_body_start == recv_buffer.begin); TORRENT_ASSERT(m_received_body <= range_end - range_start); cut_receive_buffer(r.length + m_body_start, t->block_size() + 1024); + if (m_chunk_pos > 0) + { + TORRENT_ASSERT(m_chunk_pos >= r.length); + m_chunk_pos -= r.length; + } m_body_start = 0; recv_buffer = receive_buffer(); } @@ -608,19 +744,38 @@ namespace libtorrent TORRENT_ASSERT(m_received_body <= range_end - range_start); if (m_received_body == range_end - range_start) { - cut_receive_buffer(recv_buffer.begin - receive_buffer().begin - , t->block_size() + 1024); + int size_to_cut = recv_buffer.begin - receive_buffer().begin; + cut_receive_buffer(size_to_cut, t->block_size() + 1024); + if (m_chunk_pos > 0) + { + TORRENT_ASSERT(m_chunk_pos >= size_to_cut); + m_chunk_pos -= size_to_cut; + } recv_buffer = receive_buffer(); m_file_requests.pop_front(); m_parser.reset(); m_body_start = 0; m_received_body = 0; + m_chunk_pos = 0; + m_partial_chunk_header = 0; continue; } - if (bytes_transferred == 0) break; + if (bytes_transferred == 0) + { +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() + == dl_target); +#endif + break; + } TORRENT_ASSERT(payload_transferred > 0); } TORRENT_ASSERT(bytes_transferred == 0); +#ifdef TORRENT_DEBUG + TORRENT_ASSERT(m_statistics.last_payload_downloaded() + + m_statistics.last_protocol_downloaded() == dl_target); +#endif } void web_peer_connection::get_specific_peer_info(peer_info& p) const diff --git a/test/setup_transfer.cpp b/test/setup_transfer.cpp index ad531dd1f..45f37bae9 100644 --- a/test/setup_transfer.cpp +++ b/test/setup_transfer.cpp @@ -511,9 +511,9 @@ void stop_web_server() } } -void web_server_thread(int* port, bool ssl); +void web_server_thread(int* port, bool ssl, bool chunked); -int start_web_server(bool ssl) +int start_web_server(bool ssl, bool chunked_encoding) { stop_web_server(); @@ -537,7 +537,8 @@ int start_web_server(bool ssl) int port = 0; - web_server.reset(new libtorrent::thread(boost::bind(&web_server_thread, &port, ssl))); + web_server.reset(new libtorrent::thread(boost::bind( + &web_server_thread, &port, ssl, chunked_encoding))); { libtorrent::mutex::scoped_lock l(web_lock); @@ -553,16 +554,22 @@ int start_web_server(bool ssl) } void send_response(socket_type& s, error_code& ec - , int code, char const* status_message, char const* extra_header + , int code, char const* status_message, char const** extra_header , int len) { - char msg[400]; + char msg[600]; int pkt_len = snprintf(msg, sizeof(msg), "HTTP/1.0 %d %s\r\n" "content-length: %d\r\n" "%s" + "%s" + "%s" + "%s" "\r\n" , code, status_message, len - , extra_header ? extra_header : ""); + , extra_header[0] + , extra_header[1] + , extra_header[2] + , extra_header[3]); // fprintf(stderr, ">> %s\n", msg); write(s, boost::asio::buffer(msg, pkt_len), boost::asio::transfer_all(), ec); } @@ -583,7 +590,42 @@ void on_accept(error_code const& ec) } } -void web_server_thread(int* port, bool ssl) +void send_content(socket_type& s, char const* file, int size, bool chunked) +{ + error_code ec; + if (chunked) + { + int chunk_size = 13; + char head[20]; + std::vector bufs(3); + bufs[2] = asio::const_buffer("\r\n", 2); + while (chunk_size > 0) + { + chunk_size = std::min(chunk_size, size); + int len = snprintf(head, sizeof(head), "%x\r\n", chunk_size); + bufs[0] = asio::const_buffer(head, len); + if (chunk_size == 0) + { + // terminate + bufs.erase(bufs.begin()+1); + } + else + { + bufs[1] = asio::const_buffer(file, chunk_size); + } + write(s, bufs, boost::asio::transfer_all(), ec); + size -= chunk_size; + file += chunk_size; + chunk_size *= 2; + } + } + else + { + write(s, boost::asio::buffer(file, size), boost::asio::transfer_all(), ec); + } +} + +void web_server_thread(int* port, bool ssl, bool chunked) { io_service ios; socket_acceptor acceptor(ios); @@ -699,6 +741,8 @@ void web_server_thread(int* port, bool ssl) p.incoming(buffer::const_interval(buf + offset, buf + len), error); + char const* extra_header[4] = {"","","",""}; + TEST_CHECK(error == false); if (error) { @@ -766,19 +810,22 @@ void web_server_thread(int* port, bool ssl) if (path == "/redirect") { - send_response(s, ec, 301, "Moved Permanently", "Location: /test_file\r\n", 0); + extra_header[0] = "Location: /test_file\r\n"; + send_response(s, ec, 301, "Moved Permanently", extra_header, 0); break; } if (path == "/infinite_redirect") { - send_response(s, ec, 301, "Moved Permanently", "Location: /infinite_redirect\r\n", 0); + extra_header[0] = "Location: /infinite_redirext\r\n"; + send_response(s, ec, 301, "Moved Permanently", extra_header, 0); break; } if (path == "/relative/redirect") { - send_response(s, ec, 301, "Moved Permanently", "Location: ../test_file\r\n", 0); + extra_header[0] = "Location: ../test_file\r\n"; + send_response(s, ec, 301, "Moved Permanently", extra_header, 0); break; } @@ -792,7 +839,7 @@ void web_server_thread(int* port, bool ssl) std::vector buf; bencode(std::back_inserter(buf), announce); - send_response(s, ec, 200, "OK", 0, buf.size()); + send_response(s, ec, 200, "OK", extra_header, buf.size()); write(s, boost::asio::buffer(&buf[0], buf.size()), boost::asio::transfer_all(), ec); } @@ -834,10 +881,10 @@ void web_server_thread(int* port, bool ssl) error_code ec; if (res == -1 || file_buf.empty()) { - send_response(s, ec, 404, "Not Found", 0, 0); + send_response(s, ec, 404, "Not Found", extra_header, 0); continue; } - send_response(s, ec, 200, "OK", 0, size); + send_response(s, ec, 200, "OK", extra_header, size); // fprintf(stderr, "sending %d bytes of payload [%d, %d)\n" // , size, int(off), int(off + size)); write(s, boost::asio::buffer(&file_buf[0] + off, size) @@ -856,24 +903,27 @@ void web_server_thread(int* port, bool ssl) int res = load_file(path, file_buf); if (res == -1) { - send_response(s, ec, 404, "Not Found", 0, 0); + send_response(s, ec, 404, "Not Found", extra_header, 0); continue; } if (res != 0) { // this means the file was either too big or couldn't be read - send_response(s, ec, 503, "Internal Error", 0, 0); + send_response(s, ec, 503, "Internal Error", extra_header, 0); continue; } // serve file - char const* extra_header = 0; - if (extension(path) == ".gz") { - extra_header = "Content-Encoding: gzip\r\n"; + extra_header[0] = "Content-Encoding: gzip\r\n"; + } + + if (chunked) + { + extra_header[2] = "Transfer-Encoding: chunked\r\n"; } if (!p.header("range").empty()) @@ -881,14 +931,13 @@ void web_server_thread(int* port, bool ssl) std::string range = p.header("range"); int start, end; sscanf(range.c_str(), "bytes=%d-%d", &start, &end); - char eh[200]; - snprintf(eh, sizeof(eh), "%sContent-Range: bytes %d-%d\r\n" - , extra_header ? extra_header : "", start, end); - send_response(s, ec, 206, "Partial", eh, end - start + 1); + char eh[400]; + snprintf(eh, sizeof(eh), "Content-Range: bytes %d-%d\r\n", start, end); + extra_header[1] = eh; + send_response(s, ec, 206, "Partial", extra_header, end - start + 1); if (!file_buf.empty()) { - write(s, boost::asio::buffer(&file_buf[0] + start, end - start + 1) - , boost::asio::transfer_all(), ec); + send_content(s, &file_buf[0] + start, end - start + 1, chunked); } // fprintf(stderr, "send %d bytes of payload\n", end - start + 1); } @@ -896,7 +945,7 @@ void web_server_thread(int* port, bool ssl) { send_response(s, ec, 200, "OK", extra_header, file_buf.size()); if (!file_buf.empty()) - write(s, boost::asio::buffer(&file_buf[0], file_buf.size()), boost::asio::transfer_all(), ec); + send_content(s, &file_buf[0], file_buf.size(), chunked); } // fprintf(stderr, "%d bytes left in receive buffer. offset: %d\n", len - offset, offset); memmove(buf, buf + offset, len - offset); diff --git a/test/setup_transfer.hpp b/test/setup_transfer.hpp index 1f07b8e88..17153710b 100644 --- a/test/setup_transfer.hpp +++ b/test/setup_transfer.hpp @@ -62,7 +62,7 @@ setup_transfer(libtorrent::session* ses1, libtorrent::session* ses2 , boost::intrusive_ptr* torrent = 0, bool super_seeding = false , libtorrent::add_torrent_params const* p = 0); -int start_web_server(bool ssl = false); +int start_web_server(bool ssl = false, bool chunked = false); void stop_web_server(); void start_proxy(int port, int type); void stop_proxy(int port); diff --git a/test/test_primitives.cpp b/test/test_primitives.cpp index b2d6cfbad..88c1f98a4 100644 --- a/test/test_primitives.cpp +++ b/test/test_primitives.cpp @@ -815,7 +815,6 @@ int test_main() std::cerr << unescape_string(escape_string(test_string, strlen(test_string)), ec) << std::endl; // verify_encoding - test = "\b?filename=4"; TEST_CHECK(!verify_encoding(test)); #ifdef TORRENT_WINDOWS @@ -827,8 +826,8 @@ int test_main() test = "filename=4"; TEST_CHECK(verify_encoding(test)); TEST_CHECK(test == "filename=4"); - // HTTP request parser + // HTTP request parser http_parser parser; boost::tuple received; @@ -939,8 +938,38 @@ int test_main() TEST_CHECK(received == make_tuple(5, int(strlen(web_seed_response) - 5), false)); TEST_CHECK(parser.content_range() == (std::pair(0, 4))); TEST_CHECK(parser.content_length() == 5); - // test xml parser + { + // test chunked encoding parser + char const chunk_header1[] = "f;this is a comment\r\n"; + size_type chunk_size; + int header_size; + bool ret = parser.parse_chunk_header(buffer::const_interval(chunk_header1, chunk_header1 + 10) + , &chunk_size, &header_size); + TEST_EQUAL(ret, false); + ret = parser.parse_chunk_header(buffer::const_interval(chunk_header1, chunk_header1 + sizeof(chunk_header1)) + , &chunk_size, &header_size); + TEST_EQUAL(ret, true); + TEST_EQUAL(chunk_size, 15); + TEST_EQUAL(header_size, sizeof(chunk_header1) - 1); + + char const chunk_header2[] = + "0;this is a comment\r\n" + "test1: foo\r\n" + "test2: bar\r\n" + "\r\n"; + + ret = parser.parse_chunk_header(buffer::const_interval(chunk_header2, chunk_header2 + sizeof(chunk_header2)) + , &chunk_size, &header_size); + TEST_EQUAL(ret, true); + TEST_EQUAL(chunk_size, 0); + TEST_EQUAL(header_size, sizeof(chunk_header2) - 1); + + TEST_EQUAL(parser.headers().find("test1")->second, "foo"); + TEST_EQUAL(parser.headers().find("test2")->second, "bar"); + } + + // test xml parser char xml1[] = "foobar"; std::string out1; diff --git a/test/test_web_seed.cpp b/test/test_web_seed.cpp index ab9d3f938..d2a073a68 100644 --- a/test/test_web_seed.cpp +++ b/test/test_web_seed.cpp @@ -48,7 +48,7 @@ using namespace libtorrent; // proxy: 0=none, 1=socks4, 2=socks5, 3=socks5_pw 4=http 5=http_pw void test_transfer(boost::intrusive_ptr torrent_file - , int proxy, int port, char const* protocol, bool url_seed) + , int proxy, int port, char const* protocol, bool url_seed, bool chunked_encoding) { using namespace libtorrent; @@ -63,8 +63,8 @@ void test_transfer(boost::intrusive_ptr torrent_file char const* test_name[] = {"no", "SOCKS4", "SOCKS5", "SOCKS5 password", "HTTP", "HTTP password"}; - fprintf(stderr, "\n\n ==== TESTING %s proxy ==== %s ==== %s ===\n\n\n" - , test_name[proxy], protocol, url_seed ? "URL seed" : "HTTP seed"); + fprintf(stderr, "\n\n ==== TESTING === proxy: %s ==== protocol: %s ==== seed: %s === transfer-encoding: %s\n\n\n" + , test_name[proxy], protocol, url_seed ? "URL seed" : "HTTP seed", chunked_encoding ? "chunked": "none"); if (proxy) { @@ -187,7 +187,7 @@ sha1_hash file_hash(std::string const& name) } // test_url_seed determines whether to use url-seed or http-seed -int run_suite(char const* protocol, bool test_url_seed) +int run_suite(char const* protocol, bool test_url_seed, bool chunked_encoding) { using namespace libtorrent; @@ -221,7 +221,7 @@ int run_suite(char const* protocol, bool test_url_seed) fs.add_file("seed", sizeof(random_data)); } - int port = start_web_server(strcmp(protocol, "https") == 0); + int port = start_web_server(strcmp(protocol, "https") == 0, chunked_encoding); libtorrent::create_torrent t(fs, 16, 0, libtorrent::create_torrent::calculate_file_hashes); char tmp[512]; @@ -262,12 +262,12 @@ int run_suite(char const* protocol, bool test_url_seed) } for (int i = 0; i < 6; ++i) - test_transfer(torrent_file, i, port, protocol, test_url_seed); + test_transfer(torrent_file, i, port, protocol, test_url_seed, chunked_encoding); if (test_url_seed) { torrent_file->rename_file(0, "./tmp2_web_seed/test_torrent_dir/renamed_test1"); - test_transfer(torrent_file, 0, port, protocol, test_url_seed); + test_transfer(torrent_file, 0, port, protocol, test_url_seed, chunked_encoding); } stop_web_server(); @@ -280,10 +280,16 @@ int test_main() int ret = 0; for (int i = 0; i < 2; ++i) { + // we only support chunked encoding for + // URL seeds (not HTTP seeds). + // that's why the variable limit on this loop + for (int j = 0; j < (i==0?2:1); ++j) + { #ifdef TORRENT_USE_OPENSSL - run_suite("https", i); + run_suite("https", i, j); #endif - run_suite("http", i); + run_suite("http", i, j); + } } return ret; }