diff --git a/docs/manual.rst b/docs/manual.rst index 211dfde74..9a5bf731d 100644 --- a/docs/manual.rst +++ b/docs/manual.rst @@ -1613,6 +1613,10 @@ Note that by the time this function returns, the resume data may already be inva is still downloading! The recommended practice is to first pause the torrent, then generate the fast resume data, and then close it down. +It is still a good idea to save resume data periodically during download as well as when +closing down. In full allocation mode the reume data is never invalidated by subsequent +writes to the files, since pieces won't move around. + status() -------- @@ -3208,6 +3212,12 @@ The file format is a bencoded dictionary containing the following fields: | | to consider the resume data as current. Otherwise a full | | | re-check is issued. | +----------------------+--------------------------------------------------------------+ +| ``allocation`` | The allocation mode for the storage. Can be either ``full`` | +| | or ``compact``. If this is full, the file sizes and | +| | timestamps are disregarded. Pieces are assumed not to have | +| | moved around even if the files have been modified after the | +| | last resume data checkpoint. | ++----------------------+--------------------------------------------------------------+ threads ======= @@ -3235,7 +3245,10 @@ storage allocation There are two modes in which storage (files on disk) are allocated in libtorrent. * The traditional *full allocation* mode, where the entire files are filled up with - zeros before anything is downloaded. + zeros before anything is downloaded. libtorrent will look for sparse files support + in the filesystem that is used for storage, and use sparse files or file system + zaero fill support if present. This means that on NTFS, full allocation mode will + only allocate storage for the downloaded pieces. * And the *compact allocation* mode, where only files are allocated for actual pieces that have been downloaded. This is the default allocation mode in libtorrent. @@ -3249,13 +3262,16 @@ full allocation When a torrent is started in full allocation mode, the checker thread (see threads_) will make sure that the entire storage is allocated, and fill any gaps with zeros. +This will be skipped if the filesystem supports sparse files or automatic zero filling. It will of course still check for existing pieces and fast resume data. The main drawbacks of this mode are: - * It will take longer to start the torrent, since it will need to fill the files - with zeros. This delay is linearly dependent on the size of the download. + * It may take longer to start the torrent, since it will need to fill the files + with zeros on some systems. This delay is linearly dependent on the size of + the download. - * The download will occupy unnecessary disk space between download sessions. + * The download may occupy unnecessary disk space between download sessions. In case + sparse files are not supported. * Disk caches usually perform extremely poorly with random access to large files and may slow down a download considerably. @@ -3266,8 +3282,13 @@ The benefits of this mode are: total number of disk operations will be fewer and may also play nicer to filesystems' file allocation, and reduce fragmentation. - * No risk of a download failing because of a full disk during download. + * No risk of a download failing because of a full disk during download. Unless + sparse files are being used. + * The fast resume data will be more likely to be usable, regardless of crashes or + out of date data, since pieces won't move around. + + * Can be used with the filter files feature. compact allocation ------------------ @@ -3281,6 +3302,8 @@ download has all its pieces in the correct place). So, the main drawbacks are: * Potentially more fragmentation in the filesystem. + * Cannot be used while filtering files. + The benefits though, are: * No startup delay, since the files doesn't need allocating. @@ -3290,6 +3313,8 @@ The benefits though, are: * Disk caches perform much better than in full allocation and raises the download speed limit imposed by the disk. + * Works well on filesystems that doesn't support sparse files. + The algorithm that is used when allocating pieces and slots isn't very complicated. For the interested, a description follows. diff --git a/include/libtorrent/storage.hpp b/include/libtorrent/storage.hpp index e56908c56..4a0395274 100755 --- a/include/libtorrent/storage.hpp +++ b/include/libtorrent/storage.hpp @@ -135,7 +135,7 @@ namespace libtorrent bool verify_resume_data(entry& rd, std::string& error); bool is_allocating() const; - void allocate_slots(int num_slots); + bool allocate_slots(int num_slots); void mark_failed(int index); unsigned long piece_crc( @@ -165,6 +165,8 @@ namespace libtorrent // of unassigned pieces and -1 is unallocated void export_piece_map(std::vector& pieces) const; + bool compact_allocation() const; + private: class impl; std::auto_ptr m_pimpl; diff --git a/src/file_win.cpp b/src/file_win.cpp index 833c2124a..45928c5a6 100644 --- a/src/file_win.cpp +++ b/src/file_win.cpp @@ -39,6 +39,7 @@ POSSIBILITY OF SUCH DAMAGE. #include #include +#include namespace { @@ -172,6 +173,10 @@ namespace libtorrent std::stringstream s; throw_exception(file_name); } + // try to make the file sparse if supported + DWORD temp; + ::DeviceIoControl(new_handle, FSCTL_SET_SPARSE, 0, 0 + , 0, 0, &temp, 0); // will only close old file if the open succeeded close(); m_file_handle = new_handle; diff --git a/src/piece_picker.cpp b/src/piece_picker.cpp index 8106af45c..44f860282 100755 --- a/src/piece_picker.cpp +++ b/src/piece_picker.cpp @@ -712,7 +712,7 @@ namespace libtorrent m_piece_info.resize(3); } int last_index = m_piece_info.size() - 1; - if (m_piece_info.size() & 1 == 0) + if ((m_piece_info.size() & 1) == 0) { // if there's an even number of vectors, swap // the last two to get the same layout in both cases diff --git a/src/storage.cpp b/src/storage.cpp index f9fc3b795..5f5ad023c 100755 --- a/src/storage.cpp +++ b/src/storage.cpp @@ -75,6 +75,15 @@ POSSIBILITY OF SUCH DAMAGE. #include #endif +#if defined(__APPLE__) +// for getattrlist() +#include +#include +// for statfs() +#include +#include +#endif + #if defined(_WIN32) && defined(UNICODE) #include @@ -211,6 +220,7 @@ using boost::bind; using namespace ::boost::multi_index; using boost::multi_index::multi_index_container; +#if !defined(NDEBUG) && defined(TORRENT_STORAGE_DEBUG) namespace { using namespace libtorrent; @@ -221,8 +231,8 @@ namespace log << s; log.flush(); } - } +#endif namespace libtorrent { @@ -400,6 +410,13 @@ namespace libtorrent , boost::bind((size_type const& (entry::*)() const) &entry::integer, _1), 0)) == slots.end(); + bool full_allocation_mode = false; + try + { + full_allocation_mode = rd["allocation"].string() == "full"; + } + catch (std::exception&) {} + if (seed) { if (m_info.num_files() != (int)file_sizes.size()) @@ -428,6 +445,8 @@ namespace libtorrent return true; } + if (full_allocation_mode) return true; + return match_filesizes(m_info, m_save_path, file_sizes, &error); } @@ -773,7 +792,7 @@ namespace libtorrent void release_files(); - void allocate_slots(int num_slots); + bool allocate_slots(int num_slots); void mark_failed(int index); unsigned long piece_crc( int slot_index @@ -920,6 +939,61 @@ namespace libtorrent , m_allocating(false) { assert(m_save_path.is_complete()); + // try to figure out if the filesystem supports sparse files +#if defined(WIN32) + // assume windows API is available + DWORD max_component_len = 0; + DWORD volume_flags = 0; + std::string root_device = m_save_path.root_name() + "\\"; + bool ret = ::GetVolumeInformation(root_device.c_str(), 0 + , 0, 0, &max_component_len, &volume_flags, 0, 0); + + if (ret) + { + if (volume_flags & FILE_SUPPORTS_SPARSE_FILES) + m_fill_mode = false; + } +#elif defined(__APPLE__) + + // find the last existing directory of the save path + path query_path = m_save_path; + while (!query_path.empty() && !exists(query_path)) + query_path = query_path.branch_path(); + + struct statfs fsinfo; + int ret = statfs(query_path.native_directory_string().c_str(), &fsinfo); + if (ret != 0) return; + + attrlist request; + request.bitmapcount = ATTR_BIT_MAP_COUNT; + request.reserved = 0; + request.commonattr = 0; + request.volattr = ATTR_VOL_CAPABILITIES; + request.dirattr = 0; + request.fileattr = 0; + request.forkattr = 0; + + struct vol_capabilities_attr_buf + { + unsigned long length; + vol_capabilities_attr_t info; + } vol_cap; + + ret = getattrlist(fsinfo.f_mntonname, &request, &vol_cap + , sizeof(vol_cap), 0); + + if (ret == 0 && vol_cap.info.capabilities[VOL_CAPABILITIES_FORMAT] + & (VOL_CAP_FMT_SPARSE_FILES | VOL_CAP_FMT_ZERO_RUNS)) + { + m_fill_mode = false; + } + // TODO: None of those flags are set for HFS+, which + // is supposed to support zero filling + m_fill_mode = false; +#else + //TODO: posix implementation + m_fill_mode = false; +#endif } piece_manager::piece_manager( @@ -975,6 +1049,9 @@ namespace libtorrent } } + bool piece_manager::compact_allocation() const + { return m_pimpl->m_compact_mode; } + void piece_manager::export_piece_map( std::vector& p) const { @@ -1381,7 +1458,7 @@ namespace libtorrent m_state = state_finished; return std::make_pair(true, 1.f); } - + if (m_unallocated_slots.empty()) { m_state = state_finished; @@ -1392,7 +1469,14 @@ namespace libtorrent // pieces are spread out and placed at their // final position. assert(!m_unallocated_slots.empty()); - allocate_slots(1); + + // if we're not filling the allocation + // just make sure we move the current pieces + // into place, and just skip all other + // allocation + // allocate_slots returns true if it had to + // move any data + while (!m_unallocated_slots.empty() && !allocate_slots(1)); return std::make_pair(false, 1.f - (float)m_unallocated_slots.size() / (float)m_slot_to_piece.size()); @@ -1884,7 +1968,7 @@ namespace libtorrent } - void piece_manager::impl::allocate_slots(int num_slots) + bool piece_manager::impl::allocate_slots(int num_slots) { assert(num_slots > 0); @@ -1907,6 +1991,7 @@ namespace libtorrent std::vector& buffer = m_scratch_buffer; buffer.resize(piece_size); + bool written = false; for (int i = 0; i < num_slots && !m_unallocated_slots.empty(); ++i) { @@ -1918,7 +2003,8 @@ namespace libtorrent if (m_piece_to_slot[pos] != has_no_slot) { assert(m_piece_to_slot[pos] >= 0); - m_storage->read(&buffer[0], m_piece_to_slot[pos], 0, static_cast(m_info.piece_size(pos))); + m_storage->read(&buffer[0], m_piece_to_slot[pos], 0 + , static_cast(m_info.piece_size(pos))); new_free_slot = m_piece_to_slot[pos]; m_slot_to_piece[pos] = pos; m_piece_to_slot[pos] = pos; @@ -1929,15 +2015,19 @@ namespace libtorrent m_free_slots.push_back(new_free_slot); if (write_back || m_fill_mode) + { m_storage->write(&buffer[0], pos, 0, static_cast(m_info.piece_size(pos))); + written = true; + } } assert(m_free_slots.size() > 0); + return written; } - void piece_manager::allocate_slots(int num_slots) + bool piece_manager::allocate_slots(int num_slots) { - m_pimpl->allocate_slots(num_slots); + return m_pimpl->allocate_slots(num_slots); } path const& piece_manager::save_path() const diff --git a/src/torrent_handle.cpp b/src/torrent_handle.cpp index 5cf14955d..eccc4971e 100755 --- a/src/torrent_handle.cpp +++ b/src/torrent_handle.cpp @@ -511,6 +511,8 @@ namespace libtorrent ret["file-format"] = "libtorrent resume file"; ret["file-version"] = 1; + ret["allocation"] = t->filesystem().compact_allocation()?"compact":"full"; + const sha1_hash& info_hash = t->torrent_file().info_hash(); ret["info-hash"] = std::string((char*)info_hash.begin(), (char*)info_hash.end());