file_view file_view_pool::open_file(storage_index_t st, std::string const& p , file_index_t const file_index, file_storage const& fs , open_mode_t const m) { // potentially used to hold a reference to a file object that's // about to be destructed. If we have such object we assign it to // this member to be destructed after we release the std::mutex. On some // operating systems (such as OSX) closing a file may take a long // time. We don't want to hold the std::mutex for that. std::shared_ptr<file_mapping> defer_destruction; std::unique_lock<std::mutex> l(m_mutex); TORRENT_ASSERT(is_complete(p)); auto& key_view = m_files.get<0>(); auto const i = key_view.find(file_id{st, file_index}); if (i != key_view.end()) { key_view.modify(i, [&](file_entry& e) { e.last_use = aux::time_now(); // make sure the write bit is set if we asked for it // it's OK to use a read-write file if we just asked for read. But if // we asked for write, the file we serve back must be opened in write // mode if (!(e.mode & open_mode::write) && (m & open_mode::write)) { defer_destruction = std::move(e.mapping); e.mapping = std::make_shared<file_mapping>( file_handle(fs.file_path(file_index, p) , fs.file_size(file_index), m), m , fs.file_size(file_index)); e.mode = m; } }); auto& lru_view = m_files.get<1>(); lru_view.relocate(m_files.project<1>(i), lru_view.begin()); return i->mapping->view(); } if (int(m_files.size()) >= m_size - 1) { // the file cache is at its maximum size, close // the least recently used file remove_oldest(l); } l.unlock(); file_entry e({st, file_index}, fs.file_path(file_index, p), m , fs.file_size(file_index)); auto ret = e.mapping->view(); l.lock(); auto& key_view2 = m_files.get<0>(); key_view2.insert(std::move(e)); return ret; }
std::int64_t stat_cache::get_filesize(int const i, file_storage const& fs , std::string const& save_path, error_code& ec) { TORRENT_ASSERT(i < int(fs.num_files())); if (i >= int(m_stat_cache.size())) m_stat_cache.resize(i + 1, not_in_cache); std::int64_t sz = m_stat_cache[i].file_size; if (sz < not_in_cache) { ec = m_errors[-sz + file_error]; return file_error; } else if (sz == not_in_cache) { // query the filesystem file_status s; std::string const file_path = fs.file_path(i, save_path); stat_file(file_path, &s, ec); if (ec) { set_error(i, ec); sz = file_error; } else { set_cache(i, s.file_size); sz = s.file_size; } } return sz; }
void add_files_impl(file_storage& fs, std::string const& p , std::string const& l, boost::function<bool(std::string)> pred, boost::uint32_t flags) { std::string f = combine_path(p, l); if (!pred(f)) return; error_code ec; file_status s; stat_file(f, &s, ec, (flags & create_torrent::symlinks) ? dont_follow_links : 0); if (ec) return; // recurse into directories bool recurse = (s.mode & file_status::directory) != 0; // if the file is not a link or we're following links, and it's a directory // only then should we recurse #ifndef TORRENT_WINDOWS if ((s.mode & file_status::link) && (flags & create_torrent::symlinks)) recurse = false; #endif if (recurse) { fs.add_path(parent_path(combine_path(l, "x")), s.mode & 0777); for (directory i(f, ec); !i.done(); i.next(ec)) { std::string leaf = i.file(); if (ignore_subdir(leaf)) continue; add_files_impl(fs, p, combine_path(l, leaf), pred, flags); } } else if (s.mode & (file_status::regular_file | file_status::link)) { // #error use the fields from s int file_flags = get_file_attributes(f, (flags & create_torrent::symlinks) ? dont_follow_links : 0); // mask all bits to check if the file is a symlink if ((file_flags & file_storage::attribute_symlink) && (flags & create_torrent::symlinks)) { std::string sym_path = get_symlink_path(f); fs.add_file(l, 0, file_flags, s.mtime, sym_path, s.mode & 0777); } else { fs.add_file(l, s.file_size, file_flags, s.mtime, "", s.mode & 0777); } } if (fs.num_files() == 0) fs.set_name(l); }
void transfer_info::remap_files(file_storage const& f) { INVARIANT_CHECK; // the new specified file storage must have the exact // same size as the current file storage LIBED2K_ASSERT(m_files.total_size() == f.total_size()); if (m_files.total_size() != f.total_size()) return; copy_on_write(); m_files = f; m_files.set_num_pieces(m_orig_files->num_pieces()); m_files.set_piece_length(m_orig_files->piece_length()); }
void file_pool::mark_deleted(file_storage const& fs) { mutex::scoped_lock l(m_mutex); m_deleted_storages.push_back(std::make_pair(fs.name(), (void const*)&fs)); if(m_deleted_storages.size() > 100) m_deleted_storages.erase(m_deleted_storages.begin()); }
void file_pool::mark_deleted(file_storage const& fs) { std::unique_lock<std::mutex> l(m_mutex); m_deleted_storages.push_back(std::make_pair(fs.name() , static_cast<void const*>(&fs))); if(m_deleted_storages.size() > 100) m_deleted_storages.erase(m_deleted_storages.begin()); }
// update the file progress now that we just completed downloading piece // 'index' void file_progress::update(file_storage const& fs, int const index , alert_manager* alerts, torrent_handle const& h) { INVARIANT_CHECK; if (m_file_progress.empty()) return; #if TORRENT_USE_INVARIANT_CHECKS && defined TORRENT_DEBUG // if this assert fires, we've told the file_progress object that we have // a piece twice. That violates its precondition and will cause incorect // accounting TORRENT_ASSERT(m_have_pieces.get_bit(index) == false); #endif int const piece_size = fs.piece_length(); std::int64_t off = std::int64_t(index) * piece_size; int file_index = fs.file_index_at_offset(off); int size = fs.piece_size(index); for (; size > 0; ++file_index) { std::int64_t file_offset = off - fs.file_offset(file_index); TORRENT_ASSERT(file_index != fs.num_files()); TORRENT_ASSERT(file_offset <= fs.file_size(file_index)); int add = (std::min)(fs.file_size(file_index) - file_offset, std::int64_t(size)); m_file_progress[file_index] += add; TORRENT_ASSERT(m_file_progress[file_index] <= fs.file_size(file_index)); // TODO: it would be nice to not depend on alert_manager here if (m_file_progress[file_index] >= fs.file_size(file_index) && alerts) { if (!fs.pad_file_at(file_index)) { if (alerts->should_post<file_completed_alert>()) { // this file just completed, post alert alerts->emplace_alert<file_completed_alert>(h, file_index); } } } size -= add; off += add; TORRENT_ASSERT(size >= 0); } }
QString StaticHelpers::GetBaseSuffix(const file_storage& storrage) { QString base_suffix; int maxSuffix = 0; QMap<QString, int> suffixesCount; for(int i = 0; i < storrage.num_files(); i++) { QFileInfo curfile(QString::fromUtf8(storrage.file_path(i).c_str())); if(StyleEngene::suffixes[StyleEngene::DISK].contains(curfile.suffix())) { base_suffix = curfile.suffix(); break; } if(StyleEngene::suffixes[StyleEngene::VIDEO].contains(curfile.suffix())) { base_suffix = curfile.suffix(); break; } if(!suffixesCount.contains(curfile.suffix())) { suffixesCount.insert(curfile.suffix(), 1); } else { suffixesCount[curfile.suffix()]++; } if(suffixesCount[curfile.suffix()] > maxSuffix) { maxSuffix = suffixesCount[curfile.suffix()]; base_suffix = curfile.suffix(); } } return base_suffix; }
// root_dir is the name of the torrent, unless this is a single file // torrent, in which case it's empty. bool extract_files(bdecode_node const& list, file_storage& target , std::string const& root_dir, ptrdiff_t info_ptr_diff, error_code& ec) { if (list.type() != bdecode_node::list_t) { ec = errors::torrent_file_parse_failed; return false; } target.reserve(list.list_size()); // this is the counter used to name pad files int pad_file_cnt = 0; for (int i = 0, end(list.list_size()); i < end; ++i) { if (!extract_single_file(list.list_at(i), target, root_dir , info_ptr_diff, false, pad_file_cnt, ec)) return false; } return true; }
void setup_test_storage(file_storage& st) { st.add_file(combine_path("test", "a"), 10000); st.add_file(combine_path("test", "b"), 20000); st.add_file(combine_path("test", combine_path("c", "a")), 30000); st.add_file(combine_path("test", combine_path("c", "b")), 40000); st.set_piece_length(0x4000); st.set_num_pieces((int(st.total_size()) + st.piece_length() - 1) / 0x4000); TEST_EQUAL(st.file_name(0), "a"); TEST_EQUAL(st.file_name(1), "b"); TEST_EQUAL(st.file_name(2), "a"); TEST_EQUAL(st.file_name(3), "b"); TEST_EQUAL(st.name(), "test"); TEST_EQUAL(st.file_path(0), combine_path("test", "a")); TEST_EQUAL(st.file_path(1), combine_path("test", "b")); TEST_EQUAL(st.file_path(2), combine_path("test", combine_path("c", "a"))); TEST_EQUAL(st.file_path(3), combine_path("test", combine_path("c", "b"))); TEST_EQUAL(st.file_size(0), 10000); TEST_EQUAL(st.file_size(1), 20000); TEST_EQUAL(st.file_size(2), 30000); TEST_EQUAL(st.file_size(3), 40000); TEST_EQUAL(st.file_offset(0), 0); TEST_EQUAL(st.file_offset(1), 10000); TEST_EQUAL(st.file_offset(2), 30000); TEST_EQUAL(st.file_offset(3), 60000); TEST_EQUAL(st.total_size(), 100000); TEST_EQUAL(st.piece_length(), 0x4000); printf("%d\n", st.num_pieces()); TEST_EQUAL(st.num_pieces(), (100000 + 0x3fff) / 0x4000); }
file_handle file_pool::open_file(void* st, std::string const& p , int file_index, file_storage const& fs, int m, error_code& ec) { // potentially used to hold a reference to a file object that's // about to be destructed. If we have such object we assign it to // this member to be destructed after we release the mutex. On some // operating systems (such as OSX) closing a file may take a long // time. We don't want to hold the mutex for that. file_handle defer_destruction; mutex::scoped_lock l(m_mutex); #if TORRENT_USE_ASSERTS // we're not allowed to open a file // from a deleted storage! TORRENT_ASSERT(std::find(m_deleted_storages.begin(), m_deleted_storages.end(), std::make_pair(fs.name(), (void const*)&fs)) == m_deleted_storages.end()); #endif TORRENT_ASSERT(st != 0); TORRENT_ASSERT(is_complete(p)); TORRENT_ASSERT((m & file::rw_mask) == file::read_only || (m & file::rw_mask) == file::read_write); file_set::iterator i = m_files.find(std::make_pair(st, file_index)); if (i != m_files.end()) { lru_file_entry& e = i->second; e.last_use = aux::time_now(); if (e.key != st && ((e.mode & file::rw_mask) != file::read_only || (m & file::rw_mask) != file::read_only)) { // this means that another instance of the storage // is using the exact same file. ec = errors::file_collision; return file_handle(); } e.key = st; // if we asked for a file in write mode, // and the cached file is is not opened in // write mode, re-open it if ((((e.mode & file::rw_mask) != file::read_write) && ((m & file::rw_mask) == file::read_write)) || (e.mode & file::random_access) != (m & file::random_access)) { // close the file before we open it with // the new read/write privilages, since windows may // file opening a file twice. However, since there may // be outstanding operations on it, we can't close the // file, we can only delete our reference to it. // if this is the only reference to the file, it will be closed defer_destruction = e.file_ptr; e.file_ptr = boost::make_shared<file>(); std::string full_path = fs.file_path(file_index, p); if (!e.file_ptr->open(full_path, m, ec)) { m_files.erase(i); return file_handle(); } #ifdef TORRENT_WINDOWS if (m_low_prio_io) set_low_priority(e.file_ptr); #endif TORRENT_ASSERT(e.file_ptr->is_open()); e.mode = m; } return e.file_ptr; } lru_file_entry e; e.file_ptr = boost::make_shared<file>(); if (!e.file_ptr) { ec = error_code(ENOMEM, get_posix_category()); return e.file_ptr; } std::string full_path = fs.file_path(file_index, p); if (!e.file_ptr->open(full_path, m, ec)) return file_handle(); #ifdef TORRENT_WINDOWS if (m_low_prio_io) set_low_priority(e.file_ptr); #endif e.mode = m; e.key = st; m_files.insert(std::make_pair(std::make_pair(st, file_index), e)); TORRENT_ASSERT(e.file_ptr->is_open()); file_handle file_ptr = e.file_ptr; // the file is not in our cache if ((int)m_files.size() >= m_size) { // the file cache is at its maximum size, close // the least recently used (lru) file from it remove_oldest(l); } return file_ptr; }
boost::intrusive_ptr<file> file_pool::open_file(void* st, std::string const& p , file_storage::iterator fe, file_storage const& fs, int m, error_code& ec) { TORRENT_ASSERT(st != 0); TORRENT_ASSERT(is_complete(p)); TORRENT_ASSERT((m & file::rw_mask) == file::read_only || (m & file::rw_mask) == file::read_write); mutex::scoped_lock l(m_mutex); file_set::iterator i = m_files.find(std::make_pair(st, fs.file_index(*fe))); if (i != m_files.end()) { lru_file_entry& e = i->second; e.last_use = time_now(); if (e.key != st && ((e.mode & file::rw_mask) != file::read_only || (m & file::rw_mask) != file::read_only)) { // this means that another instance of the storage // is using the exact same file. #if BOOST_VERSION >= 103500 ec = errors::file_collision; #endif return boost::intrusive_ptr<file>(); } e.key = st; // if we asked for a file in write mode, // and the cached file is is not opened in // write mode, re-open it if ((((e.mode & file::rw_mask) != file::read_write) && ((m & file::rw_mask) == file::read_write)) || (e.mode & file::no_buffer) != (m & file::no_buffer) || (e.mode & file::random_access) != (m & file::random_access)) { // close the file before we open it with // the new read/write privilages TORRENT_ASSERT(e.file_ptr->refcount() == 1); #if TORRENT_CLOSE_MAY_BLOCK mutex::scoped_lock l(m_closer_mutex); m_queued_for_close.push_back(e.file_ptr); l.unlock(); e.file_ptr = new file; #else e.file_ptr->close(); #endif std::string full_path = combine_path(p, fs.file_path(*fe)); if (!e.file_ptr->open(full_path, m, ec)) { m_files.erase(i); return boost::intrusive_ptr<file>(); } #ifdef TORRENT_WINDOWS // file prio is supported on vista and up #if _WIN32_WINNT >= 0x0600 if (m_low_prio_io) { // TODO: load this function dynamically from Kernel32.dll FILE_IO_PRIORITY_HINT_INFO priorityHint; priorityHint.PriorityHint = IoPriorityHintLow; SetFileInformationByHandle(e.file_ptr->native_handle(), FileIoPriorityHintInfo, &priorityHint, sizeof(PriorityHint)); } #endif #endif TORRENT_ASSERT(e.file_ptr->is_open()); e.mode = m; } TORRENT_ASSERT((e.mode & file::no_buffer) == (m & file::no_buffer)); return e.file_ptr; } // the file is not in our cache if ((int)m_files.size() >= m_size) { // the file cache is at its maximum size, close // the least recently used (lru) file from it remove_oldest(); } lru_file_entry e; e.file_ptr.reset(new (std::nothrow)file); if (!e.file_ptr) { ec = error_code(ENOMEM, get_posix_category()); return e.file_ptr; } std::string full_path = combine_path(p, fs.file_path(*fe)); if (!e.file_ptr->open(full_path, m, ec)) return boost::intrusive_ptr<file>(); e.mode = m; e.key = st; m_files.insert(std::make_pair(std::make_pair(st, fs.file_index(*fe)), e)); TORRENT_ASSERT(e.file_ptr->is_open()); return e.file_ptr; }
void file_progress::init(piece_picker const& picker, file_storage const& fs) { INVARIANT_CHECK; if (!m_file_progress.empty()) return; int const num_pieces = fs.num_pieces(); int const num_files = fs.num_files(); #if TORRENT_USE_INVARIANT_CHECKS && defined TORRENT_DEBUG m_have_pieces.clear(); m_have_pieces.resize(num_pieces, false); m_file_sizes.clear(); m_file_sizes.reserve(num_files); for (int i = 0; i < num_files; ++i) m_file_sizes.push_back(fs.file_size(i)); #endif m_file_progress.resize(num_files, 0); std::fill(m_file_progress.begin(), m_file_progress.end(), 0); // initialize the progress of each file int const piece_size = fs.piece_length(); std::uint64_t off = 0; std::uint64_t const total_size = fs.total_size(); int file_index = 0; for (int piece = 0; piece < num_pieces; ++piece, off += piece_size) { TORRENT_ASSERT(file_index < fs.num_files()); std::int64_t file_offset = off - fs.file_offset(file_index); TORRENT_ASSERT(file_offset >= 0); while (file_offset >= fs.file_size(file_index)) { ++file_index; TORRENT_ASSERT(file_index < fs.num_files()); file_offset = off - fs.file_offset(file_index); TORRENT_ASSERT(file_offset >= 0); } TORRENT_ASSERT(file_offset <= fs.file_size(file_index)); if (!picker.have_piece(piece)) continue; #if TORRENT_USE_INVARIANT_CHECKS && defined TORRENT_DEBUG m_have_pieces.set_bit(piece); #endif int size = (std::min)(std::uint64_t(piece_size), total_size - off); TORRENT_ASSERT(size >= 0); while (size) { int add = (std::min)(std::int64_t(size), fs.file_size(file_index) - file_offset); TORRENT_ASSERT(add >= 0); m_file_progress[file_index] += add; TORRENT_ASSERT(m_file_progress[file_index] <= fs.file_size(file_index)); size -= add; TORRENT_ASSERT(size >= 0); if (size > 0) { ++file_index; TORRENT_ASSERT(file_index < fs.num_files()); file_offset = 0; } } } }
file_handle file_pool::open_file(storage_index_t st, std::string const& p , file_index_t const file_index, file_storage const& fs, int m, error_code& ec) { // potentially used to hold a reference to a file object that's // about to be destructed. If we have such object we assign it to // this member to be destructed after we release the std::mutex. On some // operating systems (such as OSX) closing a file may take a long // time. We don't want to hold the std::mutex for that. file_handle defer_destruction; std::unique_lock<std::mutex> l(m_mutex); #if TORRENT_USE_ASSERTS // we're not allowed to open a file // from a deleted storage! TORRENT_ASSERT(std::find(m_deleted_storages.begin(), m_deleted_storages.end() , std::make_pair(fs.name(), static_cast<void const*>(&fs))) == m_deleted_storages.end()); #endif TORRENT_ASSERT(is_complete(p)); TORRENT_ASSERT((m & file::rw_mask) == file::read_only || (m & file::rw_mask) == file::read_write); auto const i = m_files.find(std::make_pair(st, file_index)); if (i != m_files.end()) { lru_file_entry& e = i->second; e.last_use = aux::time_now(); // if we asked for a file in write mode, // and the cached file is is not opened in // write mode, re-open it if ((((e.mode & file::rw_mask) != file::read_write) && ((m & file::rw_mask) == file::read_write)) || (e.mode & file::random_access) != (m & file::random_access)) { file_handle new_file = std::make_shared<file>(); std::string full_path = fs.file_path(file_index, p); if (!new_file->open(full_path, m, ec)) return file_handle(); #ifdef TORRENT_WINDOWS if (m_low_prio_io) set_low_priority(new_file); #endif TORRENT_ASSERT(new_file->is_open()); defer_destruction = std::move(e.file_ptr); e.file_ptr = std::move(new_file); e.mode = m; } return e.file_ptr; } lru_file_entry e; e.file_ptr = std::make_shared<file>(); if (!e.file_ptr) { ec = error_code(boost::system::errc::not_enough_memory, generic_category()); return file_handle(); } std::string full_path = fs.file_path(file_index, p); if (!e.file_ptr->open(full_path, m, ec)) return file_handle(); #ifdef TORRENT_WINDOWS if (m_low_prio_io) set_low_priority(e.file_ptr); #endif e.mode = m; file_handle file_ptr = e.file_ptr; m_files.insert(std::make_pair(std::make_pair(st, file_index), e)); TORRENT_ASSERT(file_ptr->is_open()); if (int(m_files.size()) >= m_size) { // the file cache is at its maximum size, close // the least recently used (lru) file from it remove_oldest(l); } return file_ptr; }
// 'top_level' is extracting the file for a single-file torrent. The // distinction is that the filename is found in "name" rather than // "path" // root_dir is the name of the torrent, unless this is a single file // torrent, in which case it's empty. bool extract_single_file(bdecode_node const& dict, file_storage& files , std::string const& root_dir, ptrdiff_t info_ptr_diff, bool top_level , int& pad_file_cnt, error_code& ec) { if (dict.type() != bdecode_node::dict_t) return false; boost::uint32_t file_flags = get_file_attributes(dict); // symlinks have an implied "size" of zero. i.e. they use up 0 bytes of // the torrent payload space boost::int64_t const file_size = (file_flags & file_storage::flag_symlink) ? 0 : dict.dict_find_int_value("length", -1); if (file_size < 0 ) { ec = errors::torrent_invalid_length; return false; } boost::int64_t const mtime = dict.dict_find_int_value("mtime", 0); std::string path = root_dir; std::string path_element; char const* filename = NULL; int filename_len = 0; if (top_level) { // prefer the name.utf-8 because if it exists, it is more likely to be // correctly encoded bdecode_node p = dict.dict_find_string("name.utf-8"); if (!p) p = dict.dict_find_string("name"); if (!p || p.string_length() == 0) { ec = errors::torrent_missing_name; return false; } filename = p.string_ptr() + info_ptr_diff; filename_len = p.string_length(); while (filename_len > 0 && filename[0] == TORRENT_SEPARATOR) { filename += 1; filename_len -= 1; } sanitize_append_path_element(path, p.string_ptr(), p.string_length()); } else { bdecode_node p = dict.dict_find_list("path.utf-8"); if (!p) p = dict.dict_find_list("path"); if (p && p.list_size() > 0) { std::size_t const orig_path_len = path.size(); int const preallocate = path.size() + path_length(p, ec); if (ec) return false; path.reserve(preallocate); for (int i = 0, end(p.list_size()); i < end; ++i) { bdecode_node e = p.list_at(i); if (i == end - 1) { filename = e.string_ptr() + info_ptr_diff; filename_len = e.string_length(); } while (filename_len > 0 && filename[0] == TORRENT_SEPARATOR) { filename += 1; filename_len -= 1; } sanitize_append_path_element(path, e.string_ptr(), e.string_length()); } // if all path elements were sanitized away, we need to use another // name instead if (path.size() == orig_path_len) { path += TORRENT_SEPARATOR; path += "_"; } } else if (file_flags & file_storage::flag_pad_file) { // pad files don't need a path element, we'll just store them // under the .pad directory char cnt[10]; snprintf(cnt, sizeof(cnt), "%d", pad_file_cnt); path = combine_path(".pad", cnt); ++pad_file_cnt; } else { ec = errors::torrent_missing_name; return false; } } // bitcomet pad file if (path.find("_____padding_file_") != std::string::npos) file_flags = file_storage::flag_pad_file; bdecode_node fh = dict.dict_find_string("sha1"); char const* filehash = NULL; if (fh && fh.string_length() == 20) filehash = fh.string_ptr() + info_ptr_diff; std::string symlink_path; if (file_flags & file_storage::flag_symlink) { if (bdecode_node s_p = dict.dict_find_list("symlink path")) { int const preallocate = path_length(s_p, ec); if (ec) return false; symlink_path.reserve(preallocate); for (int i = 0, end(s_p.list_size()); i < end; ++i) { bdecode_node const& n = s_p.list_at(i); sanitize_append_path_element(symlink_path, n.string_ptr() , n.string_length()); } } } else { file_flags &= ~file_storage::flag_symlink; } if (filename_len > path.length() || path.compare(path.size() - filename_len, filename_len, filename , filename_len) != 0) { // if the filename was sanitized and differ, clear it to just use path filename = NULL; filename_len = 0; } files.add_file_borrow(filename, filename_len, path, file_size, file_flags, filehash , mtime, symlink_path); return true; }